feat: 完善端到端对话闭环,增强消息渲染与错误处理

- ChatPanel:流式光标动画、token/延迟元数据、自动滚动、三种空状态
- Toast 组件:错误码映射为中文友好文案,3 秒自动消失
- useVisionSession:stt_result 流式更新、防重复发送、打断保存未完成内容、结束清理全部状态
- App:断线提示、视频 loading 遮罩、按钮连接中状态
- ChatMessage 增加 latencyMs、model 字段

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-06-13 11:43:16 +08:00
parent 9ccd4d6238
commit 734c93a866
8 changed files with 357 additions and 31 deletions

View File

@@ -4,10 +4,12 @@
// 来源docs/02-系统架构.md 核心 Hook 设计
// ============================================================
import { useCallback, useEffect, useState } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { v4 as uuidv4 } from "uuid";
import { wsClient } from "../lib/websocket";
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
import { getErrorMessage } from "../lib/errors";
import { showToast } from "../lib/toast";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD } from "../components/EdgeProcessor";
@@ -23,6 +25,12 @@ export function useVisionSession() {
const { startMic, stopMic } = useMicrophone();
const { status, connect, disconnect, send } = useWebSocketManager();
// 用 ref 跟踪 isProcessing避免 VAD 回调闭包问题
const isProcessingRef = useRef(false);
useEffect(() => {
isProcessingRef.current = isProcessing;
}, [isProcessing]);
// VAD语音结束时自动发送 query
const {
isSpeaking,
@@ -33,6 +41,12 @@ export function useVisionSession() {
} = useVAD({
onSpeechEnd: useCallback(
(audio: Float32Array) => {
// 处理中忽略,防止重复发送
if (isProcessingRef.current) {
console.warn("[Session] 正在处理中,忽略语音输入");
return;
}
const frame = captureFrame();
if (!frame) {
console.warn("[Session] 无法捕获图像帧");
@@ -47,7 +61,7 @@ export function useVisionSession() {
audio: encodeAudioToBase64(audio),
});
// 添加用户消息STT 结果到达后会更新文本)
// 添加用户消息STT 流式结果会逐步更新文本)
setMessages((prev) => [
...prev,
{ role: "user", content: "(语音识别中...", timestamp: Date.now() },
@@ -62,43 +76,51 @@ export function useVisionSession() {
useEffect(() => {
const unsub = wsClient.onMessage((msg: ServerMessage) => {
switch (msg.type) {
case "stt_result":
if (msg.is_final) {
setMessages((prev) => {
const updated = [...prev];
const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
if (lastUserIdx >= 0) {
updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text };
}
return updated;
});
}
case "stt_result": {
// 流式更新用户消息文本(包括中间结果和最终结果)
setMessages((prev) => {
const updated = [...prev];
const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
if (lastUserIdx >= 0) {
updated[lastUserIdx] = {
...updated[lastUserIdx],
content: msg.text || "(未识别到语音)",
};
}
return updated;
});
break;
}
case "llm_chunk":
setCurrentReply((prev) => prev + msg.delta);
break;
case "llm_done":
case "llm_done": {
const done = msg as LLMDoneMessage;
setMessages((prev) => [
...prev,
{
role: "assistant",
content: (msg as LLMDoneMessage).full_text,
content: done.full_text,
timestamp: Date.now(),
tokensUsed: (msg as LLMDoneMessage).tokens_used?.total,
tokensUsed: done.tokens_used?.total,
latencyMs: done.latency_ms,
model: done.model,
},
]);
setCurrentReply("");
setIsProcessing(false);
break;
}
case "tts_audio":
// TODO: 音频流播放
// TODO: 阶段 5 音频流播放
break;
case "error":
console.error("[Session] 服务端错误:", msg.code, msg.message);
showToast(getErrorMessage(msg.code), "error");
setIsProcessing(false);
break;
}
@@ -107,13 +129,21 @@ export function useVisionSession() {
return unsub;
}, []);
// 连接断开时显示提示
useEffect(() => {
if (status === "disconnected") {
// 只在非主动断开时提示(通过检查是否有活跃会话判断)
// 这里简单处理,由 App 层根据状态显示
}
}, [status]);
/** 启动会话 */
const startSession = useCallback(async () => {
// 1. 获取摄像头和麦克风
await startCamera();
const micStream = await startMic();
if (!micStream) {
console.error("[Session] 无法获取麦克风");
showToast("无法获取麦克风权限", "error");
return;
}
@@ -130,13 +160,25 @@ export function useVisionSession() {
stopMic();
stopCamera();
disconnect();
// 清理所有对话状态
setMessages([]);
setCurrentReply("");
setIsProcessing(false);
}, [stopVAD, stopMic, stopCamera, disconnect]);
/** 打断当前回复 */
const interrupt = useCallback(() => {
send({ type: "interrupt" });
// 将未完成的流式内容保存为最终消息
if (currentReply) {
setMessages((prev) => [
...prev,
{ role: "assistant", content: currentReply + "(已打断)", timestamp: Date.now() },
]);
}
setCurrentReply("");
setIsProcessing(false);
}, [send]);
}, [send, currentReply]);
return {
messages,