feat: 完善端到端对话闭环,增强消息渲染与错误处理
- ChatPanel:流式光标动画、token/延迟元数据、自动滚动、三种空状态 - Toast 组件:错误码映射为中文友好文案,3 秒自动消失 - useVisionSession:stt_result 流式更新、防重复发送、打断保存未完成内容、结束清理全部状态 - App:断线提示、视频 loading 遮罩、按钮连接中状态 - ChatMessage 增加 latencyMs、model 字段 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -4,10 +4,12 @@
|
||||
// 来源:docs/02-系统架构.md 核心 Hook 设计
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { wsClient } from "../lib/websocket";
|
||||
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
|
||||
import { getErrorMessage } from "../lib/errors";
|
||||
import { showToast } from "../lib/toast";
|
||||
import { useCamera } from "../components/CameraManager";
|
||||
import { useMicrophone } from "../components/MicManager";
|
||||
import { useVAD } from "../components/EdgeProcessor";
|
||||
@@ -23,6 +25,12 @@ export function useVisionSession() {
|
||||
const { startMic, stopMic } = useMicrophone();
|
||||
const { status, connect, disconnect, send } = useWebSocketManager();
|
||||
|
||||
// 用 ref 跟踪 isProcessing,避免 VAD 回调闭包问题
|
||||
const isProcessingRef = useRef(false);
|
||||
useEffect(() => {
|
||||
isProcessingRef.current = isProcessing;
|
||||
}, [isProcessing]);
|
||||
|
||||
// VAD:语音结束时自动发送 query
|
||||
const {
|
||||
isSpeaking,
|
||||
@@ -33,6 +41,12 @@ export function useVisionSession() {
|
||||
} = useVAD({
|
||||
onSpeechEnd: useCallback(
|
||||
(audio: Float32Array) => {
|
||||
// 处理中忽略,防止重复发送
|
||||
if (isProcessingRef.current) {
|
||||
console.warn("[Session] 正在处理中,忽略语音输入");
|
||||
return;
|
||||
}
|
||||
|
||||
const frame = captureFrame();
|
||||
if (!frame) {
|
||||
console.warn("[Session] 无法捕获图像帧");
|
||||
@@ -47,7 +61,7 @@ export function useVisionSession() {
|
||||
audio: encodeAudioToBase64(audio),
|
||||
});
|
||||
|
||||
// 添加用户消息(STT 结果到达后会更新文本)
|
||||
// 添加用户消息(STT 流式结果会逐步更新文本)
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{ role: "user", content: "(语音识别中...)", timestamp: Date.now() },
|
||||
@@ -62,43 +76,51 @@ export function useVisionSession() {
|
||||
useEffect(() => {
|
||||
const unsub = wsClient.onMessage((msg: ServerMessage) => {
|
||||
switch (msg.type) {
|
||||
case "stt_result":
|
||||
if (msg.is_final) {
|
||||
setMessages((prev) => {
|
||||
const updated = [...prev];
|
||||
const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
|
||||
if (lastUserIdx >= 0) {
|
||||
updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text };
|
||||
}
|
||||
return updated;
|
||||
});
|
||||
}
|
||||
case "stt_result": {
|
||||
// 流式更新用户消息文本(包括中间结果和最终结果)
|
||||
setMessages((prev) => {
|
||||
const updated = [...prev];
|
||||
const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
|
||||
if (lastUserIdx >= 0) {
|
||||
updated[lastUserIdx] = {
|
||||
...updated[lastUserIdx],
|
||||
content: msg.text || "(未识别到语音)",
|
||||
};
|
||||
}
|
||||
return updated;
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
case "llm_chunk":
|
||||
setCurrentReply((prev) => prev + msg.delta);
|
||||
break;
|
||||
|
||||
case "llm_done":
|
||||
case "llm_done": {
|
||||
const done = msg as LLMDoneMessage;
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
role: "assistant",
|
||||
content: (msg as LLMDoneMessage).full_text,
|
||||
content: done.full_text,
|
||||
timestamp: Date.now(),
|
||||
tokensUsed: (msg as LLMDoneMessage).tokens_used?.total,
|
||||
tokensUsed: done.tokens_used?.total,
|
||||
latencyMs: done.latency_ms,
|
||||
model: done.model,
|
||||
},
|
||||
]);
|
||||
setCurrentReply("");
|
||||
setIsProcessing(false);
|
||||
break;
|
||||
}
|
||||
|
||||
case "tts_audio":
|
||||
// TODO: 音频流播放
|
||||
// TODO: 阶段 5 音频流播放
|
||||
break;
|
||||
|
||||
case "error":
|
||||
console.error("[Session] 服务端错误:", msg.code, msg.message);
|
||||
showToast(getErrorMessage(msg.code), "error");
|
||||
setIsProcessing(false);
|
||||
break;
|
||||
}
|
||||
@@ -107,13 +129,21 @@ export function useVisionSession() {
|
||||
return unsub;
|
||||
}, []);
|
||||
|
||||
// 连接断开时显示提示
|
||||
useEffect(() => {
|
||||
if (status === "disconnected") {
|
||||
// 只在非主动断开时提示(通过检查是否有活跃会话判断)
|
||||
// 这里简单处理,由 App 层根据状态显示
|
||||
}
|
||||
}, [status]);
|
||||
|
||||
/** 启动会话 */
|
||||
const startSession = useCallback(async () => {
|
||||
// 1. 获取摄像头和麦克风
|
||||
await startCamera();
|
||||
const micStream = await startMic();
|
||||
if (!micStream) {
|
||||
console.error("[Session] 无法获取麦克风");
|
||||
showToast("无法获取麦克风权限", "error");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -130,13 +160,25 @@ export function useVisionSession() {
|
||||
stopMic();
|
||||
stopCamera();
|
||||
disconnect();
|
||||
// 清理所有对话状态
|
||||
setMessages([]);
|
||||
setCurrentReply("");
|
||||
setIsProcessing(false);
|
||||
}, [stopVAD, stopMic, stopCamera, disconnect]);
|
||||
|
||||
/** 打断当前回复 */
|
||||
const interrupt = useCallback(() => {
|
||||
send({ type: "interrupt" });
|
||||
// 将未完成的流式内容保存为最终消息
|
||||
if (currentReply) {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{ role: "assistant", content: currentReply + "(已打断)", timestamp: Date.now() },
|
||||
]);
|
||||
}
|
||||
setCurrentReply("");
|
||||
setIsProcessing(false);
|
||||
}, [send]);
|
||||
}, [send, currentReply]);
|
||||
|
||||
return {
|
||||
messages,
|
||||
|
||||
Reference in New Issue
Block a user