From 9ccd4d6238291dd45f92579e1406ba6405c6f961 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Sat, 13 Jun 2026 11:10:29 +0800 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20=E9=9B=86=E6=88=90=20VAD=20?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E6=B4=BB=E5=8A=A8=E6=A3=80=E6=B5=8B=EF=BC=8C?= =?UTF-8?q?=E6=89=93=E9=80=9A=E6=A0=B8=E5=BF=83=E4=BA=A4=E4=BA=92=E9=93=BE?= =?UTF-8?q?=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 重写 useVAD Hook，接入 @ricky0123/vad-web 的 MicVAD - 支持 onSpeechStart/onSpeechEnd/onVADMisfire 回调 - useVisionSession 中串联摄像头→麦克风→WebSocket→VAD 完整流程 - App.tsx 新增 VAD 加载中和错误状态的 UI 指示 - VAD 参数对齐设计文档：positiveSpeechThreshold=0.5, minSpeechMs=250 Co-Authored-By: Claude --- frontend/src/App.css | 9 ++ frontend/src/App.tsx | 22 +++- .../src/components/EdgeProcessor/index.tsx | 105 ++++++++++++++---- frontend/src/hooks/useVisionSession.ts | 33 +++++- 4 files changed, 140 insertions(+), 29 deletions(-) diff --git a/frontend/src/App.css b/frontend/src/App.css index 6f12e3a..cc89700 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -103,6 +103,15 @@ body { animation: pulse 1.5s ease-in-out infinite; } +.vad-indicator--loading { + color: var(--color-warning); +} + +.vad-indicator--error { + color: var(--color-error); + animation: none; +} + @keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.6; } diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6e8e52d..13174ec 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -13,6 +13,8 @@ function App() { currentReply, isProcessing, isSpeaking, + isVADReady, + vadError, connectionStatus, videoRef, stream, @@ -21,12 +23,18 @@ function App() { interrupt, } = useVisionSession(); + const isConnected = connectionStatus === "connected"; + return (

CamTalk

- {connectionStatus === "connected" ? "已连接" : connectionStatus === "connecting" ? "连接中..." : "未连接"} + {isConnected + ? "已连接" + : connectionStatus === "connecting" + ? "连接中..." + : "未连接"}
@@ -34,6 +42,16 @@ function App() {
{isSpeaking &&
🎤 正在聆听...
} + {isConnected && !isVADReady && !vadError && ( +
+ 正在初始化语音检测... +
+ )} + {vadError && ( +
+ ⚠️ {vadError} +
+ )}
@@ -48,7 +66,7 @@ function App() {
- {connectionStatus !== "connected" ? ( + {!isConnected ? ( diff --git a/frontend/src/components/EdgeProcessor/index.tsx b/frontend/src/components/EdgeProcessor/index.tsx index 75acf9e..36978b7 100644 --- a/frontend/src/components/EdgeProcessor/index.tsx +++ b/frontend/src/components/EdgeProcessor/index.tsx @@ -4,35 +4,103 @@ // 技术：@ricky0123/vad-web（VAD）、ONNX Runtime Web（关键帧检测） // ============================================================ -import { useCallback, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { MicVAD } from "@ricky0123/vad-web"; export interface VADOptions { - /** 语音结束回调，携带录音 Float32Array */ + /** 语音结束回调，携带录音 Float32Array（16kHz） */ onSpeechEnd?: (audio: Float32Array) => void; /** 语音开始回调 */ onSpeechStart?: () => void; + /** 语音过短被忽略回调 */ + onVADMisfire?: () => void; } -export function useVAD(_options?: VADOptions) { - const [isSpeaking] = useState(false); - // TODO: 初始化 @ricky0123/vad-web，加载后设为 true - const isReady = false; +/** + * 语音活动检测 Hook + * 基于 @ricky0123/vad-web 的 MicVAD，检测用户说话并回调 + */ +export function useVAD(options?: VADOptions) { + const [isSpeaking, setIsSpeaking] = useState(false); + const [isReady, setIsReady] = useState(false); + const [error, setError] = useState(null); + const vadRef = useRef(null); + const optionsRef = useRef(options); - // TODO: 实现 VAD 初始化 - // 1. 加载 @ricky0123/vad-web - // 2. 配置 VAD 参数（阈值、最小语音时长等） - // 3. 连接麦克风 stream - // 4. 在 onSpeechEnd 时收集音频并回调 _options.onSpeechEnd + // 保持 options 引用最新，避免回调闭包问题 + useEffect(() => { + optionsRef.current = options; + }, [options]); - const start = useCallback(() => { - // TODO: 启动 VAD 监听 + /** + * 初始化 VAD 并开始监听 + * @param stream 麦克风 MediaStream（由外部管理） + */ + const start = useCallback(async (stream: MediaStream) => { + // 如果已有实例，先销毁 + if (vadRef.current) { + await vadRef.current.destroy(); + vadRef.current = null; + } + + try { + const vad = await MicVAD.new({ + getStream: () => Promise.resolve(stream), + startOnLoad: true, + model: "legacy", + + onSpeechStart: () => { + setIsSpeaking(true); + optionsRef.current?.onSpeechStart?.(); + }, + + onSpeechEnd: (audio: Float32Array) => { + setIsSpeaking(false); + optionsRef.current?.onSpeechEnd?.(audio); + }, + + onVADMisfire: () => { + setIsSpeaking(false); + optionsRef.current?.onVADMisfire?.(); + }, + + // VAD 参数（对齐 docs/06-语音交互.md 推荐值） + positiveSpeechThreshold: 0.5, + negativeSpeechThreshold: 0.35, + redemptionMs: 300, + preSpeechPadMs: 300, + minSpeechMs: 250, + submitUserSpeechOnPause: false, + }); + + vadRef.current = vad; + setIsReady(true); + setError(null); + } catch (err) { + const message = err instanceof Error ? err.message : "VAD 初始化失败"; + setError(message); + console.error("[VAD] 初始化失败:", err); + } }, []); - const stop = useCallback(() => { - // TODO: 停止 VAD 监听 + /** 停止 VAD 并销毁实例 */ + const stop = useCallback(async () => { + if (vadRef.current) { + await vadRef.current.destroy(); + vadRef.current = null; + } + setIsReady(false); + setIsSpeaking(false); }, []); - return { isSpeaking, isReady, start, stop }; + // 组件卸载时清理 + useEffect(() => { + return () => { + vadRef.current?.destroy(); + }; + }, []); + + return { isSpeaking, isReady, error, start, stop }; } // ---- 关键帧检测（ONNX Runtime Web）---- @@ -41,11 +109,6 @@ export function useKeyframeDetection() { // TODO: 加载 ONNX 模型后设为 true const isReady = false; - // TODO: 实现关键帧检测 - // 1. 加载 ONNX 模型 - // 2. 对比当前帧与上一帧的像素差异 - // 3. 超过阈值则判定为关键帧 - const isKeyframe = useCallback( (_currentFrame: ImageData, _previousFrame: ImageData): boolean => { // TODO: 实现像素差异对比 diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts index d187e8d..e53b91f 100644 --- a/frontend/src/hooks/useVisionSession.ts +++ b/frontend/src/hooks/useVisionSession.ts @@ -9,6 +9,7 @@ import { v4 as uuidv4 } from "uuid"; import { wsClient } from "../lib/websocket"; import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio"; import { useCamera } from "../components/CameraManager"; +import { useMicrophone } from "../components/MicManager"; import { useVAD } from "../components/EdgeProcessor"; import { useWebSocketManager } from "../components/WebSocketManager"; import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types"; @@ -19,10 +20,17 @@ export function useVisionSession() { const [isProcessing, setIsProcessing] = useState(false); const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); + const { startMic, stopMic } = useMicrophone(); const { status, connect, disconnect, send } = useWebSocketManager(); // VAD：语音结束时自动发送 query - const { isSpeaking, start: startVAD, stop: stopVAD } = useVAD({ + const { + isSpeaking, + isReady: isVADReady, + error: vadError, + start: startVAD, + stop: stopVAD, + } = useVAD({ onSpeechEnd: useCallback( (audio: Float32Array) => { const frame = captureFrame(); @@ -101,17 +109,28 @@ export function useVisionSession() { /** 启动会话 */ const startSession = useCallback(async () => { + // 1. 获取摄像头和麦克风 await startCamera(); + const micStream = await startMic(); + if (!micStream) { + console.error("[Session] 无法获取麦克风"); + return; + } + + // 2. 连接 WebSocket connect(); - startVAD(); - }, [startCamera, connect, startVAD]); + + // 3. 启动 VAD（传入麦克风 stream） + await startVAD(micStream); + }, [startCamera, startMic, connect, startVAD]); /** 结束会话 */ - const stopSession = useCallback(() => { - stopVAD(); + const stopSession = useCallback(async () => { + await stopVAD(); + stopMic(); stopCamera(); disconnect(); - }, [stopVAD, stopCamera, disconnect]); + }, [stopVAD, stopMic, stopCamera, disconnect]); /** 打断当前回复 */ const interrupt = useCallback(() => { @@ -124,6 +143,8 @@ export function useVisionSession() { currentReply, isProcessing, isSpeaking, + isVADReady, + vadError, connectionStatus: status, videoRef, stream, -- 2.49.1 From 734c93a8662f2df3842c0fc6b09ed70cb21db7a1 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Sat, 13 Jun 2026 11:43:16 +0800 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20=E5=AE=8C=E5=96=84=E7=AB=AF?= =?UTF-8?q?=E5=88=B0=E7=AB=AF=E5=AF=B9=E8=AF=9D=E9=97=AD=E7=8E=AF=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=BC=BA=E6=B6=88=E6=81=AF=E6=B8=B2=E6=9F=93=E4=B8=8E?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ChatPanel：流式光标动画、token/延迟元数据、自动滚动、三种空状态 - Toast 组件：错误码映射为中文友好文案，3 秒自动消失 - useVisionSession：stt_result 流式更新、防重复发送、打断保存未完成内容、结束清理全部状态 - App：断线提示、视频 loading 遮罩、按钮连接中状态 - ChatMessage 增加 latencyMs、model 字段 Co-Authored-By: Claude --- frontend/src/App.css | 112 +++++++++++++++++++- frontend/src/App.tsx | 23 ++-- frontend/src/components/ChatPanel/index.tsx | 67 +++++++++++- frontend/src/components/Toast/index.tsx | 51 +++++++++ frontend/src/hooks/useVisionSession.ts | 80 ++++++++++---- frontend/src/lib/errors.ts | 24 +++++ frontend/src/lib/toast.ts | 29 +++++ frontend/src/types/index.ts | 2 + 8 files changed, 357 insertions(+), 31 deletions(-) create mode 100644 frontend/src/components/Toast/index.tsx create mode 100644 frontend/src/lib/errors.ts create mode 100644 frontend/src/lib/toast.ts diff --git a/frontend/src/App.css b/frontend/src/App.css index cc89700..4cf2215 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -119,10 +119,10 @@ body { .chat-section { flex: 1; - overflow-y: auto; display: flex; flex-direction: column; gap: 12px; + overflow: hidden; } /* ---- Video Preview ---- */ @@ -157,6 +157,8 @@ body { display: flex; flex-direction: column; gap: 12px; + overflow-y: auto; + flex: 1; } .chat-panel--empty { @@ -248,3 +250,111 @@ body { .btn--warning:hover { opacity: 0.9; } + +/* ---- Streaming Cursor ---- */ + +.cursor { + display: inline; + animation: blink 0.8s step-end infinite; + color: var(--color-success); +} + +@keyframes blink { + 0%, 100% { opacity: 1; } + 50% { opacity: 0; } +} + +/* ---- Message Meta ---- */ + +.chat-message__meta { + margin-top: 6px; + font-size: 0.75rem; + color: var(--color-text-muted); +} + +/* ---- System Message ---- */ + +.system-message { + text-align: center; + padding: 8px 16px; + border-radius: var(--radius); + font-size: 0.85rem; +} + +.system-message--warning { + background: rgba(245, 158, 11, 0.15); + color: var(--color-warning); + border: 1px solid rgba(245, 158, 11, 0.3); +} + +/* ---- Toast ---- */ + +.toast-container { + position: fixed; + top: 16px; + right: 16px; + z-index: 1000; + display: flex; + flex-direction: column; + gap: 8px; +} + +.toast { + padding: 10px 16px; + border-radius: var(--radius); + font-size: 0.85rem; + cursor: pointer; + animation: slideIn 0.3s ease-out; + max-width: 320px; +} + +.toast--error { + background: rgba(239, 68, 68, 0.9); + color: white; +} + +.toast--warning { + background: rgba(245, 158, 11, 0.9); + color: #000; +} + +.toast--info { + background: rgba(37, 99, 235, 0.9); + color: white; +} + +@keyframes slideIn { + from { + opacity: 0; + transform: translateX(20px); + } + to { + opacity: 1; + transform: translateX(0); + } +} + +/* ---- Video Overlay ---- */ + +.video-overlay { + position: absolute; + inset: 0; + background: rgba(0, 0, 0, 0.3); + display: flex; + align-items: center; + justify-content: center; + border-radius: var(--radius); +} + +.video-overlay__spinner { + width: 32px; + height: 32px; + border: 3px solid rgba(255, 255, 255, 0.3); + border-top-color: white; + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 13174ec..9bdb7b4 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -5,6 +5,7 @@ import { useVisionSession } from "./hooks/useVisionSession"; import { VideoPreview } from "./components/VideoPreview"; import { ChatPanel } from "./components/ChatPanel"; +import { ToastContainer } from "./components/Toast"; import "./App.css"; function App() { @@ -52,23 +53,31 @@ function App() { ⚠️ {vadError}
)} + {isProcessing && ( +
+
+
+ )}
- - {currentReply && ( -
-
AI
-
{currentReply}
+ {connectionStatus === "disconnected" && messages.length > 0 && ( +
+ 连接已断开，正在重连...
)} +
{!isConnected ? ( ) : ( <> @@ -83,6 +92,8 @@ function App() { )}
+ +
); } diff --git a/frontend/src/components/ChatPanel/index.tsx b/frontend/src/components/ChatPanel/index.tsx index 0db5641..bed57de 100644 --- a/frontend/src/components/ChatPanel/index.tsx +++ b/frontend/src/components/ChatPanel/index.tsx @@ -1,33 +1,90 @@ // ============================================================ // ChatPanel — 消息展示面板 -// 职责：渲染对话消息列表（用户提问 + AI 回复） +// 职责：渲染对话消息列表、流式光标、元数据、自动滚动 // ============================================================ +import { useEffect, useRef } from "react"; import type { ChatMessage } from "../../types"; +import type { ConnectionStatus } from "../../lib/websocket"; interface ChatPanelProps { messages: ChatMessage[]; + currentReply?: string; + connectionStatus: ConnectionStatus; } -export function ChatPanel({ messages }: ChatPanelProps) { - if (messages.length === 0) { +export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPanelProps) { + const bottomRef = useRef(null); + const containerRef = useRef(null); + const isAutoScroll = useRef(true); + + // 用户上滚时暂停自动滚动，滚到底部时恢复 + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const handleScroll = () => { + const { scrollTop, scrollHeight, clientHeight } = container; + isAutoScroll.current = scrollHeight - scrollTop - clientHeight < 60; + }; + + container.addEventListener("scroll", handleScroll); + return () => container.removeEventListener("scroll", handleScroll); + }, []); + + // 新消息或流式更新时自动滚动 + useEffect(() => { + if (isAutoScroll.current) { + bottomRef.current?.scrollIntoView({ behavior: "smooth" }); + } + }, [messages, currentReply]); + + // 空状态 + if (messages.length === 0 && !currentReply) { + if (connectionStatus !== "connected") { + return ( +
+

点击下方按钮开始对话

+
+ ); + } return (
-

开始对话：对着摄像头说话即可

+

对着摄像头说话即可

); } return ( -
+
{messages.map((msg, index) => (
{msg.role === "user" ? "你" : "AI"}
{msg.content}
+ {msg.role === "assistant" && msg.tokensUsed !== undefined && ( +
+ {msg.tokensUsed} tokens + {msg.latencyMs !== undefined && ` · ${(msg.latencyMs / 1000).toFixed(1)}s`} + {msg.model && ` · ${msg.model}`} +
+ )}
))} + + {/* 流式回复（尚未完成） */} + {currentReply && ( +
+
AI
+
+ {currentReply} + ▌ +
+
+ )} + +
); } diff --git a/frontend/src/components/Toast/index.tsx b/frontend/src/components/Toast/index.tsx new file mode 100644 index 0000000..e4f371b --- /dev/null +++ b/frontend/src/components/Toast/index.tsx @@ -0,0 +1,51 @@ +// ============================================================ +// Toast — 轻量通知组件 +// 职责：3 秒自动消失的通知提示 +// ============================================================ + +import { useCallback, useEffect, useState } from "react"; +import { registerToastSetter, type ToastItem } from "../../lib/toast"; + +export type { ToastItem }; + +/** Toast 容器组件，放在 App 根部 */ +export function ToastContainer() { + const [toasts, setToasts] = useState([]); + + // 注册全局 setter + useEffect(() => { + registerToastSetter(setToasts); + return () => registerToastSetter(null); + }, []); + + const dismiss = useCallback((id: number) => { + setToasts((prev) => prev.filter((t) => t.id !== id)); + }, []); + + return ( +
+ {toasts.map((t) => ( + + ))} +
+ ); +} + +function ToastItemView({ + item, + onDismiss, +}: { + item: ToastItem; + onDismiss: (id: number) => void; +}) { + useEffect(() => { + const timer = setTimeout(() => onDismiss(item.id), 3000); + return () => clearTimeout(timer); + }, [item.id, onDismiss]); + + return ( +
onDismiss(item.id)}> + {item.message} +
+ ); +} diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts index e53b91f..a9112f6 100644 --- a/frontend/src/hooks/useVisionSession.ts +++ b/frontend/src/hooks/useVisionSession.ts @@ -4,10 +4,12 @@ // 来源：docs/02-系统架构.md 核心 Hook 设计 // ============================================================ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { v4 as uuidv4 } from "uuid"; import { wsClient } from "../lib/websocket"; import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio"; +import { getErrorMessage } from "../lib/errors"; +import { showToast } from "../lib/toast"; import { useCamera } from "../components/CameraManager"; import { useMicrophone } from "../components/MicManager"; import { useVAD } from "../components/EdgeProcessor"; @@ -23,6 +25,12 @@ export function useVisionSession() { const { startMic, stopMic } = useMicrophone(); const { status, connect, disconnect, send } = useWebSocketManager(); + // 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题 + const isProcessingRef = useRef(false); + useEffect(() => { + isProcessingRef.current = isProcessing; + }, [isProcessing]); + // VAD：语音结束时自动发送 query const { isSpeaking, @@ -33,6 +41,12 @@ export function useVisionSession() { } = useVAD({ onSpeechEnd: useCallback( (audio: Float32Array) => { + // 处理中忽略，防止重复发送 + if (isProcessingRef.current) { + console.warn("[Session] 正在处理中，忽略语音输入"); + return; + } + const frame = captureFrame(); if (!frame) { console.warn("[Session] 无法捕获图像帧"); @@ -47,7 +61,7 @@ export function useVisionSession() { audio: encodeAudioToBase64(audio), }); - // 添加用户消息（STT 结果到达后会更新文本） + // 添加用户消息（STT 流式结果会逐步更新文本） setMessages((prev) => [ ...prev, { role: "user", content: "（语音识别中...）", timestamp: Date.now() }, @@ -62,43 +76,51 @@ export function useVisionSession() { useEffect(() => { const unsub = wsClient.onMessage((msg: ServerMessage) => { switch (msg.type) { - case "stt_result": - if (msg.is_final) { - setMessages((prev) => { - const updated = [...prev]; - const lastUserIdx = updated.findLastIndex((m) => m.role === "user"); - if (lastUserIdx >= 0) { - updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text }; - } - return updated; - }); - } + case "stt_result": { + // 流式更新用户消息文本（包括中间结果和最终结果） + setMessages((prev) => { + const updated = [...prev]; + const lastUserIdx = updated.findLastIndex((m) => m.role === "user"); + if (lastUserIdx >= 0) { + updated[lastUserIdx] = { + ...updated[lastUserIdx], + content: msg.text || "（未识别到语音）", + }; + } + return updated; + }); break; + } case "llm_chunk": setCurrentReply((prev) => prev + msg.delta); break; - case "llm_done": + case "llm_done": { + const done = msg as LLMDoneMessage; setMessages((prev) => [ ...prev, { role: "assistant", - content: (msg as LLMDoneMessage).full_text, + content: done.full_text, timestamp: Date.now(), - tokensUsed: (msg as LLMDoneMessage).tokens_used?.total, + tokensUsed: done.tokens_used?.total, + latencyMs: done.latency_ms, + model: done.model, }, ]); setCurrentReply(""); setIsProcessing(false); break; + } case "tts_audio": - // TODO: 音频流播放 + // TODO: 阶段 5 音频流播放 break; case "error": console.error("[Session] 服务端错误:", msg.code, msg.message); + showToast(getErrorMessage(msg.code), "error"); setIsProcessing(false); break; } @@ -107,13 +129,21 @@ export function useVisionSession() { return unsub; }, []); + // 连接断开时显示提示 + useEffect(() => { + if (status === "disconnected") { + // 只在非主动断开时提示（通过检查是否有活跃会话判断） + // 这里简单处理，由 App 层根据状态显示 + } + }, [status]); + /** 启动会话 */ const startSession = useCallback(async () => { // 1. 获取摄像头和麦克风 await startCamera(); const micStream = await startMic(); if (!micStream) { - console.error("[Session] 无法获取麦克风"); + showToast("无法获取麦克风权限", "error"); return; } @@ -130,13 +160,25 @@ export function useVisionSession() { stopMic(); stopCamera(); disconnect(); + // 清理所有对话状态 + setMessages([]); + setCurrentReply(""); + setIsProcessing(false); }, [stopVAD, stopMic, stopCamera, disconnect]); /** 打断当前回复 */ const interrupt = useCallback(() => { send({ type: "interrupt" }); + // 将未完成的流式内容保存为最终消息 + if (currentReply) { + setMessages((prev) => [ + ...prev, + { role: "assistant", content: currentReply + "（已打断）", timestamp: Date.now() }, + ]); + } + setCurrentReply(""); setIsProcessing(false); - }, [send]); + }, [send, currentReply]); return { messages, diff --git a/frontend/src/lib/errors.ts b/frontend/src/lib/errors.ts new file mode 100644 index 0000000..9aa56e2 --- /dev/null +++ b/frontend/src/lib/errors.ts @@ -0,0 +1,24 @@ +// ============================================================ +// 错误码 → 用户友好文案映射 +// 来源：docs/03-接口文档.md §五 错误码 +// ============================================================ + +import type { ErrorCode } from "../types"; + +const ERROR_MESSAGES: Record = { + INVALID_MESSAGE: "消息格式异常，请重试", + SESSION_NOT_FOUND: "会话已过期，请重新连接", + RATE_LIMITED: "请求太频繁，请稍后再试", + IMAGE_TOO_LARGE: "图像过大，请降低分辨率", + AUDIO_TOO_SHORT: "语音太短，请再说一句", + LLM_TIMEOUT: "AI 响应超时，请重试", + LLM_ERROR: "AI 服务异常，请稍后重试", + STT_ERROR: "语音识别失败，请重试", + TTS_ERROR: "语音合成失败", + INTERNAL_ERROR: "服务内部错误，请重试", +}; + +/** 将错误码转为用户友好文案 */ +export function getErrorMessage(code: string): string { + return ERROR_MESSAGES[code as ErrorCode] ?? `未知错误: ${code}`; +} diff --git a/frontend/src/lib/toast.ts b/frontend/src/lib/toast.ts new file mode 100644 index 0000000..3074e90 --- /dev/null +++ b/frontend/src/lib/toast.ts @@ -0,0 +1,29 @@ +// ============================================================ +// Toast 全局状态管理 +// 与 Toast 组件配合使用 +// ============================================================ + +export type ToastType = "error" | "warning" | "info"; + +let nextId = 0; +let _setToasts: React.Dispatch> | null = null; + +export interface ToastItem { + id: number; + type: ToastType; + message: string; +} + +/** 注册 Toast state setter（由 ToastContainer 组件调用） */ +export function registerToastSetter( + setter: React.Dispatch> | null, +) { + _setToasts = setter; +} + +/** 显示一条 Toast */ +export function showToast(message: string, type: ToastType = "info") { + if (!_setToasts) return; + const id = nextId++; + _setToasts((prev) => [...prev, { id, type, message }]); +} diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 2755ae3..4cad08a 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -25,6 +25,8 @@ export interface ChatMessage { imageUrl?: string; timestamp: number; tokensUsed?: number; + latencyMs?: number; + model?: string; } // ---- WebSocket 通用信封 ---- -- 2.49.1 From 87b6407e2e6204fd5581d35d77dd24611c2ec46e Mon Sep 17 00:00:00 2001 From: hhs <386998068@qq.com> Date: Sat, 13 Jun 2026 13:18:17 +0800 Subject: [PATCH 3/5] =?UTF-8?q?docs:=20=E8=A1=A5=E5=85=85=20AI=20=E7=BC=96?= =?UTF-8?q?=E6=8E=92=E5=B1=82=E6=8E=A5=E5=8F=A3=E8=A7=84=E8=8C=83=E4=B8=8E?= =?UTF-8?q?=20tts=5Faudio=20=E9=9F=B3=E9=A2=91=E6=92=AD=E6=94=BE=E6=96=B9?= =?UTF-8?q?=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 03-接口文档: 新增第三章 AI 服务层接口（STT/LLM/TTS 三个 Service interface + 接入约定） - 03-接口文档: 新增第四章 AI 编排器（句子级流式并行策略、Orchestrator 实现、错误降级表） - 03-接口文档: 锁定 tts_audio 音频格式为 audio/mpeg（MP3 24kHz），新增前端 AudioPlayer 播放方案 - 02-系统架构: 更新 Orchestrator 代码为句子级流式并行实现 - README.md: 更新 03-接口文档描述，补充 AI 服务层和编排器关键词 --- docs/02-系统架构.md | 46 ++++--- docs/03-接口文档.md | 295 +++++++++++++++++++++++++++++++++++++++++++- docs/README.md | 2 +- 3 files changed, 322 insertions(+), 21 deletions(-) diff --git a/docs/02-系统架构.md b/docs/02-系统架构.md index cd904f5..b0f65a5 100644 --- a/docs/02-系统架构.md +++ b/docs/02-系统架构.md @@ -84,32 +84,48 @@ Browser Go Gateway STT LLM TTS | AI Orchestrator | 编排多路 AI 调用（并行/串行） | context 取消 + 超时控制 | | Rate Limiter | 防止单用户过度消耗 API 额度 | 令牌桶算法 | -AI Orchestrator 核心代码： +AI Orchestrator 核心代码（句子级流式并行）： ```go -func (o *Orchestrator) ProcessQuery(ctx context.Context, req *QueryRequest) (*QueryResponse, error) { +func (o *Orchestrator) ProcessQuery(ctx context.Context, client MessageSender, req *QueryRequest) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - // 并行：LLM 推理 + 准备 TTS - llmCh := make(chan string, 1) + // Step 1: STT — 识别用户语音（串行） + text, err := o.stt.Recognize(ctx, req.Audio, STTOptions{...}) + if err != nil { + client.SendError(req.RequestID, "STT_ERROR", err.Error()) + return + } + client.SendSTTResult(req.RequestID, text, true) + + // Step 2: LLM 流式输出 + 句子切分（并行） + llmStream, _ := o.llm.ChatStream(ctx, LLMRequest{Image: req.Image, Text: text, ...}) + sentenceCh := make(chan string, 4) go func() { - resp, _ := o.llm.Chat(ctx, req.Image, req.Text, req.History) - llmCh <- resp + defer close(sentenceCh) + var buf strings.Builder + for chunk := range llmStream { + client.SendLLMChunk(req.RequestID, chunk.Delta) // 逐 token 推送文字 + buf.WriteString(chunk.Delta) + if isSentenceEnd(chunk.Delta) { // 按 。！？\n 切分 + sentenceCh <- buf.String() + buf.Reset() + } + } + if buf.Len() > 0 { sentenceCh <- buf.String() } }() - llmText := <-llmCh - // LLM 返回后，流式推送给客户端，同时启动 TTS - ttsCh := make(chan []byte, 1) - go func() { - audio, _ := o.tts.Synthesize(ctx, llmText) - ttsCh <- audio - }() - - return &QueryResponse{Text: llmText, Audio: <-ttsCh}, nil + // Step 3: TTS 并行消费句子流 + ttsStream, _ := o.tts.SynthesizeStream(ctx, sentenceCh, TTSOptions{...}) + for chunk := range ttsStream { + client.SendTTSAudio(req.RequestID, chunk.Audio, chunk.IsLast) + } } ``` +> **关键优化**：LLM 文本流和 TTS 音频流**并行推送**——客户端先逐 token 展示文字，同时 TTS 逐句子合成并推送音频，用户感知延迟大幅降低。详细的 AI 服务层接口和编排策略见 `03-接口文档.md` 第三、四章。 + ## 前端组件 | 组件 | 职责 | diff --git a/docs/03-接口文档.md b/docs/03-接口文档.md index 3aa530e..f3a0e94 100644 --- a/docs/03-接口文档.md +++ b/docs/03-接口文档.md @@ -143,11 +143,53 @@ interface TTSAudioMessage { type: "tts_audio"; request_id: string; audio: string; // Base64 编码的音频片段 - mime_type: string; // "audio/mp3" 或 "audio/pcm" + mime_type: string; // "audio/mpeg" is_last: boolean; // 是否为最后一片 } ``` +**音频格式规范**（前端播放依赖此约定）： + +| 属性 | 值 | 说明 | +|------|------|------| +| 编码 | `audio/mpeg`（MP3） | 浏览器 `