feat: 初始化前端项目，搭建 React + TypeScript + Vite 工程结构

- 创建 Vite + React 18 + TypeScript 项目（严格模式） - 安装 @ricky0123/vad-web、onnxruntime-web、uuid 依赖 - 定义全部 WebSocket 消息类型和数据模型（对齐接口文档） - 实现 WebSocket 连接管理（心跳保活、指数退避重连） - 实现音频编码工具（PCM↔Base64、DataURL 转换） - 创建组件骨架：CameraManager、MicManager、EdgeProcessor、WebSocketManager、ChatPanel、VideoPreview - 实现核心 useVisionSession Hook 骨架 - 配置 ESLint（_前缀变量忽略规则） Co-Authored-By: Claude <noreply@anthropic.com>
2026-06-12 17:46:25 +08:00
parent 10f676f3b4
commit b5c0d40609
26 changed files with 4236 additions and 0 deletions
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -0,0 +1,134 @@
+// ============================================================
+// useVisionSession — 核心视觉对话会话 Hook
+// 职责：封装一次完整的视觉对话会话（摄像头、VAD、WebSocket、消息状态）
+// 来源：docs/02-系统架构.md 核心 Hook 设计
+// ============================================================
+
+import { useCallback, useEffect, useState } from "react";
+import { v4 as uuidv4 } from "uuid";
+import { wsClient } from "../lib/websocket";
+import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
+import { useCamera } from "../components/CameraManager";
+import { useVAD } from "../components/EdgeProcessor";
+import { useWebSocketManager } from "../components/WebSocketManager";
+import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";
+
+export function useVisionSession() {
+  const [messages, setMessages] = useState<ChatMessage[]>([]);
+  const [currentReply, setCurrentReply] = useState<string>("");
+  const [isProcessing, setIsProcessing] = useState(false);
+
+  const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
+  const { status, connect, disconnect, send } = useWebSocketManager();
+
+  // VAD：语音结束时自动发送 query
+  const { isSpeaking, start: startVAD, stop: stopVAD } = useVAD({
+    onSpeechEnd: useCallback(
+      (audio: Float32Array) => {
+        const frame = captureFrame();
+        if (!frame) {
+          console.warn("[Session] 无法捕获图像帧");
+          return;
+        }
+
+        const requestId = uuidv4();
+        send({
+          type: "query",
+          request_id: requestId,
+          image: dataUrlToBase64(frame),
+          audio: encodeAudioToBase64(audio),
+        });
+
+        // 添加用户消息（STT 结果到达后会更新文本）
+        setMessages((prev) => [
+          ...prev,
+          { role: "user", content: "（语音识别中...）", timestamp: Date.now() },
+        ]);
+        setIsProcessing(true);
+      },
+      [captureFrame, send]
+    ),
+  });
+
+  // 处理服务端消息
+  useEffect(() => {
+    const unsub = wsClient.onMessage((msg: ServerMessage) => {
+      switch (msg.type) {
+        case "stt_result":
+          if (msg.is_final) {
+            setMessages((prev) => {
+              const updated = [...prev];
+              const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
+              if (lastUserIdx >= 0) {
+                updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text };
+              }
+              return updated;
+            });
+          }
+          break;
+
+        case "llm_chunk":
+          setCurrentReply((prev) => prev + msg.delta);
+          break;
+
+        case "llm_done":
+          setMessages((prev) => [
+            ...prev,
+            {
+              role: "assistant",
+              content: (msg as LLMDoneMessage).full_text,
+              timestamp: Date.now(),
+              tokensUsed: (msg as LLMDoneMessage).tokens_used?.total,
+            },
+          ]);
+          setCurrentReply("");
+          setIsProcessing(false);
+          break;
+
+        case "tts_audio":
+          // TODO: 音频流播放
+          break;
+
+        case "error":
+          console.error("[Session] 服务端错误:", msg.code, msg.message);
+          setIsProcessing(false);
+          break;
+      }
+    });
+
+    return unsub;
+  }, []);
+
+  /** 启动会话 */
+  const startSession = useCallback(async () => {
+    await startCamera();
+    connect();
+    startVAD();
+  }, [startCamera, connect, startVAD]);
+
+  /** 结束会话 */
+  const stopSession = useCallback(() => {
+    stopVAD();
+    stopCamera();
+    disconnect();
+  }, [stopVAD, stopCamera, disconnect]);
+
+  /** 打断当前回复 */
+  const interrupt = useCallback(() => {
+    send({ type: "interrupt" });
+    setIsProcessing(false);
+  }, [send]);
+
+  return {
+    messages,
+    currentReply,
+    isProcessing,
+    isSpeaking,
+    connectionStatus: status,
+    videoRef,
+    stream,
+    startSession,
+    stopSession,
+    interrupt,
+  };
+}