From 9ccd4d6238291dd45f92579e1406ba6405c6f961 Mon Sep 17 00:00:00 2001
From: cfy666 <3087823110@qq.com>
Date: Sat, 13 Jun 2026 11:10:29 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E9=9B=86=E6=88=90=20VAD=20=E8=AF=AD?=
 =?UTF-8?q?=E9=9F=B3=E6=B4=BB=E5=8A=A8=E6=A3=80=E6=B5=8B=EF=BC=8C=E6=89=93?=
 =?UTF-8?q?=E9=80=9A=E6=A0=B8=E5=BF=83=E4=BA=A4=E4=BA=92=E9=93=BE=E8=B7=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- 重写 useVAD Hook，接入 @ricky0123/vad-web 的 MicVAD
- 支持 onSpeechStart/onSpeechEnd/onVADMisfire 回调
- useVisionSession 中串联摄像头→麦克风→WebSocket→VAD 完整流程
- App.tsx 新增 VAD 加载中和错误状态的 UI 指示
- VAD 参数对齐设计文档：positiveSpeechThreshold=0.5, minSpeechMs=250

Co-Authored-By: Claude <noreply@anthropic.com>
---
 frontend/src/App.css                          |   9 ++
 frontend/src/App.tsx                          |  22 +++-
 .../src/components/EdgeProcessor/index.tsx    | 105 ++++++++++++++----
 frontend/src/hooks/useVisionSession.ts        |  33 +++++-
 4 files changed, 140 insertions(+), 29 deletions(-)

diff --git a/frontend/src/App.css b/frontend/src/App.css
index 6f12e3a..cc89700 100644
--- a/frontend/src/App.css
+++ b/frontend/src/App.css
@@ -103,6 +103,15 @@ body {
   animation: pulse 1.5s ease-in-out infinite;
 }
 
+.vad-indicator--loading {
+  color: var(--color-warning);
+}
+
+.vad-indicator--error {
+  color: var(--color-error);
+  animation: none;
+}
+
 @keyframes pulse {
   0%, 100% { opacity: 1; }
   50% { opacity: 0.6; }
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 6e8e52d..13174ec 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -13,6 +13,8 @@ function App() {
     currentReply,
     isProcessing,
     isSpeaking,
+    isVADReady,
+    vadError,
     connectionStatus,
     videoRef,
     stream,
@@ -21,12 +23,18 @@ function App() {
     interrupt,
   } = useVisionSession();
 
+  const isConnected = connectionStatus === "connected";
+
   return (
     <div className="app">
       <header className="app-header">
         <h1>CamTalk</h1>
         <span className={`status status--${connectionStatus}`}>
-          {connectionStatus === "connected" ? "已连接" : connectionStatus === "connecting" ? "连接中..." : "未连接"}
+          {isConnected
+            ? "已连接"
+            : connectionStatus === "connecting"
+              ? "连接中..."
+              : "未连接"}
         </span>
       </header>
 
@@ -34,6 +42,16 @@ function App() {
         <div className="video-section">
           <VideoPreview ref={videoRef} isStreaming={!!stream} />
           {isSpeaking && <div className="vad-indicator">🎤 正在聆听...</div>}
+          {isConnected && !isVADReady && !vadError && (
+            <div className="vad-indicator vad-indicator--loading">
+              正在初始化语音检测...
+            </div>
+          )}
+          {vadError && (
+            <div className="vad-indicator vad-indicator--error">
+              ⚠️ {vadError}
+            </div>
+          )}
         </div>
 
         <div className="chat-section">
@@ -48,7 +66,7 @@ function App() {
       </main>
 
       <footer className="app-footer">
-        {connectionStatus !== "connected" ? (
+        {!isConnected ? (
           <button className="btn btn--primary" onClick={startSession}>
             开始对话
           </button>
diff --git a/frontend/src/components/EdgeProcessor/index.tsx b/frontend/src/components/EdgeProcessor/index.tsx
index 75acf9e..36978b7 100644
--- a/frontend/src/components/EdgeProcessor/index.tsx
+++ b/frontend/src/components/EdgeProcessor/index.tsx
@@ -4,35 +4,103 @@
 // 技术：@ricky0123/vad-web（VAD）、ONNX Runtime Web（关键帧检测）
 // ============================================================
 
-import { useCallback, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { MicVAD } from "@ricky0123/vad-web";
 
 export interface VADOptions {
-  /** 语音结束回调，携带录音 Float32Array */
+  /** 语音结束回调，携带录音 Float32Array（16kHz） */
   onSpeechEnd?: (audio: Float32Array) => void;
   /** 语音开始回调 */
   onSpeechStart?: () => void;
+  /** 语音过短被忽略回调 */
+  onVADMisfire?: () => void;
 }
 
-export function useVAD(_options?: VADOptions) {
-  const [isSpeaking] = useState(false);
-  // TODO: 初始化 @ricky0123/vad-web，加载后设为 true
-  const isReady = false;
+/**
+ * 语音活动检测 Hook
+ * 基于 @ricky0123/vad-web 的 MicVAD，检测用户说话并回调
+ */
+export function useVAD(options?: VADOptions) {
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const [isReady, setIsReady] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const vadRef = useRef<MicVAD | null>(null);
+  const optionsRef = useRef(options);
 
-  // TODO: 实现 VAD 初始化
-  // 1. 加载 @ricky0123/vad-web
-  // 2. 配置 VAD 参数（阈值、最小语音时长等）
-  // 3. 连接麦克风 stream
-  // 4. 在 onSpeechEnd 时收集音频并回调 _options.onSpeechEnd
+  // 保持 options 引用最新，避免回调闭包问题
+  useEffect(() => {
+    optionsRef.current = options;
+  }, [options]);
 
-  const start = useCallback(() => {
-    // TODO: 启动 VAD 监听
+  /**
+   * 初始化 VAD 并开始监听
+   * @param stream 麦克风 MediaStream（由外部管理）
+   */
+  const start = useCallback(async (stream: MediaStream) => {
+    // 如果已有实例，先销毁
+    if (vadRef.current) {
+      await vadRef.current.destroy();
+      vadRef.current = null;
+    }
+
+    try {
+      const vad = await MicVAD.new({
+        getStream: () => Promise.resolve(stream),
+        startOnLoad: true,
+        model: "legacy",
+
+        onSpeechStart: () => {
+          setIsSpeaking(true);
+          optionsRef.current?.onSpeechStart?.();
+        },
+
+        onSpeechEnd: (audio: Float32Array) => {
+          setIsSpeaking(false);
+          optionsRef.current?.onSpeechEnd?.(audio);
+        },
+
+        onVADMisfire: () => {
+          setIsSpeaking(false);
+          optionsRef.current?.onVADMisfire?.();
+        },
+
+        // VAD 参数（对齐 docs/06-语音交互.md 推荐值）
+        positiveSpeechThreshold: 0.5,
+        negativeSpeechThreshold: 0.35,
+        redemptionMs: 300,
+        preSpeechPadMs: 300,
+        minSpeechMs: 250,
+        submitUserSpeechOnPause: false,
+      });
+
+      vadRef.current = vad;
+      setIsReady(true);
+      setError(null);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : "VAD 初始化失败";
+      setError(message);
+      console.error("[VAD] 初始化失败:", err);
+    }
   }, []);
 
-  const stop = useCallback(() => {
-    // TODO: 停止 VAD 监听
+  /** 停止 VAD 并销毁实例 */
+  const stop = useCallback(async () => {
+    if (vadRef.current) {
+      await vadRef.current.destroy();
+      vadRef.current = null;
+    }
+    setIsReady(false);
+    setIsSpeaking(false);
   }, []);
 
-  return { isSpeaking, isReady, start, stop };
+  // 组件卸载时清理
+  useEffect(() => {
+    return () => {
+      vadRef.current?.destroy();
+    };
+  }, []);
+
+  return { isSpeaking, isReady, error, start, stop };
 }
 
 // ---- 关键帧检测（ONNX Runtime Web）----
@@ -41,11 +109,6 @@ export function useKeyframeDetection() {
   // TODO: 加载 ONNX 模型后设为 true
   const isReady = false;
 
-  // TODO: 实现关键帧检测
-  // 1. 加载 ONNX 模型
-  // 2. 对比当前帧与上一帧的像素差异
-  // 3. 超过阈值则判定为关键帧
-
   const isKeyframe = useCallback(
     (_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
       // TODO: 实现像素差异对比
diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts
index d187e8d..e53b91f 100644
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -9,6 +9,7 @@ import { v4 as uuidv4 } from "uuid";
 import { wsClient } from "../lib/websocket";
 import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
 import { useCamera } from "../components/CameraManager";
+import { useMicrophone } from "../components/MicManager";
 import { useVAD } from "../components/EdgeProcessor";
 import { useWebSocketManager } from "../components/WebSocketManager";
 import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";
@@ -19,10 +20,17 @@ export function useVisionSession() {
   const [isProcessing, setIsProcessing] = useState(false);
 
   const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
+  const { startMic, stopMic } = useMicrophone();
   const { status, connect, disconnect, send } = useWebSocketManager();
 
   // VAD：语音结束时自动发送 query
-  const { isSpeaking, start: startVAD, stop: stopVAD } = useVAD({
+  const {
+    isSpeaking,
+    isReady: isVADReady,
+    error: vadError,
+    start: startVAD,
+    stop: stopVAD,
+  } = useVAD({
     onSpeechEnd: useCallback(
       (audio: Float32Array) => {
         const frame = captureFrame();
@@ -101,17 +109,28 @@ export function useVisionSession() {
 
   /** 启动会话 */
   const startSession = useCallback(async () => {
+    // 1. 获取摄像头和麦克风
     await startCamera();
+    const micStream = await startMic();
+    if (!micStream) {
+      console.error("[Session] 无法获取麦克风");
+      return;
+    }
+
+    // 2. 连接 WebSocket
     connect();
-    startVAD();
-  }, [startCamera, connect, startVAD]);
+
+    // 3. 启动 VAD（传入麦克风 stream）
+    await startVAD(micStream);
+  }, [startCamera, startMic, connect, startVAD]);
 
   /** 结束会话 */
-  const stopSession = useCallback(() => {
-    stopVAD();
+  const stopSession = useCallback(async () => {
+    await stopVAD();
+    stopMic();
     stopCamera();
     disconnect();
-  }, [stopVAD, stopCamera, disconnect]);
+  }, [stopVAD, stopMic, stopCamera, disconnect]);
 
   /** 打断当前回复 */
   const interrupt = useCallback(() => {
@@ -124,6 +143,8 @@ export function useVisionSession() {
     currentReply,
     isProcessing,
     isSpeaking,
+    isVADReady,
+    vadError,
     connectionStatus: status,
     videoRef,
     stream,