feat: 集成 VAD 语音活动检测,打通核心交互链路

- 重写 useVAD Hook,接入 @ricky0123/vad-web 的 MicVAD
- 支持 onSpeechStart/onSpeechEnd/onVADMisfire 回调
- useVisionSession 中串联摄像头→麦克风→WebSocket→VAD 完整流程
- App.tsx 新增 VAD 加载中和错误状态的 UI 指示
- VAD 参数对齐设计文档:positiveSpeechThreshold=0.5, minSpeechMs=250

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-06-13 11:10:29 +08:00
parent 14550d3aa2
commit 9ccd4d6238
4 changed files with 140 additions and 29 deletions

View File

@@ -9,6 +9,7 @@ import { v4 as uuidv4 } from "uuid";
import { wsClient } from "../lib/websocket";
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";
@@ -19,10 +20,17 @@ export function useVisionSession() {
const [isProcessing, setIsProcessing] = useState(false);
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
const { startMic, stopMic } = useMicrophone();
const { status, connect, disconnect, send } = useWebSocketManager();
// VAD语音结束时自动发送 query
const { isSpeaking, start: startVAD, stop: stopVAD } = useVAD({
const {
isSpeaking,
isReady: isVADReady,
error: vadError,
start: startVAD,
stop: stopVAD,
} = useVAD({
onSpeechEnd: useCallback(
(audio: Float32Array) => {
const frame = captureFrame();
@@ -101,17 +109,28 @@ export function useVisionSession() {
/** 启动会话 */
const startSession = useCallback(async () => {
// 1. 获取摄像头和麦克风
await startCamera();
const micStream = await startMic();
if (!micStream) {
console.error("[Session] 无法获取麦克风");
return;
}
// 2. 连接 WebSocket
connect();
startVAD();
}, [startCamera, connect, startVAD]);
// 3. 启动 VAD传入麦克风 stream
await startVAD(micStream);
}, [startCamera, startMic, connect, startVAD]);
/** 结束会话 */
const stopSession = useCallback(() => {
stopVAD();
const stopSession = useCallback(async () => {
await stopVAD();
stopMic();
stopCamera();
disconnect();
}, [stopVAD, stopCamera, disconnect]);
}, [stopVAD, stopMic, stopCamera, disconnect]);
/** 打断当前回复 */
const interrupt = useCallback(() => {
@@ -124,6 +143,8 @@ export function useVisionSession() {
currentReply,
isProcessing,
isSpeaking,
isVADReady,
vadError,
connectionStatus: status,
videoRef,
stream,