feat: 集成 VAD 语音活动检测,打通核心交互链路
- 重写 useVAD Hook,接入 @ricky0123/vad-web 的 MicVAD - 支持 onSpeechStart/onSpeechEnd/onVADMisfire 回调 - useVisionSession 中串联摄像头→麦克风→WebSocket→VAD 完整流程 - App.tsx 新增 VAD 加载中和错误状态的 UI 指示 - VAD 参数对齐设计文档:positiveSpeechThreshold=0.5, minSpeechMs=250 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import { v4 as uuidv4 } from "uuid";
|
||||
import { wsClient } from "../lib/websocket";
|
||||
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
|
||||
import { useCamera } from "../components/CameraManager";
|
||||
import { useMicrophone } from "../components/MicManager";
|
||||
import { useVAD } from "../components/EdgeProcessor";
|
||||
import { useWebSocketManager } from "../components/WebSocketManager";
|
||||
import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";
|
||||
@@ -19,10 +20,17 @@ export function useVisionSession() {
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
|
||||
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
|
||||
const { startMic, stopMic } = useMicrophone();
|
||||
const { status, connect, disconnect, send } = useWebSocketManager();
|
||||
|
||||
// VAD:语音结束时自动发送 query
|
||||
const { isSpeaking, start: startVAD, stop: stopVAD } = useVAD({
|
||||
const {
|
||||
isSpeaking,
|
||||
isReady: isVADReady,
|
||||
error: vadError,
|
||||
start: startVAD,
|
||||
stop: stopVAD,
|
||||
} = useVAD({
|
||||
onSpeechEnd: useCallback(
|
||||
(audio: Float32Array) => {
|
||||
const frame = captureFrame();
|
||||
@@ -101,17 +109,28 @@ export function useVisionSession() {
|
||||
|
||||
/** 启动会话 */
|
||||
const startSession = useCallback(async () => {
|
||||
// 1. 获取摄像头和麦克风
|
||||
await startCamera();
|
||||
const micStream = await startMic();
|
||||
if (!micStream) {
|
||||
console.error("[Session] 无法获取麦克风");
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. 连接 WebSocket
|
||||
connect();
|
||||
startVAD();
|
||||
}, [startCamera, connect, startVAD]);
|
||||
|
||||
// 3. 启动 VAD(传入麦克风 stream)
|
||||
await startVAD(micStream);
|
||||
}, [startCamera, startMic, connect, startVAD]);
|
||||
|
||||
/** 结束会话 */
|
||||
const stopSession = useCallback(() => {
|
||||
stopVAD();
|
||||
const stopSession = useCallback(async () => {
|
||||
await stopVAD();
|
||||
stopMic();
|
||||
stopCamera();
|
||||
disconnect();
|
||||
}, [stopVAD, stopCamera, disconnect]);
|
||||
}, [stopVAD, stopMic, stopCamera, disconnect]);
|
||||
|
||||
/** 打断当前回复 */
|
||||
const interrupt = useCallback(() => {
|
||||
@@ -124,6 +143,8 @@ export function useVisionSession() {
|
||||
currentReply,
|
||||
isProcessing,
|
||||
isSpeaking,
|
||||
isVADReady,
|
||||
vadError,
|
||||
connectionStatus: status,
|
||||
videoRef,
|
||||
stream,
|
||||
|
||||
Reference in New Issue
Block a user