feat: 集成 VAD 语音活动检测,打通核心交互链路
- 重写 useVAD Hook,接入 @ricky0123/vad-web 的 MicVAD - 支持 onSpeechStart/onSpeechEnd/onVADMisfire 回调 - useVisionSession 中串联摄像头→麦克风→WebSocket→VAD 完整流程 - App.tsx 新增 VAD 加载中和错误状态的 UI 指示 - VAD 参数对齐设计文档:positiveSpeechThreshold=0.5, minSpeechMs=250 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -4,35 +4,103 @@
|
||||
// 技术:@ricky0123/vad-web(VAD)、ONNX Runtime Web(关键帧检测)
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useState } from "react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { MicVAD } from "@ricky0123/vad-web";
|
||||
|
||||
export interface VADOptions {
|
||||
/** 语音结束回调,携带录音 Float32Array */
|
||||
/** 语音结束回调,携带录音 Float32Array(16kHz) */
|
||||
onSpeechEnd?: (audio: Float32Array) => void;
|
||||
/** 语音开始回调 */
|
||||
onSpeechStart?: () => void;
|
||||
/** 语音过短被忽略回调 */
|
||||
onVADMisfire?: () => void;
|
||||
}
|
||||
|
||||
export function useVAD(_options?: VADOptions) {
|
||||
const [isSpeaking] = useState(false);
|
||||
// TODO: 初始化 @ricky0123/vad-web,加载后设为 true
|
||||
const isReady = false;
|
||||
/**
|
||||
* 语音活动检测 Hook
|
||||
* 基于 @ricky0123/vad-web 的 MicVAD,检测用户说话并回调
|
||||
*/
|
||||
export function useVAD(options?: VADOptions) {
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isReady, setIsReady] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const vadRef = useRef<MicVAD | null>(null);
|
||||
const optionsRef = useRef(options);
|
||||
|
||||
// TODO: 实现 VAD 初始化
|
||||
// 1. 加载 @ricky0123/vad-web
|
||||
// 2. 配置 VAD 参数(阈值、最小语音时长等)
|
||||
// 3. 连接麦克风 stream
|
||||
// 4. 在 onSpeechEnd 时收集音频并回调 _options.onSpeechEnd
|
||||
// 保持 options 引用最新,避免回调闭包问题
|
||||
useEffect(() => {
|
||||
optionsRef.current = options;
|
||||
}, [options]);
|
||||
|
||||
const start = useCallback(() => {
|
||||
// TODO: 启动 VAD 监听
|
||||
/**
|
||||
* 初始化 VAD 并开始监听
|
||||
* @param stream 麦克风 MediaStream(由外部管理)
|
||||
*/
|
||||
const start = useCallback(async (stream: MediaStream) => {
|
||||
// 如果已有实例,先销毁
|
||||
if (vadRef.current) {
|
||||
await vadRef.current.destroy();
|
||||
vadRef.current = null;
|
||||
}
|
||||
|
||||
try {
|
||||
const vad = await MicVAD.new({
|
||||
getStream: () => Promise.resolve(stream),
|
||||
startOnLoad: true,
|
||||
model: "legacy",
|
||||
|
||||
onSpeechStart: () => {
|
||||
setIsSpeaking(true);
|
||||
optionsRef.current?.onSpeechStart?.();
|
||||
},
|
||||
|
||||
onSpeechEnd: (audio: Float32Array) => {
|
||||
setIsSpeaking(false);
|
||||
optionsRef.current?.onSpeechEnd?.(audio);
|
||||
},
|
||||
|
||||
onVADMisfire: () => {
|
||||
setIsSpeaking(false);
|
||||
optionsRef.current?.onVADMisfire?.();
|
||||
},
|
||||
|
||||
// VAD 参数(对齐 docs/06-语音交互.md 推荐值)
|
||||
positiveSpeechThreshold: 0.5,
|
||||
negativeSpeechThreshold: 0.35,
|
||||
redemptionMs: 300,
|
||||
preSpeechPadMs: 300,
|
||||
minSpeechMs: 250,
|
||||
submitUserSpeechOnPause: false,
|
||||
});
|
||||
|
||||
vadRef.current = vad;
|
||||
setIsReady(true);
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "VAD 初始化失败";
|
||||
setError(message);
|
||||
console.error("[VAD] 初始化失败:", err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
// TODO: 停止 VAD 监听
|
||||
/** 停止 VAD 并销毁实例 */
|
||||
const stop = useCallback(async () => {
|
||||
if (vadRef.current) {
|
||||
await vadRef.current.destroy();
|
||||
vadRef.current = null;
|
||||
}
|
||||
setIsReady(false);
|
||||
setIsSpeaking(false);
|
||||
}, []);
|
||||
|
||||
return { isSpeaking, isReady, start, stop };
|
||||
// 组件卸载时清理
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
vadRef.current?.destroy();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return { isSpeaking, isReady, error, start, stop };
|
||||
}
|
||||
|
||||
// ---- 关键帧检测(ONNX Runtime Web)----
|
||||
@@ -41,11 +109,6 @@ export function useKeyframeDetection() {
|
||||
// TODO: 加载 ONNX 模型后设为 true
|
||||
const isReady = false;
|
||||
|
||||
// TODO: 实现关键帧检测
|
||||
// 1. 加载 ONNX 模型
|
||||
// 2. 对比当前帧与上一帧的像素差异
|
||||
// 3. 超过阈值则判定为关键帧
|
||||
|
||||
const isKeyframe = useCallback(
|
||||
(_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
|
||||
// TODO: 实现像素差异对比
|
||||
|
||||
Reference in New Issue
Block a user