CamTalk/frontend/src/components/EdgeProcessor/index.tsx

// ============================================================
// EdgeProcessor — 边缘预处理（VAD + 关键帧检测）
// 职责：浏览器端语音活动检测、关键帧筛选
// 技术：@ricky0123/vad-web（VAD）、ONNX Runtime Web（关键帧检测）
// ============================================================

import { useCallback, useEffect, useRef, useState } from "react";
import { MicVAD } from "@ricky0123/vad-web";

export interface VADOptions {
  /** 语音结束回调，携带录音 Float32Array（16kHz） */
  onSpeechEnd?: (audio: Float32Array) => void;
  /** 语音开始回调 */
  onSpeechStart?: () => void;
  /** 语音过短被忽略回调 */
  onVADMisfire?: () => void;
}

/**
 * 语音活动检测 Hook
 * 基于 @ricky0123/vad-web 的 MicVAD，检测用户说话并回调
 */
export function useVAD(options?: VADOptions) {
  const [isSpeaking, setIsSpeaking] = useState(false);
  const [isReady, setIsReady] = useState(false);
  const [error, setError] = useState<string | null>(null);
  const vadRef = useRef<MicVAD | null>(null);
  const optionsRef = useRef(options);

  // 保持 options 引用最新，避免回调闭包问题
  useEffect(() => {
    optionsRef.current = options;
  }, [options]);

  /**
   * 初始化 VAD 并开始监听
   * @param stream 麦克风 MediaStream（由外部管理）
   */
  const start = useCallback(async (stream: MediaStream) => {
    // 如果已有实例，先销毁
    if (vadRef.current) {
      await vadRef.current.destroy();
      vadRef.current = null;
    }

    try {
      const vad = await MicVAD.new({
        getStream: () => Promise.resolve(stream),
        startOnLoad: true,
        model: "legacy",
        // 指向 node_modules 中的 WASM 文件（由 Vite 中间件提供）
        onnxWASMBasePath: "/node_modules/onnxruntime-web/dist/",

        onSpeechStart: () => {
          setIsSpeaking(true);
          optionsRef.current?.onSpeechStart?.();
        },

        onSpeechEnd: (audio: Float32Array) => {
          setIsSpeaking(false);
          optionsRef.current?.onSpeechEnd?.(audio);
        },

        onVADMisfire: () => {
          setIsSpeaking(false);
          optionsRef.current?.onVADMisfire?.();
        },

        // VAD 参数（对齐 docs/06-语音交互.md 推荐值）
        positiveSpeechThreshold: 0.5,
        negativeSpeechThreshold: 0.35,
        redemptionMs: 300,
        preSpeechPadMs: 300,
        minSpeechMs: 250,
        submitUserSpeechOnPause: false,
      });

      vadRef.current = vad;
      setIsReady(true);
      setError(null);
    } catch (err) {
      const message = err instanceof Error ? err.message : "VAD 初始化失败";
      setError(message);
      console.error("[VAD] 初始化失败:", err);
    }
  }, []);

  /** 停止 VAD 并销毁实例 */
  const stop = useCallback(async () => {
    if (vadRef.current) {
      await vadRef.current.destroy();
      vadRef.current = null;
    }
    setIsReady(false);
    setIsSpeaking(false);
  }, []);

  // 组件卸载时清理
  useEffect(() => {
    return () => {
      vadRef.current?.destroy();
    };
  }, []);

  return { isSpeaking, isReady, error, start, stop };
}

// ---- 关键帧检测 ----

const DETECT_WIDTH = 160;
const DETECT_HEIGHT = 120;
const DIFF_THRESHOLD = 30;

/** 离屏 canvas，用于降采样 */
let offscreen: HTMLCanvasElement | null = null;

function getOffscreen(): HTMLCanvasElement {
  if (!offscreen) {
    offscreen = document.createElement("canvas");
    offscreen.width = DETECT_WIDTH;
    offscreen.height = DETECT_HEIGHT;
  }
  return offscreen;
}

/**
 * 将 video 元素缩放采样为 Uint8ClampedArray（RGBA）
 * 返回 null 如果 video 未就绪
 */
export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null {
  if (video.readyState < 2) return null;
  const canvas = getOffscreen();
  const ctx = canvas.getContext("2d");
  if (!ctx) return null;
  ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT);
  return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data;
}

/**
 * 对比两帧像素差异
 * @returns { isKeyframe: boolean, similarity: number }
 */
export function compareFrames(
  prev: Uint8ClampedArray,
  curr: Uint8ClampedArray,
): { isKeyframe: boolean; similarity: number } {
  let diffSum = 0;
  const len = Math.min(prev.length, curr.length);
  const pixelCount = len / 4;

  for (let i = 0; i < len; i += 4) {
    // 只比较 RGB，跳过 Alpha
    diffSum += Math.abs(prev[i] - curr[i]);
    diffSum += Math.abs(prev[i + 1] - curr[i + 1]);
    diffSum += Math.abs(prev[i + 2] - curr[i + 2]);
  }

  const avgDiff = diffSum / (pixelCount * 3);
  const similarity = 1 - avgDiff / 255;

  return {
    isKeyframe: avgDiff > DIFF_THRESHOLD,
    similarity,
  };
}