CamTalk/frontend/src/hooks/useVisionSession.ts

// ============================================================
// useVisionSession — 核心视觉对话会话 Hook
// 职责：封装一次完整的视觉对话会话（摄像头、VAD、WebSocket、消息状态）
// 来源：docs/02-系统架构.md 核心 Hook 设计
// ============================================================

import { useCallback, useEffect, useRef, useState } from "react";
import { v4 as uuidv4 } from "uuid";
import { wsClient } from "../lib/websocket";
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
import { getErrorMessage } from "../lib/errors";
import { TTSPlayer } from "../lib/ttsPlayer";
import { showToast } from "../lib/toast";
import { loadConfig, saveConfig } from "../lib/storage";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import { useObservationMode } from "./useObservationMode";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";

export type SessionMode = "dialogue" | "observation";

const MAX_HISTORY_ROUNDS = 10;

export interface SessionStats {
  queryCount: number;
  totalTokens: number;
}

export function useVisionSession() {
  const [messages, setMessages] = useState<ChatMessage[]>([]);
  const [currentReply, setCurrentReply] = useState<string>("");
  const [isProcessing, setIsProcessing] = useState(false);
  const [isAudioPlaying, setIsAudioPlaying] = useState(false);
  const [config, setConfig] = useState<SessionConfig>(loadConfig);
  const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
  const [mode, setMode] = useState<SessionMode>("dialogue");

  // 上一帧采样数据（用于关键帧检测）
  const prevFrameRef = useRef<Uint8ClampedArray | null>(null);

  // 对话历史（role + content），用于多轮上下文
  const historyRef = useRef<Array<{ role: string; content: string }>>([]);

  // TTS 播放器
  const ttsPlayerRef = useRef<TTSPlayer | null>(null);
  const getTTSPlayer = useCallback(() => {
    if (!ttsPlayerRef.current) {
      const player = new TTSPlayer();
      player.onEnd(() => setIsAudioPlaying(false));
      ttsPlayerRef.current = player;
    }
    return ttsPlayerRef.current;
  }, []);

  const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
  const { startMic, stopMic } = useMicrophone();
  const { status, connect, disconnect, send } = useWebSocketManager();

  // 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题
  const isProcessingRef = useRef(false);
  useEffect(() => {
    isProcessingRef.current = isProcessing;
  }, [isProcessing]);

  // 观察模式：画面变化时自动发送 query
  const { isObserving, startObserving, stopObserving } = useObservationMode({
    onChange: useCallback(
      (frameDataUrl: string) => {
        if (isProcessingRef.current) return;

        const requestId = uuidv4();
        send({
          type: "query",
          request_id: requestId,
          image: dataUrlToBase64(frameDataUrl),
          audio: "", // 观察模式无音频
        });

        setMessages((prev) => [
          ...prev,
          {
            role: "user",
            content: "👁️ 画面变化检测",
            timestamp: Date.now(),
          },
        ]);
        setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
        setIsProcessing(true);
      },
      [send],
    ),
  });

  /** 切换对话/观察模式 */
  const toggleMode = useCallback(() => {
    setMode((prev) => {
      const next = prev === "dialogue" ? "observation" : "dialogue";
      if (next === "observation") {
        startObserving(videoRef.current, captureFrame);
      } else {
        stopObserving();
      }
      return next;
    });
  }, [videoRef, captureFrame, startObserving, stopObserving]);

  // WebSocket 连接成功后发送 config
  useEffect(() => {
    if (status === "connected") {
      send({
        type: "config",
        payload: {
          tts_enabled: config.ttsEnabled,
          detail_level: config.detailLevel,
          language: config.language,
        },
      });
    }
  }, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送

  /** 更新会话配置 */
  const updateConfig = useCallback((partial: Partial<SessionConfig>) => {
    setConfig((prev) => {
      const next = { ...prev, ...partial };
      saveConfig(next);
      // 如果已连接，立即发送更新
      if (status === "connected") {
        send({
          type: "config",
          payload: {
            tts_enabled: next.ttsEnabled,
            detail_level: next.detailLevel,
            language: next.language,
          },
        });
      }
      return next;
    });
  }, [status, send]);

  // VAD：语音结束时自动发送 query
  const {
    isSpeaking,
    isReady: isVADReady,
    error: vadError,
    start: startVAD,
    stop: stopVAD,
  } = useVAD({
    onSpeechEnd: useCallback(
      (audio: Float32Array) => {
        // 处理中忽略，防止重复发送
        if (isProcessingRef.current) {
          console.warn("[Session] 正在处理中，忽略语音输入");
          return;
        }

        const frame = captureFrame();
        if (!frame) {
          console.warn("[Session] 无法捕获图像帧");
          return;
        }

        // 关键帧检测：与上一帧对比，相似度过高则跳过
        const video = videoRef.current;
        if (video) {
          const currentSample = sampleFrame(video);
          if (currentSample && prevFrameRef.current) {
            const { similarity } = compareFrames(prevFrameRef.current, currentSample);
            if (similarity > 0.9) {
              console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)})，跳过`);
              prevFrameRef.current = currentSample;
              return;
            }
          }
          if (currentSample) {
            prevFrameRef.current = currentSample;
          }
        }

        const requestId = uuidv4();
        send({
          type: "query",
          request_id: requestId,
          image: dataUrlToBase64(frame),
          audio: encodeAudioToBase64(audio),
        });

        // 更新请求统计
        setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));

        // 添加用户消息（STT 流式结果会逐步更新文本）
        setMessages((prev) => [
          ...prev,
          { role: "user", content: "（语音识别中...）", timestamp: Date.now() },
        ]);
        setIsProcessing(true);
      },
      [captureFrame, send, videoRef]
    ),
  });

  // 处理服务端消息
  useEffect(() => {
    const unsub = wsClient.onMessage((msg: ServerMessage) => {
      switch (msg.type) {
        case "stt_result": {
          // 流式更新用户消息文本（包括中间结果和最终结果）
          setMessages((prev) => {
            const updated = [...prev];
            const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
            if (lastUserIdx >= 0) {
              updated[lastUserIdx] = {
                ...updated[lastUserIdx],
                content: msg.text || "（未识别到语音）",
              };
            }
            return updated;
          });
          break;
        }

        case "llm_chunk":
          setCurrentReply((prev) => prev + msg.delta);
          break;

        case "llm_done": {
          const done = msg as LLMDoneMessage;
          // 记录到对话历史
          historyRef.current.push({ role: "assistant", content: done.full_text });
          // 裁剪历史到最近 N 轮
          if (historyRef.current.length > MAX_HISTORY_ROUNDS * 2) {
            historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
          }

          // 累计 token 统计
          if (done.tokens_used?.total) {
            setStats((prev) => ({
              ...prev,
              totalTokens: prev.totalTokens + done.tokens_used.total,
            }));
          }

          setMessages((prev) => [
            ...prev,
            {
              role: "assistant",
              content: done.full_text,
              timestamp: Date.now(),
              tokensUsed: done.tokens_used?.total,
              latencyMs: done.latency_ms,
              model: done.model,
            },
          ]);
          setCurrentReply("");
          setIsProcessing(false);
          break;
        }

        case "tts_audio":
          getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
          if (!msg.is_last) {
            setIsAudioPlaying(true);
          }
          break;

        case "error":
          console.error("[Session] 服务端错误:", msg.code, msg.message);
          showToast(getErrorMessage(msg.code), "error");
          setIsProcessing(false);
          break;
      }
    });

    return unsub;
  }, [getTTSPlayer]);

  /** 启动会话 */
  const startSession = useCallback(async () => {
    // 1. 获取摄像头和麦克风
    await startCamera();
    const micStream = await startMic();
    if (!micStream) {
      showToast("无法获取麦克风权限", "error");
      return;
    }

    // 2. 连接 WebSocket
    connect();

    // 3. 启动 VAD（传入麦克风 stream）
    await startVAD(micStream);
  }, [startCamera, startMic, connect, startVAD]);

  /** 结束会话 */
  const stopSession = useCallback(async () => {
    stopObserving();
    setMode("dialogue");
    await stopVAD();
    stopMic();
    stopCamera();
    disconnect();
    // 停止 TTS 并清理状态
    ttsPlayerRef.current?.stop();
    setIsAudioPlaying(false);
    setMessages([]);
    setCurrentReply("");
    setIsProcessing(false);
    setStats({ queryCount: 0, totalTokens: 0 });
    historyRef.current = [];
    prevFrameRef.current = null;
  }, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);

  /** 打断当前回复 */
  const interrupt = useCallback(() => {
    send({ type: "interrupt" });
    // 停止 TTS 播放
    ttsPlayerRef.current?.stop();
    setIsAudioPlaying(false);
    // 将未完成的流式内容保存为最终消息
    if (currentReply) {
      const interrupted = currentReply + "（已打断）";
      historyRef.current.push({ role: "assistant", content: interrupted });
      setMessages((prev) => [
        ...prev,
        { role: "assistant", content: interrupted, timestamp: Date.now() },
      ]);
    }
    setCurrentReply("");
    setIsProcessing(false);
  }, [send, currentReply]);

  return {
    messages,
    currentReply,
    isProcessing,
    isAudioPlaying,
    isSpeaking,
    isVADReady,
    vadError,
    connectionStatus: status,
    videoRef,
    stream,
    config,
    updateConfig,
    stats,
    mode,
    isObserving,
    toggleMode,
    startSession,
    stopSession,
    interrupt,
  };
}