// ============================================================ // useVisionSession — 核心视觉对话会话 Hook // 职责:封装一次完整的视觉对话会话(摄像头、VAD、WebSocket、消息状态) // 来源:docs/02-系统架构.md 核心 Hook 设计 // ============================================================ import { useCallback, useEffect, useRef, useState } from "react"; import { v4 as uuidv4 } from "uuid"; import { wsClient } from "../lib/websocket"; import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio"; import { getErrorMessage } from "../lib/errors"; import { TTSPlayer } from "../lib/ttsPlayer"; import { showToast } from "../lib/toast"; import { loadConfig, saveConfig } from "../lib/storage"; import { useI18n } from "../lib/i18n"; import { useCamera } from "../components/CameraManager"; import { useMicrophone } from "../components/MicManager"; import { useVAD, sampleFrame } from "../components/EdgeProcessor"; import { useWebSocketManager } from "../components/WebSocketManager"; import { useObservationMode } from "./useObservationMode"; import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types"; export type SessionMode = "dialogue" | "observation"; const MAX_HISTORY_ROUNDS = 10; export interface SessionStats { queryCount: number; totalTokens: number; } export function useVisionSession(accessToken?: string | null) { const { t } = useI18n(); const [messages, setMessages] = useState([]); const [currentReply, setCurrentReply] = useState(""); const [isProcessing, setIsProcessing] = useState(false); const [isAudioPlaying, setIsAudioPlaying] = useState(false); const [config, setConfig] = useState(loadConfig); const [stats, setStats] = useState({ queryCount: 0, totalTokens: 0 }); const [mode, setMode] = useState("dialogue"); const [isCameraOn, setIsCameraOn] = useState(false); const [isMicOn, setIsMicOn] = useState(false); // 上一帧采样数据(用于关键帧检测) const prevFrameRef = useRef(null); // 对话历史(role + content),用于多轮上下文 const historyRef = useRef>([]); // 待发消息队列(未连接时暂存,连接后自动发送) const pendingMessagesRef = useRef>([]); // TTS 播放器 const ttsPlayerRef = useRef(null); const getTTSPlayer = useCallback(() => { if (!ttsPlayerRef.current) { const player = new TTSPlayer(); player.onEnd(() => setIsAudioPlaying(false)); ttsPlayerRef.current = player; } return ttsPlayerRef.current; }, []); const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); const { startMic, stopMic } = useMicrophone(); const { status, connect, disconnect, send } = useWebSocketManager(); // 用 ref 跟踪 isProcessing,避免 VAD 回调闭包问题 const isProcessingRef = useRef(false); useEffect(() => { isProcessingRef.current = isProcessing; }, [isProcessing]); // 用 ref 跟踪连接状态,避免回调闭包问题 const statusRef = useRef(status); useEffect(() => { statusRef.current = status; }, [status]); // 观察模式:画面变化时自动发送 query const { isObserving, startObserving, stopObserving } = useObservationMode({ onChange: useCallback( (frameDataUrl: string) => { if (isProcessingRef.current) return; const requestId = uuidv4(); send({ type: "query", request_id: requestId, image: dataUrlToBase64(frameDataUrl), audio: "", // 观察模式无音频 }); setMessages((prev) => [ ...prev, { role: "user", content: t("session.changeDetected"), timestamp: Date.now(), }, ]); setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 })); setIsProcessing(true); }, [send], ), }); /** 切换对话/观察模式 */ const toggleMode = useCallback(() => { setMode((prev) => { const next = prev === "dialogue" ? "observation" : "dialogue"; if (next === "observation") { startObserving(videoRef.current, captureFrame); } else { stopObserving(); } return next; }); }, [videoRef, captureFrame, startObserving, stopObserving]); // WebSocket 连接成功后发送 config + flush 待发消息 useEffect(() => { if (status === "connected") { send({ type: "config", payload: { tts_enabled: config.ttsEnabled, detail_level: config.detailLevel, language: config.language, }, }); // flush 待发消息队列 const pending = pendingMessagesRef.current; pendingMessagesRef.current = []; for (const msg of pending) { const frame = captureFrame(); send({ type: "query", request_id: msg.requestId, image: frame ? dataUrlToBase64(frame) : "", audio: "", text: msg.text, }); setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 })); setMessages((prev) => [ ...prev, { role: "user", content: msg.text, timestamp: Date.now() }, ]); historyRef.current.push({ role: "user", content: msg.text }); setIsProcessing(true); } } }, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送 /** 更新会话配置 */ const updateConfig = useCallback((partial: Partial) => { setConfig((prev) => { const next = { ...prev, ...partial }; saveConfig(next); // 如果已连接,立即发送更新 if (status === "connected") { send({ type: "config", payload: { tts_enabled: next.ttsEnabled, detail_level: next.detailLevel, language: next.language, }, }); } return next; }); }, [status, send]); // VAD:语音结束时自动发送 query const { isSpeaking, isReady: isVADReady, error: vadError, start: startVAD, stop: stopVAD, } = useVAD({ onSpeechEnd: useCallback( (audio: Float32Array) => { // 处理中忽略,防止重复发送 if (isProcessingRef.current) { console.warn("[Session] 正在处理中,忽略语音输入"); return; } // 停止上一轮的 TTS 播放,防止新旧音频重叠 ttsPlayerRef.current?.stop(); setIsAudioPlaying(false); // 尝试捕获图像帧(允许为 null,纯语音场景无需画面) const frame = captureFrame(); // 更新帧参考(供观察模式使用,但不以此阻断语音请求) const video = videoRef.current; if (video) { const currentSample = sampleFrame(video); if (currentSample) { prevFrameRef.current = currentSample; } } const requestId = uuidv4(); send({ type: "query", request_id: requestId, image: frame ? dataUrlToBase64(frame) : "", audio: encodeAudioToBase64(audio), }); // 更新请求统计 setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 })); // 添加用户消息(STT 流式结果会逐步更新文本) setMessages((prev) => [ ...prev, { role: "user", content: t("session.recognizing"), timestamp: Date.now() }, ]); setIsProcessing(true); }, [captureFrame, send, videoRef] ), }); // 处理服务端消息 useEffect(() => { const unsub = wsClient.onMessage((msg: ServerMessage) => { switch (msg.type) { case "stt_result": { // 流式更新用户消息文本(包括中间结果和最终结果) setMessages((prev) => { const updated = [...prev]; const lastUserIdx = updated.findLastIndex((m) => m.role === "user"); if (lastUserIdx >= 0) { updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text || t("session.noSpeech"), }; } return updated; }); break; } case "llm_chunk": setCurrentReply((prev) => prev + msg.delta); break; case "llm_done": { const done = msg as LLMDoneMessage; // 记录到对话历史 historyRef.current.push({ role: "assistant", content: done.full_text }); // 裁剪历史到最近 N 轮 if (historyRef.current.length > MAX_HISTORY_ROUNDS * 2) { historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2); } // 累计 token 统计 if (done.tokens_used?.total) { setStats((prev) => ({ ...prev, totalTokens: prev.totalTokens + done.tokens_used.total, })); } setMessages((prev) => [ ...prev, { role: "assistant", content: done.full_text, timestamp: Date.now(), tokensUsed: done.tokens_used?.total, latencyMs: done.latency_ms, model: done.model, }, ]); setCurrentReply(""); setIsProcessing(false); break; } case "tts_audio": getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last); setIsAudioPlaying(true); break; case "error": console.error("[Session] 服务端错误:", msg.code, msg.message); showToast(getErrorMessage(msg.code, t), "error"); // STT 失败时:将"语音识别中..."占位消息替换为失败提示 if (msg.code === "STT_ERROR") { setMessages((prev) => { const updated = [...prev]; const lastUserIdx = updated.findLastIndex((m) => m.role === "user"); if (lastUserIdx >= 0 && updated[lastUserIdx].content === t("session.recognizing")) { updated[lastUserIdx] = { ...updated[lastUserIdx], content: t("session.sttFailed"), }; } return updated; }); } setIsProcessing(false); break; } }); return unsub; }, [getTTSPlayer]); /** 启动视频通话(摄像头 + 麦克风 + VAD) */ const startSession = useCallback(async () => { // 1. 确保 WebSocket 已连接 if (statusRef.current !== "connected") { connect(accessToken || undefined); // 等待连接完成(通过 status 变化触发后续流程,这里直接继续) } // 2. 尝试获取摄像头(可选) try { await startCamera(); setIsCameraOn(true); } catch { console.warn("[Session] 无法获取摄像头权限,将以纯文本模式运行"); } // 3. 尝试获取麦克风(可选) const micStream = await startMic(); if (micStream) { setIsMicOn(true); // 4. 启动 VAD(仅在麦克风可用时) await startVAD(micStream); } else { console.warn("[Session] 无法获取麦克风权限,将以文本输入模式运行"); } }, [startCamera, startMic, connect, startVAD, accessToken]); /** 结束会话 */ const stopSession = useCallback(async () => { stopObserving(); setMode("dialogue"); await stopVAD(); stopMic(); stopCamera(); disconnect(); // 停止 TTS 并清理状态 ttsPlayerRef.current?.stop(); setIsAudioPlaying(false); setMessages([]); setCurrentReply(""); setIsProcessing(false); setStats({ queryCount: 0, totalTokens: 0 }); historyRef.current = []; prevFrameRef.current = null; setIsCameraOn(false); setIsMicOn(false); }, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]); /** 结束视频,保留聊天和连接 */ const stopVideo = useCallback(async () => { stopObserving(); setMode("dialogue"); await stopVAD(); stopMic(); stopCamera(); ttsPlayerRef.current?.stop(); setIsAudioPlaying(false); setCurrentReply(""); setIsProcessing(false); setIsCameraOn(false); setIsMicOn(false); // 不断开 WebSocket,不清空消息、历史、统计 }, [stopObserving, stopVAD, stopMic, stopCamera]); /** 摄像头开关 */ const toggleCamera = useCallback(async () => { if (isCameraOn) { stopCamera(); setIsCameraOn(false); } else { await startCamera(); setIsCameraOn(true); } }, [isCameraOn, startCamera, stopCamera]); /** 麦克风开关 */ const toggleMic = useCallback(async () => { if (isMicOn) { await stopVAD(); stopMic(); setIsMicOn(false); } else { const micStream = await startMic(); if (micStream) { await startVAD(micStream); setIsMicOn(true); } } }, [isMicOn, startMic, stopMic, startVAD, stopVAD]); /** 打断当前回复 */ const interrupt = useCallback(() => { send({ type: "interrupt" }); // 停止 TTS 播放 ttsPlayerRef.current?.stop(); setIsAudioPlaying(false); // 将未完成的流式内容保存为最终消息 if (currentReply) { const interrupted = currentReply + t("session.interrupted"); historyRef.current.push({ role: "assistant", content: interrupted }); setMessages((prev) => [ ...prev, { role: "assistant", content: interrupted, timestamp: Date.now() }, ]); } setCurrentReply(""); setIsProcessing(false); }, [send, currentReply]); /** 发送文本消息(手动输入) */ const sendTextMessage = useCallback( (text: string) => { if (!text.trim() || isProcessingRef.current) return; // 停止上一轮的 TTS 播放 ttsPlayerRef.current?.stop(); setIsAudioPlaying(false); const requestId = uuidv4(); // 未连接时:自动连接,消息加入待发队列 if (statusRef.current !== "connected") { pendingMessagesRef.current.push({ text: text.trim(), requestId }); // 自动连接 WebSocket(消息在连接成功后由 flush 统一添加到 UI,避免重复) connect(accessToken || undefined); return; } // 已连接:直接发送 const frame = captureFrame(); send({ type: "query", request_id: requestId, image: frame ? dataUrlToBase64(frame) : "", audio: "", // 文本输入无音频 text: text.trim(), }); // 更新请求统计 setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 })); // 添加用户消息 setMessages((prev) => [ ...prev, { role: "user", content: text.trim(), timestamp: Date.now() }, ]); // 记录到对话历史 historyRef.current.push({ role: "user", content: text.trim() }); setIsProcessing(true); }, [captureFrame, send, connect, accessToken], ); return { messages, setMessages, currentReply, isProcessing, isAudioPlaying, isSpeaking, isVADReady, vadError, connectionStatus: status, videoRef, stream, config, updateConfig, stats, mode, isObserving, toggleMode, startSession, stopSession, stopVideo, interrupt, isCameraOn, isMicOn, toggleCamera, toggleMic, sendTextMessage, captureFrame, }; }