cfy666
50c84fce88
根因：sendTextMessage 断开状态时立即 setMessages 添加用户消息到 UI， 同时 connect() 触发连接成功后 flush useEffect 再次 setMessages 添加同一条消息， 导致消息重复、ChatPanel 渲染出两个输入区域。 修复：断开状态时只入队不立即显示，由 flush 统一处理。
503 lines
16 KiB
TypeScript
503 lines
16 KiB
TypeScript
// ============================================================
|
||
// useVisionSession — 核心视觉对话会话 Hook
|
||
// 职责：封装一次完整的视觉对话会话（摄像头、VAD、WebSocket、消息状态）
|
||
// 来源：docs/02-系统架构.md 核心 Hook 设计
|
||
// ============================================================
|
||
|
||
import { useCallback, useEffect, useRef, useState } from "react";
|
||
import { v4 as uuidv4 } from "uuid";
|
||
import { wsClient } from "../lib/websocket";
|
||
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
|
||
import { getErrorMessage } from "../lib/errors";
|
||
import { TTSPlayer } from "../lib/ttsPlayer";
|
||
import { showToast } from "../lib/toast";
|
||
import { loadConfig, saveConfig } from "../lib/storage";
|
||
import { useI18n } from "../lib/i18n";
|
||
import { useCamera } from "../components/CameraManager";
|
||
import { useMicrophone } from "../components/MicManager";
|
||
import { useVAD, sampleFrame } from "../components/EdgeProcessor";
|
||
import { useWebSocketManager } from "../components/WebSocketManager";
|
||
import { useObservationMode } from "./useObservationMode";
|
||
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
||
|
||
export type SessionMode = "dialogue" | "observation";
|
||
|
||
const MAX_HISTORY_ROUNDS = 10;
|
||
|
||
export interface SessionStats {
|
||
queryCount: number;
|
||
totalTokens: number;
|
||
}
|
||
|
||
export function useVisionSession(accessToken?: string | null) {
|
||
const { t } = useI18n();
|
||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||
const [currentReply, setCurrentReply] = useState<string>("");
|
||
const [isProcessing, setIsProcessing] = useState(false);
|
||
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
|
||
const [config, setConfig] = useState<SessionConfig>(loadConfig);
|
||
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
|
||
const [mode, setMode] = useState<SessionMode>("dialogue");
|
||
const [isCameraOn, setIsCameraOn] = useState(false);
|
||
const [isMicOn, setIsMicOn] = useState(false);
|
||
|
||
// 上一帧采样数据（用于关键帧检测）
|
||
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
|
||
|
||
// 对话历史（role + content），用于多轮上下文
|
||
const historyRef = useRef<Array<{ role: string; content: string }>>([]);
|
||
|
||
// 待发消息队列（未连接时暂存，连接后自动发送）
|
||
const pendingMessagesRef = useRef<Array<{ text: string; requestId: string }>>([]);
|
||
|
||
// TTS 播放器
|
||
const ttsPlayerRef = useRef<TTSPlayer | null>(null);
|
||
const getTTSPlayer = useCallback(() => {
|
||
if (!ttsPlayerRef.current) {
|
||
const player = new TTSPlayer();
|
||
player.onEnd(() => setIsAudioPlaying(false));
|
||
ttsPlayerRef.current = player;
|
||
}
|
||
return ttsPlayerRef.current;
|
||
}, []);
|
||
|
||
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
|
||
const { startMic, stopMic } = useMicrophone();
|
||
const { status, connect, disconnect, send } = useWebSocketManager();
|
||
|
||
// 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题
|
||
const isProcessingRef = useRef(false);
|
||
useEffect(() => {
|
||
isProcessingRef.current = isProcessing;
|
||
}, [isProcessing]);
|
||
|
||
// 用 ref 跟踪连接状态，避免回调闭包问题
|
||
const statusRef = useRef(status);
|
||
useEffect(() => {
|
||
statusRef.current = status;
|
||
}, [status]);
|
||
|
||
// 观察模式：画面变化时自动发送 query
|
||
const { isObserving, startObserving, stopObserving } = useObservationMode({
|
||
onChange: useCallback(
|
||
(frameDataUrl: string) => {
|
||
if (isProcessingRef.current) return;
|
||
|
||
const requestId = uuidv4();
|
||
send({
|
||
type: "query",
|
||
request_id: requestId,
|
||
image: dataUrlToBase64(frameDataUrl),
|
||
audio: "", // 观察模式无音频
|
||
});
|
||
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{
|
||
role: "user",
|
||
content: t("session.changeDetected"),
|
||
timestamp: Date.now(),
|
||
},
|
||
]);
|
||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||
setIsProcessing(true);
|
||
},
|
||
[send],
|
||
),
|
||
});
|
||
|
||
/** 切换对话/观察模式 */
|
||
const toggleMode = useCallback(() => {
|
||
setMode((prev) => {
|
||
const next = prev === "dialogue" ? "observation" : "dialogue";
|
||
if (next === "observation") {
|
||
startObserving(videoRef.current, captureFrame);
|
||
} else {
|
||
stopObserving();
|
||
}
|
||
return next;
|
||
});
|
||
}, [videoRef, captureFrame, startObserving, stopObserving]);
|
||
|
||
// WebSocket 连接成功后发送 config + flush 待发消息
|
||
useEffect(() => {
|
||
if (status === "connected") {
|
||
send({
|
||
type: "config",
|
||
payload: {
|
||
tts_enabled: config.ttsEnabled,
|
||
detail_level: config.detailLevel,
|
||
language: config.language,
|
||
},
|
||
});
|
||
|
||
// flush 待发消息队列
|
||
const pending = pendingMessagesRef.current;
|
||
pendingMessagesRef.current = [];
|
||
for (const msg of pending) {
|
||
const frame = captureFrame();
|
||
send({
|
||
type: "query",
|
||
request_id: msg.requestId,
|
||
image: frame ? dataUrlToBase64(frame) : "",
|
||
audio: "",
|
||
text: msg.text,
|
||
});
|
||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{ role: "user", content: msg.text, timestamp: Date.now() },
|
||
]);
|
||
historyRef.current.push({ role: "user", content: msg.text });
|
||
setIsProcessing(true);
|
||
}
|
||
}
|
||
}, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送
|
||
|
||
/** 更新会话配置 */
|
||
const updateConfig = useCallback((partial: Partial<SessionConfig>) => {
|
||
setConfig((prev) => {
|
||
const next = { ...prev, ...partial };
|
||
saveConfig(next);
|
||
// 如果已连接，立即发送更新
|
||
if (status === "connected") {
|
||
send({
|
||
type: "config",
|
||
payload: {
|
||
tts_enabled: next.ttsEnabled,
|
||
detail_level: next.detailLevel,
|
||
language: next.language,
|
||
},
|
||
});
|
||
}
|
||
return next;
|
||
});
|
||
}, [status, send]);
|
||
|
||
// VAD：语音结束时自动发送 query
|
||
const {
|
||
isSpeaking,
|
||
isReady: isVADReady,
|
||
error: vadError,
|
||
start: startVAD,
|
||
stop: stopVAD,
|
||
} = useVAD({
|
||
onSpeechEnd: useCallback(
|
||
(audio: Float32Array) => {
|
||
// 处理中忽略，防止重复发送
|
||
if (isProcessingRef.current) {
|
||
console.warn("[Session] 正在处理中，忽略语音输入");
|
||
return;
|
||
}
|
||
|
||
// 停止上一轮的 TTS 播放，防止新旧音频重叠
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
|
||
// 尝试捕获图像帧（允许为 null，纯语音场景无需画面）
|
||
const frame = captureFrame();
|
||
|
||
// 更新帧参考（供观察模式使用，但不以此阻断语音请求）
|
||
const video = videoRef.current;
|
||
if (video) {
|
||
const currentSample = sampleFrame(video);
|
||
if (currentSample) {
|
||
prevFrameRef.current = currentSample;
|
||
}
|
||
}
|
||
|
||
const requestId = uuidv4();
|
||
send({
|
||
type: "query",
|
||
request_id: requestId,
|
||
image: frame ? dataUrlToBase64(frame) : "",
|
||
audio: encodeAudioToBase64(audio),
|
||
});
|
||
|
||
// 更新请求统计
|
||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||
|
||
// 添加用户消息（STT 流式结果会逐步更新文本）
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{ role: "user", content: t("session.recognizing"), timestamp: Date.now() },
|
||
]);
|
||
setIsProcessing(true);
|
||
},
|
||
[captureFrame, send, videoRef]
|
||
),
|
||
});
|
||
|
||
// 处理服务端消息
|
||
useEffect(() => {
|
||
const unsub = wsClient.onMessage((msg: ServerMessage) => {
|
||
switch (msg.type) {
|
||
case "stt_result": {
|
||
// 流式更新用户消息文本（包括中间结果和最终结果）
|
||
setMessages((prev) => {
|
||
const updated = [...prev];
|
||
const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
|
||
if (lastUserIdx >= 0) {
|
||
updated[lastUserIdx] = {
|
||
...updated[lastUserIdx],
|
||
content: msg.text || t("session.noSpeech"),
|
||
};
|
||
}
|
||
return updated;
|
||
});
|
||
break;
|
||
}
|
||
|
||
case "llm_chunk":
|
||
setCurrentReply((prev) => prev + msg.delta);
|
||
break;
|
||
|
||
case "llm_done": {
|
||
const done = msg as LLMDoneMessage;
|
||
// 记录到对话历史
|
||
historyRef.current.push({ role: "assistant", content: done.full_text });
|
||
// 裁剪历史到最近 N 轮
|
||
if (historyRef.current.length > MAX_HISTORY_ROUNDS * 2) {
|
||
historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
|
||
}
|
||
|
||
// 累计 token 统计
|
||
if (done.tokens_used?.total) {
|
||
setStats((prev) => ({
|
||
...prev,
|
||
totalTokens: prev.totalTokens + done.tokens_used.total,
|
||
}));
|
||
}
|
||
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{
|
||
role: "assistant",
|
||
content: done.full_text,
|
||
timestamp: Date.now(),
|
||
tokensUsed: done.tokens_used?.total,
|
||
latencyMs: done.latency_ms,
|
||
model: done.model,
|
||
},
|
||
]);
|
||
setCurrentReply("");
|
||
setIsProcessing(false);
|
||
break;
|
||
}
|
||
|
||
case "tts_audio":
|
||
getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
|
||
setIsAudioPlaying(true);
|
||
break;
|
||
|
||
case "error":
|
||
console.error("[Session] 服务端错误:", msg.code, msg.message);
|
||
showToast(getErrorMessage(msg.code, t), "error");
|
||
// STT 失败时：将"语音识别中..."占位消息替换为失败提示
|
||
if (msg.code === "STT_ERROR") {
|
||
setMessages((prev) => {
|
||
const updated = [...prev];
|
||
const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
|
||
if (lastUserIdx >= 0 && updated[lastUserIdx].content === t("session.recognizing")) {
|
||
updated[lastUserIdx] = {
|
||
...updated[lastUserIdx],
|
||
content: t("session.sttFailed"),
|
||
};
|
||
}
|
||
return updated;
|
||
});
|
||
}
|
||
setIsProcessing(false);
|
||
break;
|
||
}
|
||
});
|
||
|
||
return unsub;
|
||
}, [getTTSPlayer]);
|
||
|
||
/** 启动视频通话（摄像头 + 麦克风 + VAD） */
|
||
const startSession = useCallback(async () => {
|
||
// 1. 确保 WebSocket 已连接
|
||
if (statusRef.current !== "connected") {
|
||
connect(accessToken || undefined);
|
||
// 等待连接完成（通过 status 变化触发后续流程，这里直接继续）
|
||
}
|
||
|
||
// 2. 尝试获取摄像头（可选）
|
||
try {
|
||
await startCamera();
|
||
setIsCameraOn(true);
|
||
} catch {
|
||
console.warn("[Session] 无法获取摄像头权限，将以纯文本模式运行");
|
||
}
|
||
|
||
// 3. 尝试获取麦克风（可选）
|
||
const micStream = await startMic();
|
||
if (micStream) {
|
||
setIsMicOn(true);
|
||
// 4. 启动 VAD（仅在麦克风可用时）
|
||
await startVAD(micStream);
|
||
} else {
|
||
console.warn("[Session] 无法获取麦克风权限，将以文本输入模式运行");
|
||
}
|
||
}, [startCamera, startMic, connect, startVAD, accessToken]);
|
||
|
||
/** 结束会话 */
|
||
const stopSession = useCallback(async () => {
|
||
stopObserving();
|
||
setMode("dialogue");
|
||
await stopVAD();
|
||
stopMic();
|
||
stopCamera();
|
||
disconnect();
|
||
// 停止 TTS 并清理状态
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
setMessages([]);
|
||
setCurrentReply("");
|
||
setIsProcessing(false);
|
||
setStats({ queryCount: 0, totalTokens: 0 });
|
||
historyRef.current = [];
|
||
prevFrameRef.current = null;
|
||
setIsCameraOn(false);
|
||
setIsMicOn(false);
|
||
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
|
||
|
||
/** 结束视频，保留聊天和连接 */
|
||
const stopVideo = useCallback(async () => {
|
||
stopObserving();
|
||
setMode("dialogue");
|
||
await stopVAD();
|
||
stopMic();
|
||
stopCamera();
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
setCurrentReply("");
|
||
setIsProcessing(false);
|
||
setIsCameraOn(false);
|
||
setIsMicOn(false);
|
||
// 不断开 WebSocket，不清空消息、历史、统计
|
||
}, [stopObserving, stopVAD, stopMic, stopCamera]);
|
||
|
||
/** 摄像头开关 */
|
||
const toggleCamera = useCallback(async () => {
|
||
if (isCameraOn) {
|
||
stopCamera();
|
||
setIsCameraOn(false);
|
||
} else {
|
||
await startCamera();
|
||
setIsCameraOn(true);
|
||
}
|
||
}, [isCameraOn, startCamera, stopCamera]);
|
||
|
||
/** 麦克风开关 */
|
||
const toggleMic = useCallback(async () => {
|
||
if (isMicOn) {
|
||
await stopVAD();
|
||
stopMic();
|
||
setIsMicOn(false);
|
||
} else {
|
||
const micStream = await startMic();
|
||
if (micStream) {
|
||
await startVAD(micStream);
|
||
setIsMicOn(true);
|
||
}
|
||
}
|
||
}, [isMicOn, startMic, stopMic, startVAD, stopVAD]);
|
||
|
||
/** 打断当前回复 */
|
||
const interrupt = useCallback(() => {
|
||
send({ type: "interrupt" });
|
||
// 停止 TTS 播放
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
// 将未完成的流式内容保存为最终消息
|
||
if (currentReply) {
|
||
const interrupted = currentReply + t("session.interrupted");
|
||
historyRef.current.push({ role: "assistant", content: interrupted });
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{ role: "assistant", content: interrupted, timestamp: Date.now() },
|
||
]);
|
||
}
|
||
setCurrentReply("");
|
||
setIsProcessing(false);
|
||
}, [send, currentReply]);
|
||
|
||
/** 发送文本消息（手动输入） */
|
||
const sendTextMessage = useCallback(
|
||
(text: string) => {
|
||
if (!text.trim() || isProcessingRef.current) return;
|
||
|
||
// 停止上一轮的 TTS 播放
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
|
||
const requestId = uuidv4();
|
||
|
||
// 未连接时：自动连接，消息加入待发队列
|
||
if (statusRef.current !== "connected") {
|
||
pendingMessagesRef.current.push({ text: text.trim(), requestId });
|
||
// 自动连接 WebSocket（消息在连接成功后由 flush 统一添加到 UI，避免重复）
|
||
connect(accessToken || undefined);
|
||
return;
|
||
}
|
||
|
||
// 已连接：直接发送
|
||
const frame = captureFrame();
|
||
send({
|
||
type: "query",
|
||
request_id: requestId,
|
||
image: frame ? dataUrlToBase64(frame) : "",
|
||
audio: "", // 文本输入无音频
|
||
text: text.trim(),
|
||
});
|
||
|
||
// 更新请求统计
|
||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||
|
||
// 添加用户消息
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{ role: "user", content: text.trim(), timestamp: Date.now() },
|
||
]);
|
||
|
||
// 记录到对话历史
|
||
historyRef.current.push({ role: "user", content: text.trim() });
|
||
|
||
setIsProcessing(true);
|
||
},
|
||
[captureFrame, send, connect, accessToken],
|
||
);
|
||
|
||
return {
|
||
messages,
|
||
setMessages,
|
||
currentReply,
|
||
isProcessing,
|
||
isAudioPlaying,
|
||
isSpeaking,
|
||
isVADReady,
|
||
vadError,
|
||
connectionStatus: status,
|
||
videoRef,
|
||
stream,
|
||
config,
|
||
updateConfig,
|
||
stats,
|
||
mode,
|
||
isObserving,
|
||
toggleMode,
|
||
startSession,
|
||
stopSession,
|
||
stopVideo,
|
||
interrupt,
|
||
isCameraOn,
|
||
isMicOn,
|
||
toggleCamera,
|
||
toggleMic,
|
||
sendTextMessage,
|
||
captureFrame,
|
||
};
|
||
}
|