聊天框与会话历史栏优化 #91

Merged
cfy777 merged 4 commits from fix/language into develop 2026-06-14 15:46:41 +08:00
6 changed files with 73 additions and 31 deletions
Showing only changes of commit 15b1dd56fa - Show all commits

View File

@@ -175,7 +175,7 @@ function AppContent() {
<path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z" /> <path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z" />
<circle cx="12" cy="13" r="4" /> <circle cx="12" cy="13" r="4" />
</svg> </svg>
<span>{tr("video.clickToStart")}</span> <span>{tr("video.placeholder")}</span>
</div> </div>
)} )}
{isConnected && !isCameraOn && ( {isConnected && !isCameraOn && (
@@ -194,7 +194,7 @@ function AppContent() {
<div className="video-controls"> <div className="video-controls">
{!isConnected ? ( {!isConnected ? (
<button className="btn btn--primary btn--lg" onClick={startSession}> <button className="btn btn--primary btn--lg" onClick={startSession}>
{connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.start")} {connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")}
</button> </button>
) : ( ) : (
<div className="video-controls__row"> <div className="video-controls__row">

View File

@@ -53,17 +53,6 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
setInputText(""); setInputText("");
}; };
// 未连接时的空状态
if (!isConnected) {
return (
<div className="chat-panel chat-panel--empty">
<span className="chat-panel--empty-icon">💬</span>
<p>{t("chat.empty.prompt")}</p>
<span className="chat-panel--empty-hint">{t("chat.empty.hint")}</span>
</div>
);
}
return ( return (
<div className="chat-panel"> <div className="chat-panel">
<div className="chat-panel__messages" ref={containerRef}> <div className="chat-panel__messages" ref={containerRef}>
@@ -71,8 +60,8 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
{messages.length === 0 && !currentReply && ( {messages.length === 0 && !currentReply && (
<div className="chat-panel__welcome"> <div className="chat-panel__welcome">
<span className="chat-panel__welcome-icon">💬</span> <span className="chat-panel__welcome-icon">💬</span>
<p>{t("chat.welcome.prompt")}</p> <p>{isConnected ? t("chat.welcome.prompt") : t("chat.empty.prompt")}</p>
<span className="chat-panel__welcome-hint">{t("chat.welcome.hint")}</span> <span className="chat-panel__welcome-hint">{isConnected ? t("chat.welcome.hint") : t("chat.empty.hint")}</span>
</div> </div>
)} )}
@@ -106,7 +95,7 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
<div ref={bottomRef} /> <div ref={bottomRef} />
</div> </div>
{/* 文本输入框 - 连接后始终显示 */} {/* 文本输入框 - 始终显示 */}
{onSendText && ( {onSendText && (
<form className="chat-input" onSubmit={handleSubmit}> <form className="chat-input" onSubmit={handleSubmit}>
<input <input
@@ -115,11 +104,12 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
placeholder={t("chat.input.placeholder")} placeholder={t("chat.input.placeholder")}
value={inputText} value={inputText}
onChange={(e) => setInputText(e.target.value)} onChange={(e) => setInputText(e.target.value)}
disabled={connectionStatus === "connecting"}
/> />
<button <button
type="submit" type="submit"
className="chat-input__send" className="chat-input__send"
disabled={!inputText.trim()} disabled={!inputText.trim() || connectionStatus === "connecting"}
title={t("chat.send")} title={t("chat.send")}
> >

View File

@@ -47,6 +47,9 @@ export function useVisionSession() {
// 对话历史role + content用于多轮上下文 // 对话历史role + content用于多轮上下文
const historyRef = useRef<Array<{ role: string; content: string }>>([]); const historyRef = useRef<Array<{ role: string; content: string }>>([]);
// 待发消息队列(未连接时暂存,连接后自动发送)
const pendingMessagesRef = useRef<Array<{ text: string; requestId: string }>>([]);
// TTS 播放器 // TTS 播放器
const ttsPlayerRef = useRef<TTSPlayer | null>(null); const ttsPlayerRef = useRef<TTSPlayer | null>(null);
const getTTSPlayer = useCallback(() => { const getTTSPlayer = useCallback(() => {
@@ -68,6 +71,12 @@ export function useVisionSession() {
isProcessingRef.current = isProcessing; isProcessingRef.current = isProcessing;
}, [isProcessing]); }, [isProcessing]);
// 用 ref 跟踪连接状态,避免回调闭包问题
const statusRef = useRef(status);
useEffect(() => {
statusRef.current = status;
}, [status]);
// 观察模式:画面变化时自动发送 query // 观察模式:画面变化时自动发送 query
const { isObserving, startObserving, stopObserving } = useObservationMode({ const { isObserving, startObserving, stopObserving } = useObservationMode({
onChange: useCallback( onChange: useCallback(
@@ -110,7 +119,7 @@ export function useVisionSession() {
}); });
}, [videoRef, captureFrame, startObserving, stopObserving]); }, [videoRef, captureFrame, startObserving, stopObserving]);
// WebSocket 连接成功后发送 config // WebSocket 连接成功后发送 config + flush 待发消息
useEffect(() => { useEffect(() => {
if (status === "connected") { if (status === "connected") {
send({ send({
@@ -121,6 +130,27 @@ export function useVisionSession() {
language: config.language, language: config.language,
}, },
}); });
// flush 待发消息队列
const pending = pendingMessagesRef.current;
pendingMessagesRef.current = [];
for (const msg of pending) {
const frame = captureFrame();
send({
type: "query",
request_id: msg.requestId,
image: frame ? dataUrlToBase64(frame) : "",
audio: "",
text: msg.text,
});
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
setMessages((prev) => [
...prev,
{ role: "user", content: msg.text, timestamp: Date.now() },
]);
historyRef.current.push({ role: "user", content: msg.text });
setIsProcessing(true);
}
} }
}, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送 }, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送
@@ -282,10 +312,13 @@ export function useVisionSession() {
return unsub; return unsub;
}, [getTTSPlayer]); }, [getTTSPlayer]);
/** 启动会话 */ /** 启动视频通话(摄像头 + 麦克风 + VAD */
const startSession = useCallback(async () => { const startSession = useCallback(async () => {
// 1. 连接 WebSocket(必须) // 1. 确保 WebSocket 已连接
connect(); if (statusRef.current !== "connected") {
connect();
// 等待连接完成(通过 status 变化触发后续流程,这里直接继续)
}
// 2. 尝试获取摄像头(可选) // 2. 尝试获取摄像头(可选)
try { try {
@@ -381,10 +414,23 @@ export function useVisionSession() {
ttsPlayerRef.current?.stop(); ttsPlayerRef.current?.stop();
setIsAudioPlaying(false); setIsAudioPlaying(false);
// 捕获当前摄像头画面
const frame = captureFrame();
const requestId = uuidv4(); const requestId = uuidv4();
// 未连接时:自动连接,消息加入待发队列
if (statusRef.current !== "connected") {
pendingMessagesRef.current.push({ text: text.trim(), requestId });
// 添加用户消息到 UI立即反馈
setMessages((prev) => [
...prev,
{ role: "user", content: text.trim(), timestamp: Date.now() }],
);
// 自动连接 WebSocket
connect();
return;
}
// 已连接:直接发送
const frame = captureFrame();
send({ send({
type: "query", type: "query",
request_id: requestId, request_id: requestId,
@@ -407,7 +453,7 @@ export function useVisionSession() {
setIsProcessing(true); setIsProcessing(true);
}, },
[captureFrame, send], [captureFrame, send, connect],
); );
return { return {

View File

@@ -33,12 +33,14 @@ export const enUS: TranslationMap = {
"video.playing": "🔊 Playing...", "video.playing": "🔊 Playing...",
"video.initVad": "Initializing voice detection...", "video.initVad": "Initializing voice detection...",
"video.clickToStart": "Click the button below to start", "video.clickToStart": "Click the button below to start",
"video.placeholder": "Type in the chat panel to start",
"video.cameraOff": "Camera is off", "video.cameraOff": "Camera is off",
"video.cameraOff.hint": "You can type in the chat panel", "video.cameraOff.hint": "You can type in the chat panel",
// Controls // Controls
"controls.connecting": "Connecting...", "controls.connecting": "Connecting...",
"controls.start": "🎙️ Start Session", "controls.start": "🎙️ Start Session",
"controls.startVideo": "🎙️ Start Video Call",
"controls.cameraOff": "Turn off camera", "controls.cameraOff": "Turn off camera",
"controls.cameraOn": "Turn on camera", "controls.cameraOn": "Turn on camera",
"controls.micOff": "Turn off microphone", "controls.micOff": "Turn off microphone",
@@ -52,8 +54,8 @@ export const enUS: TranslationMap = {
"chat.title": "Chat", "chat.title": "Chat",
"chat.mode.observation": "Observing", "chat.mode.observation": "Observing",
"chat.reconnecting": "Connection lost, reconnecting...", "chat.reconnecting": "Connection lost, reconnecting...",
"chat.empty.prompt": "Click the button below to start", "chat.empty.prompt": "Type below to start chatting",
"chat.empty.hint": "Type or speak with AI after connecting", "chat.empty.hint": "Type to chat with AI, or click the button on the left to start video",
"chat.welcome.prompt": "Type below to start chatting", "chat.welcome.prompt": "Type below to start chatting",
"chat.welcome.hint": "Or enable microphone for voice chat", "chat.welcome.hint": "Or enable microphone for voice chat",
"chat.userLabel": "You", "chat.userLabel": "You",

View File

@@ -33,12 +33,14 @@ export const jaJP: TranslationMap = {
"video.playing": "🔊 再生中...", "video.playing": "🔊 再生中...",
"video.initVad": "音声検出を初期化中...", "video.initVad": "音声検出を初期化中...",
"video.clickToStart": "下のボタンをクリックして開始", "video.clickToStart": "下のボタンをクリックして開始",
"video.placeholder": "右側のチャットに入力して開始",
"video.cameraOff": "カメラがオフです", "video.cameraOff": "カメラがオフです",
"video.cameraOff.hint": "右側のチャットでテキスト対話ができます", "video.cameraOff.hint": "右側のチャットでテキスト対話ができます",
// Controls // Controls
"controls.connecting": "接続中...", "controls.connecting": "接続中...",
"controls.start": "🎙️ 対話を開始", "controls.start": "🎙️ 対話を開始",
"controls.startVideo": "🎙️ ビデオ通話を開始",
"controls.cameraOff": "カメラをオフ", "controls.cameraOff": "カメラをオフ",
"controls.cameraOn": "カメラをオン", "controls.cameraOn": "カメラをオン",
"controls.micOff": "マイクをオフ", "controls.micOff": "マイクをオフ",
@@ -52,8 +54,8 @@ export const jaJP: TranslationMap = {
"chat.title": "チャット", "chat.title": "チャット",
"chat.mode.observation": "観察モード", "chat.mode.observation": "観察モード",
"chat.reconnecting": "接続が切断されました。再接続中...", "chat.reconnecting": "接続が切断されました。再接続中...",
"chat.empty.prompt": "下のボタンをクリックして開始", "chat.empty.prompt": "下にテキストを入力して対話を開始",
"chat.empty.hint": "接続後にテキストまたは音声でAIと対話できます", "chat.empty.hint": "テキストでAIと対話、または左のボタンでビデオ通話を開始",
"chat.welcome.prompt": "下にテキストを入力して対話を開始", "chat.welcome.prompt": "下にテキストを入力して対話を開始",
"chat.welcome.hint": "マイクを有効にして音声対話もできます", "chat.welcome.hint": "マイクを有効にして音声対話もできます",
"chat.userLabel": "あなた", "chat.userLabel": "あなた",

View File

@@ -33,12 +33,14 @@ export const zhCN: TranslationMap = {
"video.playing": "🔊 正在播放...", "video.playing": "🔊 正在播放...",
"video.initVad": "正在初始化语音检测...", "video.initVad": "正在初始化语音检测...",
"video.clickToStart": "点击下方按钮开始对话", "video.clickToStart": "点击下方按钮开始对话",
"video.placeholder": "在右侧聊天框输入即可开始对话",
"video.cameraOff": "摄像头未开启", "video.cameraOff": "摄像头未开启",
"video.cameraOff.hint": "可在右侧聊天框打字对话", "video.cameraOff.hint": "可在右侧聊天框打字对话",
// Controls // Controls
"controls.connecting": "连接中...", "controls.connecting": "连接中...",
"controls.start": "🎙️ 开始对话", "controls.start": "🎙️ 开始对话",
"controls.startVideo": "🎙️ 开始视频通话",
"controls.cameraOff": "关闭摄像头", "controls.cameraOff": "关闭摄像头",
"controls.cameraOn": "开启摄像头", "controls.cameraOn": "开启摄像头",
"controls.micOff": "关闭麦克风", "controls.micOff": "关闭麦克风",
@@ -52,8 +54,8 @@ export const zhCN: TranslationMap = {
"chat.title": "对话", "chat.title": "对话",
"chat.mode.observation": "观察模式", "chat.mode.observation": "观察模式",
"chat.reconnecting": "连接已断开,正在重连...", "chat.reconnecting": "连接已断开,正在重连...",
"chat.empty.prompt": "点击下方按钮开始对话", "chat.empty.prompt": "在下方输入文字开始对话",
"chat.empty.hint": "连接后可打字或语音与 AI 交互", "chat.empty.hint": "输入文字即可与 AI 交互,也可点击左侧按钮开启视频",
"chat.welcome.prompt": "在下方输入文字开始对话", "chat.welcome.prompt": "在下方输入文字开始对话",
"chat.welcome.hint": "也可以开启麦克风用语音对话", "chat.welcome.hint": "也可以开启麦克风用语音对话",
"chat.userLabel": "你", "chat.userLabel": "你",