feat: 聊天框与视频功能解耦,输入文字即可开始对话
之前必须点击「开始对话」连接 WebSocket 后才能使用聊天框。 现在聊天输入框始终可用,输入文字自动连接 WebSocket 并发送消息; 左侧按钮改为「开始视频通话」,单独控制摄像头/麦克风。 - ChatPanel 移除 !isConnected early return,始终显示输入框 - useVisionSession 新增待发消息队列,sendTextMessage 支持自动连接 - startSession 改为仅负责视频(摄像头 + 麦克风 + VAD) - 修复 statusRef 在 useWebSocketManager 之前声明导致的 TDZ 错误 - 更新 i18n 翻译(新增 controls.startVideo、video.placeholder)
This commit is contained in:
@@ -175,7 +175,7 @@ function AppContent() {
|
|||||||
<path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z" />
|
<path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z" />
|
||||||
<circle cx="12" cy="13" r="4" />
|
<circle cx="12" cy="13" r="4" />
|
||||||
</svg>
|
</svg>
|
||||||
<span>{tr("video.clickToStart")}</span>
|
<span>{tr("video.placeholder")}</span>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{isConnected && !isCameraOn && (
|
{isConnected && !isCameraOn && (
|
||||||
@@ -194,7 +194,7 @@ function AppContent() {
|
|||||||
<div className="video-controls">
|
<div className="video-controls">
|
||||||
{!isConnected ? (
|
{!isConnected ? (
|
||||||
<button className="btn btn--primary btn--lg" onClick={startSession}>
|
<button className="btn btn--primary btn--lg" onClick={startSession}>
|
||||||
{connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.start")}
|
{connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")}
|
||||||
</button>
|
</button>
|
||||||
) : (
|
) : (
|
||||||
<div className="video-controls__row">
|
<div className="video-controls__row">
|
||||||
|
|||||||
@@ -53,17 +53,6 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
|
|||||||
setInputText("");
|
setInputText("");
|
||||||
};
|
};
|
||||||
|
|
||||||
// 未连接时的空状态
|
|
||||||
if (!isConnected) {
|
|
||||||
return (
|
|
||||||
<div className="chat-panel chat-panel--empty">
|
|
||||||
<span className="chat-panel--empty-icon">💬</span>
|
|
||||||
<p>{t("chat.empty.prompt")}</p>
|
|
||||||
<span className="chat-panel--empty-hint">{t("chat.empty.hint")}</span>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="chat-panel">
|
<div className="chat-panel">
|
||||||
<div className="chat-panel__messages" ref={containerRef}>
|
<div className="chat-panel__messages" ref={containerRef}>
|
||||||
@@ -71,8 +60,8 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
|
|||||||
{messages.length === 0 && !currentReply && (
|
{messages.length === 0 && !currentReply && (
|
||||||
<div className="chat-panel__welcome">
|
<div className="chat-panel__welcome">
|
||||||
<span className="chat-panel__welcome-icon">💬</span>
|
<span className="chat-panel__welcome-icon">💬</span>
|
||||||
<p>{t("chat.welcome.prompt")}</p>
|
<p>{isConnected ? t("chat.welcome.prompt") : t("chat.empty.prompt")}</p>
|
||||||
<span className="chat-panel__welcome-hint">{t("chat.welcome.hint")}</span>
|
<span className="chat-panel__welcome-hint">{isConnected ? t("chat.welcome.hint") : t("chat.empty.hint")}</span>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@@ -106,7 +95,7 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
|
|||||||
<div ref={bottomRef} />
|
<div ref={bottomRef} />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* 文本输入框 - 连接后始终显示 */}
|
{/* 文本输入框 - 始终显示 */}
|
||||||
{onSendText && (
|
{onSendText && (
|
||||||
<form className="chat-input" onSubmit={handleSubmit}>
|
<form className="chat-input" onSubmit={handleSubmit}>
|
||||||
<input
|
<input
|
||||||
@@ -115,11 +104,12 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
|
|||||||
placeholder={t("chat.input.placeholder")}
|
placeholder={t("chat.input.placeholder")}
|
||||||
value={inputText}
|
value={inputText}
|
||||||
onChange={(e) => setInputText(e.target.value)}
|
onChange={(e) => setInputText(e.target.value)}
|
||||||
|
disabled={connectionStatus === "connecting"}
|
||||||
/>
|
/>
|
||||||
<button
|
<button
|
||||||
type="submit"
|
type="submit"
|
||||||
className="chat-input__send"
|
className="chat-input__send"
|
||||||
disabled={!inputText.trim()}
|
disabled={!inputText.trim() || connectionStatus === "connecting"}
|
||||||
title={t("chat.send")}
|
title={t("chat.send")}
|
||||||
>
|
>
|
||||||
➤
|
➤
|
||||||
|
|||||||
@@ -47,6 +47,9 @@ export function useVisionSession() {
|
|||||||
// 对话历史(role + content),用于多轮上下文
|
// 对话历史(role + content),用于多轮上下文
|
||||||
const historyRef = useRef<Array<{ role: string; content: string }>>([]);
|
const historyRef = useRef<Array<{ role: string; content: string }>>([]);
|
||||||
|
|
||||||
|
// 待发消息队列(未连接时暂存,连接后自动发送)
|
||||||
|
const pendingMessagesRef = useRef<Array<{ text: string; requestId: string }>>([]);
|
||||||
|
|
||||||
// TTS 播放器
|
// TTS 播放器
|
||||||
const ttsPlayerRef = useRef<TTSPlayer | null>(null);
|
const ttsPlayerRef = useRef<TTSPlayer | null>(null);
|
||||||
const getTTSPlayer = useCallback(() => {
|
const getTTSPlayer = useCallback(() => {
|
||||||
@@ -68,6 +71,12 @@ export function useVisionSession() {
|
|||||||
isProcessingRef.current = isProcessing;
|
isProcessingRef.current = isProcessing;
|
||||||
}, [isProcessing]);
|
}, [isProcessing]);
|
||||||
|
|
||||||
|
// 用 ref 跟踪连接状态,避免回调闭包问题
|
||||||
|
const statusRef = useRef(status);
|
||||||
|
useEffect(() => {
|
||||||
|
statusRef.current = status;
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
// 观察模式:画面变化时自动发送 query
|
// 观察模式:画面变化时自动发送 query
|
||||||
const { isObserving, startObserving, stopObserving } = useObservationMode({
|
const { isObserving, startObserving, stopObserving } = useObservationMode({
|
||||||
onChange: useCallback(
|
onChange: useCallback(
|
||||||
@@ -110,7 +119,7 @@ export function useVisionSession() {
|
|||||||
});
|
});
|
||||||
}, [videoRef, captureFrame, startObserving, stopObserving]);
|
}, [videoRef, captureFrame, startObserving, stopObserving]);
|
||||||
|
|
||||||
// WebSocket 连接成功后发送 config
|
// WebSocket 连接成功后发送 config + flush 待发消息
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (status === "connected") {
|
if (status === "connected") {
|
||||||
send({
|
send({
|
||||||
@@ -121,6 +130,27 @@ export function useVisionSession() {
|
|||||||
language: config.language,
|
language: config.language,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// flush 待发消息队列
|
||||||
|
const pending = pendingMessagesRef.current;
|
||||||
|
pendingMessagesRef.current = [];
|
||||||
|
for (const msg of pending) {
|
||||||
|
const frame = captureFrame();
|
||||||
|
send({
|
||||||
|
type: "query",
|
||||||
|
request_id: msg.requestId,
|
||||||
|
image: frame ? dataUrlToBase64(frame) : "",
|
||||||
|
audio: "",
|
||||||
|
text: msg.text,
|
||||||
|
});
|
||||||
|
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||||||
|
setMessages((prev) => [
|
||||||
|
...prev,
|
||||||
|
{ role: "user", content: msg.text, timestamp: Date.now() },
|
||||||
|
]);
|
||||||
|
historyRef.current.push({ role: "user", content: msg.text });
|
||||||
|
setIsProcessing(true);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送
|
}, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送
|
||||||
|
|
||||||
@@ -282,10 +312,13 @@ export function useVisionSession() {
|
|||||||
return unsub;
|
return unsub;
|
||||||
}, [getTTSPlayer]);
|
}, [getTTSPlayer]);
|
||||||
|
|
||||||
/** 启动会话 */
|
/** 启动视频通话(摄像头 + 麦克风 + VAD) */
|
||||||
const startSession = useCallback(async () => {
|
const startSession = useCallback(async () => {
|
||||||
// 1. 连接 WebSocket(必须)
|
// 1. 确保 WebSocket 已连接
|
||||||
connect();
|
if (statusRef.current !== "connected") {
|
||||||
|
connect();
|
||||||
|
// 等待连接完成(通过 status 变化触发后续流程,这里直接继续)
|
||||||
|
}
|
||||||
|
|
||||||
// 2. 尝试获取摄像头(可选)
|
// 2. 尝试获取摄像头(可选)
|
||||||
try {
|
try {
|
||||||
@@ -381,10 +414,23 @@ export function useVisionSession() {
|
|||||||
ttsPlayerRef.current?.stop();
|
ttsPlayerRef.current?.stop();
|
||||||
setIsAudioPlaying(false);
|
setIsAudioPlaying(false);
|
||||||
|
|
||||||
// 捕获当前摄像头画面
|
|
||||||
const frame = captureFrame();
|
|
||||||
|
|
||||||
const requestId = uuidv4();
|
const requestId = uuidv4();
|
||||||
|
|
||||||
|
// 未连接时:自动连接,消息加入待发队列
|
||||||
|
if (statusRef.current !== "connected") {
|
||||||
|
pendingMessagesRef.current.push({ text: text.trim(), requestId });
|
||||||
|
// 添加用户消息到 UI(立即反馈)
|
||||||
|
setMessages((prev) => [
|
||||||
|
...prev,
|
||||||
|
{ role: "user", content: text.trim(), timestamp: Date.now() }],
|
||||||
|
);
|
||||||
|
// 自动连接 WebSocket
|
||||||
|
connect();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 已连接:直接发送
|
||||||
|
const frame = captureFrame();
|
||||||
send({
|
send({
|
||||||
type: "query",
|
type: "query",
|
||||||
request_id: requestId,
|
request_id: requestId,
|
||||||
@@ -407,7 +453,7 @@ export function useVisionSession() {
|
|||||||
|
|
||||||
setIsProcessing(true);
|
setIsProcessing(true);
|
||||||
},
|
},
|
||||||
[captureFrame, send],
|
[captureFrame, send, connect],
|
||||||
);
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -33,12 +33,14 @@ export const enUS: TranslationMap = {
|
|||||||
"video.playing": "🔊 Playing...",
|
"video.playing": "🔊 Playing...",
|
||||||
"video.initVad": "Initializing voice detection...",
|
"video.initVad": "Initializing voice detection...",
|
||||||
"video.clickToStart": "Click the button below to start",
|
"video.clickToStart": "Click the button below to start",
|
||||||
|
"video.placeholder": "Type in the chat panel to start",
|
||||||
"video.cameraOff": "Camera is off",
|
"video.cameraOff": "Camera is off",
|
||||||
"video.cameraOff.hint": "You can type in the chat panel",
|
"video.cameraOff.hint": "You can type in the chat panel",
|
||||||
|
|
||||||
// Controls
|
// Controls
|
||||||
"controls.connecting": "Connecting...",
|
"controls.connecting": "Connecting...",
|
||||||
"controls.start": "🎙️ Start Session",
|
"controls.start": "🎙️ Start Session",
|
||||||
|
"controls.startVideo": "🎙️ Start Video Call",
|
||||||
"controls.cameraOff": "Turn off camera",
|
"controls.cameraOff": "Turn off camera",
|
||||||
"controls.cameraOn": "Turn on camera",
|
"controls.cameraOn": "Turn on camera",
|
||||||
"controls.micOff": "Turn off microphone",
|
"controls.micOff": "Turn off microphone",
|
||||||
@@ -52,8 +54,8 @@ export const enUS: TranslationMap = {
|
|||||||
"chat.title": "Chat",
|
"chat.title": "Chat",
|
||||||
"chat.mode.observation": "Observing",
|
"chat.mode.observation": "Observing",
|
||||||
"chat.reconnecting": "Connection lost, reconnecting...",
|
"chat.reconnecting": "Connection lost, reconnecting...",
|
||||||
"chat.empty.prompt": "Click the button below to start",
|
"chat.empty.prompt": "Type below to start chatting",
|
||||||
"chat.empty.hint": "Type or speak with AI after connecting",
|
"chat.empty.hint": "Type to chat with AI, or click the button on the left to start video",
|
||||||
"chat.welcome.prompt": "Type below to start chatting",
|
"chat.welcome.prompt": "Type below to start chatting",
|
||||||
"chat.welcome.hint": "Or enable microphone for voice chat",
|
"chat.welcome.hint": "Or enable microphone for voice chat",
|
||||||
"chat.userLabel": "You",
|
"chat.userLabel": "You",
|
||||||
|
|||||||
@@ -33,12 +33,14 @@ export const jaJP: TranslationMap = {
|
|||||||
"video.playing": "🔊 再生中...",
|
"video.playing": "🔊 再生中...",
|
||||||
"video.initVad": "音声検出を初期化中...",
|
"video.initVad": "音声検出を初期化中...",
|
||||||
"video.clickToStart": "下のボタンをクリックして開始",
|
"video.clickToStart": "下のボタンをクリックして開始",
|
||||||
|
"video.placeholder": "右側のチャットに入力して開始",
|
||||||
"video.cameraOff": "カメラがオフです",
|
"video.cameraOff": "カメラがオフです",
|
||||||
"video.cameraOff.hint": "右側のチャットでテキスト対話ができます",
|
"video.cameraOff.hint": "右側のチャットでテキスト対話ができます",
|
||||||
|
|
||||||
// Controls
|
// Controls
|
||||||
"controls.connecting": "接続中...",
|
"controls.connecting": "接続中...",
|
||||||
"controls.start": "🎙️ 対話を開始",
|
"controls.start": "🎙️ 対話を開始",
|
||||||
|
"controls.startVideo": "🎙️ ビデオ通話を開始",
|
||||||
"controls.cameraOff": "カメラをオフ",
|
"controls.cameraOff": "カメラをオフ",
|
||||||
"controls.cameraOn": "カメラをオン",
|
"controls.cameraOn": "カメラをオン",
|
||||||
"controls.micOff": "マイクをオフ",
|
"controls.micOff": "マイクをオフ",
|
||||||
@@ -52,8 +54,8 @@ export const jaJP: TranslationMap = {
|
|||||||
"chat.title": "チャット",
|
"chat.title": "チャット",
|
||||||
"chat.mode.observation": "観察モード",
|
"chat.mode.observation": "観察モード",
|
||||||
"chat.reconnecting": "接続が切断されました。再接続中...",
|
"chat.reconnecting": "接続が切断されました。再接続中...",
|
||||||
"chat.empty.prompt": "下のボタンをクリックして開始",
|
"chat.empty.prompt": "下にテキストを入力して対話を開始",
|
||||||
"chat.empty.hint": "接続後にテキストまたは音声でAIと対話できます",
|
"chat.empty.hint": "テキストでAIと対話、または左のボタンでビデオ通話を開始",
|
||||||
"chat.welcome.prompt": "下にテキストを入力して対話を開始",
|
"chat.welcome.prompt": "下にテキストを入力して対話を開始",
|
||||||
"chat.welcome.hint": "マイクを有効にして音声対話もできます",
|
"chat.welcome.hint": "マイクを有効にして音声対話もできます",
|
||||||
"chat.userLabel": "あなた",
|
"chat.userLabel": "あなた",
|
||||||
|
|||||||
@@ -33,12 +33,14 @@ export const zhCN: TranslationMap = {
|
|||||||
"video.playing": "🔊 正在播放...",
|
"video.playing": "🔊 正在播放...",
|
||||||
"video.initVad": "正在初始化语音检测...",
|
"video.initVad": "正在初始化语音检测...",
|
||||||
"video.clickToStart": "点击下方按钮开始对话",
|
"video.clickToStart": "点击下方按钮开始对话",
|
||||||
|
"video.placeholder": "在右侧聊天框输入即可开始对话",
|
||||||
"video.cameraOff": "摄像头未开启",
|
"video.cameraOff": "摄像头未开启",
|
||||||
"video.cameraOff.hint": "可在右侧聊天框打字对话",
|
"video.cameraOff.hint": "可在右侧聊天框打字对话",
|
||||||
|
|
||||||
// Controls
|
// Controls
|
||||||
"controls.connecting": "连接中...",
|
"controls.connecting": "连接中...",
|
||||||
"controls.start": "🎙️ 开始对话",
|
"controls.start": "🎙️ 开始对话",
|
||||||
|
"controls.startVideo": "🎙️ 开始视频通话",
|
||||||
"controls.cameraOff": "关闭摄像头",
|
"controls.cameraOff": "关闭摄像头",
|
||||||
"controls.cameraOn": "开启摄像头",
|
"controls.cameraOn": "开启摄像头",
|
||||||
"controls.micOff": "关闭麦克风",
|
"controls.micOff": "关闭麦克风",
|
||||||
@@ -52,8 +54,8 @@ export const zhCN: TranslationMap = {
|
|||||||
"chat.title": "对话",
|
"chat.title": "对话",
|
||||||
"chat.mode.observation": "观察模式",
|
"chat.mode.observation": "观察模式",
|
||||||
"chat.reconnecting": "连接已断开,正在重连...",
|
"chat.reconnecting": "连接已断开,正在重连...",
|
||||||
"chat.empty.prompt": "点击下方按钮开始对话",
|
"chat.empty.prompt": "在下方输入文字开始对话",
|
||||||
"chat.empty.hint": "连接后可打字或语音与 AI 交互",
|
"chat.empty.hint": "输入文字即可与 AI 交互,也可点击左侧按钮开启视频",
|
||||||
"chat.welcome.prompt": "在下方输入文字开始对话",
|
"chat.welcome.prompt": "在下方输入文字开始对话",
|
||||||
"chat.welcome.hint": "也可以开启麦克风用语音对话",
|
"chat.welcome.hint": "也可以开启麦克风用语音对话",
|
||||||
"chat.userLabel": "你",
|
"chat.userLabel": "你",
|
||||||
|
|||||||
Reference in New Issue
Block a user