feat: 聊天框与视频功能解耦，输入文字即可开始对话

之前必须点击「开始对话」连接 WebSocket 后才能使用聊天框。现在聊天输入框始终可用，输入文字自动连接 WebSocket 并发送消息；左侧按钮改为「开始视频通话」，单独控制摄像头/麦克风。 - ChatPanel 移除 !isConnected early return，始终显示输入框 - useVisionSession 新增待发消息队列，sendTextMessage 支持自动连接 - startSession 改为仅负责视频（摄像头 + 麦克风 + VAD） - 修复 statusRef 在 useWebSocketManager 之前声明导致的 TDZ 错误 - 更新 i18n 翻译（新增 controls.startVideo、video.placeholder）
2026-06-14 15:07:39 +08:00
parent e95b1603c1
commit 15b1dd56fa
6 changed files with 73 additions and 31 deletions
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -175,7 +175,7 @@ function AppContent() {
                  <path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z" />
                  <circle cx="12" cy="13" r="4" />
                </svg>
-                <span>{tr("video.clickToStart")}</span>
+                <span>{tr("video.placeholder")}</span>
              </div>
            )}
            {isConnected && !isCameraOn && (
@@ -194,7 +194,7 @@ function AppContent() {
          <div className="video-controls">
            {!isConnected ? (
              <button className="btn btn--primary btn--lg" onClick={startSession}>
-                {connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.start")}
+                {connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")}
              </button>
            ) : (
              <div className="video-controls__row">
--- a/frontend/src/components/ChatPanel/index.tsx
+++ b/frontend/src/components/ChatPanel/index.tsx
@@ -53,17 +53,6 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
    setInputText("");
  };

-  // 未连接时的空状态
-  if (!isConnected) {
-    return (
-      <div className="chat-panel chat-panel--empty">
-        <span className="chat-panel--empty-icon">💬</span>
-        <p>{t("chat.empty.prompt")}</p>
-        <span className="chat-panel--empty-hint">{t("chat.empty.hint")}</span>
-      </div>
-    );
-  }
-
  return (
    <div className="chat-panel">
      <div className="chat-panel__messages" ref={containerRef}>
@@ -71,8 +60,8 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
        {messages.length === 0 && !currentReply && (
          <div className="chat-panel__welcome">
            <span className="chat-panel__welcome-icon">💬</span>
-            <p>{t("chat.welcome.prompt")}</p>
-            <span className="chat-panel__welcome-hint">{t("chat.welcome.hint")}</span>
+            <p>{isConnected ? t("chat.welcome.prompt") : t("chat.empty.prompt")}</p>
+            <span className="chat-panel__welcome-hint">{isConnected ? t("chat.welcome.hint") : t("chat.empty.hint")}</span>
          </div>
        )}

@@ -106,7 +95,7 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
        <div ref={bottomRef} />
      </div>

-      {/* 文本输入框 - 连接后始终显示 */}
+      {/* 文本输入框 - 始终显示 */}
      {onSendText && (
        <form className="chat-input" onSubmit={handleSubmit}>
          <input
@@ -115,11 +104,12 @@ export function ChatPanel({ messages, currentReply, connectionStatus, onSendText
            placeholder={t("chat.input.placeholder")}
            value={inputText}
            onChange={(e) => setInputText(e.target.value)}
+            disabled={connectionStatus === "connecting"}
          />
          <button
            type="submit"
            className="chat-input__send"
-            disabled={!inputText.trim()}
+            disabled={!inputText.trim() || connectionStatus === "connecting"}
            title={t("chat.send")}
          >
            ➤
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -47,6 +47,9 @@ export function useVisionSession() {
  // 对话历史（role + content），用于多轮上下文
  const historyRef = useRef<Array<{ role: string; content: string }>>([]);

+  // 待发消息队列（未连接时暂存，连接后自动发送）
+  const pendingMessagesRef = useRef<Array<{ text: string; requestId: string }>>([]);
+
  // TTS 播放器
  const ttsPlayerRef = useRef<TTSPlayer | null>(null);
  const getTTSPlayer = useCallback(() => {
@@ -68,6 +71,12 @@ export function useVisionSession() {
    isProcessingRef.current = isProcessing;
  }, [isProcessing]);

+  // 用 ref 跟踪连接状态，避免回调闭包问题
+  const statusRef = useRef(status);
+  useEffect(() => {
+    statusRef.current = status;
+  }, [status]);
+
  // 观察模式：画面变化时自动发送 query
  const { isObserving, startObserving, stopObserving } = useObservationMode({
    onChange: useCallback(
@@ -110,7 +119,7 @@ export function useVisionSession() {
    });
  }, [videoRef, captureFrame, startObserving, stopObserving]);

-  // WebSocket 连接成功后发送 config
+  // WebSocket 连接成功后发送 config + flush 待发消息
  useEffect(() => {
    if (status === "connected") {
      send({
@@ -121,6 +130,27 @@ export function useVisionSession() {
          language: config.language,
        },
      });
+
+      // flush 待发消息队列
+      const pending = pendingMessagesRef.current;
+      pendingMessagesRef.current = [];
+      for (const msg of pending) {
+        const frame = captureFrame();
+        send({
+          type: "query",
+          request_id: msg.requestId,
+          image: frame ? dataUrlToBase64(frame) : "",
+          audio: "",
+          text: msg.text,
+        });
+        setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
+        setMessages((prev) => [
+          ...prev,
+          { role: "user", content: msg.text, timestamp: Date.now() },
+        ]);
+        historyRef.current.push({ role: "user", content: msg.text });
+        setIsProcessing(true);
+      }
    }
  }, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送

@@ -282,10 +312,13 @@ export function useVisionSession() {
    return unsub;
  }, [getTTSPlayer]);

-  /** 启动会话 */
+  /** 启动视频通话（摄像头 + 麦克风 + VAD） */
  const startSession = useCallback(async () => {
-    // 1. 连接 WebSocket（必须）
+    // 1. 确保 WebSocket 已连接
+    if (statusRef.current !== "connected") {
      connect();
+      // 等待连接完成（通过 status 变化触发后续流程，这里直接继续）
+    }

    // 2. 尝试获取摄像头（可选）
    try {
@@ -381,10 +414,23 @@ export function useVisionSession() {
      ttsPlayerRef.current?.stop();
      setIsAudioPlaying(false);

-      // 捕获当前摄像头画面
-      const frame = captureFrame();
-
      const requestId = uuidv4();
+
+      // 未连接时：自动连接，消息加入待发队列
+      if (statusRef.current !== "connected") {
+        pendingMessagesRef.current.push({ text: text.trim(), requestId });
+        // 添加用户消息到 UI（立即反馈）
+        setMessages((prev) => [
+          ...prev,
+          { role: "user", content: text.trim(), timestamp: Date.now() }],
+        );
+        // 自动连接 WebSocket
+        connect();
+        return;
+      }
+
+      // 已连接：直接发送
+      const frame = captureFrame();
      send({
        type: "query",
        request_id: requestId,
@@ -407,7 +453,7 @@ export function useVisionSession() {

      setIsProcessing(true);
    },
-    [captureFrame, send],
+    [captureFrame, send, connect],
  );

  return {
--- a/frontend/src/lib/i18n/en-US.ts
+++ b/frontend/src/lib/i18n/en-US.ts
@@ -33,12 +33,14 @@ export const enUS: TranslationMap = {
  "video.playing": "🔊 Playing...",
  "video.initVad": "Initializing voice detection...",
  "video.clickToStart": "Click the button below to start",
+  "video.placeholder": "Type in the chat panel to start",
  "video.cameraOff": "Camera is off",
  "video.cameraOff.hint": "You can type in the chat panel",

  // Controls
  "controls.connecting": "Connecting...",
  "controls.start": "🎙️ Start Session",
+  "controls.startVideo": "🎙️ Start Video Call",
  "controls.cameraOff": "Turn off camera",
  "controls.cameraOn": "Turn on camera",
  "controls.micOff": "Turn off microphone",
@@ -52,8 +54,8 @@ export const enUS: TranslationMap = {
  "chat.title": "Chat",
  "chat.mode.observation": "Observing",
  "chat.reconnecting": "Connection lost, reconnecting...",
-  "chat.empty.prompt": "Click the button below to start",
-  "chat.empty.hint": "Type or speak with AI after connecting",
+  "chat.empty.prompt": "Type below to start chatting",
+  "chat.empty.hint": "Type to chat with AI, or click the button on the left to start video",
  "chat.welcome.prompt": "Type below to start chatting",
  "chat.welcome.hint": "Or enable microphone for voice chat",
  "chat.userLabel": "You",
--- a/frontend/src/lib/i18n/ja-JP.ts
+++ b/frontend/src/lib/i18n/ja-JP.ts
@@ -33,12 +33,14 @@ export const jaJP: TranslationMap = {
  "video.playing": "🔊 再生中...",
  "video.initVad": "音声検出を初期化中...",
  "video.clickToStart": "下のボタンをクリックして開始",
+  "video.placeholder": "右側のチャットに入力して開始",
  "video.cameraOff": "カメラがオフです",
  "video.cameraOff.hint": "右側のチャットでテキスト対話ができます",

  // Controls
  "controls.connecting": "接続中...",
  "controls.start": "🎙️ 対話を開始",
+  "controls.startVideo": "🎙️ ビデオ通話を開始",
  "controls.cameraOff": "カメラをオフ",
  "controls.cameraOn": "カメラをオン",
  "controls.micOff": "マイクをオフ",
@@ -52,8 +54,8 @@ export const jaJP: TranslationMap = {
  "chat.title": "チャット",
  "chat.mode.observation": "観察モード",
  "chat.reconnecting": "接続が切断されました。再接続中...",
-  "chat.empty.prompt": "下のボタンをクリックして開始",
-  "chat.empty.hint": "接続後にテキストまたは音声でAIと対話できます",
+  "chat.empty.prompt": "下にテキストを入力して対話を開始",
+  "chat.empty.hint": "テキストでAIと対話、または左のボタンでビデオ通話を開始",
  "chat.welcome.prompt": "下にテキストを入力して対話を開始",
  "chat.welcome.hint": "マイクを有効にして音声対話もできます",
  "chat.userLabel": "あなた",
--- a/frontend/src/lib/i18n/zh-CN.ts
+++ b/frontend/src/lib/i18n/zh-CN.ts
@@ -33,12 +33,14 @@ export const zhCN: TranslationMap = {
  "video.playing": "🔊 正在播放...",
  "video.initVad": "正在初始化语音检测...",
  "video.clickToStart": "点击下方按钮开始对话",
+  "video.placeholder": "在右侧聊天框输入即可开始对话",
  "video.cameraOff": "摄像头未开启",
  "video.cameraOff.hint": "可在右侧聊天框打字对话",

  // Controls
  "controls.connecting": "连接中...",
  "controls.start": "🎙️ 开始对话",
+  "controls.startVideo": "🎙️ 开始视频通话",
  "controls.cameraOff": "关闭摄像头",
  "controls.cameraOn": "开启摄像头",
  "controls.micOff": "关闭麦克风",
@@ -52,8 +54,8 @@ export const zhCN: TranslationMap = {
  "chat.title": "对话",
  "chat.mode.observation": "观察模式",
  "chat.reconnecting": "连接已断开，正在重连...",
-  "chat.empty.prompt": "点击下方按钮开始对话",
-  "chat.empty.hint": "连接后可打字或语音与 AI 交互",
+  "chat.empty.prompt": "在下方输入文字开始对话",
+  "chat.empty.hint": "输入文字即可与 AI 交互，也可点击左侧按钮开启视频",
  "chat.welcome.prompt": "在下方输入文字开始对话",
  "chat.welcome.hint": "也可以开启麦克风用语音对话",
  "chat.userLabel": "你",