Merge pull request '修复对话麦克风问题' (#64) from frontend-10 into develop

Reviewed-on: http://8.161.227.145:3000/XEngineers/CamTalk/pulls/64

frontend/src/App.css

@@ -473,6 +473,13 @@ body {
 /* ---- Chat Panel (覆盖子组件样式) ---- */
 
 .chat-panel {
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  min-height: 0;
+}
+
+.chat-panel__messages {
   flex: 1;
   overflow-y: auto;
   padding: 16px 24px;
@@ -560,6 +567,56 @@ body {
   letter-spacing: 0.01em;
 }
 
+/* ---- Chat Input ---- */
+
+.chat-input {
+  display: flex;
+  gap: 8px;
+  padding: 12px 24px 16px;
+  border-top: 1px solid var(--color-border);
+  background: var(--color-surface);
+}
+
+.chat-input__field {
+  flex: 1;
+  padding: 10px 14px;
+  border: 1px solid var(--color-border);
+  border-radius: var(--radius-sm);
+  background: var(--color-surface-2);
+  color: var(--color-text);
+  font-size: 0.82rem;
+  outline: none;
+  transition: border-color var(--transition-fast);
+}
+
+.chat-input__field::placeholder {
+  color: var(--color-text-muted);
+}
+
+.chat-input__field:focus {
+  border-color: var(--color-primary);
+}
+
+.chat-input__send {
+  padding: 10px 16px;
+  border: none;
+  border-radius: var(--radius-sm);
+  background: var(--color-primary);
+  color: white;
+  font-size: 0.9rem;
+  cursor: pointer;
+  transition: opacity var(--transition-fast);
+}
+
+.chat-input__send:hover:not(:disabled) {
+  opacity: 0.9;
+}
+
+.chat-input__send:disabled {
+  opacity: 0.4;
+  cursor: not-allowed;
+}
+
 /* ---- Streaming Cursor ---- */
 
 .cursor {

frontend/src/App.tsx

@@ -58,6 +58,7 @@ function App() {
     isMicOn,
     toggleCamera,
     toggleMic,
+    sendTextMessage,
   } = useVisionSession();
 
   const isConnected = connectionStatus === "connected";
@@ -233,6 +234,7 @@ function App() {
               messages={messages}
               currentReply={currentReply}
               connectionStatus={connectionStatus}
+              onSendText={sendTextMessage}
             />
           </div>
         </div>

frontend/src/components/ChatPanel/index.tsx

@@ -1,9 +1,9 @@
 // ============================================================
 // ChatPanel — 消息展示面板
-// 职责：渲染对话消息列表、流式光标、元数据、自动滚动
+// 职责：渲染对话消息列表、流式光标、元数据、自动滚动、文本输入
 // ============================================================
 
-import { useEffect, useRef } from "react";
+import { useEffect, useRef, useState } from "react";
 import type { ChatMessage } from "../../types";
 import type { ConnectionStatus } from "../../lib/websocket";
 
@@ -11,12 +11,16 @@ interface ChatPanelProps {
   messages: ChatMessage[];
   currentReply?: string;
   connectionStatus: ConnectionStatus;
+  onSendText?: (text: string) => void;
 }
 
-export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPanelProps) {
+export function ChatPanel({ messages, currentReply, connectionStatus, onSendText }: ChatPanelProps) {
   const bottomRef = useRef<HTMLDivElement>(null);
   const containerRef = useRef<HTMLDivElement>(null);
   const isAutoScroll = useRef(true);
+  const [inputText, setInputText] = useState("");
+
+  const isConnected = connectionStatus === "connected";
 
   // 用户上滚时暂停自动滚动，滚到底部时恢复
   useEffect(() => {
@@ -39,9 +43,17 @@ export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPane
     }
   }, [messages, currentReply]);
 
+  // 提交文本消息
+  const handleSubmit = (e: React.FormEvent) => {
+    e.preventDefault();
+    if (!inputText.trim() || !onSendText) return;
+    onSendText(inputText);
+    setInputText("");
+  };
+
   // 空状态
   if (messages.length === 0 && !currentReply) {
-    if (connectionStatus !== "connected") {
+    if (!isConnected) {
       return (
         <div className="chat-panel chat-panel--empty">
           <span className="chat-panel--empty-icon">💬</span>
@@ -60,35 +72,58 @@ export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPane
   }
 
   return (
-    <div className="chat-panel" ref={containerRef}>
-      {messages.map((msg, index) => (
-        <div key={index} className={`chat-message chat-message--${msg.role}`}>
-          <div className="chat-message__role">
-            {msg.role === "user" ? "你" : "AI"}
-          </div>
-          <div className="chat-message__content">{msg.content}</div>
-          {msg.role === "assistant" && msg.tokensUsed !== undefined && (
-            <div className="chat-message__meta">
-              {msg.tokensUsed} tokens
-              {msg.latencyMs !== undefined && ` · ${(msg.latencyMs / 1000).toFixed(1)}s`}
-              {msg.model && ` · ${msg.model}`}
+    <div className="chat-panel">
+      <div className="chat-panel__messages" ref={containerRef}>
+        {messages.map((msg, index) => (
+          <div key={index} className={`chat-message chat-message--${msg.role}`}>
+            <div className="chat-message__role">
+              {msg.role === "user" ? "你" : "AI"}
             </div>
-          )}
-        </div>
-      ))}
-
-      {/* 流式回复（尚未完成） */}
-      {currentReply && (
-        <div className="chat-message chat-message--assistant chat-message--streaming">
-          <div className="chat-message__role">AI</div>
-          <div className="chat-message__content">
-            {currentReply}
-            <span className="cursor">▌</span>
+            <div className="chat-message__content">{msg.content}</div>
+            {msg.role === "assistant" && msg.tokensUsed !== undefined && (
+              <div className="chat-message__meta">
+                {msg.tokensUsed} tokens
+                {msg.latencyMs !== undefined && ` · ${(msg.latencyMs / 1000).toFixed(1)}s`}
+                {msg.model && ` · ${msg.model}`}
+              </div>
+            )}
           </div>
-        </div>
-      )}
+        ))}
 
-      <div ref={bottomRef} />
+        {/* 流式回复（尚未完成） */}
+        {currentReply && (
+          <div className="chat-message chat-message--assistant chat-message--streaming">
+            <div className="chat-message__role">AI</div>
+            <div className="chat-message__content">
+              {currentReply}
+              <span className="cursor">▌</span>
+            </div>
+          </div>
+        )}
+
+        <div ref={bottomRef} />
+      </div>
+
+      {/* 文本输入框 */}
+      {isConnected && onSendText && (
+        <form className="chat-input" onSubmit={handleSubmit}>
+          <input
+            type="text"
+            className="chat-input__field"
+            placeholder="输入文字对话..."
+            value={inputText}
+            onChange={(e) => setInputText(e.target.value)}
+          />
+          <button
+            type="submit"
+            className="chat-input__send"
+            disabled={!inputText.trim()}
+            title="发送"
+          >
+            ➤
+          </button>
+        </form>
+      )}
     </div>
   );
 }

frontend/src/hooks/useVisionSession.ts

@@ -336,13 +336,17 @@ export function useVisionSession() {
   /** 麦克风开关 */
   const toggleMic = useCallback(async () => {
     if (isMicOn) {
+      await stopVAD();
       stopMic();
       setIsMicOn(false);
     } else {
       const micStream = await startMic();
-      setIsMicOn(!!micStream);
+      if (micStream) {
+        await startVAD(micStream);
+        setIsMicOn(true);
+      }
     }
-  }, [isMicOn, startMic, stopMic]);
+  }, [isMicOn, startMic, stopMic, startVAD, stopVAD]);
 
   /** 打断当前回复 */
   const interrupt = useCallback(() => {
@@ -363,6 +367,44 @@ export function useVisionSession() {
     setIsProcessing(false);
   }, [send, currentReply]);
 
+  /** 发送文本消息（手动输入） */
+  const sendTextMessage = useCallback(
+    (text: string) => {
+      if (!text.trim() || isProcessingRef.current) return;
+
+      // 停止上一轮的 TTS 播放
+      ttsPlayerRef.current?.stop();
+      setIsAudioPlaying(false);
+
+      // 捕获当前摄像头画面
+      const frame = captureFrame();
+
+      const requestId = uuidv4();
+      send({
+        type: "query",
+        request_id: requestId,
+        image: frame ? dataUrlToBase64(frame) : "",
+        audio: "", // 文本输入无音频
+        text: text.trim(),
+      });
+
+      // 更新请求统计
+      setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
+
+      // 添加用户消息
+      setMessages((prev) => [
+        ...prev,
+        { role: "user", content: text.trim(), timestamp: Date.now() },
+      ]);
+
+      // 记录到对话历史
+      historyRef.current.push({ role: "user", content: text.trim() });
+
+      setIsProcessing(true);
+    },
+    [captureFrame, send],
+  );
+
   return {
     messages,
     currentReply,
@@ -387,5 +429,6 @@ export function useVisionSession() {
     isMicOn,
     toggleCamera,
     toggleMic,
+    sendTextMessage,
   };
 }

frontend/src/types/index.ts

@@ -46,7 +46,8 @@ export interface QueryMessage {
   type: "query";
   request_id: string;
   image: string;       // Base64 JPEG（不含 data: 前缀）
-  audio: string;       // Base64 PCM 16kHz
+  audio: string;       // Base64 PCM 16kHz（文本输入时为空字符串）
+  text?: string;       // 用户手动输入的文本（有值时跳过 STT）
   mime_type?: string;  // 默认 "audio/pcm"
 }