2026-06-13 13:35:48 +08:00
8 changed files with 357 additions and 31 deletions
--- a/frontend/src/App.css
+++ b/frontend/src/App.css
@@ -119,10 +119,10 @@ body {

 .chat-section {
  flex: 1;
-  overflow-y: auto;
  display: flex;
  flex-direction: column;
  gap: 12px;
+  overflow: hidden;
 }

 /* ---- Video Preview ---- */
@@ -157,6 +157,8 @@ body {
  display: flex;
  flex-direction: column;
  gap: 12px;
+  overflow-y: auto;
+  flex: 1;
 }

 .chat-panel--empty {
@@ -248,3 +250,111 @@ body {
 .btn--warning:hover {
  opacity: 0.9;
 }
+
+/* ---- Streaming Cursor ---- */
+
+.cursor {
+  display: inline;
+  animation: blink 0.8s step-end infinite;
+  color: var(--color-success);
+}
+
+@keyframes blink {
+  0%, 100% { opacity: 1; }
+  50% { opacity: 0; }
+}
+
+/* ---- Message Meta ---- */
+
+.chat-message__meta {
+  margin-top: 6px;
+  font-size: 0.75rem;
+  color: var(--color-text-muted);
+}
+
+/* ---- System Message ---- */
+
+.system-message {
+  text-align: center;
+  padding: 8px 16px;
+  border-radius: var(--radius);
+  font-size: 0.85rem;
+}
+
+.system-message--warning {
+  background: rgba(245, 158, 11, 0.15);
+  color: var(--color-warning);
+  border: 1px solid rgba(245, 158, 11, 0.3);
+}
+
+/* ---- Toast ---- */
+
+.toast-container {
+  position: fixed;
+  top: 16px;
+  right: 16px;
+  z-index: 1000;
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+}
+
+.toast {
+  padding: 10px 16px;
+  border-radius: var(--radius);
+  font-size: 0.85rem;
+  cursor: pointer;
+  animation: slideIn 0.3s ease-out;
+  max-width: 320px;
+}
+
+.toast--error {
+  background: rgba(239, 68, 68, 0.9);
+  color: white;
+}
+
+.toast--warning {
+  background: rgba(245, 158, 11, 0.9);
+  color: #000;
+}
+
+.toast--info {
+  background: rgba(37, 99, 235, 0.9);
+  color: white;
+}
+
+@keyframes slideIn {
+  from {
+    opacity: 0;
+    transform: translateX(20px);
+  }
+  to {
+    opacity: 1;
+    transform: translateX(0);
+  }
+}
+
+/* ---- Video Overlay ---- */
+
+.video-overlay {
+  position: absolute;
+  inset: 0;
+  background: rgba(0, 0, 0, 0.3);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  border-radius: var(--radius);
+}
+
+.video-overlay__spinner {
+  width: 32px;
+  height: 32px;
+  border: 3px solid rgba(255, 255, 255, 0.3);
+  border-top-color: white;
+  border-radius: 50%;
+  animation: spin 0.8s linear infinite;
+}
+
+@keyframes spin {
+  to { transform: rotate(360deg); }
+}
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -5,6 +5,7 @@
 import { useVisionSession } from "./hooks/useVisionSession";
 import { VideoPreview } from "./components/VideoPreview";
 import { ChatPanel } from "./components/ChatPanel";
+import { ToastContainer } from "./components/Toast";
 import "./App.css";

 function App() {
@@ -52,23 +53,31 @@ function App() {
              ⚠️ {vadError}
            </div>
          )}
+          {isProcessing && (
+            <div className="video-overlay">
+              <div className="video-overlay__spinner" />
+            </div>
+          )}
        </div>

        <div className="chat-section">
-          <ChatPanel messages={messages} />
-          {currentReply && (
-            <div className="chat-message chat-message--assistant chat-message--streaming">
-              <div className="chat-message__role">AI</div>
-              <div className="chat-message__content">{currentReply}</div>
+          {connectionStatus === "disconnected" && messages.length > 0 && (
+            <div className="system-message system-message--warning">
+              连接已断开，正在重连...
            </div>
          )}
+          <ChatPanel
+            messages={messages}
+            currentReply={currentReply}
+            connectionStatus={connectionStatus}
+          />
        </div>
      </main>

      <footer className="app-footer">
        {!isConnected ? (
          <button className="btn btn--primary" onClick={startSession}>
-            开始对话
+            {connectionStatus === "connecting" ? "连接中..." : "开始对话"}
          </button>
        ) : (
          <>
@@ -83,6 +92,8 @@ function App() {
          </>
        )}
      </footer>
+
+      <ToastContainer />
    </div>
  );
 }
--- a/frontend/src/components/ChatPanel/index.tsx
+++ b/frontend/src/components/ChatPanel/index.tsx
@@ -1,33 +1,90 @@
 // ============================================================
 // ChatPanel — 消息展示面板
-// 职责：渲染对话消息列表（用户提问 + AI 回复）
+// 职责：渲染对话消息列表、流式光标、元数据、自动滚动
 // ============================================================

+import { useEffect, useRef } from "react";
 import type { ChatMessage } from "../../types";
+import type { ConnectionStatus } from "../../lib/websocket";

 interface ChatPanelProps {
  messages: ChatMessage[];
+  currentReply?: string;
+  connectionStatus: ConnectionStatus;
 }

-export function ChatPanel({ messages }: ChatPanelProps) {
-  if (messages.length === 0) {
+export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPanelProps) {
+  const bottomRef = useRef<HTMLDivElement>(null);
+  const containerRef = useRef<HTMLDivElement>(null);
+  const isAutoScroll = useRef(true);
+
+  // 用户上滚时暂停自动滚动，滚到底部时恢复
+  useEffect(() => {
+    const container = containerRef.current;
+    if (!container) return;
+
+    const handleScroll = () => {
+      const { scrollTop, scrollHeight, clientHeight } = container;
+      isAutoScroll.current = scrollHeight - scrollTop - clientHeight < 60;
+    };
+
+    container.addEventListener("scroll", handleScroll);
+    return () => container.removeEventListener("scroll", handleScroll);
+  }, []);
+
+  // 新消息或流式更新时自动滚动
+  useEffect(() => {
+    if (isAutoScroll.current) {
+      bottomRef.current?.scrollIntoView({ behavior: "smooth" });
+    }
+  }, [messages, currentReply]);
+
+  // 空状态
+  if (messages.length === 0 && !currentReply) {
+    if (connectionStatus !== "connected") {
      return (
        <div className="chat-panel chat-panel--empty">
-        <p>开始对话：对着摄像头说话即可</p>
+          <p>点击下方按钮开始对话</p>
+        </div>
+      );
+    }
+    return (
+      <div className="chat-panel chat-panel--empty">
+        <p>对着摄像头说话即可</p>
      </div>
    );
  }

  return (
-    <div className="chat-panel">
+    <div className="chat-panel" ref={containerRef}>
      {messages.map((msg, index) => (
        <div key={index} className={`chat-message chat-message--${msg.role}`}>
          <div className="chat-message__role">
            {msg.role === "user" ? "你" : "AI"}
          </div>
          <div className="chat-message__content">{msg.content}</div>
+          {msg.role === "assistant" && msg.tokensUsed !== undefined && (
+            <div className="chat-message__meta">
+              {msg.tokensUsed} tokens
+              {msg.latencyMs !== undefined && ` · ${(msg.latencyMs / 1000).toFixed(1)}s`}
+              {msg.model && ` · ${msg.model}`}
+            </div>
+          )}
        </div>
      ))}
+
+      {/* 流式回复（尚未完成） */}
+      {currentReply && (
+        <div className="chat-message chat-message--assistant chat-message--streaming">
+          <div className="chat-message__role">AI</div>
+          <div className="chat-message__content">
+            {currentReply}
+            <span className="cursor">▌</span>
+          </div>
+        </div>
+      )}
+
+      <div ref={bottomRef} />
    </div>
  );
 }
--- a/frontend/src/components/Toast/index.tsx
+++ b/frontend/src/components/Toast/index.tsx
@@ -0,0 +1,51 @@
+// ============================================================
+// Toast — 轻量通知组件
+// 职责：3 秒自动消失的通知提示
+// ============================================================
+
+import { useCallback, useEffect, useState } from "react";
+import { registerToastSetter, type ToastItem } from "../../lib/toast";
+
+export type { ToastItem };
+
+/** Toast 容器组件，放在 App 根部 */
+export function ToastContainer() {
+  const [toasts, setToasts] = useState<ToastItem[]>([]);
+
+  // 注册全局 setter
+  useEffect(() => {
+    registerToastSetter(setToasts);
+    return () => registerToastSetter(null);
+  }, []);
+
+  const dismiss = useCallback((id: number) => {
+    setToasts((prev) => prev.filter((t) => t.id !== id));
+  }, []);
+
+  return (
+    <div className="toast-container">
+      {toasts.map((t) => (
+        <ToastItemView key={t.id} item={t} onDismiss={dismiss} />
+      ))}
+    </div>
+  );
+}
+
+function ToastItemView({
+  item,
+  onDismiss,
+}: {
+  item: ToastItem;
+  onDismiss: (id: number) => void;
+}) {
+  useEffect(() => {
+    const timer = setTimeout(() => onDismiss(item.id), 3000);
+    return () => clearTimeout(timer);
+  }, [item.id, onDismiss]);
+
+  return (
+    <div className={`toast toast--${item.type}`} onClick={() => onDismiss(item.id)}>
+      {item.message}
+    </div>
+  );
+}
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -4,10 +4,12 @@
 // 来源：docs/02-系统架构.md 核心 Hook 设计
 // ============================================================

-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import { v4 as uuidv4 } from "uuid";
 import { wsClient } from "../lib/websocket";
 import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
+import { getErrorMessage } from "../lib/errors";
+import { showToast } from "../lib/toast";
 import { useCamera } from "../components/CameraManager";
 import { useMicrophone } from "../components/MicManager";
 import { useVAD } from "../components/EdgeProcessor";
@@ -23,6 +25,12 @@ export function useVisionSession() {
  const { startMic, stopMic } = useMicrophone();
  const { status, connect, disconnect, send } = useWebSocketManager();

+  // 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题
+  const isProcessingRef = useRef(false);
+  useEffect(() => {
+    isProcessingRef.current = isProcessing;
+  }, [isProcessing]);
+
  // VAD：语音结束时自动发送 query
  const {
    isSpeaking,
@@ -33,6 +41,12 @@ export function useVisionSession() {
  } = useVAD({
    onSpeechEnd: useCallback(
      (audio: Float32Array) => {
+        // 处理中忽略，防止重复发送
+        if (isProcessingRef.current) {
+          console.warn("[Session] 正在处理中，忽略语音输入");
+          return;
+        }
+
        const frame = captureFrame();
        if (!frame) {
          console.warn("[Session] 无法捕获图像帧");
@@ -47,7 +61,7 @@ export function useVisionSession() {
          audio: encodeAudioToBase64(audio),
        });

-        // 添加用户消息（STT 结果到达后会更新文本）
+        // 添加用户消息（STT 流式结果会逐步更新文本）
        setMessages((prev) => [
          ...prev,
          { role: "user", content: "（语音识别中...）", timestamp: Date.now() },
@@ -62,43 +76,51 @@ export function useVisionSession() {
  useEffect(() => {
    const unsub = wsClient.onMessage((msg: ServerMessage) => {
      switch (msg.type) {
-        case "stt_result":
-          if (msg.is_final) {
+        case "stt_result": {
+          // 流式更新用户消息文本（包括中间结果和最终结果）
          setMessages((prev) => {
            const updated = [...prev];
            const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
            if (lastUserIdx >= 0) {
-                updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text };
+              updated[lastUserIdx] = {
+                ...updated[lastUserIdx],
+                content: msg.text || "（未识别到语音）",
+              };
            }
            return updated;
          });
-          }
          break;
+        }

        case "llm_chunk":
          setCurrentReply((prev) => prev + msg.delta);
          break;

-        case "llm_done":
+        case "llm_done": {
+          const done = msg as LLMDoneMessage;
          setMessages((prev) => [
            ...prev,
            {
              role: "assistant",
-              content: (msg as LLMDoneMessage).full_text,
+              content: done.full_text,
              timestamp: Date.now(),
-              tokensUsed: (msg as LLMDoneMessage).tokens_used?.total,
+              tokensUsed: done.tokens_used?.total,
+              latencyMs: done.latency_ms,
+              model: done.model,
            },
          ]);
          setCurrentReply("");
          setIsProcessing(false);
          break;
+        }

        case "tts_audio":
-          // TODO: 音频流播放
+          // TODO: 阶段 5 音频流播放
          break;

        case "error":
          console.error("[Session] 服务端错误:", msg.code, msg.message);
+          showToast(getErrorMessage(msg.code), "error");
          setIsProcessing(false);
          break;
      }
@@ -107,13 +129,21 @@ export function useVisionSession() {
    return unsub;
  }, []);

+  // 连接断开时显示提示
+  useEffect(() => {
+    if (status === "disconnected") {
+      // 只在非主动断开时提示（通过检查是否有活跃会话判断）
+      // 这里简单处理，由 App 层根据状态显示
+    }
+  }, [status]);
+
  /** 启动会话 */
  const startSession = useCallback(async () => {
    // 1. 获取摄像头和麦克风
    await startCamera();
    const micStream = await startMic();
    if (!micStream) {
-      console.error("[Session] 无法获取麦克风");
+      showToast("无法获取麦克风权限", "error");
      return;
    }

@@ -130,13 +160,25 @@ export function useVisionSession() {
    stopMic();
    stopCamera();
    disconnect();
+    // 清理所有对话状态
+    setMessages([]);
+    setCurrentReply("");
+    setIsProcessing(false);
  }, [stopVAD, stopMic, stopCamera, disconnect]);

  /** 打断当前回复 */
  const interrupt = useCallback(() => {
    send({ type: "interrupt" });
+    // 将未完成的流式内容保存为最终消息
+    if (currentReply) {
+      setMessages((prev) => [
+        ...prev,
+        { role: "assistant", content: currentReply + "（已打断）", timestamp: Date.now() },
+      ]);
+    }
+    setCurrentReply("");
    setIsProcessing(false);
-  }, [send]);
+  }, [send, currentReply]);

  return {
    messages,
--- a/frontend/src/lib/errors.ts
+++ b/frontend/src/lib/errors.ts
@@ -0,0 +1,24 @@
+// ============================================================
+// 错误码 → 用户友好文案映射
+// 来源：docs/03-接口文档.md §五 错误码
+// ============================================================
+
+import type { ErrorCode } from "../types";
+
+const ERROR_MESSAGES: Record<ErrorCode, string> = {
+  INVALID_MESSAGE: "消息格式异常，请重试",
+  SESSION_NOT_FOUND: "会话已过期，请重新连接",
+  RATE_LIMITED: "请求太频繁，请稍后再试",
+  IMAGE_TOO_LARGE: "图像过大，请降低分辨率",
+  AUDIO_TOO_SHORT: "语音太短，请再说一句",
+  LLM_TIMEOUT: "AI 响应超时，请重试",
+  LLM_ERROR: "AI 服务异常，请稍后重试",
+  STT_ERROR: "语音识别失败，请重试",
+  TTS_ERROR: "语音合成失败",
+  INTERNAL_ERROR: "服务内部错误，请重试",
+};
+
+/** 将错误码转为用户友好文案 */
+export function getErrorMessage(code: string): string {
+  return ERROR_MESSAGES[code as ErrorCode] ?? `未知错误: ${code}`;
+}
--- a/frontend/src/lib/toast.ts
+++ b/frontend/src/lib/toast.ts
@@ -0,0 +1,29 @@
+// ============================================================
+// Toast 全局状态管理
+// 与 Toast 组件配合使用
+// ============================================================
+
+export type ToastType = "error" | "warning" | "info";
+
+let nextId = 0;
+let _setToasts: React.Dispatch<React.SetStateAction<ToastItem[]>> | null = null;
+
+export interface ToastItem {
+  id: number;
+  type: ToastType;
+  message: string;
+}
+
+/** 注册 Toast state setter（由 ToastContainer 组件调用） */
+export function registerToastSetter(
+  setter: React.Dispatch<React.SetStateAction<ToastItem[]>> | null,
+) {
+  _setToasts = setter;
+}
+
+/** 显示一条 Toast */
+export function showToast(message: string, type: ToastType = "info") {
+  if (!_setToasts) return;
+  const id = nextId++;
+  _setToasts((prev) => [...prev, { id, type, message }]);
+}
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -25,6 +25,8 @@ export interface ChatMessage {
  imageUrl?: string;
  timestamp: number;
  tokensUsed?: number;
+  latencyMs?: number;
+  model?: string;
 }

 // ---- WebSocket 通用信封 ----