Merge pull request 'feat：语音对话优化' (#106) from feat/onload into develop

Reviewed-on: http://8.161.227.145:3000/XEngineers/CamTalk/pulls/106
2026-06-14 19:55:02 +08:00
parent 8ff3c519ac c3118dd72a
commit 44f4f3ce1a
11 changed files with 156 additions and 52 deletions
--- a/backend/internal/ai/stt/mimo.go
+++ b/backend/internal/ai/stt/mimo.go
@@ -158,7 +158,8 @@ func (m *MiMoService) Recognize(ctx context.Context, audio []byte, opts Options)
 	}

 	if len(mResp.Choices) == 0 {
-		return "", fmt.Errorf("stt: mimo returned empty choices")
+		// MiMo 返回空结果，视为无法识别（非错误），返回空文本
+		return "", nil
 	}

 	text := strings.TrimSpace(mResp.Choices[0].Message.Content)
--- a/backend/internal/ai/stt/mimo_test.go
+++ b/backend/internal/ai/stt/mimo_test.go
@@ -103,9 +103,12 @@ func TestMiMoService_Recognize_EmptyChoices(t *testing.T) {
 	defer srv.Close()

 	wav := makeValidWAV([]byte{0x00, 0x00})
-	_, err := s.Recognize(context.Background(), wav, Options{})
-	if err == nil {
-		t.Fatal("expected error for empty choices")
+	text, err := s.Recognize(context.Background(), wav, Options{})
+	if err != nil {
+		t.Fatalf("unexpected error for empty choices: %v", err)
+	}
+	if text != "" {
+		t.Errorf("expected empty string for empty choices, got %q", text)
 	}
 }

--- a/backend/internal/orchestrator/pipeline.go
+++ b/backend/internal/orchestrator/pipeline.go
@@ -133,24 +133,48 @@ func (p *Pipeline) ProcessQuery(
 		}
 	} else {
 		// 语音模式：执行 STT 语音识别
-		log.Infow("开始语音识别", "request_id", req.RequestID)
+		log.Infow("开始语音识别", "request_id", req.RequestID, "audio_bytes", len(audio))
 		sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{
 			Encoding:   "pcm_s16le",
 			SampleRate: 16000,
 			Language:   sess.Config.Language,
 		})
 		if err != nil {
-			log.Errorw("语音识别失败", "error", err)
+			log.Errorw("语音识别失败", "error", err, "audio_bytes", len(audio))
 			sender.SendError(models.WsError{
 				Type:      "error",
 				RequestID: req.RequestID,
 				Code:      "STT_ERROR",
-				Message:   "语音识别失败",
+				Message:   "语音识别失败: " + err.Error(),
 			})
 			return err
 		}
 		userText = sttResult

+		// STT 返回空文本：未识别到语音，发送结果后直接返回（不调 LLM）
+		if strings.TrimSpace(userText) == "" {
+			log.Infow("语音识别结果为空", "request_id", req.RequestID)
+			userText = "（未识别到语音）"
+			if err := sender.SendSTTResult(models.WsSTTResult{
+				Type:      "stt_result",
+				RequestID: req.RequestID,
+				Text:      userText,
+				IsFinal:   true,
+			}); err != nil {
+				log.Errorw("发送 STT 结果失败", "error", err)
+			}
+			// 发送空的 llm_done 以结束本轮处理
+			latency := time.Since(startTime).Milliseconds()
+			_ = sender.SendLLMDone(models.WsLLMDone{
+				Type:      "llm_done",
+				RequestID: req.RequestID,
+				FullText:  "",
+				Model:     p.model,
+				LatencyMs: latency,
+			})
+			return nil
+		}
+
 		// 发送 STT 结果
 		if err := sender.SendSTTResult(models.WsSTTResult{
 			Type:      "stt_result",
--- a/frontend/src/App.css
+++ b/frontend/src/App.css
@@ -590,7 +590,6 @@ body {
  animation: pulse 1.5s cubic-bezier(0.25, 0.46, 0.45, 0.94) infinite;
 }

-.video-indicator--loading { color: var(--color-warning); }
 .video-indicator--audio { left: auto; right: 16px; color: var(--color-primary); }
 .video-indicator--error { color: var(--color-error); animation: none; }

@@ -651,26 +650,6 @@ body {
  animation: pulse 2s cubic-bezier(0.25, 0.46, 0.45, 0.94) infinite;
 }

-.video-overlay {
-  position: absolute;
-  inset: 0;
-  background: rgba(10, 10, 15, 0.4);
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  backdrop-filter: blur(3px);
-  -webkit-backdrop-filter: blur(3px);
-}
-
-.video-overlay__spinner {
-  width: 36px;
-  height: 36px;
-  border: 2.5px solid rgba(255, 255, 255, 0.15);
-  border-top-color: rgba(255, 255, 255, 0.8);
-  border-radius: 50%;
-  animation: spin 0.8s linear infinite;
-}
-
 /* ---- 视频控制栏 ---- */

 .video-controls {
@@ -998,6 +977,52 @@ body {
  border: 1px solid rgba(251, 191, 36, 0.15);
 }

+.system-message--info {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 8px;
+  background: rgba(59, 130, 246, 0.06);
+  color: var(--color-text-muted);
+  border: 1px solid rgba(59, 130, 246, 0.12);
+}
+
+/* ---- Typing Indicator（跳动点） ---- */
+
+.typing-indicator {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  padding: 2px 0;
+}
+
+.typing-indicator__dot {
+  width: 6px;
+  height: 6px;
+  border-radius: 50%;
+  background: var(--color-text-muted);
+  animation: typing-bounce 1.2s ease-in-out infinite;
+}
+
+.typing-indicator__dot:nth-child(2) {
+  animation-delay: 0.2s;
+}
+
+.typing-indicator__dot:nth-child(3) {
+  animation-delay: 0.4s;
+}
+
+@keyframes typing-bounce {
+  0%, 60%, 100% {
+    opacity: 0.3;
+    transform: translateY(0);
+  }
+  30% {
+    opacity: 1;
+    transform: translateY(-4px);
+  }
+}
+
 /* ---- Drawer (Config Panel) ---- */

 .drawer-overlay {
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -306,17 +306,9 @@ function AppContent() {
            {isAudioPlaying && config.ttsEnabled && (
              <div className="video-indicator video-indicator--audio">{tr("video.playing")}</div>
            )}
-            {isConnected && !isVADReady && !vadError && (
-              <div className="video-indicator video-indicator--loading">{tr("video.initVad")}</div>
-            )}
            {vadError && (
              <div className="video-indicator video-indicator--error">⚠️ {vadError}</div>
            )}
-            {isProcessing && (
-              <div className="video-overlay">
-                <div className="video-overlay__spinner" />
-              </div>
-            )}
            {!isConnected && !stream && (
              <div className="video-placeholder">
                <svg className="video-placeholder__icon" width="48" height="48" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.2" strokeLinecap="round" strokeLinejoin="round">
@@ -473,6 +465,9 @@ function AppContent() {
              messages={messages}
              currentReply={currentReply}
              connectionStatus={connectionStatus}
+              isProcessing={isProcessing}
+              isVADReady={isVADReady}
+              vadError={vadError}
              isMicOn={isMicOn}
              isSpeaking={isSpeaking}
              onSendText={sendTextMessage}
--- a/frontend/src/components/ChatPanel/index.tsx
+++ b/frontend/src/components/ChatPanel/index.tsx
@@ -13,6 +13,9 @@ interface ChatPanelProps {
  messages: ChatMessage[];
  currentReply?: string;
  connectionStatus: ConnectionStatus;
+  isProcessing?: boolean;
+  isVADReady?: boolean;
+  vadError?: string;
  isMicOn?: boolean;
  isSpeaking?: boolean;
  onSendText?: (text: string) => void;
@@ -34,6 +37,9 @@ export function ChatPanel({
  messages,
  currentReply,
  connectionStatus,
+  isProcessing,
+  isVADReady,
+  vadError,
  isMicOn,
  isSpeaking,
  onSendText,
@@ -144,6 +150,44 @@ export function ChatPanel({
          </div>
        )}

+        {/* 连接中状态 */}
+        {connectionStatus === "connecting" && (
+          <div className="system-message system-message--info">
+            <span className="typing-indicator">
+              <span className="typing-indicator__dot" />
+              <span className="typing-indicator__dot" />
+              <span className="typing-indicator__dot" />
+            </span>
+            <span>{t("chat.connecting")}</span>
+          </div>
+        )}
+
+        {/* VAD 初始化中 */}
+        {isConnected && isVADReady === false && !vadError && (
+          <div className="system-message system-message--info">
+            <span className="typing-indicator">
+              <span className="typing-indicator__dot" />
+              <span className="typing-indicator__dot" />
+              <span className="typing-indicator__dot" />
+            </span>
+            <span>{t("chat.vadInit")}</span>
+          </div>
+        )}
+
+        {/* AI 处理中（尚未开始流式输出） */}
+        {isProcessing && !currentReply && (
+          <div className="chat-message chat-message--assistant">
+            <div className="chat-message__role">AI</div>
+            <div className="chat-message__content">
+              <span className="typing-indicator">
+                <span className="typing-indicator__dot" />
+                <span className="typing-indicator__dot" />
+                <span className="typing-indicator__dot" />
+              </span>
+            </div>
+          </div>
+        )}
+
        <div ref={bottomRef} />
      </div>

--- a/frontend/src/components/EdgeProcessor/index.tsx
+++ b/frontend/src/components/EdgeProcessor/index.tsx
@@ -132,7 +132,7 @@ function getOffscreen(): HTMLCanvasElement {
 export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null {
  if (video.readyState < 2) return null;
  const canvas = getOffscreen();
-  const ctx = canvas.getContext("2d");
+  const ctx = canvas.getContext("2d", { willReadFrequently: true });
  if (!ctx) return null;
  ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT);
  return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data;
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -15,7 +15,7 @@ import { loadConfig, saveConfig } from "../lib/storage";
 import { useI18n } from "../lib/i18n";
 import { useCamera } from "../components/CameraManager";
 import { useMicrophone } from "../components/MicManager";
-import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
+import { useVAD, sampleFrame } from "../components/EdgeProcessor";
 import { useWebSocketManager } from "../components/WebSocketManager";
 import { useObservationMode } from "./useObservationMode";
 import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
@@ -194,24 +194,13 @@ export function useVisionSession(accessToken?: string | null) {
        ttsPlayerRef.current?.stop();
        setIsAudioPlaying(false);

+        // 尝试捕获图像帧（允许为 null，纯语音场景无需画面）
        const frame = captureFrame();
-        if (!frame) {
-          console.warn("[Session] 无法捕获图像帧");
-          return;
-        }

-        // 关键帧检测：与上一帧对比，相似度过高则跳过
+        // 更新帧参考（供观察模式使用，但不以此阻断语音请求）
        const video = videoRef.current;
        if (video) {
          const currentSample = sampleFrame(video);
-          if (currentSample && prevFrameRef.current) {
-            const { similarity } = compareFrames(prevFrameRef.current, currentSample);
-            if (similarity > 0.9) {
-              console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)})，跳过`);
-              prevFrameRef.current = currentSample;
-              return;
-            }
-          }
          if (currentSample) {
            prevFrameRef.current = currentSample;
          }
@@ -221,7 +210,7 @@ export function useVisionSession(accessToken?: string | null) {
        send({
          type: "query",
          request_id: requestId,
-          image: dataUrlToBase64(frame),
+          image: frame ? dataUrlToBase64(frame) : "",
          audio: encodeAudioToBase64(audio),
        });

@@ -304,6 +293,20 @@ export function useVisionSession(accessToken?: string | null) {
        case "error":
          console.error("[Session] 服务端错误:", msg.code, msg.message);
          showToast(getErrorMessage(msg.code, t), "error");
+          // STT 失败时：将"语音识别中..."占位消息替换为失败提示
+          if (msg.code === "STT_ERROR") {
+            setMessages((prev) => {
+              const updated = [...prev];
+              const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
+              if (lastUserIdx >= 0 && updated[lastUserIdx].content === t("session.recognizing")) {
+                updated[lastUserIdx] = {
+                  ...updated[lastUserIdx],
+                  content: t("session.sttFailed"),
+                };
+              }
+              return updated;
+            });
+          }
          setIsProcessing(false);
          break;
      }
--- a/frontend/src/lib/i18n/en-US.ts
+++ b/frontend/src/lib/i18n/en-US.ts
@@ -54,6 +54,8 @@ export const enUS: TranslationMap = {
  "chat.title": "Chat",
  "chat.mode.observation": "Observing",
  "chat.reconnecting": "Connection lost, reconnecting...",
+  "chat.connecting": "Connecting to server...",
+  "chat.vadInit": "Initializing voice detection...",
  "chat.empty.prompt": "Type below to start chatting",
  "chat.empty.hint": "Type to chat with AI, or click the button on the left to start video",
  "chat.welcome.prompt": "Type below to start chatting",
@@ -65,6 +67,7 @@ export const enUS: TranslationMap = {
  // Session messages
  "session.changeDetected": "👁️ Scene change detected",
  "session.recognizing": "(Recognizing speech...)",
+  "session.sttFailed": "(Speech recognition failed, please try again)",
  "session.noSpeech": "(No speech detected)",
  "session.interrupted": "(Interrupted)",

--- a/frontend/src/lib/i18n/ja-JP.ts
+++ b/frontend/src/lib/i18n/ja-JP.ts
@@ -54,6 +54,8 @@ export const jaJP: TranslationMap = {
  "chat.title": "チャット",
  "chat.mode.observation": "観察モード",
  "chat.reconnecting": "接続が切断されました。再接続中...",
+  "chat.connecting": "サーバーに接続中...",
+  "chat.vadInit": "音声検出を初期化中...",
  "chat.empty.prompt": "下にテキストを入力して対話を開始",
  "chat.empty.hint": "テキストでAIと対話、または左のボタンでビデオ通話を開始",
  "chat.welcome.prompt": "下にテキストを入力して対話を開始",
@@ -65,6 +67,7 @@ export const jaJP: TranslationMap = {
  // Session messages
  "session.changeDetected": "👁️ シーン変化を検出",
  "session.recognizing": "（音声認識中...）",
+  "session.sttFailed": "（音声認識に失敗しました。もう一度お試しください）",
  "session.noSpeech": "（音声が検出されませんでした）",
  "session.interrupted": "（中断済み）",

--- a/frontend/src/lib/i18n/zh-CN.ts
+++ b/frontend/src/lib/i18n/zh-CN.ts
@@ -54,6 +54,8 @@ export const zhCN: TranslationMap = {
  "chat.title": "对话",
  "chat.mode.observation": "观察模式",
  "chat.reconnecting": "连接已断开，正在重连...",
+  "chat.connecting": "正在连接服务...",
+  "chat.vadInit": "正在初始化语音检测...",
  "chat.empty.prompt": "在下方输入文字开始对话",
  "chat.empty.hint": "输入文字即可与 AI 交互，也可点击左侧按钮开启视频",
  "chat.welcome.prompt": "在下方输入文字开始对话",
@@ -65,6 +67,7 @@ export const zhCN: TranslationMap = {
  // Session messages
  "session.changeDetected": "👁️ 画面变化检测",
  "session.recognizing": "（语音识别中...）",
+  "session.sttFailed": "（语音识别失败，请重试）",
  "session.noSpeech": "（未识别到语音）",
  "session.interrupted": "（已打断）",