diff --git a/backend/internal/ai/stt/mimo.go b/backend/internal/ai/stt/mimo.go
index 68e0c0d..a2da512 100644
--- a/backend/internal/ai/stt/mimo.go
+++ b/backend/internal/ai/stt/mimo.go
@@ -158,7 +158,8 @@ func (m *MiMoService) Recognize(ctx context.Context, audio []byte, opts Options)
}
if len(mResp.Choices) == 0 {
- return "", fmt.Errorf("stt: mimo returned empty choices")
+ // MiMo 返回空结果，视为无法识别（非错误），返回空文本
+ return "", nil
}
text := strings.TrimSpace(mResp.Choices[0].Message.Content)
diff --git a/backend/internal/ai/stt/mimo_test.go b/backend/internal/ai/stt/mimo_test.go
index 78ab038..e4bda09 100644
--- a/backend/internal/ai/stt/mimo_test.go
+++ b/backend/internal/ai/stt/mimo_test.go
@@ -103,9 +103,12 @@ func TestMiMoService_Recognize_EmptyChoices(t *testing.T) {
defer srv.Close()
wav := makeValidWAV([]byte{0x00, 0x00})
- _, err := s.Recognize(context.Background(), wav, Options{})
- if err == nil {
- t.Fatal("expected error for empty choices")
+ text, err := s.Recognize(context.Background(), wav, Options{})
+ if err != nil {
+ t.Fatalf("unexpected error for empty choices: %v", err)
+ }
+ if text != "" {
+ t.Errorf("expected empty string for empty choices, got %q", text)
}
}
diff --git a/backend/internal/orchestrator/pipeline.go b/backend/internal/orchestrator/pipeline.go
index 914b70e..87d8146 100644
--- a/backend/internal/orchestrator/pipeline.go
+++ b/backend/internal/orchestrator/pipeline.go
@@ -133,24 +133,48 @@ func (p *Pipeline) ProcessQuery(
}
} else {
// 语音模式：执行 STT 语音识别
- log.Infow("开始语音识别", "request_id", req.RequestID)
+ log.Infow("开始语音识别", "request_id", req.RequestID, "audio_bytes", len(audio))
sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{
Encoding: "pcm_s16le",
SampleRate: 16000,
Language: sess.Config.Language,
})
if err != nil {
- log.Errorw("语音识别失败", "error", err)
+ log.Errorw("语音识别失败", "error", err, "audio_bytes", len(audio))
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "STT_ERROR",
- Message: "语音识别失败",
+ Message: "语音识别失败: " + err.Error(),
})
return err
}
userText = sttResult
+ // STT 返回空文本：未识别到语音，发送结果后直接返回（不调 LLM）
+ if strings.TrimSpace(userText) == "" {
+ log.Infow("语音识别结果为空", "request_id", req.RequestID)
+ userText = "（未识别到语音）"
+ if err := sender.SendSTTResult(models.WsSTTResult{
+ Type: "stt_result",
+ RequestID: req.RequestID,
+ Text: userText,
+ IsFinal: true,
+ }); err != nil {
+ log.Errorw("发送 STT 结果失败", "error", err)
+ }
+ // 发送空的 llm_done 以结束本轮处理
+ latency := time.Since(startTime).Milliseconds()
+ _ = sender.SendLLMDone(models.WsLLMDone{
+ Type: "llm_done",
+ RequestID: req.RequestID,
+ FullText: "",
+ Model: p.model,
+ LatencyMs: latency,
+ })
+ return nil
+ }
+
// 发送 STT 结果
if err := sender.SendSTTResult(models.WsSTTResult{
Type: "stt_result",
diff --git a/frontend/src/App.css b/frontend/src/App.css
index 6d1d39c..7264e5b 100644
--- a/frontend/src/App.css
+++ b/frontend/src/App.css
@@ -590,7 +590,6 @@ body {
animation: pulse 1.5s cubic-bezier(0.25, 0.46, 0.45, 0.94) infinite;
}
-.video-indicator--loading { color: var(--color-warning); }
.video-indicator--audio { left: auto; right: 16px; color: var(--color-primary); }
.video-indicator--error { color: var(--color-error); animation: none; }
@@ -651,26 +650,6 @@ body {
animation: pulse 2s cubic-bezier(0.25, 0.46, 0.45, 0.94) infinite;
}
-.video-overlay {
- position: absolute;
- inset: 0;
- background: rgba(10, 10, 15, 0.4);
- display: flex;
- align-items: center;
- justify-content: center;
- backdrop-filter: blur(3px);
- -webkit-backdrop-filter: blur(3px);
-}
-
-.video-overlay__spinner {
- width: 36px;
- height: 36px;
- border: 2.5px solid rgba(255, 255, 255, 0.15);
- border-top-color: rgba(255, 255, 255, 0.8);
- border-radius: 50%;
- animation: spin 0.8s linear infinite;
-}
-
/* ---- 视频控制栏 ---- */
.video-controls {
@@ -998,6 +977,52 @@ body {
border: 1px solid rgba(251, 191, 36, 0.15);
}
+.system-message--info {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ gap: 8px;
+ background: rgba(59, 130, 246, 0.06);
+ color: var(--color-text-muted);
+ border: 1px solid rgba(59, 130, 246, 0.12);
+}
+
+/* ---- Typing Indicator（跳动点） ---- */
+
+.typing-indicator {
+ display: inline-flex;
+ align-items: center;
+ gap: 4px;
+ padding: 2px 0;
+}
+
+.typing-indicator__dot {
+ width: 6px;
+ height: 6px;
+ border-radius: 50%;
+ background: var(--color-text-muted);
+ animation: typing-bounce 1.2s ease-in-out infinite;
+}
+
+.typing-indicator__dot:nth-child(2) {
+ animation-delay: 0.2s;
+}
+
+.typing-indicator__dot:nth-child(3) {
+ animation-delay: 0.4s;
+}
+
+@keyframes typing-bounce {
+ 0%, 60%, 100% {
+ opacity: 0.3;
+ transform: translateY(0);
+ }
+ 30% {
+ opacity: 1;
+ transform: translateY(-4px);
+ }
+}
+
/* ---- Drawer (Config Panel) ---- */
.drawer-overlay {
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index d0e9473..b82bdca 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -306,17 +306,9 @@ function AppContent() {
{isAudioPlaying && config.ttsEnabled && (
{tr("video.playing")}
)}
- {isConnected && !isVADReady && !vadError && (
- {tr("video.initVad")}
- )}
{vadError && (
⚠️ {vadError}
)}
- {isProcessing && (
-
- )}
{!isConnected && !stream && (
)}
+ {/* 连接中状态 */}
+ {connectionStatus === "connecting" && (
+
+
+
+
+
+
+ {t("chat.connecting")}
+
+ )}
+
+ {/* VAD 初始化中 */}
+ {isConnected && isVADReady === false && !vadError && (
+
+
+
+
+
+
+ {t("chat.vadInit")}
+
+ )}
+
+ {/* AI 处理中（尚未开始流式输出） */}
+ {isProcessing && !currentReply && (
+
+ )}
+
diff --git a/frontend/src/components/EdgeProcessor/index.tsx b/frontend/src/components/EdgeProcessor/index.tsx
index 4fd9b26..b03e3f9 100644
--- a/frontend/src/components/EdgeProcessor/index.tsx
+++ b/frontend/src/components/EdgeProcessor/index.tsx
@@ -132,7 +132,7 @@ function getOffscreen(): HTMLCanvasElement {
export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null {
if (video.readyState < 2) return null;
const canvas = getOffscreen();
- const ctx = canvas.getContext("2d");
+ const ctx = canvas.getContext("2d", { willReadFrequently: true });
if (!ctx) return null;
ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT);
return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data;
diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts
index 0b68bf6..6b474a1 100644
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -15,7 +15,7 @@ import { loadConfig, saveConfig } from "../lib/storage";
import { useI18n } from "../lib/i18n";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
-import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
+import { useVAD, sampleFrame } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import { useObservationMode } from "./useObservationMode";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
@@ -194,24 +194,13 @@ export function useVisionSession(accessToken?: string | null) {
ttsPlayerRef.current?.stop();
setIsAudioPlaying(false);
+ // 尝试捕获图像帧（允许为 null，纯语音场景无需画面）
const frame = captureFrame();
- if (!frame) {
- console.warn("[Session] 无法捕获图像帧");
- return;
- }
- // 关键帧检测：与上一帧对比，相似度过高则跳过
+ // 更新帧参考（供观察模式使用，但不以此阻断语音请求）
const video = videoRef.current;
if (video) {
const currentSample = sampleFrame(video);
- if (currentSample && prevFrameRef.current) {
- const { similarity } = compareFrames(prevFrameRef.current, currentSample);
- if (similarity > 0.9) {
- console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)})，跳过`);
- prevFrameRef.current = currentSample;
- return;
- }
- }
if (currentSample) {
prevFrameRef.current = currentSample;
}
@@ -221,7 +210,7 @@ export function useVisionSession(accessToken?: string | null) {
send({
type: "query",
request_id: requestId,
- image: dataUrlToBase64(frame),
+ image: frame ? dataUrlToBase64(frame) : "",
audio: encodeAudioToBase64(audio),
});
@@ -304,6 +293,20 @@ export function useVisionSession(accessToken?: string | null) {
case "error":
console.error("[Session] 服务端错误:", msg.code, msg.message);
showToast(getErrorMessage(msg.code, t), "error");
+ // STT 失败时：将"语音识别中..."占位消息替换为失败提示
+ if (msg.code === "STT_ERROR") {
+ setMessages((prev) => {
+ const updated = [...prev];
+ const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
+ if (lastUserIdx >= 0 && updated[lastUserIdx].content === t("session.recognizing")) {
+ updated[lastUserIdx] = {
+ ...updated[lastUserIdx],
+ content: t("session.sttFailed"),
+ };
+ }
+ return updated;
+ });
+ }
setIsProcessing(false);
break;
}
diff --git a/frontend/src/lib/i18n/en-US.ts b/frontend/src/lib/i18n/en-US.ts
index 835c5fa..0ac2f10 100644
--- a/frontend/src/lib/i18n/en-US.ts
+++ b/frontend/src/lib/i18n/en-US.ts
@@ -54,6 +54,8 @@ export const enUS: TranslationMap = {
"chat.title": "Chat",
"chat.mode.observation": "Observing",
"chat.reconnecting": "Connection lost, reconnecting...",
+ "chat.connecting": "Connecting to server...",
+ "chat.vadInit": "Initializing voice detection...",
"chat.empty.prompt": "Type below to start chatting",
"chat.empty.hint": "Type to chat with AI, or click the button on the left to start video",
"chat.welcome.prompt": "Type below to start chatting",
@@ -65,6 +67,7 @@ export const enUS: TranslationMap = {
// Session messages
"session.changeDetected": "👁️ Scene change detected",
"session.recognizing": "(Recognizing speech...)",
+ "session.sttFailed": "(Speech recognition failed, please try again)",
"session.noSpeech": "(No speech detected)",
"session.interrupted": "(Interrupted)",
diff --git a/frontend/src/lib/i18n/ja-JP.ts b/frontend/src/lib/i18n/ja-JP.ts
index 74f4764..8eb152f 100644
--- a/frontend/src/lib/i18n/ja-JP.ts
+++ b/frontend/src/lib/i18n/ja-JP.ts
@@ -54,6 +54,8 @@ export const jaJP: TranslationMap = {
"chat.title": "チャット",
"chat.mode.observation": "観察モード",
"chat.reconnecting": "接続が切断されました。再接続中...",
+ "chat.connecting": "サーバーに接続中...",
+ "chat.vadInit": "音声検出を初期化中...",
"chat.empty.prompt": "下にテキストを入力して対話を開始",
"chat.empty.hint": "テキストでAIと対話、または左のボタンでビデオ通話を開始",
"chat.welcome.prompt": "下にテキストを入力して対話を開始",
@@ -65,6 +67,7 @@ export const jaJP: TranslationMap = {
// Session messages
"session.changeDetected": "👁️ シーン変化を検出",
"session.recognizing": "（音声認識中...）",
+ "session.sttFailed": "（音声認識に失敗しました。もう一度お試しください）",
"session.noSpeech": "（音声が検出されませんでした）",
"session.interrupted": "（中断済み）",
diff --git a/frontend/src/lib/i18n/zh-CN.ts b/frontend/src/lib/i18n/zh-CN.ts
index 8fc5716..a851b89 100644
--- a/frontend/src/lib/i18n/zh-CN.ts
+++ b/frontend/src/lib/i18n/zh-CN.ts
@@ -54,6 +54,8 @@ export const zhCN: TranslationMap = {
"chat.title": "对话",
"chat.mode.observation": "观察模式",
"chat.reconnecting": "连接已断开，正在重连...",
+ "chat.connecting": "正在连接服务...",
+ "chat.vadInit": "正在初始化语音检测...",
"chat.empty.prompt": "在下方输入文字开始对话",
"chat.empty.hint": "输入文字即可与 AI 交互，也可点击左侧按钮开启视频",
"chat.welcome.prompt": "在下方输入文字开始对话",
@@ -65,6 +67,7 @@ export const zhCN: TranslationMap = {
// Session messages
"session.changeDetected": "👁️ 画面变化检测",
"session.recognizing": "（语音识别中...）",
+ "session.sttFailed": "（语音识别失败，请重试）",
"session.noSpeech": "（未识别到语音）",
"session.interrupted": "（已打断）",