refactor: 删除实时分析/按需识别/纯聊天三种模式切换

三种模式仅为前端 UI 区分,后端无感知,实际使用中价值不大:
- 纯聊天模式名不副实(VAD 仍附带摄像头帧)
- 实时分析增加 API 成本且体验不佳
- 按需识别已是默认且最自然的交互方式

删除后保留按需识别行为:语音带图 + 手动识别按钮,UI 更简洁。
This commit is contained in:
2026-06-21 16:24:09 +08:00
parent 1079e22699
commit eea5c07eaa
7 changed files with 13 additions and 280 deletions

View File

@@ -644,12 +644,6 @@ body {
transition: all var(--transition-fast);
}
.ai-vision-indicator--active {
color: var(--color-success);
background: rgba(52, 211, 153, 0.15);
animation: pulse 2s cubic-bezier(0.25, 0.46, 0.45, 0.94) infinite;
}
/* ---- 视频控制栏 ---- */
.video-controls {
@@ -750,16 +744,6 @@ body {
gap: 10px;
}
.chat-panel-header__mode {
font-size: 0.68rem;
font-weight: 500;
color: var(--color-success);
background: rgba(52, 211, 153, 0.08);
padding: 3px 10px;
border-radius: 12px;
letter-spacing: 0.02em;
}
/* ---- Scenario Quick Selector (chat header) ---- */
.scenario-selector {
appearance: none;
@@ -1473,38 +1457,6 @@ body {
outline-offset: 1px;
}
/* Mode switcher */
.video-controls__mode {
display: flex;
gap: 2px;
background: var(--color-surface-2);
border-radius: var(--radius-sm);
padding: 2px;
justify-content: center;
}
.mode-btn {
background: none;
border: none;
color: var(--color-text-muted);
font-size: 0.7rem;
font-weight: 500;
padding: 5px 12px;
border-radius: 6px;
cursor: pointer;
transition: all var(--transition-fast);
white-space: nowrap;
}
.mode-btn:hover {
color: var(--color-text);
}
.mode-btn--active {
background: var(--color-primary);
color: white;
}
/* ---- Text-only mode (video ended, chat preserved) ---- */
.video-controls__text-only {

View File

@@ -24,9 +24,6 @@ import type { Theme } from "./types";
import type { UserScenario } from "./lib/api/scenarios";
import "./App.css";
/** AI 视觉模式 */
type VisionMode = "realtime" | "ondemand" | "chat";
/** 内部组件,确保在 I18nContext.Provider 内部使用 hooks */
function AppContent() {
const { isAuthenticated, isLoading, user, logout, accessToken } = useAuth();
@@ -36,7 +33,6 @@ function AppContent() {
const [theme, setTheme] = useState<Theme>(loadTheme);
const [elapsed, setElapsed] = useState(0);
const [sidebarOpen, setSidebarOpen] = useState(false);
const [visionMode, setVisionMode] = useState<VisionMode>("ondemand");
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
// ---- 自建情景管理 ----
@@ -91,8 +87,6 @@ function AppContent() {
config,
updateConfig,
stats,
mode,
isObserving,
startSession,
stopSession,
stopVideo,
@@ -365,13 +359,13 @@ function AppContent() {
<div className="detail-badge">HD</div>
)}
{/* AI 视觉状态指示 */}
{isConnected && visionMode !== "chat" && (
<div className={`ai-vision-indicator ${isObserving ? "ai-vision-indicator--active" : ""}`}>
{isConnected && (
<div className="ai-vision-indicator">
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
<path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z" />
<circle cx="12" cy="12" r="3" />
</svg>
<span>{isObserving ? tr("video.observing") : "AI"}</span>
<span>AI</span>
</div>
)}
{isSpeaking && (
@@ -427,27 +421,6 @@ function AppContent() {
</select>
</div>
</div>
{/* 模式切换器 */}
<div className="video-controls__mode">
<button
className={`mode-btn ${visionMode === "realtime" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("realtime")}
>
{tr("controls.mode.realtime")}
</button>
<button
className={`mode-btn ${visionMode === "ondemand" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("ondemand")}
>
{tr("controls.mode.ondemand")}
</button>
<button
className={`mode-btn ${visionMode === "chat" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("chat")}
>
{tr("controls.mode.chat")}
</button>
</div>
</>
) : isCameraOn ? (
<>
@@ -467,16 +440,14 @@ function AppContent() {
>
🎤
</button>
{/* 识别画面按钮(按需模式下显示) */}
{visionMode === "ondemand" && (
<button
className="btn--recognize"
onClick={handleRecognize}
disabled={isProcessing}
>
🔍 {tr("controls.recognize")}
</button>
)}
{/* 识别画面按钮 */}
<button
className="btn--recognize"
onClick={handleRecognize}
disabled={isProcessing}
>
🔍 {tr("controls.recognize")}
</button>
{isProcessing && (
<button className="btn btn--warning" onClick={interrupt}>
{tr("controls.interrupt")}
@@ -486,27 +457,6 @@ function AppContent() {
{tr("controls.stopVideo")}
</button>
</div>
{/* 通话态模式切换 */}
<div className="video-controls__mode">
<button
className={`mode-btn ${visionMode === "realtime" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("realtime")}
>
{tr("controls.mode.realtime")}
</button>
<button
className={`mode-btn ${visionMode === "ondemand" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("ondemand")}
>
{tr("controls.mode.ondemand")}
</button>
<button
className={`mode-btn ${visionMode === "chat" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("chat")}
>
{tr("controls.mode.chat")}
</button>
</div>
</>
) : (
<>
@@ -547,9 +497,6 @@ function AppContent() {
</option>
))}
</select>
{isConnected && mode === "observation" && (
<span className="chat-panel-header__mode">{tr("chat.mode.observation")}</span>
)}
{isConnected && stats.queryCount > 0 && (
<span className="chat-panel-header__stats">
{stats.queryCount} {tr("statusbar.recognitions")}

View File

@@ -1,88 +0,0 @@
// ============================================================
// useObservationMode — 观察模式 Hook
// 职责:定时采帧 → 关键帧检测 → 画面变化时触发回调
// 来源docs/05-用户故事.md US-05持续场景监控
// ============================================================
import { useCallback, useEffect, useRef, useState } from "react";
import { sampleFrame, compareFrames } from "../components/EdgeProcessor";
/** 画面变化显著阈值 */
const CHANGE_THRESHOLD = 0.85;
/** 采样间隔ms */
const SAMPLE_INTERVAL = 5000;
export interface ObservationOptions {
/** 画面变化回调,携带当前帧的 DataURL */
onChange?: (frameDataUrl: string) => void;
}
export function useObservationMode(options?: ObservationOptions) {
const [isObserving, setIsObserving] = useState(false);
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
const optionsRef = useRef(options);
useEffect(() => {
optionsRef.current = options;
}, [options]);
/**
* 启动观察模式
* @param video 摄像头 video 元素
* @param captureFrame 从 video 捕获 DataURL 的函数
*/
const startObserving = useCallback(
(video: HTMLVideoElement | null, captureFrame: () => string | null) => {
if (!video) return;
// 立即采一帧作为基准
prevFrameRef.current = sampleFrame(video);
intervalRef.current = setInterval(() => {
const current = sampleFrame(video);
if (!current) return;
if (prevFrameRef.current) {
const { similarity } = compareFrames(prevFrameRef.current, current);
if (similarity < CHANGE_THRESHOLD) {
console.log(
`[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`,
);
const frameDataUrl = captureFrame();
if (frameDataUrl) {
optionsRef.current?.onChange?.(frameDataUrl);
}
}
}
prevFrameRef.current = current;
}, SAMPLE_INTERVAL);
setIsObserving(true);
},
[],
);
/** 停止观察模式 */
const stopObserving = useCallback(() => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
prevFrameRef.current = null;
setIsObserving(false);
}, []);
// 组件卸载时清理
useEffect(() => {
return () => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
}
};
}, []);
return { isObserving, startObserving, stopObserving };
}

View File

@@ -17,11 +17,8 @@ import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD, sampleFrame } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import { useObservationMode } from "./useObservationMode";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
export type SessionMode = "dialogue" | "observation";
export interface SessionStats {
queryCount: number;
totalTokens: number;
@@ -35,7 +32,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
const [config, setConfig] = useState<SessionConfig>(loadConfig);
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
const [mode, setMode] = useState<SessionMode>("dialogue");
const [isCameraOn, setIsCameraOn] = useState(false);
const [isMicOn, setIsMicOn] = useState(false);
@@ -78,49 +74,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
conversationIdRef.current = conversationId;
}, [conversationId]);
// 观察模式:画面变化时自动发送 query
const { isObserving, startObserving, stopObserving } = useObservationMode({
onChange: useCallback(
(frameDataUrl: string) => {
if (isProcessingRef.current) return;
const requestId = uuidv4();
send({
type: "query",
request_id: requestId,
image: dataUrlToBase64(frameDataUrl),
audio: "", // 观察模式无音频
});
setMessages((prev) => [
...prev,
{
id: uuidv4(),
role: "user",
content: t("session.changeDetected"),
timestamp: Date.now(),
},
]);
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
setIsProcessing(true);
},
[send],
),
});
/** 切换对话/观察模式 */
const toggleMode = useCallback(() => {
setMode((prev) => {
const next = prev === "dialogue" ? "observation" : "dialogue";
if (next === "observation") {
startObserving(videoRef.current, captureFrame);
} else {
stopObserving();
}
return next;
});
}, [videoRef, captureFrame, startObserving, stopObserving]);
// WebSocket 连接成功后发送 config + flush 待发消息
useEffect(() => {
if (status === "connected") {
@@ -343,8 +296,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
/** 结束会话 */
const stopSession = useCallback(async () => {
stopObserving();
setMode("dialogue");
await stopVAD();
stopMic();
stopCamera();
@@ -359,12 +310,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
prevFrameRef.current = null;
setIsCameraOn(false);
setIsMicOn(false);
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
}, [stopVAD, stopMic, stopCamera, disconnect]);
/** 结束视频,保留聊天和连接 */
const stopVideo = useCallback(async () => {
stopObserving();
setMode("dialogue");
await stopVAD();
stopMic();
stopCamera();
@@ -375,7 +324,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
setIsCameraOn(false);
setIsMicOn(false);
// 不断开 WebSocket不清空消息、统计
}, [stopObserving, stopVAD, stopMic, stopCamera]);
}, [stopVAD, stopMic, stopCamera]);
/** 摄像头开关 */
const toggleCamera = useCallback(async () => {
@@ -480,9 +429,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
config,
updateConfig,
stats,
mode,
isObserving,
toggleMode,
startSession,
stopSession,
stopVideo,

View File

@@ -48,7 +48,6 @@ export const enUS: TranslationMap = {
"scenario.interpreter.hint": "AI will translate your speech in real-time (Chinese-English), without explanations",
// Video indicators
"video.observing": "👁️ Observing",
"video.listening": "🎤 Listening...",
"video.playing": "🔊 Playing...",
"video.initVad": "Initializing voice detection...",
@@ -67,8 +66,6 @@ export const enUS: TranslationMap = {
"controls.cameraOn": "Turn on camera",
"controls.micOff": "Turn off microphone",
"controls.micOn": "Turn on microphone",
"controls.observing": "👁️ Observing",
"controls.observation": "👁️ Observe",
"controls.interrupt": "⏹ Interrupt",
"controls.stop": "End Session",
"controls.stopVideo": "End Video",
@@ -77,7 +74,6 @@ export const enUS: TranslationMap = {
// Chat panel
"chat.title": "Chat",
"chat.mode.observation": "Observing",
"chat.reconnecting": "Connection lost, reconnecting...",
"chat.connecting": "Connecting to server...",
"chat.vadInit": "Initializing voice detection...",
@@ -91,7 +87,6 @@ export const enUS: TranslationMap = {
"chat.scenarioSwitched": "Switched to {name} mode",
// Session messages
"session.changeDetected": "👁️ Scene change detected",
"session.recognizing": "(Recognizing speech...)",
"session.sttFailed": "(Speech recognition failed, please try again)",
"session.noSpeech": "(No speech detected)",
@@ -141,9 +136,6 @@ export const enUS: TranslationMap = {
"controls.device.camera": "Camera",
"controls.device.mic": "Microphone",
"controls.device.default": "Default",
"controls.mode.realtime": "Realtime",
"controls.mode.ondemand": "On-demand",
"controls.mode.chat": "Chat only",
// Status bar
"statusbar.ready": "Ready · Select devices to start",

View File

@@ -48,7 +48,6 @@ export const jaJP: TranslationMap = {
"scenario.interpreter.hint": "AIがあなたの発言をリアルタイムで翻訳します中日相互翻訳、説明なし",
// Video indicators
"video.observing": "👁️ 観察中",
"video.listening": "🎤 聞き取り中...",
"video.playing": "🔊 再生中...",
"video.initVad": "音声検出を初期化中...",
@@ -67,8 +66,6 @@ export const jaJP: TranslationMap = {
"controls.cameraOn": "カメラをオン",
"controls.micOff": "マイクをオフ",
"controls.micOn": "マイクをオン",
"controls.observing": "👁️ 観察中",
"controls.observation": "👁️ 観察モード",
"controls.interrupt": "⏹ 中断",
"controls.stop": "対話を終了",
"controls.stopVideo": "ビデオ終了",
@@ -77,7 +74,6 @@ export const jaJP: TranslationMap = {
// Chat panel
"chat.title": "チャット",
"chat.mode.observation": "観察モード",
"chat.reconnecting": "接続が切断されました。再接続中...",
"chat.connecting": "サーバーに接続中...",
"chat.vadInit": "音声検出を初期化中...",
@@ -91,7 +87,6 @@ export const jaJP: TranslationMap = {
"chat.scenarioSwitched": "{name} モードに切り替えました",
// Session messages
"session.changeDetected": "👁️ シーン変化を検出",
"session.recognizing": "(音声認識中...",
"session.sttFailed": "(音声認識に失敗しました。もう一度お試しください)",
"session.noSpeech": "(音声が検出されませんでした)",
@@ -141,9 +136,6 @@ export const jaJP: TranslationMap = {
"controls.device.camera": "カメラ",
"controls.device.mic": "マイク",
"controls.device.default": "デフォルト",
"controls.mode.realtime": "リアルタイム",
"controls.mode.ondemand": "オンデマンド",
"controls.mode.chat": "チャットのみ",
// Status bar
"statusbar.ready": "準備完了 · デバイスを選択して開始",

View File

@@ -48,7 +48,6 @@ export const zhCN: TranslationMap = {
"scenario.interpreter.hint": "AI 会实时翻译你的话(中英互译),无解释评论",
// Video indicators
"video.observing": "👁️ 观察中",
"video.listening": "🎤 正在聆听...",
"video.playing": "🔊 正在播放...",
"video.initVad": "正在初始化语音检测...",
@@ -67,8 +66,6 @@ export const zhCN: TranslationMap = {
"controls.cameraOn": "开启摄像头",
"controls.micOff": "关闭麦克风",
"controls.micOn": "开启麦克风",
"controls.observing": "👁️ 观察中",
"controls.observation": "👁️ 观察模式",
"controls.interrupt": "⏹ 打断",
"controls.stop": "结束对话",
"controls.stopVideo": "结束视频",
@@ -77,7 +74,6 @@ export const zhCN: TranslationMap = {
// Chat panel
"chat.title": "对话",
"chat.mode.observation": "观察模式",
"chat.reconnecting": "连接已断开,正在重连...",
"chat.connecting": "正在连接服务...",
"chat.vadInit": "正在初始化语音检测...",
@@ -91,7 +87,6 @@ export const zhCN: TranslationMap = {
"chat.scenarioSwitched": "已切换到 {name} 模式",
// Session messages
"session.changeDetected": "👁️ 画面变化检测",
"session.recognizing": "(语音识别中...",
"session.sttFailed": "(语音识别失败,请重试)",
"session.noSpeech": "(未识别到语音)",
@@ -141,9 +136,6 @@ export const zhCN: TranslationMap = {
"controls.device.camera": "摄像头",
"controls.device.mic": "麦克风",
"controls.device.default": "默认",
"controls.mode.realtime": "实时分析",
"controls.mode.ondemand": "按需识别",
"controls.mode.chat": "纯聊天",
// Status bar
"statusbar.ready": "就绪 · 选择设备后开始通话",