refactor: 删除实时分析/按需识别/纯聊天三种模式切换
三种模式仅为前端 UI 区分,后端无感知,实际使用中价值不大: - 纯聊天模式名不副实(VAD 仍附带摄像头帧) - 实时分析增加 API 成本且体验不佳 - 按需识别已是默认且最自然的交互方式 删除后保留按需识别行为:语音带图 + 手动识别按钮,UI 更简洁。
This commit is contained in:
@@ -644,12 +644,6 @@ body {
|
||||
transition: all var(--transition-fast);
|
||||
}
|
||||
|
||||
.ai-vision-indicator--active {
|
||||
color: var(--color-success);
|
||||
background: rgba(52, 211, 153, 0.15);
|
||||
animation: pulse 2s cubic-bezier(0.25, 0.46, 0.45, 0.94) infinite;
|
||||
}
|
||||
|
||||
/* ---- 视频控制栏 ---- */
|
||||
|
||||
.video-controls {
|
||||
@@ -750,16 +744,6 @@ body {
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.chat-panel-header__mode {
|
||||
font-size: 0.68rem;
|
||||
font-weight: 500;
|
||||
color: var(--color-success);
|
||||
background: rgba(52, 211, 153, 0.08);
|
||||
padding: 3px 10px;
|
||||
border-radius: 12px;
|
||||
letter-spacing: 0.02em;
|
||||
}
|
||||
|
||||
/* ---- Scenario Quick Selector (chat header) ---- */
|
||||
.scenario-selector {
|
||||
appearance: none;
|
||||
@@ -1473,38 +1457,6 @@ body {
|
||||
outline-offset: 1px;
|
||||
}
|
||||
|
||||
/* Mode switcher */
|
||||
.video-controls__mode {
|
||||
display: flex;
|
||||
gap: 2px;
|
||||
background: var(--color-surface-2);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 2px;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.mode-btn {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.7rem;
|
||||
font-weight: 500;
|
||||
padding: 5px 12px;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
transition: all var(--transition-fast);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.mode-btn:hover {
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.mode-btn--active {
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
}
|
||||
|
||||
/* ---- Text-only mode (video ended, chat preserved) ---- */
|
||||
|
||||
.video-controls__text-only {
|
||||
|
||||
@@ -24,9 +24,6 @@ import type { Theme } from "./types";
|
||||
import type { UserScenario } from "./lib/api/scenarios";
|
||||
import "./App.css";
|
||||
|
||||
/** AI 视觉模式 */
|
||||
type VisionMode = "realtime" | "ondemand" | "chat";
|
||||
|
||||
/** 内部组件,确保在 I18nContext.Provider 内部使用 hooks */
|
||||
function AppContent() {
|
||||
const { isAuthenticated, isLoading, user, logout, accessToken } = useAuth();
|
||||
@@ -36,7 +33,6 @@ function AppContent() {
|
||||
const [theme, setTheme] = useState<Theme>(loadTheme);
|
||||
const [elapsed, setElapsed] = useState(0);
|
||||
const [sidebarOpen, setSidebarOpen] = useState(false);
|
||||
const [visionMode, setVisionMode] = useState<VisionMode>("ondemand");
|
||||
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
// ---- 自建情景管理 ----
|
||||
@@ -91,8 +87,6 @@ function AppContent() {
|
||||
config,
|
||||
updateConfig,
|
||||
stats,
|
||||
mode,
|
||||
isObserving,
|
||||
startSession,
|
||||
stopSession,
|
||||
stopVideo,
|
||||
@@ -365,13 +359,13 @@ function AppContent() {
|
||||
<div className="detail-badge">HD</div>
|
||||
)}
|
||||
{/* AI 视觉状态指示 */}
|
||||
{isConnected && visionMode !== "chat" && (
|
||||
<div className={`ai-vision-indicator ${isObserving ? "ai-vision-indicator--active" : ""}`}>
|
||||
{isConnected && (
|
||||
<div className="ai-vision-indicator">
|
||||
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z" />
|
||||
<circle cx="12" cy="12" r="3" />
|
||||
</svg>
|
||||
<span>{isObserving ? tr("video.observing") : "AI"}</span>
|
||||
<span>AI</span>
|
||||
</div>
|
||||
)}
|
||||
{isSpeaking && (
|
||||
@@ -427,27 +421,6 @@ function AppContent() {
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
{/* 模式切换器 */}
|
||||
<div className="video-controls__mode">
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "realtime" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("realtime")}
|
||||
>
|
||||
{tr("controls.mode.realtime")}
|
||||
</button>
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "ondemand" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("ondemand")}
|
||||
>
|
||||
{tr("controls.mode.ondemand")}
|
||||
</button>
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "chat" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("chat")}
|
||||
>
|
||||
{tr("controls.mode.chat")}
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
) : isCameraOn ? (
|
||||
<>
|
||||
@@ -467,8 +440,7 @@ function AppContent() {
|
||||
>
|
||||
🎤
|
||||
</button>
|
||||
{/* 识别画面按钮(按需模式下显示) */}
|
||||
{visionMode === "ondemand" && (
|
||||
{/* 识别画面按钮 */}
|
||||
<button
|
||||
className="btn--recognize"
|
||||
onClick={handleRecognize}
|
||||
@@ -476,7 +448,6 @@ function AppContent() {
|
||||
>
|
||||
🔍 {tr("controls.recognize")}
|
||||
</button>
|
||||
)}
|
||||
{isProcessing && (
|
||||
<button className="btn btn--warning" onClick={interrupt}>
|
||||
{tr("controls.interrupt")}
|
||||
@@ -486,27 +457,6 @@ function AppContent() {
|
||||
{tr("controls.stopVideo")}
|
||||
</button>
|
||||
</div>
|
||||
{/* 通话态模式切换 */}
|
||||
<div className="video-controls__mode">
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "realtime" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("realtime")}
|
||||
>
|
||||
{tr("controls.mode.realtime")}
|
||||
</button>
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "ondemand" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("ondemand")}
|
||||
>
|
||||
{tr("controls.mode.ondemand")}
|
||||
</button>
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "chat" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("chat")}
|
||||
>
|
||||
{tr("controls.mode.chat")}
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
@@ -547,9 +497,6 @@ function AppContent() {
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
{isConnected && mode === "observation" && (
|
||||
<span className="chat-panel-header__mode">{tr("chat.mode.observation")}</span>
|
||||
)}
|
||||
{isConnected && stats.queryCount > 0 && (
|
||||
<span className="chat-panel-header__stats">
|
||||
{stats.queryCount} {tr("statusbar.recognitions")}
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
// ============================================================
|
||||
// useObservationMode — 观察模式 Hook
|
||||
// 职责:定时采帧 → 关键帧检测 → 画面变化时触发回调
|
||||
// 来源:docs/05-用户故事.md US-05(持续场景监控)
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { sampleFrame, compareFrames } from "../components/EdgeProcessor";
|
||||
|
||||
/** 画面变化显著阈值 */
|
||||
const CHANGE_THRESHOLD = 0.85;
|
||||
/** 采样间隔(ms) */
|
||||
const SAMPLE_INTERVAL = 5000;
|
||||
|
||||
export interface ObservationOptions {
|
||||
/** 画面变化回调,携带当前帧的 DataURL */
|
||||
onChange?: (frameDataUrl: string) => void;
|
||||
}
|
||||
|
||||
export function useObservationMode(options?: ObservationOptions) {
|
||||
const [isObserving, setIsObserving] = useState(false);
|
||||
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
|
||||
const optionsRef = useRef(options);
|
||||
|
||||
useEffect(() => {
|
||||
optionsRef.current = options;
|
||||
}, [options]);
|
||||
|
||||
/**
|
||||
* 启动观察模式
|
||||
* @param video 摄像头 video 元素
|
||||
* @param captureFrame 从 video 捕获 DataURL 的函数
|
||||
*/
|
||||
const startObserving = useCallback(
|
||||
(video: HTMLVideoElement | null, captureFrame: () => string | null) => {
|
||||
if (!video) return;
|
||||
|
||||
// 立即采一帧作为基准
|
||||
prevFrameRef.current = sampleFrame(video);
|
||||
|
||||
intervalRef.current = setInterval(() => {
|
||||
const current = sampleFrame(video);
|
||||
if (!current) return;
|
||||
|
||||
if (prevFrameRef.current) {
|
||||
const { similarity } = compareFrames(prevFrameRef.current, current);
|
||||
|
||||
if (similarity < CHANGE_THRESHOLD) {
|
||||
console.log(
|
||||
`[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`,
|
||||
);
|
||||
const frameDataUrl = captureFrame();
|
||||
if (frameDataUrl) {
|
||||
optionsRef.current?.onChange?.(frameDataUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prevFrameRef.current = current;
|
||||
}, SAMPLE_INTERVAL);
|
||||
|
||||
setIsObserving(true);
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
/** 停止观察模式 */
|
||||
const stopObserving = useCallback(() => {
|
||||
if (intervalRef.current) {
|
||||
clearInterval(intervalRef.current);
|
||||
intervalRef.current = null;
|
||||
}
|
||||
prevFrameRef.current = null;
|
||||
setIsObserving(false);
|
||||
}, []);
|
||||
|
||||
// 组件卸载时清理
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (intervalRef.current) {
|
||||
clearInterval(intervalRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
return { isObserving, startObserving, stopObserving };
|
||||
}
|
||||
@@ -17,11 +17,8 @@ import { useCamera } from "../components/CameraManager";
|
||||
import { useMicrophone } from "../components/MicManager";
|
||||
import { useVAD, sampleFrame } from "../components/EdgeProcessor";
|
||||
import { useWebSocketManager } from "../components/WebSocketManager";
|
||||
import { useObservationMode } from "./useObservationMode";
|
||||
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
||||
|
||||
export type SessionMode = "dialogue" | "observation";
|
||||
|
||||
export interface SessionStats {
|
||||
queryCount: number;
|
||||
totalTokens: number;
|
||||
@@ -35,7 +32,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
|
||||
const [config, setConfig] = useState<SessionConfig>(loadConfig);
|
||||
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
|
||||
const [mode, setMode] = useState<SessionMode>("dialogue");
|
||||
const [isCameraOn, setIsCameraOn] = useState(false);
|
||||
const [isMicOn, setIsMicOn] = useState(false);
|
||||
|
||||
@@ -78,49 +74,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
conversationIdRef.current = conversationId;
|
||||
}, [conversationId]);
|
||||
|
||||
// 观察模式:画面变化时自动发送 query
|
||||
const { isObserving, startObserving, stopObserving } = useObservationMode({
|
||||
onChange: useCallback(
|
||||
(frameDataUrl: string) => {
|
||||
if (isProcessingRef.current) return;
|
||||
|
||||
const requestId = uuidv4();
|
||||
send({
|
||||
type: "query",
|
||||
request_id: requestId,
|
||||
image: dataUrlToBase64(frameDataUrl),
|
||||
audio: "", // 观察模式无音频
|
||||
});
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: uuidv4(),
|
||||
role: "user",
|
||||
content: t("session.changeDetected"),
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
]);
|
||||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||||
setIsProcessing(true);
|
||||
},
|
||||
[send],
|
||||
),
|
||||
});
|
||||
|
||||
/** 切换对话/观察模式 */
|
||||
const toggleMode = useCallback(() => {
|
||||
setMode((prev) => {
|
||||
const next = prev === "dialogue" ? "observation" : "dialogue";
|
||||
if (next === "observation") {
|
||||
startObserving(videoRef.current, captureFrame);
|
||||
} else {
|
||||
stopObserving();
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [videoRef, captureFrame, startObserving, stopObserving]);
|
||||
|
||||
// WebSocket 连接成功后发送 config + flush 待发消息
|
||||
useEffect(() => {
|
||||
if (status === "connected") {
|
||||
@@ -343,8 +296,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
|
||||
/** 结束会话 */
|
||||
const stopSession = useCallback(async () => {
|
||||
stopObserving();
|
||||
setMode("dialogue");
|
||||
await stopVAD();
|
||||
stopMic();
|
||||
stopCamera();
|
||||
@@ -359,12 +310,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
prevFrameRef.current = null;
|
||||
setIsCameraOn(false);
|
||||
setIsMicOn(false);
|
||||
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
|
||||
}, [stopVAD, stopMic, stopCamera, disconnect]);
|
||||
|
||||
/** 结束视频,保留聊天和连接 */
|
||||
const stopVideo = useCallback(async () => {
|
||||
stopObserving();
|
||||
setMode("dialogue");
|
||||
await stopVAD();
|
||||
stopMic();
|
||||
stopCamera();
|
||||
@@ -375,7 +324,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
setIsCameraOn(false);
|
||||
setIsMicOn(false);
|
||||
// 不断开 WebSocket,不清空消息、统计
|
||||
}, [stopObserving, stopVAD, stopMic, stopCamera]);
|
||||
}, [stopVAD, stopMic, stopCamera]);
|
||||
|
||||
/** 摄像头开关 */
|
||||
const toggleCamera = useCallback(async () => {
|
||||
@@ -480,9 +429,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
config,
|
||||
updateConfig,
|
||||
stats,
|
||||
mode,
|
||||
isObserving,
|
||||
toggleMode,
|
||||
startSession,
|
||||
stopSession,
|
||||
stopVideo,
|
||||
|
||||
@@ -48,7 +48,6 @@ export const enUS: TranslationMap = {
|
||||
"scenario.interpreter.hint": "AI will translate your speech in real-time (Chinese-English), without explanations",
|
||||
|
||||
// Video indicators
|
||||
"video.observing": "👁️ Observing",
|
||||
"video.listening": "🎤 Listening...",
|
||||
"video.playing": "🔊 Playing...",
|
||||
"video.initVad": "Initializing voice detection...",
|
||||
@@ -67,8 +66,6 @@ export const enUS: TranslationMap = {
|
||||
"controls.cameraOn": "Turn on camera",
|
||||
"controls.micOff": "Turn off microphone",
|
||||
"controls.micOn": "Turn on microphone",
|
||||
"controls.observing": "👁️ Observing",
|
||||
"controls.observation": "👁️ Observe",
|
||||
"controls.interrupt": "⏹ Interrupt",
|
||||
"controls.stop": "End Session",
|
||||
"controls.stopVideo": "End Video",
|
||||
@@ -77,7 +74,6 @@ export const enUS: TranslationMap = {
|
||||
|
||||
// Chat panel
|
||||
"chat.title": "Chat",
|
||||
"chat.mode.observation": "Observing",
|
||||
"chat.reconnecting": "Connection lost, reconnecting...",
|
||||
"chat.connecting": "Connecting to server...",
|
||||
"chat.vadInit": "Initializing voice detection...",
|
||||
@@ -91,7 +87,6 @@ export const enUS: TranslationMap = {
|
||||
"chat.scenarioSwitched": "Switched to {name} mode",
|
||||
|
||||
// Session messages
|
||||
"session.changeDetected": "👁️ Scene change detected",
|
||||
"session.recognizing": "(Recognizing speech...)",
|
||||
"session.sttFailed": "(Speech recognition failed, please try again)",
|
||||
"session.noSpeech": "(No speech detected)",
|
||||
@@ -141,9 +136,6 @@ export const enUS: TranslationMap = {
|
||||
"controls.device.camera": "Camera",
|
||||
"controls.device.mic": "Microphone",
|
||||
"controls.device.default": "Default",
|
||||
"controls.mode.realtime": "Realtime",
|
||||
"controls.mode.ondemand": "On-demand",
|
||||
"controls.mode.chat": "Chat only",
|
||||
|
||||
// Status bar
|
||||
"statusbar.ready": "Ready · Select devices to start",
|
||||
|
||||
@@ -48,7 +48,6 @@ export const jaJP: TranslationMap = {
|
||||
"scenario.interpreter.hint": "AIがあなたの発言をリアルタイムで翻訳します(中日相互翻訳、説明なし)",
|
||||
|
||||
// Video indicators
|
||||
"video.observing": "👁️ 観察中",
|
||||
"video.listening": "🎤 聞き取り中...",
|
||||
"video.playing": "🔊 再生中...",
|
||||
"video.initVad": "音声検出を初期化中...",
|
||||
@@ -67,8 +66,6 @@ export const jaJP: TranslationMap = {
|
||||
"controls.cameraOn": "カメラをオン",
|
||||
"controls.micOff": "マイクをオフ",
|
||||
"controls.micOn": "マイクをオン",
|
||||
"controls.observing": "👁️ 観察中",
|
||||
"controls.observation": "👁️ 観察モード",
|
||||
"controls.interrupt": "⏹ 中断",
|
||||
"controls.stop": "対話を終了",
|
||||
"controls.stopVideo": "ビデオ終了",
|
||||
@@ -77,7 +74,6 @@ export const jaJP: TranslationMap = {
|
||||
|
||||
// Chat panel
|
||||
"chat.title": "チャット",
|
||||
"chat.mode.observation": "観察モード",
|
||||
"chat.reconnecting": "接続が切断されました。再接続中...",
|
||||
"chat.connecting": "サーバーに接続中...",
|
||||
"chat.vadInit": "音声検出を初期化中...",
|
||||
@@ -91,7 +87,6 @@ export const jaJP: TranslationMap = {
|
||||
"chat.scenarioSwitched": "{name} モードに切り替えました",
|
||||
|
||||
// Session messages
|
||||
"session.changeDetected": "👁️ シーン変化を検出",
|
||||
"session.recognizing": "(音声認識中...)",
|
||||
"session.sttFailed": "(音声認識に失敗しました。もう一度お試しください)",
|
||||
"session.noSpeech": "(音声が検出されませんでした)",
|
||||
@@ -141,9 +136,6 @@ export const jaJP: TranslationMap = {
|
||||
"controls.device.camera": "カメラ",
|
||||
"controls.device.mic": "マイク",
|
||||
"controls.device.default": "デフォルト",
|
||||
"controls.mode.realtime": "リアルタイム",
|
||||
"controls.mode.ondemand": "オンデマンド",
|
||||
"controls.mode.chat": "チャットのみ",
|
||||
|
||||
// Status bar
|
||||
"statusbar.ready": "準備完了 · デバイスを選択して開始",
|
||||
|
||||
@@ -48,7 +48,6 @@ export const zhCN: TranslationMap = {
|
||||
"scenario.interpreter.hint": "AI 会实时翻译你的话(中英互译),无解释评论",
|
||||
|
||||
// Video indicators
|
||||
"video.observing": "👁️ 观察中",
|
||||
"video.listening": "🎤 正在聆听...",
|
||||
"video.playing": "🔊 正在播放...",
|
||||
"video.initVad": "正在初始化语音检测...",
|
||||
@@ -67,8 +66,6 @@ export const zhCN: TranslationMap = {
|
||||
"controls.cameraOn": "开启摄像头",
|
||||
"controls.micOff": "关闭麦克风",
|
||||
"controls.micOn": "开启麦克风",
|
||||
"controls.observing": "👁️ 观察中",
|
||||
"controls.observation": "👁️ 观察模式",
|
||||
"controls.interrupt": "⏹ 打断",
|
||||
"controls.stop": "结束对话",
|
||||
"controls.stopVideo": "结束视频",
|
||||
@@ -77,7 +74,6 @@ export const zhCN: TranslationMap = {
|
||||
|
||||
// Chat panel
|
||||
"chat.title": "对话",
|
||||
"chat.mode.observation": "观察模式",
|
||||
"chat.reconnecting": "连接已断开,正在重连...",
|
||||
"chat.connecting": "正在连接服务...",
|
||||
"chat.vadInit": "正在初始化语音检测...",
|
||||
@@ -91,7 +87,6 @@ export const zhCN: TranslationMap = {
|
||||
"chat.scenarioSwitched": "已切换到 {name} 模式",
|
||||
|
||||
// Session messages
|
||||
"session.changeDetected": "👁️ 画面变化检测",
|
||||
"session.recognizing": "(语音识别中...)",
|
||||
"session.sttFailed": "(语音识别失败,请重试)",
|
||||
"session.noSpeech": "(未识别到语音)",
|
||||
@@ -141,9 +136,6 @@ export const zhCN: TranslationMap = {
|
||||
"controls.device.camera": "摄像头",
|
||||
"controls.device.mic": "麦克风",
|
||||
"controls.device.default": "默认",
|
||||
"controls.mode.realtime": "实时分析",
|
||||
"controls.mode.ondemand": "按需识别",
|
||||
"controls.mode.chat": "纯聊天",
|
||||
|
||||
// Status bar
|
||||
"statusbar.ready": "就绪 · 选择设备后开始通话",
|
||||
|
||||
Reference in New Issue
Block a user