feat: 实现观察模式,支持持续场景监控(US-05/US-10)

- 新增 useObservationMode Hook:定时 5s 采帧,similarity < 0.85 触发变化回调
- useVisionSession 新增 dialogue/observation 模式切换
- App Footer 加观察模式切换按钮(激活时蓝色脉冲)
- 视频区左上角显示'👁️ 观察中'绿色角标
- 观察模式下画面变化自动发送 query

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-06-13 14:46:42 +08:00
parent 2c6489b9a6
commit f5536005f6
4 changed files with 177 additions and 1 deletions

View File

@@ -16,8 +16,11 @@ import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import { useObservationMode } from "./useObservationMode";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
export type SessionMode = "dialogue" | "observation";
const MAX_HISTORY_ROUNDS = 10;
export interface SessionStats {
@@ -32,6 +35,7 @@ export function useVisionSession() {
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
const [config, setConfig] = useState<SessionConfig>(loadConfig);
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
const [mode, setMode] = useState<SessionMode>("dialogue");
// 上一帧采样数据(用于关键帧检测)
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
@@ -60,6 +64,48 @@ export function useVisionSession() {
isProcessingRef.current = isProcessing;
}, [isProcessing]);
// 观察模式:画面变化时自动发送 query
const { isObserving, startObserving, stopObserving } = useObservationMode({
onChange: useCallback(
(frameDataUrl: string) => {
if (isProcessingRef.current) return;
const requestId = uuidv4();
send({
type: "query",
request_id: requestId,
image: dataUrlToBase64(frameDataUrl),
audio: "", // 观察模式无音频
});
setMessages((prev) => [
...prev,
{
role: "user",
content: "👁️ 画面变化检测",
timestamp: Date.now(),
},
]);
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
setIsProcessing(true);
},
[send],
),
});
/** 切换对话/观察模式 */
const toggleMode = useCallback(() => {
setMode((prev) => {
const next = prev === "dialogue" ? "observation" : "dialogue";
if (next === "observation") {
startObserving(videoRef.current, captureFrame);
} else {
stopObserving();
}
return next;
});
}, [videoRef, captureFrame, startObserving, stopObserving]);
// WebSocket 连接成功后发送 config
useEffect(() => {
if (status === "connected") {
@@ -249,6 +295,8 @@ export function useVisionSession() {
/** 结束会话 */
const stopSession = useCallback(async () => {
stopObserving();
setMode("dialogue");
await stopVAD();
stopMic();
stopCamera();
@@ -262,7 +310,7 @@ export function useVisionSession() {
setStats({ queryCount: 0, totalTokens: 0 });
historyRef.current = [];
prevFrameRef.current = null;
}, [stopVAD, stopMic, stopCamera, disconnect]);
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
/** 打断当前回复 */
const interrupt = useCallback(() => {
@@ -297,6 +345,9 @@ export function useVisionSession() {
config,
updateConfig,
stats,
mode,
isObserving,
toggleMode,
startSession,
stopSession,
interrupt,