diff --git a/frontend/src/App.css b/frontend/src/App.css
index 431c149..ca328d8 100644
--- a/frontend/src/App.css
+++ b/frontend/src/App.css
@@ -287,6 +287,18 @@ body {
opacity: 0.9;
}
+.btn--active {
+ background: var(--color-primary);
+ color: white;
+ box-shadow: 0 0 8px rgba(37, 99, 235, 0.5);
+ animation: pulseBtn 2s ease-in-out infinite;
+}
+
+@keyframes pulseBtn {
+ 0%, 100% { box-shadow: 0 0 8px rgba(37, 99, 235, 0.5); }
+ 50% { box-shadow: 0 0 16px rgba(37, 99, 235, 0.8); }
+}
+
/* ---- Streaming Cursor ---- */
.cursor {
@@ -458,3 +470,16 @@ body {
font-weight: 700;
letter-spacing: 0.5px;
}
+
+.observation-badge {
+ position: absolute;
+ top: 8px;
+ left: 8px;
+ background: rgba(34, 197, 94, 0.8);
+ color: white;
+ padding: 2px 8px;
+ border-radius: 4px;
+ font-size: 0.7rem;
+ font-weight: 700;
+ animation: pulse 2s ease-in-out infinite;
+}
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index dcb8a6f..b23082c 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -27,6 +27,9 @@ function App() {
config,
updateConfig,
stats,
+ mode,
+ isObserving,
+ toggleMode,
startSession,
stopSession,
interrupt,
@@ -72,6 +75,9 @@ function App() {
{config.detailLevel === "high" && (
HD
)}
+ {isObserving && (
+ 👁️ 观察中
+ )}
{isSpeaking && 🎤 正在聆听...
}
{isAudioPlaying && config.ttsEnabled && (
🔊 正在播放...
@@ -121,6 +127,12 @@ function App() {
) : (
<>
+
diff --git a/frontend/src/hooks/useObservationMode.ts b/frontend/src/hooks/useObservationMode.ts
new file mode 100644
index 0000000..e49eb62
--- /dev/null
+++ b/frontend/src/hooks/useObservationMode.ts
@@ -0,0 +1,88 @@
+// ============================================================
+// useObservationMode — 观察模式 Hook
+// 职责：定时采帧 → 关键帧检测 → 画面变化时触发回调
+// 来源：docs/05-用户故事.md US-05（持续场景监控）
+// ============================================================
+
+import { useCallback, useEffect, useRef, useState } from "react";
+import { sampleFrame, compareFrames } from "../components/EdgeProcessor";
+
+/** 画面变化显著阈值 */
+const CHANGE_THRESHOLD = 0.85;
+/** 采样间隔（ms） */
+const SAMPLE_INTERVAL = 5000;
+
+export interface ObservationOptions {
+ /** 画面变化回调，携带当前帧的 DataURL */
+ onChange?: (frameDataUrl: string) => void;
+}
+
+export function useObservationMode(options?: ObservationOptions) {
+ const [isObserving, setIsObserving] = useState(false);
+ const intervalRef = useRef | null>(null);
+ const prevFrameRef = useRef(null);
+ const optionsRef = useRef(options);
+
+ useEffect(() => {
+ optionsRef.current = options;
+ }, [options]);
+
+ /**
+ * 启动观察模式
+ * @param video 摄像头 video 元素
+ * @param captureFrame 从 video 捕获 DataURL 的函数
+ */
+ const startObserving = useCallback(
+ (video: HTMLVideoElement | null, captureFrame: () => string | null) => {
+ if (!video) return;
+
+ // 立即采一帧作为基准
+ prevFrameRef.current = sampleFrame(video);
+
+ intervalRef.current = setInterval(() => {
+ const current = sampleFrame(video);
+ if (!current) return;
+
+ if (prevFrameRef.current) {
+ const { similarity } = compareFrames(prevFrameRef.current, current);
+
+ if (similarity < CHANGE_THRESHOLD) {
+ console.log(
+ `[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`,
+ );
+ const frameDataUrl = captureFrame();
+ if (frameDataUrl) {
+ optionsRef.current?.onChange?.(frameDataUrl);
+ }
+ }
+ }
+
+ prevFrameRef.current = current;
+ }, SAMPLE_INTERVAL);
+
+ setIsObserving(true);
+ },
+ [],
+ );
+
+ /** 停止观察模式 */
+ const stopObserving = useCallback(() => {
+ if (intervalRef.current) {
+ clearInterval(intervalRef.current);
+ intervalRef.current = null;
+ }
+ prevFrameRef.current = null;
+ setIsObserving(false);
+ }, []);
+
+ // 组件卸载时清理
+ useEffect(() => {
+ return () => {
+ if (intervalRef.current) {
+ clearInterval(intervalRef.current);
+ }
+ };
+ }, []);
+
+ return { isObserving, startObserving, stopObserving };
+}
diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts
index a37389f..fd4c005 100644
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -16,8 +16,11 @@ import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
+import { useObservationMode } from "./useObservationMode";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
+export type SessionMode = "dialogue" | "observation";
+
const MAX_HISTORY_ROUNDS = 10;
export interface SessionStats {
@@ -32,6 +35,7 @@ export function useVisionSession() {
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
const [config, setConfig] = useState(loadConfig);
const [stats, setStats] = useState({ queryCount: 0, totalTokens: 0 });
+ const [mode, setMode] = useState("dialogue");
// 上一帧采样数据（用于关键帧检测）
const prevFrameRef = useRef(null);
@@ -60,6 +64,48 @@ export function useVisionSession() {
isProcessingRef.current = isProcessing;
}, [isProcessing]);
+ // 观察模式：画面变化时自动发送 query
+ const { isObserving, startObserving, stopObserving } = useObservationMode({
+ onChange: useCallback(
+ (frameDataUrl: string) => {
+ if (isProcessingRef.current) return;
+
+ const requestId = uuidv4();
+ send({
+ type: "query",
+ request_id: requestId,
+ image: dataUrlToBase64(frameDataUrl),
+ audio: "", // 观察模式无音频
+ });
+
+ setMessages((prev) => [
+ ...prev,
+ {
+ role: "user",
+ content: "👁️ 画面变化检测",
+ timestamp: Date.now(),
+ },
+ ]);
+ setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
+ setIsProcessing(true);
+ },
+ [send],
+ ),
+ });
+
+ /** 切换对话/观察模式 */
+ const toggleMode = useCallback(() => {
+ setMode((prev) => {
+ const next = prev === "dialogue" ? "observation" : "dialogue";
+ if (next === "observation") {
+ startObserving(videoRef.current, captureFrame);
+ } else {
+ stopObserving();
+ }
+ return next;
+ });
+ }, [videoRef, captureFrame, startObserving, stopObserving]);
+
// WebSocket 连接成功后发送 config
useEffect(() => {
if (status === "connected") {
@@ -249,6 +295,8 @@ export function useVisionSession() {
/** 结束会话 */
const stopSession = useCallback(async () => {
+ stopObserving();
+ setMode("dialogue");
await stopVAD();
stopMic();
stopCamera();
@@ -262,7 +310,7 @@ export function useVisionSession() {
setStats({ queryCount: 0, totalTokens: 0 });
historyRef.current = [];
prevFrameRef.current = null;
- }, [stopVAD, stopMic, stopCamera, disconnect]);
+ }, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
/** 打断当前回复 */
const interrupt = useCallback(() => {
@@ -297,6 +345,9 @@ export function useVisionSession() {
config,
updateConfig,
stats,
+ mode,
+ isObserving,
+ toggleMode,
startSession,
stopSession,
interrupt,