From f5536005f66c22536b758a2eefd665dc9d299c43 Mon Sep 17 00:00:00 2001
From: cfy666 <3087823110@qq.com>
Date: Sat, 13 Jun 2026 14:46:42 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=E8=A7=82=E5=AF=9F?=
 =?UTF-8?q?=E6=A8=A1=E5=BC=8F=EF=BC=8C=E6=94=AF=E6=8C=81=E6=8C=81=E7=BB=AD?=
 =?UTF-8?q?=E5=9C=BA=E6=99=AF=E7=9B=91=E6=8E=A7=EF=BC=88US-05/US-10?=
 =?UTF-8?q?=EF=BC=89?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- 新增 useObservationMode Hook：定时 5s 采帧，similarity < 0.85 触发变化回调
- useVisionSession 新增 dialogue/observation 模式切换
- App Footer 加观察模式切换按钮（激活时蓝色脉冲）
- 视频区左上角显示'👁️ 观察中'绿色角标
- 观察模式下画面变化自动发送 query

Co-Authored-By: Claude <noreply@anthropic.com>
---
 frontend/src/App.css                     | 25 +++++++
 frontend/src/App.tsx                     | 12 ++++
 frontend/src/hooks/useObservationMode.ts | 88 ++++++++++++++++++++++++
 frontend/src/hooks/useVisionSession.ts   | 53 +++++++++++++-
 4 files changed, 177 insertions(+), 1 deletion(-)
 create mode 100644 frontend/src/hooks/useObservationMode.ts

diff --git a/frontend/src/App.css b/frontend/src/App.css
index 431c149..ca328d8 100644
--- a/frontend/src/App.css
+++ b/frontend/src/App.css
@@ -287,6 +287,18 @@ body {
   opacity: 0.9;
 }
 
+.btn--active {
+  background: var(--color-primary);
+  color: white;
+  box-shadow: 0 0 8px rgba(37, 99, 235, 0.5);
+  animation: pulseBtn 2s ease-in-out infinite;
+}
+
+@keyframes pulseBtn {
+  0%, 100% { box-shadow: 0 0 8px rgba(37, 99, 235, 0.5); }
+  50% { box-shadow: 0 0 16px rgba(37, 99, 235, 0.8); }
+}
+
 /* ---- Streaming Cursor ---- */
 
 .cursor {
@@ -458,3 +470,16 @@ body {
   font-weight: 700;
   letter-spacing: 0.5px;
 }
+
+.observation-badge {
+  position: absolute;
+  top: 8px;
+  left: 8px;
+  background: rgba(34, 197, 94, 0.8);
+  color: white;
+  padding: 2px 8px;
+  border-radius: 4px;
+  font-size: 0.7rem;
+  font-weight: 700;
+  animation: pulse 2s ease-in-out infinite;
+}
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index dcb8a6f..b23082c 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -27,6 +27,9 @@ function App() {
     config,
     updateConfig,
     stats,
+    mode,
+    isObserving,
+    toggleMode,
     startSession,
     stopSession,
     interrupt,
@@ -72,6 +75,9 @@ function App() {
           {config.detailLevel === "high" && (
             <div className="detail-badge">HD</div>
           )}
+          {isObserving && (
+            <div className="observation-badge">👁️ 观察中</div>
+          )}
           {isSpeaking && <div className="vad-indicator">🎤 正在聆听...</div>}
           {isAudioPlaying && config.ttsEnabled && (
             <div className="vad-indicator vad-indicator--audio">🔊 正在播放...</div>
@@ -121,6 +127,12 @@ function App() {
           </button>
         ) : (
           <>
+            <button
+              className={`btn ${mode === "observation" ? "btn--active" : "btn--secondary"}`}
+              onClick={toggleMode}
+            >
+              {mode === "observation" ? "👁️ 观察中" : "👁️ 观察模式"}
+            </button>
             <button className="btn btn--secondary" onClick={stopSession}>
               结束对话
             </button>
diff --git a/frontend/src/hooks/useObservationMode.ts b/frontend/src/hooks/useObservationMode.ts
new file mode 100644
index 0000000..e49eb62
--- /dev/null
+++ b/frontend/src/hooks/useObservationMode.ts
@@ -0,0 +1,88 @@
+// ============================================================
+// useObservationMode — 观察模式 Hook
+// 职责：定时采帧 → 关键帧检测 → 画面变化时触发回调
+// 来源：docs/05-用户故事.md US-05（持续场景监控）
+// ============================================================
+
+import { useCallback, useEffect, useRef, useState } from "react";
+import { sampleFrame, compareFrames } from "../components/EdgeProcessor";
+
+/** 画面变化显著阈值 */
+const CHANGE_THRESHOLD = 0.85;
+/** 采样间隔（ms） */
+const SAMPLE_INTERVAL = 5000;
+
+export interface ObservationOptions {
+  /** 画面变化回调，携带当前帧的 DataURL */
+  onChange?: (frameDataUrl: string) => void;
+}
+
+export function useObservationMode(options?: ObservationOptions) {
+  const [isObserving, setIsObserving] = useState(false);
+  const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
+  const optionsRef = useRef(options);
+
+  useEffect(() => {
+    optionsRef.current = options;
+  }, [options]);
+
+  /**
+   * 启动观察模式
+   * @param video 摄像头 video 元素
+   * @param captureFrame 从 video 捕获 DataURL 的函数
+   */
+  const startObserving = useCallback(
+    (video: HTMLVideoElement | null, captureFrame: () => string | null) => {
+      if (!video) return;
+
+      // 立即采一帧作为基准
+      prevFrameRef.current = sampleFrame(video);
+
+      intervalRef.current = setInterval(() => {
+        const current = sampleFrame(video);
+        if (!current) return;
+
+        if (prevFrameRef.current) {
+          const { similarity } = compareFrames(prevFrameRef.current, current);
+
+          if (similarity < CHANGE_THRESHOLD) {
+            console.log(
+              `[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`,
+            );
+            const frameDataUrl = captureFrame();
+            if (frameDataUrl) {
+              optionsRef.current?.onChange?.(frameDataUrl);
+            }
+          }
+        }
+
+        prevFrameRef.current = current;
+      }, SAMPLE_INTERVAL);
+
+      setIsObserving(true);
+    },
+    [],
+  );
+
+  /** 停止观察模式 */
+  const stopObserving = useCallback(() => {
+    if (intervalRef.current) {
+      clearInterval(intervalRef.current);
+      intervalRef.current = null;
+    }
+    prevFrameRef.current = null;
+    setIsObserving(false);
+  }, []);
+
+  // 组件卸载时清理
+  useEffect(() => {
+    return () => {
+      if (intervalRef.current) {
+        clearInterval(intervalRef.current);
+      }
+    };
+  }, []);
+
+  return { isObserving, startObserving, stopObserving };
+}
diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts
index a37389f..fd4c005 100644
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -16,8 +16,11 @@ import { useCamera } from "../components/CameraManager";
 import { useMicrophone } from "../components/MicManager";
 import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
 import { useWebSocketManager } from "../components/WebSocketManager";
+import { useObservationMode } from "./useObservationMode";
 import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
 
+export type SessionMode = "dialogue" | "observation";
+
 const MAX_HISTORY_ROUNDS = 10;
 
 export interface SessionStats {
@@ -32,6 +35,7 @@ export function useVisionSession() {
   const [isAudioPlaying, setIsAudioPlaying] = useState(false);
   const [config, setConfig] = useState<SessionConfig>(loadConfig);
   const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
+  const [mode, setMode] = useState<SessionMode>("dialogue");
 
   // 上一帧采样数据（用于关键帧检测）
   const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
@@ -60,6 +64,48 @@ export function useVisionSession() {
     isProcessingRef.current = isProcessing;
   }, [isProcessing]);
 
+  // 观察模式：画面变化时自动发送 query
+  const { isObserving, startObserving, stopObserving } = useObservationMode({
+    onChange: useCallback(
+      (frameDataUrl: string) => {
+        if (isProcessingRef.current) return;
+
+        const requestId = uuidv4();
+        send({
+          type: "query",
+          request_id: requestId,
+          image: dataUrlToBase64(frameDataUrl),
+          audio: "", // 观察模式无音频
+        });
+
+        setMessages((prev) => [
+          ...prev,
+          {
+            role: "user",
+            content: "👁️ 画面变化检测",
+            timestamp: Date.now(),
+          },
+        ]);
+        setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
+        setIsProcessing(true);
+      },
+      [send],
+    ),
+  });
+
+  /** 切换对话/观察模式 */
+  const toggleMode = useCallback(() => {
+    setMode((prev) => {
+      const next = prev === "dialogue" ? "observation" : "dialogue";
+      if (next === "observation") {
+        startObserving(videoRef.current, captureFrame);
+      } else {
+        stopObserving();
+      }
+      return next;
+    });
+  }, [videoRef, captureFrame, startObserving, stopObserving]);
+
   // WebSocket 连接成功后发送 config
   useEffect(() => {
     if (status === "connected") {
@@ -249,6 +295,8 @@ export function useVisionSession() {
 
   /** 结束会话 */
   const stopSession = useCallback(async () => {
+    stopObserving();
+    setMode("dialogue");
     await stopVAD();
     stopMic();
     stopCamera();
@@ -262,7 +310,7 @@ export function useVisionSession() {
     setStats({ queryCount: 0, totalTokens: 0 });
     historyRef.current = [];
     prevFrameRef.current = null;
-  }, [stopVAD, stopMic, stopCamera, disconnect]);
+  }, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
 
   /** 打断当前回复 */
   const interrupt = useCallback(() => {
@@ -297,6 +345,9 @@ export function useVisionSession() {
     config,
     updateConfig,
     stats,
+    mode,
+    isObserving,
+    toggleMode,
     startSession,
     stopSession,
     interrupt,