feat: 实现观察模式,支持持续场景监控(US-05/US-10)
- 新增 useObservationMode Hook:定时 5s 采帧,similarity < 0.85 触发变化回调 - useVisionSession 新增 dialogue/observation 模式切换 - App Footer 加观察模式切换按钮(激活时蓝色脉冲) - 视频区左上角显示'👁️ 观察中'绿色角标 - 观察模式下画面变化自动发送 query Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -16,8 +16,11 @@ import { useCamera } from "../components/CameraManager";
|
||||
import { useMicrophone } from "../components/MicManager";
|
||||
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
|
||||
import { useWebSocketManager } from "../components/WebSocketManager";
|
||||
import { useObservationMode } from "./useObservationMode";
|
||||
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
||||
|
||||
export type SessionMode = "dialogue" | "observation";
|
||||
|
||||
const MAX_HISTORY_ROUNDS = 10;
|
||||
|
||||
export interface SessionStats {
|
||||
@@ -32,6 +35,7 @@ export function useVisionSession() {
|
||||
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
|
||||
const [config, setConfig] = useState<SessionConfig>(loadConfig);
|
||||
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
|
||||
const [mode, setMode] = useState<SessionMode>("dialogue");
|
||||
|
||||
// 上一帧采样数据(用于关键帧检测)
|
||||
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
|
||||
@@ -60,6 +64,48 @@ export function useVisionSession() {
|
||||
isProcessingRef.current = isProcessing;
|
||||
}, [isProcessing]);
|
||||
|
||||
// 观察模式:画面变化时自动发送 query
|
||||
const { isObserving, startObserving, stopObserving } = useObservationMode({
|
||||
onChange: useCallback(
|
||||
(frameDataUrl: string) => {
|
||||
if (isProcessingRef.current) return;
|
||||
|
||||
const requestId = uuidv4();
|
||||
send({
|
||||
type: "query",
|
||||
request_id: requestId,
|
||||
image: dataUrlToBase64(frameDataUrl),
|
||||
audio: "", // 观察模式无音频
|
||||
});
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
role: "user",
|
||||
content: "👁️ 画面变化检测",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
]);
|
||||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||||
setIsProcessing(true);
|
||||
},
|
||||
[send],
|
||||
),
|
||||
});
|
||||
|
||||
/** 切换对话/观察模式 */
|
||||
const toggleMode = useCallback(() => {
|
||||
setMode((prev) => {
|
||||
const next = prev === "dialogue" ? "observation" : "dialogue";
|
||||
if (next === "observation") {
|
||||
startObserving(videoRef.current, captureFrame);
|
||||
} else {
|
||||
stopObserving();
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [videoRef, captureFrame, startObserving, stopObserving]);
|
||||
|
||||
// WebSocket 连接成功后发送 config
|
||||
useEffect(() => {
|
||||
if (status === "connected") {
|
||||
@@ -249,6 +295,8 @@ export function useVisionSession() {
|
||||
|
||||
/** 结束会话 */
|
||||
const stopSession = useCallback(async () => {
|
||||
stopObserving();
|
||||
setMode("dialogue");
|
||||
await stopVAD();
|
||||
stopMic();
|
||||
stopCamera();
|
||||
@@ -262,7 +310,7 @@ export function useVisionSession() {
|
||||
setStats({ queryCount: 0, totalTokens: 0 });
|
||||
historyRef.current = [];
|
||||
prevFrameRef.current = null;
|
||||
}, [stopVAD, stopMic, stopCamera, disconnect]);
|
||||
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
|
||||
|
||||
/** 打断当前回复 */
|
||||
const interrupt = useCallback(() => {
|
||||
@@ -297,6 +345,9 @@ export function useVisionSession() {
|
||||
config,
|
||||
updateConfig,
|
||||
stats,
|
||||
mode,
|
||||
isObserving,
|
||||
toggleMode,
|
||||
startSession,
|
||||
stopSession,
|
||||
interrupt,
|
||||
|
||||
Reference in New Issue
Block a user