feat: 实现关键帧检测与成本控制

- EdgeProcessor:sampleFrame 降采样 160x120 + compareFrames 像素差异对比
- 重复画面跳过:similarity > 0.9 时不发送 query
- 混合采样策略:静默 5s / 用户说话 1s(docs/08-成本控制.md)
- 请求统计:queryCount + totalTokens 实时显示
- App 状态栏显示成本统计

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-06-13 14:38:15 +08:00
parent 8dc59d2c09
commit 2c6489b9a6
5 changed files with 157 additions and 14 deletions

View File

@@ -103,19 +103,61 @@ export function useVAD(options?: VADOptions) {
return { isSpeaking, isReady, error, start, stop };
}
// ---- 关键帧检测ONNX Runtime Web----
// ---- 关键帧检测 ----
export function useKeyframeDetection() {
// TODO: 加载 ONNX 模型后设为 true
const isReady = false;
const DETECT_WIDTH = 160;
const DETECT_HEIGHT = 120;
const DIFF_THRESHOLD = 30;
const isKeyframe = useCallback(
(_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
// TODO: 实现像素差异对比
return true; // 暂时所有帧都视为关键帧
},
[],
);
/** 离屏 canvas用于降采样 */
let offscreen: HTMLCanvasElement | null = null;
return { isReady, isKeyframe };
function getOffscreen(): HTMLCanvasElement {
if (!offscreen) {
offscreen = document.createElement("canvas");
offscreen.width = DETECT_WIDTH;
offscreen.height = DETECT_HEIGHT;
}
return offscreen;
}
/**
* 将 video 元素缩放采样为 Uint8ClampedArrayRGBA
* 返回 null 如果 video 未就绪
*/
export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null {
if (video.readyState < 2) return null;
const canvas = getOffscreen();
const ctx = canvas.getContext("2d");
if (!ctx) return null;
ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT);
return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data;
}
/**
* 对比两帧像素差异
* @returns { isKeyframe: boolean, similarity: number }
*/
export function compareFrames(
prev: Uint8ClampedArray,
curr: Uint8ClampedArray,
): { isKeyframe: boolean; similarity: number } {
let diffSum = 0;
const len = Math.min(prev.length, curr.length);
const pixelCount = len / 4;
for (let i = 0; i < len; i += 4) {
// 只比较 RGB跳过 Alpha
diffSum += Math.abs(prev[i] - curr[i]);
diffSum += Math.abs(prev[i + 1] - curr[i + 1]);
diffSum += Math.abs(prev[i + 2] - curr[i + 2]);
}
const avgDiff = diffSum / (pixelCount * 3);
const similarity = 1 - avgDiff / 255;
return {
isKeyframe: avgDiff > DIFF_THRESHOLD,
similarity,
};
}