feat: 实现关键帧检测与成本控制
- EdgeProcessor:sampleFrame 降采样 160x120 + compareFrames 像素差异对比 - 重复画面跳过:similarity > 0.9 时不发送 query - 混合采样策略:静默 5s / 用户说话 1s(docs/08-成本控制.md) - 请求统计:queryCount + totalTokens 实时显示 - App 状态栏显示成本统计 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -103,19 +103,61 @@ export function useVAD(options?: VADOptions) {
|
||||
return { isSpeaking, isReady, error, start, stop };
|
||||
}
|
||||
|
||||
// ---- 关键帧检测(ONNX Runtime Web)----
|
||||
// ---- 关键帧检测 ----
|
||||
|
||||
export function useKeyframeDetection() {
|
||||
// TODO: 加载 ONNX 模型后设为 true
|
||||
const isReady = false;
|
||||
const DETECT_WIDTH = 160;
|
||||
const DETECT_HEIGHT = 120;
|
||||
const DIFF_THRESHOLD = 30;
|
||||
|
||||
const isKeyframe = useCallback(
|
||||
(_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
|
||||
// TODO: 实现像素差异对比
|
||||
return true; // 暂时所有帧都视为关键帧
|
||||
},
|
||||
[],
|
||||
);
|
||||
/** 离屏 canvas,用于降采样 */
|
||||
let offscreen: HTMLCanvasElement | null = null;
|
||||
|
||||
return { isReady, isKeyframe };
|
||||
function getOffscreen(): HTMLCanvasElement {
|
||||
if (!offscreen) {
|
||||
offscreen = document.createElement("canvas");
|
||||
offscreen.width = DETECT_WIDTH;
|
||||
offscreen.height = DETECT_HEIGHT;
|
||||
}
|
||||
return offscreen;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 video 元素缩放采样为 Uint8ClampedArray(RGBA)
|
||||
* 返回 null 如果 video 未就绪
|
||||
*/
|
||||
export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null {
|
||||
if (video.readyState < 2) return null;
|
||||
const canvas = getOffscreen();
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return null;
|
||||
ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT);
|
||||
return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data;
|
||||
}
|
||||
|
||||
/**
|
||||
* 对比两帧像素差异
|
||||
* @returns { isKeyframe: boolean, similarity: number }
|
||||
*/
|
||||
export function compareFrames(
|
||||
prev: Uint8ClampedArray,
|
||||
curr: Uint8ClampedArray,
|
||||
): { isKeyframe: boolean; similarity: number } {
|
||||
let diffSum = 0;
|
||||
const len = Math.min(prev.length, curr.length);
|
||||
const pixelCount = len / 4;
|
||||
|
||||
for (let i = 0; i < len; i += 4) {
|
||||
// 只比较 RGB,跳过 Alpha
|
||||
diffSum += Math.abs(prev[i] - curr[i]);
|
||||
diffSum += Math.abs(prev[i + 1] - curr[i + 1]);
|
||||
diffSum += Math.abs(prev[i + 2] - curr[i + 2]);
|
||||
}
|
||||
|
||||
const avgDiff = diffSum / (pixelCount * 3);
|
||||
const similarity = 1 - avgDiff / 255;
|
||||
|
||||
return {
|
||||
isKeyframe: avgDiff > DIFF_THRESHOLD,
|
||||
similarity,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user