From 2c6489b9a6ecc5241f25e2ee729b043055568f0c Mon Sep 17 00:00:00 2001
From: cfy666 <3087823110@qq.com>
Date: Sat, 13 Jun 2026 14:38:15 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=E5=85=B3=E9=94=AE?=
 =?UTF-8?q?=E5=B8=A7=E6=A3=80=E6=B5=8B=E4=B8=8E=E6=88=90=E6=9C=AC=E6=8E=A7?=
 =?UTF-8?q?=E5=88=B6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- EdgeProcessor：sampleFrame 降采样 160x120 + compareFrames 像素差异对比
- 重复画面跳过：similarity > 0.9 时不发送 query
- 混合采样策略：静默 5s / 用户说话 1s（docs/08-成本控制.md）
- 请求统计：queryCount + totalTokens 实时显示
- App 状态栏显示成本统计

Co-Authored-By: Claude <noreply@anthropic.com>
---
 frontend/src/App.css                          | 10 +++
 frontend/src/App.tsx                          |  8 +++
 .../src/components/EdgeProcessor/index.tsx    | 66 +++++++++++++++----
 frontend/src/hooks/useVisionSession.ts        | 44 ++++++++++++-
 frontend/src/lib/sampling.ts                  | 43 ++++++++++++
 5 files changed, 157 insertions(+), 14 deletions(-)
 create mode 100644 frontend/src/lib/sampling.ts

diff --git a/frontend/src/App.css b/frontend/src/App.css
index bdbe21e..431c149 100644
--- a/frontend/src/App.css
+++ b/frontend/src/App.css
@@ -237,6 +237,16 @@ body {
   border-top: 1px solid var(--color-border);
 }
 
+/* ---- Stats Bar ---- */
+
+.stats-bar {
+  text-align: center;
+  padding: 4px 0;
+  font-size: 0.75rem;
+  color: var(--color-text-muted);
+  border-top: 1px solid var(--color-border);
+}
+
 /* ---- Buttons ---- */
 
 .btn {
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 0fcb64d..dcb8a6f 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -26,6 +26,7 @@ function App() {
     stream,
     config,
     updateConfig,
+    stats,
     startSession,
     stopSession,
     interrupt,
@@ -106,6 +107,13 @@ function App() {
         </div>
       </main>
 
+      {isConnected && (stats.queryCount > 0 || stats.totalTokens > 0) && (
+        <div className="stats-bar">
+          {stats.queryCount > 0 && <span>{stats.queryCount} 次请求</span>}
+          {stats.totalTokens > 0 && <span>· {stats.totalTokens} tokens</span>}
+        </div>
+      )}
+
       <footer className="app-footer">
         {!isConnected ? (
           <button className="btn btn--primary" onClick={startSession}>
diff --git a/frontend/src/components/EdgeProcessor/index.tsx b/frontend/src/components/EdgeProcessor/index.tsx
index 36978b7..a7c83a6 100644
--- a/frontend/src/components/EdgeProcessor/index.tsx
+++ b/frontend/src/components/EdgeProcessor/index.tsx
@@ -103,19 +103,61 @@ export function useVAD(options?: VADOptions) {
   return { isSpeaking, isReady, error, start, stop };
 }
 
-// ---- 关键帧检测（ONNX Runtime Web）----
+// ---- 关键帧检测 ----
 
-export function useKeyframeDetection() {
-  // TODO: 加载 ONNX 模型后设为 true
-  const isReady = false;
+const DETECT_WIDTH = 160;
+const DETECT_HEIGHT = 120;
+const DIFF_THRESHOLD = 30;
 
-  const isKeyframe = useCallback(
-    (_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
-      // TODO: 实现像素差异对比
-      return true; // 暂时所有帧都视为关键帧
-    },
-    [],
-  );
+/** 离屏 canvas，用于降采样 */
+let offscreen: HTMLCanvasElement | null = null;
 
-  return { isReady, isKeyframe };
+function getOffscreen(): HTMLCanvasElement {
+  if (!offscreen) {
+    offscreen = document.createElement("canvas");
+    offscreen.width = DETECT_WIDTH;
+    offscreen.height = DETECT_HEIGHT;
+  }
+  return offscreen;
+}
+
+/**
+ * 将 video 元素缩放采样为 Uint8ClampedArray（RGBA）
+ * 返回 null 如果 video 未就绪
+ */
+export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null {
+  if (video.readyState < 2) return null;
+  const canvas = getOffscreen();
+  const ctx = canvas.getContext("2d");
+  if (!ctx) return null;
+  ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT);
+  return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data;
+}
+
+/**
+ * 对比两帧像素差异
+ * @returns { isKeyframe: boolean, similarity: number }
+ */
+export function compareFrames(
+  prev: Uint8ClampedArray,
+  curr: Uint8ClampedArray,
+): { isKeyframe: boolean; similarity: number } {
+  let diffSum = 0;
+  const len = Math.min(prev.length, curr.length);
+  const pixelCount = len / 4;
+
+  for (let i = 0; i < len; i += 4) {
+    // 只比较 RGB，跳过 Alpha
+    diffSum += Math.abs(prev[i] - curr[i]);
+    diffSum += Math.abs(prev[i + 1] - curr[i + 1]);
+    diffSum += Math.abs(prev[i + 2] - curr[i + 2]);
+  }
+
+  const avgDiff = diffSum / (pixelCount * 3);
+  const similarity = 1 - avgDiff / 255;
+
+  return {
+    isKeyframe: avgDiff > DIFF_THRESHOLD,
+    similarity,
+  };
 }
diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts
index 9217c55..a37389f 100644
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -14,18 +14,27 @@ import { showToast } from "../lib/toast";
 import { loadConfig, saveConfig } from "../lib/storage";
 import { useCamera } from "../components/CameraManager";
 import { useMicrophone } from "../components/MicManager";
-import { useVAD } from "../components/EdgeProcessor";
+import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
 import { useWebSocketManager } from "../components/WebSocketManager";
 import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
 
 const MAX_HISTORY_ROUNDS = 10;
 
+export interface SessionStats {
+  queryCount: number;
+  totalTokens: number;
+}
+
 export function useVisionSession() {
   const [messages, setMessages] = useState<ChatMessage[]>([]);
   const [currentReply, setCurrentReply] = useState<string>("");
   const [isProcessing, setIsProcessing] = useState(false);
   const [isAudioPlaying, setIsAudioPlaying] = useState(false);
   const [config, setConfig] = useState<SessionConfig>(loadConfig);
+  const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
+
+  // 上一帧采样数据（用于关键帧检测）
+  const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
 
   // 对话历史（role + content），用于多轮上下文
   const historyRef = useRef<Array<{ role: string; content: string }>>([]);
@@ -107,6 +116,23 @@ export function useVisionSession() {
           return;
         }
 
+        // 关键帧检测：与上一帧对比，相似度过高则跳过
+        const video = videoRef.current;
+        if (video) {
+          const currentSample = sampleFrame(video);
+          if (currentSample && prevFrameRef.current) {
+            const { similarity } = compareFrames(prevFrameRef.current, currentSample);
+            if (similarity > 0.9) {
+              console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)})，跳过`);
+              prevFrameRef.current = currentSample;
+              return;
+            }
+          }
+          if (currentSample) {
+            prevFrameRef.current = currentSample;
+          }
+        }
+
         const requestId = uuidv4();
         send({
           type: "query",
@@ -115,6 +141,9 @@ export function useVisionSession() {
           audio: encodeAudioToBase64(audio),
         });
 
+        // 更新请求统计
+        setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
+
         // 添加用户消息（STT 流式结果会逐步更新文本）
         setMessages((prev) => [
           ...prev,
@@ -122,7 +151,7 @@ export function useVisionSession() {
         ]);
         setIsProcessing(true);
       },
-      [captureFrame, send]
+      [captureFrame, send, videoRef]
     ),
   });
 
@@ -159,6 +188,14 @@ export function useVisionSession() {
             historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
           }
 
+          // 累计 token 统计
+          if (done.tokens_used?.total) {
+            setStats((prev) => ({
+              ...prev,
+              totalTokens: prev.totalTokens + done.tokens_used.total,
+            }));
+          }
+
           setMessages((prev) => [
             ...prev,
             {
@@ -222,7 +259,9 @@ export function useVisionSession() {
     setMessages([]);
     setCurrentReply("");
     setIsProcessing(false);
+    setStats({ queryCount: 0, totalTokens: 0 });
     historyRef.current = [];
+    prevFrameRef.current = null;
   }, [stopVAD, stopMic, stopCamera, disconnect]);
 
   /** 打断当前回复 */
@@ -257,6 +296,7 @@ export function useVisionSession() {
     stream,
     config,
     updateConfig,
+    stats,
     startSession,
     stopSession,
     interrupt,
diff --git a/frontend/src/lib/sampling.ts b/frontend/src/lib/sampling.ts
new file mode 100644
index 0000000..235ec66
--- /dev/null
+++ b/frontend/src/lib/sampling.ts
@@ -0,0 +1,43 @@
+// ============================================================
+// Sampling — 混合采样策略
+// 来源：docs/08-成本控制.md — "定时低频 + 事件高频"
+// ============================================================
+
+/** 静默时采样间隔（ms） */
+const IDLE_INTERVAL = 5000;
+/** 用户说话时采样间隔（ms） */
+const ACTIVE_INTERVAL = 1000;
+
+export class SamplingController {
+  private lastSampleTime = 0;
+  private _isUserSpeaking = false;
+
+  /** 设置用户是否正在说话 */
+  set speaking(value: boolean) {
+    this._isUserSpeaking = value;
+  }
+
+  get speaking(): boolean {
+    return this._isUserSpeaking;
+  }
+
+  /** 当前采样间隔 */
+  get interval(): number {
+    return this._isUserSpeaking ? ACTIVE_INTERVAL : IDLE_INTERVAL;
+  }
+
+  /** 是否应该采样（基于时间间隔） */
+  shouldSample(): boolean {
+    const now = Date.now();
+    if (now - this.lastSampleTime < this.interval) {
+      return false;
+    }
+    this.lastSampleTime = now;
+    return true;
+  }
+
+  /** 强制允许下次采样（如用户刚说完话时） */
+  resetTimer(): void {
+    this.lastSampleTime = 0;
+  }
+}
-- 
2.49.1