feat: 实现关键帧检测与成本控制

- EdgeProcessor:sampleFrame 降采样 160x120 + compareFrames 像素差异对比
- 重复画面跳过:similarity > 0.9 时不发送 query
- 混合采样策略:静默 5s / 用户说话 1s(docs/08-成本控制.md)
- 请求统计:queryCount + totalTokens 实时显示
- App 状态栏显示成本统计

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-06-13 14:38:15 +08:00
parent 8dc59d2c09
commit 2c6489b9a6
5 changed files with 157 additions and 14 deletions

View File

@@ -14,18 +14,27 @@ import { showToast } from "../lib/toast";
import { loadConfig, saveConfig } from "../lib/storage";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD } from "../components/EdgeProcessor";
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
const MAX_HISTORY_ROUNDS = 10;
export interface SessionStats {
queryCount: number;
totalTokens: number;
}
export function useVisionSession() {
const [messages, setMessages] = useState<ChatMessage[]>([]);
const [currentReply, setCurrentReply] = useState<string>("");
const [isProcessing, setIsProcessing] = useState(false);
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
const [config, setConfig] = useState<SessionConfig>(loadConfig);
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
// 上一帧采样数据(用于关键帧检测)
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
// 对话历史role + content用于多轮上下文
const historyRef = useRef<Array<{ role: string; content: string }>>([]);
@@ -107,6 +116,23 @@ export function useVisionSession() {
return;
}
// 关键帧检测:与上一帧对比,相似度过高则跳过
const video = videoRef.current;
if (video) {
const currentSample = sampleFrame(video);
if (currentSample && prevFrameRef.current) {
const { similarity } = compareFrames(prevFrameRef.current, currentSample);
if (similarity > 0.9) {
console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)}),跳过`);
prevFrameRef.current = currentSample;
return;
}
}
if (currentSample) {
prevFrameRef.current = currentSample;
}
}
const requestId = uuidv4();
send({
type: "query",
@@ -115,6 +141,9 @@ export function useVisionSession() {
audio: encodeAudioToBase64(audio),
});
// 更新请求统计
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
// 添加用户消息STT 流式结果会逐步更新文本)
setMessages((prev) => [
...prev,
@@ -122,7 +151,7 @@ export function useVisionSession() {
]);
setIsProcessing(true);
},
[captureFrame, send]
[captureFrame, send, videoRef]
),
});
@@ -159,6 +188,14 @@ export function useVisionSession() {
historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
}
// 累计 token 统计
if (done.tokens_used?.total) {
setStats((prev) => ({
...prev,
totalTokens: prev.totalTokens + done.tokens_used.total,
}));
}
setMessages((prev) => [
...prev,
{
@@ -222,7 +259,9 @@ export function useVisionSession() {
setMessages([]);
setCurrentReply("");
setIsProcessing(false);
setStats({ queryCount: 0, totalTokens: 0 });
historyRef.current = [];
prevFrameRef.current = null;
}, [stopVAD, stopMic, stopCamera, disconnect]);
/** 打断当前回复 */
@@ -257,6 +296,7 @@ export function useVisionSession() {
stream,
config,
updateConfig,
stats,
startSession,
stopSession,
interrupt,