feat: 实现关键帧检测与成本控制
- EdgeProcessor:sampleFrame 降采样 160x120 + compareFrames 像素差异对比 - 重复画面跳过:similarity > 0.9 时不发送 query - 混合采样策略:静默 5s / 用户说话 1s(docs/08-成本控制.md) - 请求统计:queryCount + totalTokens 实时显示 - App 状态栏显示成本统计 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -14,18 +14,27 @@ import { showToast } from "../lib/toast";
|
||||
import { loadConfig, saveConfig } from "../lib/storage";
|
||||
import { useCamera } from "../components/CameraManager";
|
||||
import { useMicrophone } from "../components/MicManager";
|
||||
import { useVAD } from "../components/EdgeProcessor";
|
||||
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
|
||||
import { useWebSocketManager } from "../components/WebSocketManager";
|
||||
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
||||
|
||||
const MAX_HISTORY_ROUNDS = 10;
|
||||
|
||||
export interface SessionStats {
|
||||
queryCount: number;
|
||||
totalTokens: number;
|
||||
}
|
||||
|
||||
export function useVisionSession() {
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const [currentReply, setCurrentReply] = useState<string>("");
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
|
||||
const [config, setConfig] = useState<SessionConfig>(loadConfig);
|
||||
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
|
||||
|
||||
// 上一帧采样数据(用于关键帧检测)
|
||||
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
|
||||
|
||||
// 对话历史(role + content),用于多轮上下文
|
||||
const historyRef = useRef<Array<{ role: string; content: string }>>([]);
|
||||
@@ -107,6 +116,23 @@ export function useVisionSession() {
|
||||
return;
|
||||
}
|
||||
|
||||
// 关键帧检测:与上一帧对比,相似度过高则跳过
|
||||
const video = videoRef.current;
|
||||
if (video) {
|
||||
const currentSample = sampleFrame(video);
|
||||
if (currentSample && prevFrameRef.current) {
|
||||
const { similarity } = compareFrames(prevFrameRef.current, currentSample);
|
||||
if (similarity > 0.9) {
|
||||
console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)}),跳过`);
|
||||
prevFrameRef.current = currentSample;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (currentSample) {
|
||||
prevFrameRef.current = currentSample;
|
||||
}
|
||||
}
|
||||
|
||||
const requestId = uuidv4();
|
||||
send({
|
||||
type: "query",
|
||||
@@ -115,6 +141,9 @@ export function useVisionSession() {
|
||||
audio: encodeAudioToBase64(audio),
|
||||
});
|
||||
|
||||
// 更新请求统计
|
||||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||||
|
||||
// 添加用户消息(STT 流式结果会逐步更新文本)
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
@@ -122,7 +151,7 @@ export function useVisionSession() {
|
||||
]);
|
||||
setIsProcessing(true);
|
||||
},
|
||||
[captureFrame, send]
|
||||
[captureFrame, send, videoRef]
|
||||
),
|
||||
});
|
||||
|
||||
@@ -159,6 +188,14 @@ export function useVisionSession() {
|
||||
historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
|
||||
}
|
||||
|
||||
// 累计 token 统计
|
||||
if (done.tokens_used?.total) {
|
||||
setStats((prev) => ({
|
||||
...prev,
|
||||
totalTokens: prev.totalTokens + done.tokens_used.total,
|
||||
}));
|
||||
}
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
@@ -222,7 +259,9 @@ export function useVisionSession() {
|
||||
setMessages([]);
|
||||
setCurrentReply("");
|
||||
setIsProcessing(false);
|
||||
setStats({ queryCount: 0, totalTokens: 0 });
|
||||
historyRef.current = [];
|
||||
prevFrameRef.current = null;
|
||||
}, [stopVAD, stopMic, stopCamera, disconnect]);
|
||||
|
||||
/** 打断当前回复 */
|
||||
@@ -257,6 +296,7 @@ export function useVisionSession() {
|
||||
stream,
|
||||
config,
|
||||
updateConfig,
|
||||
stats,
|
||||
startSession,
|
||||
stopSession,
|
||||
interrupt,
|
||||
|
||||
Reference in New Issue
Block a user