cfy666
da5c727d8c
之前前端 TTSPlayer 攒齐所有音频片段后才播放，导致文字全部显示后才开始语音。 改为后端每句 TTS 发送 is_last: true，前端收到每句即加入播放队列，第一句到达即开始播放。 - 后端 Chunk 结构体新增 Final 字段，区分句子结束和整轮结束 - 前端 TTSPlayer 重写为队列式播放，onended 回调自动衔接下一句 - 同步更新接口文档和测试用例
438 lines
13 KiB
TypeScript
438 lines
13 KiB
TypeScript
// ============================================================
|
||
// useVisionSession — 核心视觉对话会话 Hook
|
||
// 职责：封装一次完整的视觉对话会话（摄像头、VAD、WebSocket、消息状态）
|
||
// 来源：docs/02-系统架构.md 核心 Hook 设计
|
||
// ============================================================
|
||
|
||
import { useCallback, useEffect, useRef, useState } from "react";
|
||
import { v4 as uuidv4 } from "uuid";
|
||
import { wsClient } from "../lib/websocket";
|
||
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
|
||
import { getErrorMessage } from "../lib/errors";
|
||
import { TTSPlayer } from "../lib/ttsPlayer";
|
||
import { showToast } from "../lib/toast";
|
||
import { loadConfig, saveConfig } from "../lib/storage";
|
||
import { useCamera } from "../components/CameraManager";
|
||
import { useMicrophone } from "../components/MicManager";
|
||
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
|
||
import { useWebSocketManager } from "../components/WebSocketManager";
|
||
import { useObservationMode } from "./useObservationMode";
|
||
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
||
|
||
export type SessionMode = "dialogue" | "observation";
|
||
|
||
const MAX_HISTORY_ROUNDS = 10;
|
||
|
||
export interface SessionStats {
|
||
queryCount: number;
|
||
totalTokens: number;
|
||
}
|
||
|
||
export function useVisionSession() {
|
||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||
const [currentReply, setCurrentReply] = useState<string>("");
|
||
const [isProcessing, setIsProcessing] = useState(false);
|
||
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
|
||
const [config, setConfig] = useState<SessionConfig>(loadConfig);
|
||
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
|
||
const [mode, setMode] = useState<SessionMode>("dialogue");
|
||
const [isCameraOn, setIsCameraOn] = useState(false);
|
||
const [isMicOn, setIsMicOn] = useState(false);
|
||
|
||
// 上一帧采样数据（用于关键帧检测）
|
||
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
|
||
|
||
// 对话历史（role + content），用于多轮上下文
|
||
const historyRef = useRef<Array<{ role: string; content: string }>>([]);
|
||
|
||
// TTS 播放器
|
||
const ttsPlayerRef = useRef<TTSPlayer | null>(null);
|
||
const getTTSPlayer = useCallback(() => {
|
||
if (!ttsPlayerRef.current) {
|
||
const player = new TTSPlayer();
|
||
player.onEnd(() => setIsAudioPlaying(false));
|
||
ttsPlayerRef.current = player;
|
||
}
|
||
return ttsPlayerRef.current;
|
||
}, []);
|
||
|
||
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
|
||
const { startMic, stopMic } = useMicrophone();
|
||
const { status, connect, disconnect, send } = useWebSocketManager();
|
||
|
||
// 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题
|
||
const isProcessingRef = useRef(false);
|
||
useEffect(() => {
|
||
isProcessingRef.current = isProcessing;
|
||
}, [isProcessing]);
|
||
|
||
// 观察模式：画面变化时自动发送 query
|
||
const { isObserving, startObserving, stopObserving } = useObservationMode({
|
||
onChange: useCallback(
|
||
(frameDataUrl: string) => {
|
||
if (isProcessingRef.current) return;
|
||
|
||
const requestId = uuidv4();
|
||
send({
|
||
type: "query",
|
||
request_id: requestId,
|
||
image: dataUrlToBase64(frameDataUrl),
|
||
audio: "", // 观察模式无音频
|
||
});
|
||
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{
|
||
role: "user",
|
||
content: "👁️ 画面变化检测",
|
||
timestamp: Date.now(),
|
||
},
|
||
]);
|
||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||
setIsProcessing(true);
|
||
},
|
||
[send],
|
||
),
|
||
});
|
||
|
||
/** 切换对话/观察模式 */
|
||
const toggleMode = useCallback(() => {
|
||
setMode((prev) => {
|
||
const next = prev === "dialogue" ? "observation" : "dialogue";
|
||
if (next === "observation") {
|
||
startObserving(videoRef.current, captureFrame);
|
||
} else {
|
||
stopObserving();
|
||
}
|
||
return next;
|
||
});
|
||
}, [videoRef, captureFrame, startObserving, stopObserving]);
|
||
|
||
// WebSocket 连接成功后发送 config
|
||
useEffect(() => {
|
||
if (status === "connected") {
|
||
send({
|
||
type: "config",
|
||
payload: {
|
||
tts_enabled: config.ttsEnabled,
|
||
detail_level: config.detailLevel,
|
||
language: config.language,
|
||
},
|
||
});
|
||
}
|
||
}, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送
|
||
|
||
/** 更新会话配置 */
|
||
const updateConfig = useCallback((partial: Partial<SessionConfig>) => {
|
||
setConfig((prev) => {
|
||
const next = { ...prev, ...partial };
|
||
saveConfig(next);
|
||
// 如果已连接，立即发送更新
|
||
if (status === "connected") {
|
||
send({
|
||
type: "config",
|
||
payload: {
|
||
tts_enabled: next.ttsEnabled,
|
||
detail_level: next.detailLevel,
|
||
language: next.language,
|
||
},
|
||
});
|
||
}
|
||
return next;
|
||
});
|
||
}, [status, send]);
|
||
|
||
// VAD：语音结束时自动发送 query
|
||
const {
|
||
isSpeaking,
|
||
isReady: isVADReady,
|
||
error: vadError,
|
||
start: startVAD,
|
||
stop: stopVAD,
|
||
} = useVAD({
|
||
onSpeechEnd: useCallback(
|
||
(audio: Float32Array) => {
|
||
// 处理中忽略，防止重复发送
|
||
if (isProcessingRef.current) {
|
||
console.warn("[Session] 正在处理中，忽略语音输入");
|
||
return;
|
||
}
|
||
|
||
// 停止上一轮的 TTS 播放，防止新旧音频重叠
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
|
||
const frame = captureFrame();
|
||
if (!frame) {
|
||
console.warn("[Session] 无法捕获图像帧");
|
||
return;
|
||
}
|
||
|
||
// 关键帧检测：与上一帧对比，相似度过高则跳过
|
||
const video = videoRef.current;
|
||
if (video) {
|
||
const currentSample = sampleFrame(video);
|
||
if (currentSample && prevFrameRef.current) {
|
||
const { similarity } = compareFrames(prevFrameRef.current, currentSample);
|
||
if (similarity > 0.9) {
|
||
console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)})，跳过`);
|
||
prevFrameRef.current = currentSample;
|
||
return;
|
||
}
|
||
}
|
||
if (currentSample) {
|
||
prevFrameRef.current = currentSample;
|
||
}
|
||
}
|
||
|
||
const requestId = uuidv4();
|
||
send({
|
||
type: "query",
|
||
request_id: requestId,
|
||
image: dataUrlToBase64(frame),
|
||
audio: encodeAudioToBase64(audio),
|
||
});
|
||
|
||
// 更新请求统计
|
||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||
|
||
// 添加用户消息（STT 流式结果会逐步更新文本）
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{ role: "user", content: "（语音识别中...）", timestamp: Date.now() },
|
||
]);
|
||
setIsProcessing(true);
|
||
},
|
||
[captureFrame, send, videoRef]
|
||
),
|
||
});
|
||
|
||
// 处理服务端消息
|
||
useEffect(() => {
|
||
const unsub = wsClient.onMessage((msg: ServerMessage) => {
|
||
switch (msg.type) {
|
||
case "stt_result": {
|
||
// 流式更新用户消息文本（包括中间结果和最终结果）
|
||
setMessages((prev) => {
|
||
const updated = [...prev];
|
||
const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
|
||
if (lastUserIdx >= 0) {
|
||
updated[lastUserIdx] = {
|
||
...updated[lastUserIdx],
|
||
content: msg.text || "（未识别到语音）",
|
||
};
|
||
}
|
||
return updated;
|
||
});
|
||
break;
|
||
}
|
||
|
||
case "llm_chunk":
|
||
setCurrentReply((prev) => prev + msg.delta);
|
||
break;
|
||
|
||
case "llm_done": {
|
||
const done = msg as LLMDoneMessage;
|
||
// 记录到对话历史
|
||
historyRef.current.push({ role: "assistant", content: done.full_text });
|
||
// 裁剪历史到最近 N 轮
|
||
if (historyRef.current.length > MAX_HISTORY_ROUNDS * 2) {
|
||
historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
|
||
}
|
||
|
||
// 累计 token 统计
|
||
if (done.tokens_used?.total) {
|
||
setStats((prev) => ({
|
||
...prev,
|
||
totalTokens: prev.totalTokens + done.tokens_used.total,
|
||
}));
|
||
}
|
||
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{
|
||
role: "assistant",
|
||
content: done.full_text,
|
||
timestamp: Date.now(),
|
||
tokensUsed: done.tokens_used?.total,
|
||
latencyMs: done.latency_ms,
|
||
model: done.model,
|
||
},
|
||
]);
|
||
setCurrentReply("");
|
||
setIsProcessing(false);
|
||
break;
|
||
}
|
||
|
||
case "tts_audio":
|
||
getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
|
||
setIsAudioPlaying(true);
|
||
break;
|
||
|
||
case "error":
|
||
console.error("[Session] 服务端错误:", msg.code, msg.message);
|
||
showToast(getErrorMessage(msg.code), "error");
|
||
setIsProcessing(false);
|
||
break;
|
||
}
|
||
});
|
||
|
||
return unsub;
|
||
}, [getTTSPlayer]);
|
||
|
||
/** 启动会话 */
|
||
const startSession = useCallback(async () => {
|
||
// 1. 连接 WebSocket（必须）
|
||
connect();
|
||
|
||
// 2. 尝试获取摄像头（可选）
|
||
try {
|
||
await startCamera();
|
||
setIsCameraOn(true);
|
||
} catch {
|
||
console.warn("[Session] 无法获取摄像头权限，将以纯文本模式运行");
|
||
}
|
||
|
||
// 3. 尝试获取麦克风（可选）
|
||
const micStream = await startMic();
|
||
if (micStream) {
|
||
setIsMicOn(true);
|
||
// 4. 启动 VAD（仅在麦克风可用时）
|
||
await startVAD(micStream);
|
||
} else {
|
||
console.warn("[Session] 无法获取麦克风权限，将以文本输入模式运行");
|
||
}
|
||
}, [startCamera, startMic, connect, startVAD]);
|
||
|
||
/** 结束会话 */
|
||
const stopSession = useCallback(async () => {
|
||
stopObserving();
|
||
setMode("dialogue");
|
||
await stopVAD();
|
||
stopMic();
|
||
stopCamera();
|
||
disconnect();
|
||
// 停止 TTS 并清理状态
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
setMessages([]);
|
||
setCurrentReply("");
|
||
setIsProcessing(false);
|
||
setStats({ queryCount: 0, totalTokens: 0 });
|
||
historyRef.current = [];
|
||
prevFrameRef.current = null;
|
||
setIsCameraOn(false);
|
||
setIsMicOn(false);
|
||
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
|
||
|
||
/** 摄像头开关 */
|
||
const toggleCamera = useCallback(async () => {
|
||
if (isCameraOn) {
|
||
stopCamera();
|
||
setIsCameraOn(false);
|
||
} else {
|
||
await startCamera();
|
||
setIsCameraOn(true);
|
||
}
|
||
}, [isCameraOn, startCamera, stopCamera]);
|
||
|
||
/** 麦克风开关 */
|
||
const toggleMic = useCallback(async () => {
|
||
if (isMicOn) {
|
||
await stopVAD();
|
||
stopMic();
|
||
setIsMicOn(false);
|
||
} else {
|
||
const micStream = await startMic();
|
||
if (micStream) {
|
||
await startVAD(micStream);
|
||
setIsMicOn(true);
|
||
}
|
||
}
|
||
}, [isMicOn, startMic, stopMic, startVAD, stopVAD]);
|
||
|
||
/** 打断当前回复 */
|
||
const interrupt = useCallback(() => {
|
||
send({ type: "interrupt" });
|
||
// 停止 TTS 播放
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
// 将未完成的流式内容保存为最终消息
|
||
if (currentReply) {
|
||
const interrupted = currentReply + "（已打断）";
|
||
historyRef.current.push({ role: "assistant", content: interrupted });
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{ role: "assistant", content: interrupted, timestamp: Date.now() },
|
||
]);
|
||
}
|
||
setCurrentReply("");
|
||
setIsProcessing(false);
|
||
}, [send, currentReply]);
|
||
|
||
/** 发送文本消息（手动输入） */
|
||
const sendTextMessage = useCallback(
|
||
(text: string) => {
|
||
if (!text.trim() || isProcessingRef.current) return;
|
||
|
||
// 停止上一轮的 TTS 播放
|
||
ttsPlayerRef.current?.stop();
|
||
setIsAudioPlaying(false);
|
||
|
||
// 捕获当前摄像头画面
|
||
const frame = captureFrame();
|
||
|
||
const requestId = uuidv4();
|
||
send({
|
||
type: "query",
|
||
request_id: requestId,
|
||
image: frame ? dataUrlToBase64(frame) : "",
|
||
audio: "", // 文本输入无音频
|
||
text: text.trim(),
|
||
});
|
||
|
||
// 更新请求统计
|
||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||
|
||
// 添加用户消息
|
||
setMessages((prev) => [
|
||
...prev,
|
||
{ role: "user", content: text.trim(), timestamp: Date.now() },
|
||
]);
|
||
|
||
// 记录到对话历史
|
||
historyRef.current.push({ role: "user", content: text.trim() });
|
||
|
||
setIsProcessing(true);
|
||
},
|
||
[captureFrame, send],
|
||
);
|
||
|
||
return {
|
||
messages,
|
||
currentReply,
|
||
isProcessing,
|
||
isAudioPlaying,
|
||
isSpeaking,
|
||
isVADReady,
|
||
vadError,
|
||
connectionStatus: status,
|
||
videoRef,
|
||
stream,
|
||
config,
|
||
updateConfig,
|
||
stats,
|
||
mode,
|
||
isObserving,
|
||
toggleMode,
|
||
startSession,
|
||
stopSession,
|
||
interrupt,
|
||
isCameraOn,
|
||
isMicOn,
|
||
toggleCamera,
|
||
toggleMic,
|
||
sendTextMessage,
|
||
};
|
||
}
|