// ============================================================ // useVisionSession — 核心视觉对话会话 Hook // 职责:封装一次完整的视觉对话会话(摄像头、VAD、WebSocket、消息状态) // 来源:docs/02-系统架构.md 核心 Hook 设计 // ============================================================ import { useCallback, useEffect, useRef, useState } from "react"; import { v4 as uuidv4 } from "uuid"; import { wsClient } from "../lib/websocket"; import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio"; import { getErrorMessage } from "../lib/errors"; import { showToast } from "../lib/toast"; import { useCamera } from "../components/CameraManager"; import { useMicrophone } from "../components/MicManager"; import { useVAD } from "../components/EdgeProcessor"; import { useWebSocketManager } from "../components/WebSocketManager"; import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types"; export function useVisionSession() { const [messages, setMessages] = useState([]); const [currentReply, setCurrentReply] = useState(""); const [isProcessing, setIsProcessing] = useState(false); const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); const { startMic, stopMic } = useMicrophone(); const { status, connect, disconnect, send } = useWebSocketManager(); // 用 ref 跟踪 isProcessing,避免 VAD 回调闭包问题 const isProcessingRef = useRef(false); useEffect(() => { isProcessingRef.current = isProcessing; }, [isProcessing]); // VAD:语音结束时自动发送 query const { isSpeaking, isReady: isVADReady, error: vadError, start: startVAD, stop: stopVAD, } = useVAD({ onSpeechEnd: useCallback( (audio: Float32Array) => { // 处理中忽略,防止重复发送 if (isProcessingRef.current) { console.warn("[Session] 正在处理中,忽略语音输入"); return; } const frame = captureFrame(); if (!frame) { console.warn("[Session] 无法捕获图像帧"); return; } const requestId = uuidv4(); send({ type: "query", request_id: requestId, image: dataUrlToBase64(frame), audio: encodeAudioToBase64(audio), }); // 添加用户消息(STT 流式结果会逐步更新文本) setMessages((prev) => [ ...prev, { role: "user", content: "(语音识别中...)", timestamp: Date.now() }, ]); setIsProcessing(true); }, [captureFrame, send] ), }); // 处理服务端消息 useEffect(() => { const unsub = wsClient.onMessage((msg: ServerMessage) => { switch (msg.type) { case "stt_result": { // 流式更新用户消息文本(包括中间结果和最终结果) setMessages((prev) => { const updated = [...prev]; const lastUserIdx = updated.findLastIndex((m) => m.role === "user"); if (lastUserIdx >= 0) { updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text || "(未识别到语音)", }; } return updated; }); break; } case "llm_chunk": setCurrentReply((prev) => prev + msg.delta); break; case "llm_done": { const done = msg as LLMDoneMessage; setMessages((prev) => [ ...prev, { role: "assistant", content: done.full_text, timestamp: Date.now(), tokensUsed: done.tokens_used?.total, latencyMs: done.latency_ms, model: done.model, }, ]); setCurrentReply(""); setIsProcessing(false); break; } case "tts_audio": // TODO: 阶段 5 音频流播放 break; case "error": console.error("[Session] 服务端错误:", msg.code, msg.message); showToast(getErrorMessage(msg.code), "error"); setIsProcessing(false); break; } }); return unsub; }, []); // 连接断开时显示提示 useEffect(() => { if (status === "disconnected") { // 只在非主动断开时提示(通过检查是否有活跃会话判断) // 这里简单处理,由 App 层根据状态显示 } }, [status]); /** 启动会话 */ const startSession = useCallback(async () => { // 1. 获取摄像头和麦克风 await startCamera(); const micStream = await startMic(); if (!micStream) { showToast("无法获取麦克风权限", "error"); return; } // 2. 连接 WebSocket connect(); // 3. 启动 VAD(传入麦克风 stream) await startVAD(micStream); }, [startCamera, startMic, connect, startVAD]); /** 结束会话 */ const stopSession = useCallback(async () => { await stopVAD(); stopMic(); stopCamera(); disconnect(); // 清理所有对话状态 setMessages([]); setCurrentReply(""); setIsProcessing(false); }, [stopVAD, stopMic, stopCamera, disconnect]); /** 打断当前回复 */ const interrupt = useCallback(() => { send({ type: "interrupt" }); // 将未完成的流式内容保存为最终消息 if (currentReply) { setMessages((prev) => [ ...prev, { role: "assistant", content: currentReply + "(已打断)", timestamp: Date.now() }, ]); } setCurrentReply(""); setIsProcessing(false); }, [send, currentReply]); return { messages, currentReply, isProcessing, isSpeaking, isVADReady, vadError, connectionStatus: status, videoRef, stream, startSession, stopSession, interrupt, }; }