// ============================================================ // useVisionSession — 核心视觉对话会话 Hook // 职责:封装一次完整的视觉对话会话(摄像头、VAD、WebSocket、消息状态) // 来源:docs/02-系统架构.md 核心 Hook 设计 // ============================================================ import { useCallback, useEffect, useState } from "react"; import { v4 as uuidv4 } from "uuid"; import { wsClient } from "../lib/websocket"; import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio"; import { useCamera } from "../components/CameraManager"; import { useMicrophone } from "../components/MicManager"; import { useVAD } from "../components/EdgeProcessor"; import { useWebSocketManager } from "../components/WebSocketManager"; import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types"; export function useVisionSession() { const [messages, setMessages] = useState([]); const [currentReply, setCurrentReply] = useState(""); const [isProcessing, setIsProcessing] = useState(false); const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); const { startMic, stopMic } = useMicrophone(); const { status, connect, disconnect, send } = useWebSocketManager(); // VAD:语音结束时自动发送 query const { isSpeaking, isReady: isVADReady, error: vadError, start: startVAD, stop: stopVAD, } = useVAD({ onSpeechEnd: useCallback( (audio: Float32Array) => { const frame = captureFrame(); if (!frame) { console.warn("[Session] 无法捕获图像帧"); return; } const requestId = uuidv4(); send({ type: "query", request_id: requestId, image: dataUrlToBase64(frame), audio: encodeAudioToBase64(audio), }); // 添加用户消息(STT 结果到达后会更新文本) setMessages((prev) => [ ...prev, { role: "user", content: "(语音识别中...)", timestamp: Date.now() }, ]); setIsProcessing(true); }, [captureFrame, send] ), }); // 处理服务端消息 useEffect(() => { const unsub = wsClient.onMessage((msg: ServerMessage) => { switch (msg.type) { case "stt_result": if (msg.is_final) { setMessages((prev) => { const updated = [...prev]; const lastUserIdx = updated.findLastIndex((m) => m.role === "user"); if (lastUserIdx >= 0) { updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text }; } return updated; }); } break; case "llm_chunk": setCurrentReply((prev) => prev + msg.delta); break; case "llm_done": setMessages((prev) => [ ...prev, { role: "assistant", content: (msg as LLMDoneMessage).full_text, timestamp: Date.now(), tokensUsed: (msg as LLMDoneMessage).tokens_used?.total, }, ]); setCurrentReply(""); setIsProcessing(false); break; case "tts_audio": // TODO: 音频流播放 break; case "error": console.error("[Session] 服务端错误:", msg.code, msg.message); setIsProcessing(false); break; } }); return unsub; }, []); /** 启动会话 */ const startSession = useCallback(async () => { // 1. 获取摄像头和麦克风 await startCamera(); const micStream = await startMic(); if (!micStream) { console.error("[Session] 无法获取麦克风"); return; } // 2. 连接 WebSocket connect(); // 3. 启动 VAD(传入麦克风 stream) await startVAD(micStream); }, [startCamera, startMic, connect, startVAD]); /** 结束会话 */ const stopSession = useCallback(async () => { await stopVAD(); stopMic(); stopCamera(); disconnect(); }, [stopVAD, stopMic, stopCamera, disconnect]); /** 打断当前回复 */ const interrupt = useCallback(() => { send({ type: "interrupt" }); setIsProcessing(false); }, [send]); return { messages, currentReply, isProcessing, isSpeaking, isVADReady, vadError, connectionStatus: status, videoRef, stream, startSession, stopSession, interrupt, }; }