CamTalk/frontend/src/hooks/useVisionSession.ts

// ============================================================
// useVisionSession — 核心视觉对话会话 Hook
// 职责：封装一次完整的视觉对话会话（摄像头、VAD、WebSocket、消息状态）
// 来源：docs/02-系统架构.md 核心 Hook 设计
// ============================================================

import { useCallback, useEffect, useState } from "react";
import { v4 as uuidv4 } from "uuid";
import { wsClient } from "../lib/websocket";
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
import { useCamera } from "../components/CameraManager";
import { useVAD } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";

export function useVisionSession() {
  const [messages, setMessages] = useState<ChatMessage[]>([]);
  const [currentReply, setCurrentReply] = useState<string>("");
  const [isProcessing, setIsProcessing] = useState(false);

  const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
  const { status, connect, disconnect, send } = useWebSocketManager();

  // VAD：语音结束时自动发送 query
  const { isSpeaking, start: startVAD, stop: stopVAD } = useVAD({
    onSpeechEnd: useCallback(
      (audio: Float32Array) => {
        const frame = captureFrame();
        if (!frame) {
          console.warn("[Session] 无法捕获图像帧");
          return;
        }

        const requestId = uuidv4();
        send({
          type: "query",
          request_id: requestId,
          image: dataUrlToBase64(frame),
          audio: encodeAudioToBase64(audio),
        });

        // 添加用户消息（STT 结果到达后会更新文本）
        setMessages((prev) => [
          ...prev,
          { role: "user", content: "（语音识别中...）", timestamp: Date.now() },
        ]);
        setIsProcessing(true);
      },
      [captureFrame, send]
    ),
  });

  // 处理服务端消息
  useEffect(() => {
    const unsub = wsClient.onMessage((msg: ServerMessage) => {
      switch (msg.type) {
        case "stt_result":
          if (msg.is_final) {
            setMessages((prev) => {
              const updated = [...prev];
              const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
              if (lastUserIdx >= 0) {
                updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text };
              }
              return updated;
            });
          }
          break;

        case "llm_chunk":
          setCurrentReply((prev) => prev + msg.delta);
          break;

        case "llm_done":
          setMessages((prev) => [
            ...prev,
            {
              role: "assistant",
              content: (msg as LLMDoneMessage).full_text,
              timestamp: Date.now(),
              tokensUsed: (msg as LLMDoneMessage).tokens_used?.total,
            },
          ]);
          setCurrentReply("");
          setIsProcessing(false);
          break;

        case "tts_audio":
          // TODO: 音频流播放
          break;

        case "error":
          console.error("[Session] 服务端错误:", msg.code, msg.message);
          setIsProcessing(false);
          break;
      }
    });

    return unsub;
  }, []);

  /** 启动会话 */
  const startSession = useCallback(async () => {
    await startCamera();
    connect();
    startVAD();
  }, [startCamera, connect, startVAD]);

  /** 结束会话 */
  const stopSession = useCallback(() => {
    stopVAD();
    stopCamera();
    disconnect();
  }, [stopVAD, stopCamera, disconnect]);

  /** 打断当前回复 */
  const interrupt = useCallback(() => {
    send({ type: "interrupt" });
    setIsProcessing(false);
  }, [send]);

  return {
    messages,
    currentReply,
    isProcessing,
    isSpeaking,
    connectionStatus: status,
    videoRef,
    stream,
    startSession,
    stopSession,
    interrupt,
  };
}