CamTalk/frontend/src/hooks/useVisionSession.ts

// ============================================================
// useVisionSession — 核心视觉对话会话 Hook
// 职责：封装一次完整的视觉对话会话（摄像头、VAD、WebSocket、消息状态）
// 来源：docs/02-系统架构.md 核心 Hook 设计
// ============================================================

import { useCallback, useEffect, useRef, useState } from "react";
import { v4 as uuidv4 } from "uuid";
import { wsClient } from "../lib/websocket";
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
import { getErrorMessage } from "../lib/errors";
import { TTSPlayer } from "../lib/ttsPlayer";
import { showToast } from "../lib/toast";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";

export function useVisionSession() {
  const [messages, setMessages] = useState<ChatMessage[]>([]);
  const [currentReply, setCurrentReply] = useState<string>("");
  const [isProcessing, setIsProcessing] = useState(false);
  const [isAudioPlaying, setIsAudioPlaying] = useState(false);

  // TTS 播放器
  const ttsPlayerRef = useRef<TTSPlayer | null>(null);
  const getTTSPlayer = useCallback(() => {
    if (!ttsPlayerRef.current) {
      const player = new TTSPlayer();
      player.onEnd(() => setIsAudioPlaying(false));
      ttsPlayerRef.current = player;
    }
    return ttsPlayerRef.current;
  }, []);

  const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
  const { startMic, stopMic } = useMicrophone();
  const { status, connect, disconnect, send } = useWebSocketManager();

  // 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题
  const isProcessingRef = useRef(false);
  useEffect(() => {
    isProcessingRef.current = isProcessing;
  }, [isProcessing]);

  // VAD：语音结束时自动发送 query
  const {
    isSpeaking,
    isReady: isVADReady,
    error: vadError,
    start: startVAD,
    stop: stopVAD,
  } = useVAD({
    onSpeechEnd: useCallback(
      (audio: Float32Array) => {
        // 处理中忽略，防止重复发送
        if (isProcessingRef.current) {
          console.warn("[Session] 正在处理中，忽略语音输入");
          return;
        }

        const frame = captureFrame();
        if (!frame) {
          console.warn("[Session] 无法捕获图像帧");
          return;
        }

        const requestId = uuidv4();
        send({
          type: "query",
          request_id: requestId,
          image: dataUrlToBase64(frame),
          audio: encodeAudioToBase64(audio),
        });

        // 添加用户消息（STT 流式结果会逐步更新文本）
        setMessages((prev) => [
          ...prev,
          { role: "user", content: "（语音识别中...）", timestamp: Date.now() },
        ]);
        setIsProcessing(true);
      },
      [captureFrame, send]
    ),
  });

  // 处理服务端消息
  useEffect(() => {
    const unsub = wsClient.onMessage((msg: ServerMessage) => {
      switch (msg.type) {
        case "stt_result": {
          // 流式更新用户消息文本（包括中间结果和最终结果）
          setMessages((prev) => {
            const updated = [...prev];
            const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
            if (lastUserIdx >= 0) {
              updated[lastUserIdx] = {
                ...updated[lastUserIdx],
                content: msg.text || "（未识别到语音）",
              };
            }
            return updated;
          });
          break;
        }

        case "llm_chunk":
          setCurrentReply((prev) => prev + msg.delta);
          break;

        case "llm_done": {
          const done = msg as LLMDoneMessage;
          setMessages((prev) => [
            ...prev,
            {
              role: "assistant",
              content: done.full_text,
              timestamp: Date.now(),
              tokensUsed: done.tokens_used?.total,
              latencyMs: done.latency_ms,
              model: done.model,
            },
          ]);
          setCurrentReply("");
          setIsProcessing(false);
          break;
        }

        case "tts_audio":
          getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
          if (!msg.is_last) {
            setIsAudioPlaying(true);
          }
          break;

        case "error":
          console.error("[Session] 服务端错误:", msg.code, msg.message);
          showToast(getErrorMessage(msg.code), "error");
          setIsProcessing(false);
          break;
      }
    });

    return unsub;
  }, [getTTSPlayer]);

  // 连接断开时显示提示
  useEffect(() => {
    if (status === "disconnected") {
      // 只在非主动断开时提示（通过检查是否有活跃会话判断）
      // 这里简单处理，由 App 层根据状态显示
    }
  }, [status]);

  /** 启动会话 */
  const startSession = useCallback(async () => {
    // 1. 获取摄像头和麦克风
    await startCamera();
    const micStream = await startMic();
    if (!micStream) {
      showToast("无法获取麦克风权限", "error");
      return;
    }

    // 2. 连接 WebSocket
    connect();

    // 3. 启动 VAD（传入麦克风 stream）
    await startVAD(micStream);
  }, [startCamera, startMic, connect, startVAD]);

  /** 结束会话 */
  const stopSession = useCallback(async () => {
    await stopVAD();
    stopMic();
    stopCamera();
    disconnect();
    // 停止 TTS 并清理状态
    ttsPlayerRef.current?.stop();
    setIsAudioPlaying(false);
    setMessages([]);
    setCurrentReply("");
    setIsProcessing(false);
  }, [stopVAD, stopMic, stopCamera, disconnect]);

  /** 打断当前回复 */
  const interrupt = useCallback(() => {
    send({ type: "interrupt" });
    // 停止 TTS 播放
    ttsPlayerRef.current?.stop();
    setIsAudioPlaying(false);
    // 将未完成的流式内容保存为最终消息
    if (currentReply) {
      setMessages((prev) => [
        ...prev,
        { role: "assistant", content: currentReply + "（已打断）", timestamp: Date.now() },
      ]);
    }
    setCurrentReply("");
    setIsProcessing(false);
  }, [send, currentReply]);

  return {
    messages,
    currentReply,
    isProcessing,
    isAudioPlaying,
    isSpeaking,
    isVADReady,
    vadError,
    connectionStatus: status,
    videoRef,
    stream,
    startSession,
    stopSession,
    interrupt,
  };
}