CamTalk/frontend/src/hooks/useVisionSession.ts

// ============================================================
// useVisionSession — 核心视觉对话会话 Hook
// 职责：封装一次完整的视觉对话会话（摄像头、VAD、WebSocket、消息状态）
// 来源：docs/02-系统架构.md 核心 Hook 设计
// ============================================================

import { useCallback, useEffect, useRef, useState } from "react";
import { v4 as uuidv4 } from "uuid";
import { wsClient } from "../lib/websocket";
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
import { getErrorMessage } from "../lib/errors";
import { TTSPlayer } from "../lib/ttsPlayer";
import { showToast } from "../lib/toast";
import { loadConfig, saveConfig } from "../lib/storage";
import { useI18n } from "../lib/i18n";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import { useObservationMode } from "./useObservationMode";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";

export type SessionMode = "dialogue" | "observation";

const MAX_HISTORY_ROUNDS = 10;

export interface SessionStats {
  queryCount: number;
  totalTokens: number;
}

export function useVisionSession() {
  const { t } = useI18n();
  const [messages, setMessages] = useState<ChatMessage[]>([]);
  const [currentReply, setCurrentReply] = useState<string>("");
  const [isProcessing, setIsProcessing] = useState(false);
  const [isAudioPlaying, setIsAudioPlaying] = useState(false);
  const [config, setConfig] = useState<SessionConfig>(loadConfig);
  const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
  const [mode, setMode] = useState<SessionMode>("dialogue");
  const [isCameraOn, setIsCameraOn] = useState(false);
  const [isMicOn, setIsMicOn] = useState(false);

  // 上一帧采样数据（用于关键帧检测）
  const prevFrameRef = useRef<Uint8ClampedArray | null>(null);

  // 对话历史（role + content），用于多轮上下文
  const historyRef = useRef<Array<{ role: string; content: string }>>([]);

  // 待发消息队列（未连接时暂存，连接后自动发送）
  const pendingMessagesRef = useRef<Array<{ text: string; requestId: string }>>([]);

  // TTS 播放器
  const ttsPlayerRef = useRef<TTSPlayer | null>(null);
  const getTTSPlayer = useCallback(() => {
    if (!ttsPlayerRef.current) {
      const player = new TTSPlayer();
      player.onEnd(() => setIsAudioPlaying(false));
      ttsPlayerRef.current = player;
    }
    return ttsPlayerRef.current;
  }, []);

  const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
  const { startMic, stopMic } = useMicrophone();
  const { status, connect, disconnect, send } = useWebSocketManager();

  // 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题
  const isProcessingRef = useRef(false);
  useEffect(() => {
    isProcessingRef.current = isProcessing;
  }, [isProcessing]);

  // 用 ref 跟踪连接状态，避免回调闭包问题
  const statusRef = useRef(status);
  useEffect(() => {
    statusRef.current = status;
  }, [status]);

  // 观察模式：画面变化时自动发送 query
  const { isObserving, startObserving, stopObserving } = useObservationMode({
    onChange: useCallback(
      (frameDataUrl: string) => {
        if (isProcessingRef.current) return;

        const requestId = uuidv4();
        send({
          type: "query",
          request_id: requestId,
          image: dataUrlToBase64(frameDataUrl),
          audio: "", // 观察模式无音频
        });

        setMessages((prev) => [
          ...prev,
          {
            role: "user",
            content: t("session.changeDetected"),
            timestamp: Date.now(),
          },
        ]);
        setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
        setIsProcessing(true);
      },
      [send],
    ),
  });

  /** 切换对话/观察模式 */
  const toggleMode = useCallback(() => {
    setMode((prev) => {
      const next = prev === "dialogue" ? "observation" : "dialogue";
      if (next === "observation") {
        startObserving(videoRef.current, captureFrame);
      } else {
        stopObserving();
      }
      return next;
    });
  }, [videoRef, captureFrame, startObserving, stopObserving]);

  // WebSocket 连接成功后发送 config + flush 待发消息
  useEffect(() => {
    if (status === "connected") {
      send({
        type: "config",
        payload: {
          tts_enabled: config.ttsEnabled,
          detail_level: config.detailLevel,
          language: config.language,
        },
      });

      // flush 待发消息队列
      const pending = pendingMessagesRef.current;
      pendingMessagesRef.current = [];
      for (const msg of pending) {
        const frame = captureFrame();
        send({
          type: "query",
          request_id: msg.requestId,
          image: frame ? dataUrlToBase64(frame) : "",
          audio: "",
          text: msg.text,
        });
        setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
        setMessages((prev) => [
          ...prev,
          { role: "user", content: msg.text, timestamp: Date.now() },
        ]);
        historyRef.current.push({ role: "user", content: msg.text });
        setIsProcessing(true);
      }
    }
  }, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送

  /** 更新会话配置 */
  const updateConfig = useCallback((partial: Partial<SessionConfig>) => {
    setConfig((prev) => {
      const next = { ...prev, ...partial };
      saveConfig(next);
      // 如果已连接，立即发送更新
      if (status === "connected") {
        send({
          type: "config",
          payload: {
            tts_enabled: next.ttsEnabled,
            detail_level: next.detailLevel,
            language: next.language,
          },
        });
      }
      return next;
    });
  }, [status, send]);

  // VAD：语音结束时自动发送 query
  const {
    isSpeaking,
    isReady: isVADReady,
    error: vadError,
    start: startVAD,
    stop: stopVAD,
  } = useVAD({
    onSpeechEnd: useCallback(
      (audio: Float32Array) => {
        // 处理中忽略，防止重复发送
        if (isProcessingRef.current) {
          console.warn("[Session] 正在处理中，忽略语音输入");
          return;
        }

        // 停止上一轮的 TTS 播放，防止新旧音频重叠
        ttsPlayerRef.current?.stop();
        setIsAudioPlaying(false);

        const frame = captureFrame();
        if (!frame) {
          console.warn("[Session] 无法捕获图像帧");
          return;
        }

        // 关键帧检测：与上一帧对比，相似度过高则跳过
        const video = videoRef.current;
        if (video) {
          const currentSample = sampleFrame(video);
          if (currentSample && prevFrameRef.current) {
            const { similarity } = compareFrames(prevFrameRef.current, currentSample);
            if (similarity > 0.9) {
              console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)})，跳过`);
              prevFrameRef.current = currentSample;
              return;
            }
          }
          if (currentSample) {
            prevFrameRef.current = currentSample;
          }
        }

        const requestId = uuidv4();
        send({
          type: "query",
          request_id: requestId,
          image: dataUrlToBase64(frame),
          audio: encodeAudioToBase64(audio),
        });

        // 更新请求统计
        setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));

        // 添加用户消息（STT 流式结果会逐步更新文本）
        setMessages((prev) => [
          ...prev,
          { role: "user", content: t("session.recognizing"), timestamp: Date.now() },
        ]);
        setIsProcessing(true);
      },
      [captureFrame, send, videoRef]
    ),
  });

  // 处理服务端消息
  useEffect(() => {
    const unsub = wsClient.onMessage((msg: ServerMessage) => {
      switch (msg.type) {
        case "stt_result": {
          // 流式更新用户消息文本（包括中间结果和最终结果）
          setMessages((prev) => {
            const updated = [...prev];
            const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
            if (lastUserIdx >= 0) {
              updated[lastUserIdx] = {
                ...updated[lastUserIdx],
                content: msg.text || t("session.noSpeech"),
              };
            }
            return updated;
          });
          break;
        }

        case "llm_chunk":
          setCurrentReply((prev) => prev + msg.delta);
          break;

        case "llm_done": {
          const done = msg as LLMDoneMessage;
          // 记录到对话历史
          historyRef.current.push({ role: "assistant", content: done.full_text });
          // 裁剪历史到最近 N 轮
          if (historyRef.current.length > MAX_HISTORY_ROUNDS * 2) {
            historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
          }

          // 累计 token 统计
          if (done.tokens_used?.total) {
            setStats((prev) => ({
              ...prev,
              totalTokens: prev.totalTokens + done.tokens_used.total,
            }));
          }

          setMessages((prev) => [
            ...prev,
            {
              role: "assistant",
              content: done.full_text,
              timestamp: Date.now(),
              tokensUsed: done.tokens_used?.total,
              latencyMs: done.latency_ms,
              model: done.model,
            },
          ]);
          setCurrentReply("");
          setIsProcessing(false);
          break;
        }

        case "tts_audio":
          getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
          setIsAudioPlaying(true);
          break;

        case "error":
          console.error("[Session] 服务端错误:", msg.code, msg.message);
          showToast(getErrorMessage(msg.code, t), "error");
          setIsProcessing(false);
          break;
      }
    });

    return unsub;
  }, [getTTSPlayer]);

  /** 启动视频通话（摄像头 + 麦克风 + VAD） */
  const startSession = useCallback(async () => {
    // 1. 确保 WebSocket 已连接
    if (statusRef.current !== "connected") {
      connect();
      // 等待连接完成（通过 status 变化触发后续流程，这里直接继续）
    }

    // 2. 尝试获取摄像头（可选）
    try {
      await startCamera();
      setIsCameraOn(true);
    } catch {
      console.warn("[Session] 无法获取摄像头权限，将以纯文本模式运行");
    }

    // 3. 尝试获取麦克风（可选）
    const micStream = await startMic();
    if (micStream) {
      setIsMicOn(true);
      // 4. 启动 VAD（仅在麦克风可用时）
      await startVAD(micStream);
    } else {
      console.warn("[Session] 无法获取麦克风权限，将以文本输入模式运行");
    }
  }, [startCamera, startMic, connect, startVAD]);

  /** 结束会话 */
  const stopSession = useCallback(async () => {
    stopObserving();
    setMode("dialogue");
    await stopVAD();
    stopMic();
    stopCamera();
    disconnect();
    // 停止 TTS 并清理状态
    ttsPlayerRef.current?.stop();
    setIsAudioPlaying(false);
    setMessages([]);
    setCurrentReply("");
    setIsProcessing(false);
    setStats({ queryCount: 0, totalTokens: 0 });
    historyRef.current = [];
    prevFrameRef.current = null;
    setIsCameraOn(false);
    setIsMicOn(false);
  }, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);

  /** 摄像头开关 */
  const toggleCamera = useCallback(async () => {
    if (isCameraOn) {
      stopCamera();
      setIsCameraOn(false);
    } else {
      await startCamera();
      setIsCameraOn(true);
    }
  }, [isCameraOn, startCamera, stopCamera]);

  /** 麦克风开关 */
  const toggleMic = useCallback(async () => {
    if (isMicOn) {
      await stopVAD();
      stopMic();
      setIsMicOn(false);
    } else {
      const micStream = await startMic();
      if (micStream) {
        await startVAD(micStream);
        setIsMicOn(true);
      }
    }
  }, [isMicOn, startMic, stopMic, startVAD, stopVAD]);

  /** 打断当前回复 */
  const interrupt = useCallback(() => {
    send({ type: "interrupt" });
    // 停止 TTS 播放
    ttsPlayerRef.current?.stop();
    setIsAudioPlaying(false);
    // 将未完成的流式内容保存为最终消息
    if (currentReply) {
      const interrupted = currentReply + t("session.interrupted");
      historyRef.current.push({ role: "assistant", content: interrupted });
      setMessages((prev) => [
        ...prev,
        { role: "assistant", content: interrupted, timestamp: Date.now() },
      ]);
    }
    setCurrentReply("");
    setIsProcessing(false);
  }, [send, currentReply]);

  /** 发送文本消息（手动输入） */
  const sendTextMessage = useCallback(
    (text: string) => {
      if (!text.trim() || isProcessingRef.current) return;

      // 停止上一轮的 TTS 播放
      ttsPlayerRef.current?.stop();
      setIsAudioPlaying(false);

      const requestId = uuidv4();

      // 未连接时：自动连接，消息加入待发队列
      if (statusRef.current !== "connected") {
        pendingMessagesRef.current.push({ text: text.trim(), requestId });
        // 添加用户消息到 UI（立即反馈）
        setMessages((prev) => [
          ...prev,
          { role: "user", content: text.trim(), timestamp: Date.now() }],
        );
        // 自动连接 WebSocket
        connect();
        return;
      }

      // 已连接：直接发送
      const frame = captureFrame();
      send({
        type: "query",
        request_id: requestId,
        image: frame ? dataUrlToBase64(frame) : "",
        audio: "", // 文本输入无音频
        text: text.trim(),
      });

      // 更新请求统计
      setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));

      // 添加用户消息
      setMessages((prev) => [
        ...prev,
        { role: "user", content: text.trim(), timestamp: Date.now() },
      ]);

      // 记录到对话历史
      historyRef.current.push({ role: "user", content: text.trim() });

      setIsProcessing(true);
    },
    [captureFrame, send, connect],
  );

  return {
    messages,
    setMessages,
    currentReply,
    isProcessing,
    isAudioPlaying,
    isSpeaking,
    isVADReady,
    vadError,
    connectionStatus: status,
    videoRef,
    stream,
    config,
    updateConfig,
    stats,
    mode,
    isObserving,
    toggleMode,
    startSession,
    stopSession,
    interrupt,
    isCameraOn,
    isMicOn,
    toggleCamera,
    toggleMic,
    sendTextMessage,
  };
}