Merge pull request '完善端到端对话闭环，增强消息渲染与错误处理;集成 VAD 语音活动检测，打通核心交互链路' (#24) from develop-frontend into develop

Reviewed-on: http://8.161.227.145:3000/XEngineers/CamTalk/pulls/24

frontend/src/App.css

@@ -103,6 +103,15 @@ body {
   animation: pulse 1.5s ease-in-out infinite;
 }
 
+.vad-indicator--loading {
+  color: var(--color-warning);
+}
+
+.vad-indicator--error {
+  color: var(--color-error);
+  animation: none;
+}
+
 @keyframes pulse {
   0%, 100% { opacity: 1; }
   50% { opacity: 0.6; }
@@ -110,10 +119,10 @@ body {
 
 .chat-section {
   flex: 1;
-  overflow-y: auto;
   display: flex;
   flex-direction: column;
   gap: 12px;
+  overflow: hidden;
 }
 
 /* ---- Video Preview ---- */
@@ -148,6 +157,8 @@ body {
   display: flex;
   flex-direction: column;
   gap: 12px;
+  overflow-y: auto;
+  flex: 1;
 }
 
 .chat-panel--empty {
@@ -239,3 +250,111 @@ body {
 .btn--warning:hover {
   opacity: 0.9;
 }
+
+/* ---- Streaming Cursor ---- */
+
+.cursor {
+  display: inline;
+  animation: blink 0.8s step-end infinite;
+  color: var(--color-success);
+}
+
+@keyframes blink {
+  0%, 100% { opacity: 1; }
+  50% { opacity: 0; }
+}
+
+/* ---- Message Meta ---- */
+
+.chat-message__meta {
+  margin-top: 6px;
+  font-size: 0.75rem;
+  color: var(--color-text-muted);
+}
+
+/* ---- System Message ---- */
+
+.system-message {
+  text-align: center;
+  padding: 8px 16px;
+  border-radius: var(--radius);
+  font-size: 0.85rem;
+}
+
+.system-message--warning {
+  background: rgba(245, 158, 11, 0.15);
+  color: var(--color-warning);
+  border: 1px solid rgba(245, 158, 11, 0.3);
+}
+
+/* ---- Toast ---- */
+
+.toast-container {
+  position: fixed;
+  top: 16px;
+  right: 16px;
+  z-index: 1000;
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+}
+
+.toast {
+  padding: 10px 16px;
+  border-radius: var(--radius);
+  font-size: 0.85rem;
+  cursor: pointer;
+  animation: slideIn 0.3s ease-out;
+  max-width: 320px;
+}
+
+.toast--error {
+  background: rgba(239, 68, 68, 0.9);
+  color: white;
+}
+
+.toast--warning {
+  background: rgba(245, 158, 11, 0.9);
+  color: #000;
+}
+
+.toast--info {
+  background: rgba(37, 99, 235, 0.9);
+  color: white;
+}
+
+@keyframes slideIn {
+  from {
+    opacity: 0;
+    transform: translateX(20px);
+  }
+  to {
+    opacity: 1;
+    transform: translateX(0);
+  }
+}
+
+/* ---- Video Overlay ---- */
+
+.video-overlay {
+  position: absolute;
+  inset: 0;
+  background: rgba(0, 0, 0, 0.3);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  border-radius: var(--radius);
+}
+
+.video-overlay__spinner {
+  width: 32px;
+  height: 32px;
+  border: 3px solid rgba(255, 255, 255, 0.3);
+  border-top-color: white;
+  border-radius: 50%;
+  animation: spin 0.8s linear infinite;
+}
+
+@keyframes spin {
+  to { transform: rotate(360deg); }
+}

frontend/src/App.tsx

@@ -5,6 +5,7 @@
 import { useVisionSession } from "./hooks/useVisionSession";
 import { VideoPreview } from "./components/VideoPreview";
 import { ChatPanel } from "./components/ChatPanel";
+import { ToastContainer } from "./components/Toast";
 import "./App.css";
 
 function App() {
@@ -13,6 +14,8 @@ function App() {
     currentReply,
     isProcessing,
     isSpeaking,
+    isVADReady,
+    vadError,
     connectionStatus,
     videoRef,
     stream,
@@ -21,12 +24,18 @@ function App() {
     interrupt,
   } = useVisionSession();
 
+  const isConnected = connectionStatus === "connected";
+
   return (
     <div className="app">
       <header className="app-header">
         <h1>CamTalk</h1>
         <span className={`status status--${connectionStatus}`}>
-          {connectionStatus === "connected" ? "已连接" : connectionStatus === "connecting" ? "连接中..." : "未连接"}
+          {isConnected
+            ? "已连接"
+            : connectionStatus === "connecting"
+              ? "连接中..."
+              : "未连接"}
         </span>
       </header>
 
@@ -34,23 +43,41 @@ function App() {
         <div className="video-section">
           <VideoPreview ref={videoRef} isStreaming={!!stream} />
           {isSpeaking && <div className="vad-indicator">🎤 正在聆听...</div>}
+          {isConnected && !isVADReady && !vadError && (
+            <div className="vad-indicator vad-indicator--loading">
+              正在初始化语音检测...
+            </div>
+          )}
+          {vadError && (
+            <div className="vad-indicator vad-indicator--error">
+              ⚠️ {vadError}
+            </div>
+          )}
+          {isProcessing && (
+            <div className="video-overlay">
+              <div className="video-overlay__spinner" />
+            </div>
+          )}
         </div>
 
         <div className="chat-section">
-          <ChatPanel messages={messages} />
+          {connectionStatus === "disconnected" && messages.length > 0 && (
-          {currentReply && (
+            <div className="system-message system-message--warning">
-            <div className="chat-message chat-message--assistant chat-message--streaming">
+              连接已断开，正在重连...
-              <div className="chat-message__role">AI</div>
-              <div className="chat-message__content">{currentReply}</div>
             </div>
           )}
+          <ChatPanel
+            messages={messages}
+            currentReply={currentReply}
+            connectionStatus={connectionStatus}
+          />
         </div>
       </main>
 
       <footer className="app-footer">
-        {connectionStatus !== "connected" ? (
+        {!isConnected ? (
           <button className="btn btn--primary" onClick={startSession}>
-            开始对话
+            {connectionStatus === "connecting" ? "连接中..." : "开始对话"}
           </button>
         ) : (
           <>
@@ -65,6 +92,8 @@ function App() {
           </>
         )}
       </footer>
+
+      <ToastContainer />
     </div>
   );
 }

frontend/src/components/ChatPanel/index.tsx

@@ -1,33 +1,90 @@
 // ============================================================
 // ChatPanel — 消息展示面板
-// 职责：渲染对话消息列表（用户提问 + AI 回复）
+// 职责：渲染对话消息列表、流式光标、元数据、自动滚动
 // ============================================================
 
+import { useEffect, useRef } from "react";
 import type { ChatMessage } from "../../types";
+import type { ConnectionStatus } from "../../lib/websocket";
 
 interface ChatPanelProps {
   messages: ChatMessage[];
+  currentReply?: string;
+  connectionStatus: ConnectionStatus;
 }
 
-export function ChatPanel({ messages }: ChatPanelProps) {
+export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPanelProps) {
-  if (messages.length === 0) {
+  const bottomRef = useRef<HTMLDivElement>(null);
+  const containerRef = useRef<HTMLDivElement>(null);
+  const isAutoScroll = useRef(true);
+
+  // 用户上滚时暂停自动滚动，滚到底部时恢复
+  useEffect(() => {
+    const container = containerRef.current;
+    if (!container) return;
+
+    const handleScroll = () => {
+      const { scrollTop, scrollHeight, clientHeight } = container;
+      isAutoScroll.current = scrollHeight - scrollTop - clientHeight < 60;
+    };
+
+    container.addEventListener("scroll", handleScroll);
+    return () => container.removeEventListener("scroll", handleScroll);
+  }, []);
+
+  // 新消息或流式更新时自动滚动
+  useEffect(() => {
+    if (isAutoScroll.current) {
+      bottomRef.current?.scrollIntoView({ behavior: "smooth" });
+    }
+  }, [messages, currentReply]);
+
+  // 空状态
+  if (messages.length === 0 && !currentReply) {
+    if (connectionStatus !== "connected") {
+      return (
+        <div className="chat-panel chat-panel--empty">
+          <p>点击下方按钮开始对话</p>
+        </div>
+      );
+    }
     return (
       <div className="chat-panel chat-panel--empty">
-        <p>开始对话：对着摄像头说话即可</p>
+        <p>对着摄像头说话即可</p>
       </div>
     );
   }
 
   return (
-    <div className="chat-panel">
+    <div className="chat-panel" ref={containerRef}>
       {messages.map((msg, index) => (
         <div key={index} className={`chat-message chat-message--${msg.role}`}>
           <div className="chat-message__role">
             {msg.role === "user" ? "你" : "AI"}
           </div>
           <div className="chat-message__content">{msg.content}</div>
+          {msg.role === "assistant" && msg.tokensUsed !== undefined && (
+            <div className="chat-message__meta">
+              {msg.tokensUsed} tokens
+              {msg.latencyMs !== undefined && ` · ${(msg.latencyMs / 1000).toFixed(1)}s`}
+              {msg.model && ` · ${msg.model}`}
+            </div>
+          )}
         </div>
       ))}
+
+      {/* 流式回复（尚未完成） */}
+      {currentReply && (
+        <div className="chat-message chat-message--assistant chat-message--streaming">
+          <div className="chat-message__role">AI</div>
+          <div className="chat-message__content">
+            {currentReply}
+            <span className="cursor">▌</span>
+          </div>
+        </div>
+      )}
+
+      <div ref={bottomRef} />
     </div>
   );
 }

frontend/src/components/EdgeProcessor/index.tsx

@@ -4,35 +4,103 @@
 // 技术：@ricky0123/vad-web（VAD）、ONNX Runtime Web（关键帧检测）
 // ============================================================
 
-import { useCallback, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { MicVAD } from "@ricky0123/vad-web";
 
 export interface VADOptions {
-  /** 语音结束回调，携带录音 Float32Array */
+  /** 语音结束回调，携带录音 Float32Array（16kHz） */
   onSpeechEnd?: (audio: Float32Array) => void;
   /** 语音开始回调 */
   onSpeechStart?: () => void;
+  /** 语音过短被忽略回调 */
+  onVADMisfire?: () => void;
 }
 
-export function useVAD(_options?: VADOptions) {
+/**
-  const [isSpeaking] = useState(false);
+ * 语音活动检测 Hook
-  // TODO: 初始化 @ricky0123/vad-web，加载后设为 true
+ * 基于 @ricky0123/vad-web 的 MicVAD，检测用户说话并回调
-  const isReady = false;
+ */
+export function useVAD(options?: VADOptions) {
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const [isReady, setIsReady] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const vadRef = useRef<MicVAD | null>(null);
+  const optionsRef = useRef(options);
 
-  // TODO: 实现 VAD 初始化
+  // 保持 options 引用最新，避免回调闭包问题
-  // 1. 加载 @ricky0123/vad-web
+  useEffect(() => {
-  // 2. 配置 VAD 参数（阈值、最小语音时长等）
+    optionsRef.current = options;
-  // 3. 连接麦克风 stream
+  }, [options]);
-  // 4. 在 onSpeechEnd 时收集音频并回调 _options.onSpeechEnd
 
-  const start = useCallback(() => {
+  /**
-    // TODO: 启动 VAD 监听
+   * 初始化 VAD 并开始监听
+   * @param stream 麦克风 MediaStream（由外部管理）
+   */
+  const start = useCallback(async (stream: MediaStream) => {
+    // 如果已有实例，先销毁
+    if (vadRef.current) {
+      await vadRef.current.destroy();
+      vadRef.current = null;
+    }
+
+    try {
+      const vad = await MicVAD.new({
+        getStream: () => Promise.resolve(stream),
+        startOnLoad: true,
+        model: "legacy",
+
+        onSpeechStart: () => {
+          setIsSpeaking(true);
+          optionsRef.current?.onSpeechStart?.();
+        },
+
+        onSpeechEnd: (audio: Float32Array) => {
+          setIsSpeaking(false);
+          optionsRef.current?.onSpeechEnd?.(audio);
+        },
+
+        onVADMisfire: () => {
+          setIsSpeaking(false);
+          optionsRef.current?.onVADMisfire?.();
+        },
+
+        // VAD 参数（对齐 docs/06-语音交互.md 推荐值）
+        positiveSpeechThreshold: 0.5,
+        negativeSpeechThreshold: 0.35,
+        redemptionMs: 300,
+        preSpeechPadMs: 300,
+        minSpeechMs: 250,
+        submitUserSpeechOnPause: false,
+      });
+
+      vadRef.current = vad;
+      setIsReady(true);
+      setError(null);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : "VAD 初始化失败";
+      setError(message);
+      console.error("[VAD] 初始化失败:", err);
+    }
   }, []);
 
-  const stop = useCallback(() => {
+  /** 停止 VAD 并销毁实例 */
-    // TODO: 停止 VAD 监听
+  const stop = useCallback(async () => {
+    if (vadRef.current) {
+      await vadRef.current.destroy();
+      vadRef.current = null;
+    }
+    setIsReady(false);
+    setIsSpeaking(false);
   }, []);
 
-  return { isSpeaking, isReady, start, stop };
+  // 组件卸载时清理
+  useEffect(() => {
+    return () => {
+      vadRef.current?.destroy();
+    };
+  }, []);
+
+  return { isSpeaking, isReady, error, start, stop };
 }
 
 // ---- 关键帧检测（ONNX Runtime Web）----
@@ -41,11 +109,6 @@ export function useKeyframeDetection() {
   // TODO: 加载 ONNX 模型后设为 true
   const isReady = false;
 
-  // TODO: 实现关键帧检测
-  // 1. 加载 ONNX 模型
-  // 2. 对比当前帧与上一帧的像素差异
-  // 3. 超过阈值则判定为关键帧
-
   const isKeyframe = useCallback(
     (_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
       // TODO: 实现像素差异对比

frontend/src/components/Toast/index.tsx

@@ -0,0 +1,51 @@
+// ============================================================
+// Toast — 轻量通知组件
+// 职责：3 秒自动消失的通知提示
+// ============================================================
+
+import { useCallback, useEffect, useState } from "react";
+import { registerToastSetter, type ToastItem } from "../../lib/toast";
+
+export type { ToastItem };
+
+/** Toast 容器组件，放在 App 根部 */
+export function ToastContainer() {
+  const [toasts, setToasts] = useState<ToastItem[]>([]);
+
+  // 注册全局 setter
+  useEffect(() => {
+    registerToastSetter(setToasts);
+    return () => registerToastSetter(null);
+  }, []);
+
+  const dismiss = useCallback((id: number) => {
+    setToasts((prev) => prev.filter((t) => t.id !== id));
+  }, []);
+
+  return (
+    <div className="toast-container">
+      {toasts.map((t) => (
+        <ToastItemView key={t.id} item={t} onDismiss={dismiss} />
+      ))}
+    </div>
+  );
+}
+
+function ToastItemView({
+  item,
+  onDismiss,
+}: {
+  item: ToastItem;
+  onDismiss: (id: number) => void;
+}) {
+  useEffect(() => {
+    const timer = setTimeout(() => onDismiss(item.id), 3000);
+    return () => clearTimeout(timer);
+  }, [item.id, onDismiss]);
+
+  return (
+    <div className={`toast toast--${item.type}`} onClick={() => onDismiss(item.id)}>
+      {item.message}
+    </div>
+  );
+}

frontend/src/hooks/useVisionSession.ts

@@ -4,11 +4,14 @@
 // 来源：docs/02-系统架构.md 核心 Hook 设计
 // ============================================================
 
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import { v4 as uuidv4 } from "uuid";
 import { wsClient } from "../lib/websocket";
 import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
+import { getErrorMessage } from "../lib/errors";
+import { showToast } from "../lib/toast";
 import { useCamera } from "../components/CameraManager";
+import { useMicrophone } from "../components/MicManager";
 import { useVAD } from "../components/EdgeProcessor";
 import { useWebSocketManager } from "../components/WebSocketManager";
 import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";
@@ -19,12 +22,31 @@ export function useVisionSession() {
   const [isProcessing, setIsProcessing] = useState(false);
 
   const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
+  const { startMic, stopMic } = useMicrophone();
   const { status, connect, disconnect, send } = useWebSocketManager();
 
+  // 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题
+  const isProcessingRef = useRef(false);
+  useEffect(() => {
+    isProcessingRef.current = isProcessing;
+  }, [isProcessing]);
+
   // VAD：语音结束时自动发送 query
-  const { isSpeaking, start: startVAD, stop: stopVAD } = useVAD({
+  const {
+    isSpeaking,
+    isReady: isVADReady,
+    error: vadError,
+    start: startVAD,
+    stop: stopVAD,
+  } = useVAD({
     onSpeechEnd: useCallback(
       (audio: Float32Array) => {
+        // 处理中忽略，防止重复发送
+        if (isProcessingRef.current) {
+          console.warn("[Session] 正在处理中，忽略语音输入");
+          return;
+        }
+
         const frame = captureFrame();
         if (!frame) {
           console.warn("[Session] 无法捕获图像帧");
@@ -39,7 +61,7 @@ export function useVisionSession() {
           audio: encodeAudioToBase64(audio),
         });
 
-        // 添加用户消息（STT 结果到达后会更新文本）
+        // 添加用户消息（STT 流式结果会逐步更新文本）
         setMessages((prev) => [
           ...prev,
           { role: "user", content: "（语音识别中...）", timestamp: Date.now() },
@@ -54,43 +76,51 @@ export function useVisionSession() {
   useEffect(() => {
     const unsub = wsClient.onMessage((msg: ServerMessage) => {
       switch (msg.type) {
-        case "stt_result":
+        case "stt_result": {
-          if (msg.is_final) {
+          // 流式更新用户消息文本（包括中间结果和最终结果）
-            setMessages((prev) => {
+          setMessages((prev) => {
-              const updated = [...prev];
+            const updated = [...prev];
-              const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
+            const lastUserIdx = updated.findLastIndex((m) => m.role === "user");
-              if (lastUserIdx >= 0) {
+            if (lastUserIdx >= 0) {
-                updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text };
+              updated[lastUserIdx] = {
-              }
+                ...updated[lastUserIdx],
-              return updated;
+                content: msg.text || "（未识别到语音）",
-            });
+              };
-          }
+            }
+            return updated;
+          });
           break;
+        }
 
         case "llm_chunk":
           setCurrentReply((prev) => prev + msg.delta);
           break;
 
-        case "llm_done":
+        case "llm_done": {
+          const done = msg as LLMDoneMessage;
           setMessages((prev) => [
             ...prev,
             {
               role: "assistant",
-              content: (msg as LLMDoneMessage).full_text,
+              content: done.full_text,
               timestamp: Date.now(),
-              tokensUsed: (msg as LLMDoneMessage).tokens_used?.total,
+              tokensUsed: done.tokens_used?.total,
+              latencyMs: done.latency_ms,
+              model: done.model,
             },
           ]);
           setCurrentReply("");
           setIsProcessing(false);
           break;
+        }
 
         case "tts_audio":
-          // TODO: 音频流播放
+          // TODO: 阶段 5 音频流播放
           break;
 
         case "error":
           console.error("[Session] 服务端错误:", msg.code, msg.message);
+          showToast(getErrorMessage(msg.code), "error");
           setIsProcessing(false);
           break;
       }
@@ -99,31 +129,64 @@ export function useVisionSession() {
     return unsub;
   }, []);
 
+  // 连接断开时显示提示
+  useEffect(() => {
+    if (status === "disconnected") {
+      // 只在非主动断开时提示（通过检查是否有活跃会话判断）
+      // 这里简单处理，由 App 层根据状态显示
+    }
+  }, [status]);
+
   /** 启动会话 */
   const startSession = useCallback(async () => {
+    // 1. 获取摄像头和麦克风
     await startCamera();
+    const micStream = await startMic();
+    if (!micStream) {
+      showToast("无法获取麦克风权限", "error");
+      return;
+    }
+
+    // 2. 连接 WebSocket
     connect();
-    startVAD();
+
-  }, [startCamera, connect, startVAD]);
+    // 3. 启动 VAD（传入麦克风 stream）
+    await startVAD(micStream);
+  }, [startCamera, startMic, connect, startVAD]);
 
   /** 结束会话 */
-  const stopSession = useCallback(() => {
+  const stopSession = useCallback(async () => {
-    stopVAD();
+    await stopVAD();
+    stopMic();
     stopCamera();
     disconnect();
-  }, [stopVAD, stopCamera, disconnect]);
+    // 清理所有对话状态
+    setMessages([]);
+    setCurrentReply("");
+    setIsProcessing(false);
+  }, [stopVAD, stopMic, stopCamera, disconnect]);
 
   /** 打断当前回复 */
   const interrupt = useCallback(() => {
     send({ type: "interrupt" });
+    // 将未完成的流式内容保存为最终消息
+    if (currentReply) {
+      setMessages((prev) => [
+        ...prev,
+        { role: "assistant", content: currentReply + "（已打断）", timestamp: Date.now() },
+      ]);
+    }
+    setCurrentReply("");
     setIsProcessing(false);
-  }, [send]);
+  }, [send, currentReply]);
 
   return {
     messages,
     currentReply,
     isProcessing,
     isSpeaking,
+    isVADReady,
+    vadError,
     connectionStatus: status,
     videoRef,
     stream,

frontend/src/lib/errors.ts

@@ -0,0 +1,24 @@
+// ============================================================
+// 错误码 → 用户友好文案映射
+// 来源：docs/03-接口文档.md §五 错误码
+// ============================================================
+
+import type { ErrorCode } from "../types";
+
+const ERROR_MESSAGES: Record<ErrorCode, string> = {
+  INVALID_MESSAGE: "消息格式异常，请重试",
+  SESSION_NOT_FOUND: "会话已过期，请重新连接",
+  RATE_LIMITED: "请求太频繁，请稍后再试",
+  IMAGE_TOO_LARGE: "图像过大，请降低分辨率",
+  AUDIO_TOO_SHORT: "语音太短，请再说一句",
+  LLM_TIMEOUT: "AI 响应超时，请重试",
+  LLM_ERROR: "AI 服务异常，请稍后重试",
+  STT_ERROR: "语音识别失败，请重试",
+  TTS_ERROR: "语音合成失败",
+  INTERNAL_ERROR: "服务内部错误，请重试",
+};
+
+/** 将错误码转为用户友好文案 */
+export function getErrorMessage(code: string): string {
+  return ERROR_MESSAGES[code as ErrorCode] ?? `未知错误: ${code}`;
+}

frontend/src/lib/toast.ts

@@ -0,0 +1,29 @@
+// ============================================================
+// Toast 全局状态管理
+// 与 Toast 组件配合使用
+// ============================================================
+
+export type ToastType = "error" | "warning" | "info";
+
+let nextId = 0;
+let _setToasts: React.Dispatch<React.SetStateAction<ToastItem[]>> | null = null;
+
+export interface ToastItem {
+  id: number;
+  type: ToastType;
+  message: string;
+}
+
+/** 注册 Toast state setter（由 ToastContainer 组件调用） */
+export function registerToastSetter(
+  setter: React.Dispatch<React.SetStateAction<ToastItem[]>> | null,
+) {
+  _setToasts = setter;
+}
+
+/** 显示一条 Toast */
+export function showToast(message: string, type: ToastType = "info") {
+  if (!_setToasts) return;
+  const id = nextId++;
+  _setToasts((prev) => [...prev, { id, type, message }]);
+}

frontend/src/types/index.ts

@@ -25,6 +25,8 @@ export interface ChatMessage {
   imageUrl?: string;
   timestamp: number;
   tokensUsed?: number;
+  latencyMs?: number;
+  model?: string;
 }
 
 // ---- WebSocket 通用信封 ----