diff --git a/frontend/src/App.css b/frontend/src/App.css index 6f12e3a..4cf2215 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -103,6 +103,15 @@ body { animation: pulse 1.5s ease-in-out infinite; } +.vad-indicator--loading { + color: var(--color-warning); +} + +.vad-indicator--error { + color: var(--color-error); + animation: none; +} + @keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.6; } @@ -110,10 +119,10 @@ body { .chat-section { flex: 1; - overflow-y: auto; display: flex; flex-direction: column; gap: 12px; + overflow: hidden; } /* ---- Video Preview ---- */ @@ -148,6 +157,8 @@ body { display: flex; flex-direction: column; gap: 12px; + overflow-y: auto; + flex: 1; } .chat-panel--empty { @@ -239,3 +250,111 @@ body { .btn--warning:hover { opacity: 0.9; } + +/* ---- Streaming Cursor ---- */ + +.cursor { + display: inline; + animation: blink 0.8s step-end infinite; + color: var(--color-success); +} + +@keyframes blink { + 0%, 100% { opacity: 1; } + 50% { opacity: 0; } +} + +/* ---- Message Meta ---- */ + +.chat-message__meta { + margin-top: 6px; + font-size: 0.75rem; + color: var(--color-text-muted); +} + +/* ---- System Message ---- */ + +.system-message { + text-align: center; + padding: 8px 16px; + border-radius: var(--radius); + font-size: 0.85rem; +} + +.system-message--warning { + background: rgba(245, 158, 11, 0.15); + color: var(--color-warning); + border: 1px solid rgba(245, 158, 11, 0.3); +} + +/* ---- Toast ---- */ + +.toast-container { + position: fixed; + top: 16px; + right: 16px; + z-index: 1000; + display: flex; + flex-direction: column; + gap: 8px; +} + +.toast { + padding: 10px 16px; + border-radius: var(--radius); + font-size: 0.85rem; + cursor: pointer; + animation: slideIn 0.3s ease-out; + max-width: 320px; +} + +.toast--error { + background: rgba(239, 68, 68, 0.9); + color: white; +} + +.toast--warning { + background: rgba(245, 158, 11, 0.9); + color: #000; +} + +.toast--info { + background: rgba(37, 99, 235, 0.9); + color: white; +} + +@keyframes slideIn { + from { + opacity: 0; + transform: translateX(20px); + } + to { + opacity: 1; + transform: translateX(0); + } +} + +/* ---- Video Overlay ---- */ + +.video-overlay { + position: absolute; + inset: 0; + background: rgba(0, 0, 0, 0.3); + display: flex; + align-items: center; + justify-content: center; + border-radius: var(--radius); +} + +.video-overlay__spinner { + width: 32px; + height: 32px; + border: 3px solid rgba(255, 255, 255, 0.3); + border-top-color: white; + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6e8e52d..9bdb7b4 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -5,6 +5,7 @@ import { useVisionSession } from "./hooks/useVisionSession"; import { VideoPreview } from "./components/VideoPreview"; import { ChatPanel } from "./components/ChatPanel"; +import { ToastContainer } from "./components/Toast"; import "./App.css"; function App() { @@ -13,6 +14,8 @@ function App() { currentReply, isProcessing, isSpeaking, + isVADReady, + vadError, connectionStatus, videoRef, stream, @@ -21,12 +24,18 @@ function App() { interrupt, } = useVisionSession(); + const isConnected = connectionStatus === "connected"; + return (

CamTalk

- {connectionStatus === "connected" ? "已连接" : connectionStatus === "connecting" ? "连接中..." : "未连接"} + {isConnected + ? "已连接" + : connectionStatus === "connecting" + ? "连接中..." + : "未连接"}
@@ -34,23 +43,41 @@ function App() {
{isSpeaking &&
🎤 正在聆听...
} + {isConnected && !isVADReady && !vadError && ( +
+ 正在初始化语音检测... +
+ )} + {vadError && ( +
+ ⚠️ {vadError} +
+ )} + {isProcessing && ( +
+
+
+ )}
- - {currentReply && ( -
-
AI
-
{currentReply}
+ {connectionStatus === "disconnected" && messages.length > 0 && ( +
+ 连接已断开,正在重连...
)} +
- {connectionStatus !== "connected" ? ( + {!isConnected ? ( ) : ( <> @@ -65,6 +92,8 @@ function App() { )}
+ +
); } diff --git a/frontend/src/components/ChatPanel/index.tsx b/frontend/src/components/ChatPanel/index.tsx index 0db5641..bed57de 100644 --- a/frontend/src/components/ChatPanel/index.tsx +++ b/frontend/src/components/ChatPanel/index.tsx @@ -1,33 +1,90 @@ // ============================================================ // ChatPanel — 消息展示面板 -// 职责:渲染对话消息列表(用户提问 + AI 回复) +// 职责:渲染对话消息列表、流式光标、元数据、自动滚动 // ============================================================ +import { useEffect, useRef } from "react"; import type { ChatMessage } from "../../types"; +import type { ConnectionStatus } from "../../lib/websocket"; interface ChatPanelProps { messages: ChatMessage[]; + currentReply?: string; + connectionStatus: ConnectionStatus; } -export function ChatPanel({ messages }: ChatPanelProps) { - if (messages.length === 0) { +export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPanelProps) { + const bottomRef = useRef(null); + const containerRef = useRef(null); + const isAutoScroll = useRef(true); + + // 用户上滚时暂停自动滚动,滚到底部时恢复 + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const handleScroll = () => { + const { scrollTop, scrollHeight, clientHeight } = container; + isAutoScroll.current = scrollHeight - scrollTop - clientHeight < 60; + }; + + container.addEventListener("scroll", handleScroll); + return () => container.removeEventListener("scroll", handleScroll); + }, []); + + // 新消息或流式更新时自动滚动 + useEffect(() => { + if (isAutoScroll.current) { + bottomRef.current?.scrollIntoView({ behavior: "smooth" }); + } + }, [messages, currentReply]); + + // 空状态 + if (messages.length === 0 && !currentReply) { + if (connectionStatus !== "connected") { + return ( +
+

点击下方按钮开始对话

+
+ ); + } return (
-

开始对话:对着摄像头说话即可

+

对着摄像头说话即可

); } return ( -
+
{messages.map((msg, index) => (
{msg.role === "user" ? "你" : "AI"}
{msg.content}
+ {msg.role === "assistant" && msg.tokensUsed !== undefined && ( +
+ {msg.tokensUsed} tokens + {msg.latencyMs !== undefined && ` · ${(msg.latencyMs / 1000).toFixed(1)}s`} + {msg.model && ` · ${msg.model}`} +
+ )}
))} + + {/* 流式回复(尚未完成) */} + {currentReply && ( +
+
AI
+
+ {currentReply} + +
+
+ )} + +
); } diff --git a/frontend/src/components/EdgeProcessor/index.tsx b/frontend/src/components/EdgeProcessor/index.tsx index 75acf9e..36978b7 100644 --- a/frontend/src/components/EdgeProcessor/index.tsx +++ b/frontend/src/components/EdgeProcessor/index.tsx @@ -4,35 +4,103 @@ // 技术:@ricky0123/vad-web(VAD)、ONNX Runtime Web(关键帧检测) // ============================================================ -import { useCallback, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { MicVAD } from "@ricky0123/vad-web"; export interface VADOptions { - /** 语音结束回调,携带录音 Float32Array */ + /** 语音结束回调,携带录音 Float32Array(16kHz) */ onSpeechEnd?: (audio: Float32Array) => void; /** 语音开始回调 */ onSpeechStart?: () => void; + /** 语音过短被忽略回调 */ + onVADMisfire?: () => void; } -export function useVAD(_options?: VADOptions) { - const [isSpeaking] = useState(false); - // TODO: 初始化 @ricky0123/vad-web,加载后设为 true - const isReady = false; +/** + * 语音活动检测 Hook + * 基于 @ricky0123/vad-web 的 MicVAD,检测用户说话并回调 + */ +export function useVAD(options?: VADOptions) { + const [isSpeaking, setIsSpeaking] = useState(false); + const [isReady, setIsReady] = useState(false); + const [error, setError] = useState(null); + const vadRef = useRef(null); + const optionsRef = useRef(options); - // TODO: 实现 VAD 初始化 - // 1. 加载 @ricky0123/vad-web - // 2. 配置 VAD 参数(阈值、最小语音时长等) - // 3. 连接麦克风 stream - // 4. 在 onSpeechEnd 时收集音频并回调 _options.onSpeechEnd + // 保持 options 引用最新,避免回调闭包问题 + useEffect(() => { + optionsRef.current = options; + }, [options]); - const start = useCallback(() => { - // TODO: 启动 VAD 监听 + /** + * 初始化 VAD 并开始监听 + * @param stream 麦克风 MediaStream(由外部管理) + */ + const start = useCallback(async (stream: MediaStream) => { + // 如果已有实例,先销毁 + if (vadRef.current) { + await vadRef.current.destroy(); + vadRef.current = null; + } + + try { + const vad = await MicVAD.new({ + getStream: () => Promise.resolve(stream), + startOnLoad: true, + model: "legacy", + + onSpeechStart: () => { + setIsSpeaking(true); + optionsRef.current?.onSpeechStart?.(); + }, + + onSpeechEnd: (audio: Float32Array) => { + setIsSpeaking(false); + optionsRef.current?.onSpeechEnd?.(audio); + }, + + onVADMisfire: () => { + setIsSpeaking(false); + optionsRef.current?.onVADMisfire?.(); + }, + + // VAD 参数(对齐 docs/06-语音交互.md 推荐值) + positiveSpeechThreshold: 0.5, + negativeSpeechThreshold: 0.35, + redemptionMs: 300, + preSpeechPadMs: 300, + minSpeechMs: 250, + submitUserSpeechOnPause: false, + }); + + vadRef.current = vad; + setIsReady(true); + setError(null); + } catch (err) { + const message = err instanceof Error ? err.message : "VAD 初始化失败"; + setError(message); + console.error("[VAD] 初始化失败:", err); + } }, []); - const stop = useCallback(() => { - // TODO: 停止 VAD 监听 + /** 停止 VAD 并销毁实例 */ + const stop = useCallback(async () => { + if (vadRef.current) { + await vadRef.current.destroy(); + vadRef.current = null; + } + setIsReady(false); + setIsSpeaking(false); }, []); - return { isSpeaking, isReady, start, stop }; + // 组件卸载时清理 + useEffect(() => { + return () => { + vadRef.current?.destroy(); + }; + }, []); + + return { isSpeaking, isReady, error, start, stop }; } // ---- 关键帧检测(ONNX Runtime Web)---- @@ -41,11 +109,6 @@ export function useKeyframeDetection() { // TODO: 加载 ONNX 模型后设为 true const isReady = false; - // TODO: 实现关键帧检测 - // 1. 加载 ONNX 模型 - // 2. 对比当前帧与上一帧的像素差异 - // 3. 超过阈值则判定为关键帧 - const isKeyframe = useCallback( (_currentFrame: ImageData, _previousFrame: ImageData): boolean => { // TODO: 实现像素差异对比 diff --git a/frontend/src/components/Toast/index.tsx b/frontend/src/components/Toast/index.tsx new file mode 100644 index 0000000..e4f371b --- /dev/null +++ b/frontend/src/components/Toast/index.tsx @@ -0,0 +1,51 @@ +// ============================================================ +// Toast — 轻量通知组件 +// 职责:3 秒自动消失的通知提示 +// ============================================================ + +import { useCallback, useEffect, useState } from "react"; +import { registerToastSetter, type ToastItem } from "../../lib/toast"; + +export type { ToastItem }; + +/** Toast 容器组件,放在 App 根部 */ +export function ToastContainer() { + const [toasts, setToasts] = useState([]); + + // 注册全局 setter + useEffect(() => { + registerToastSetter(setToasts); + return () => registerToastSetter(null); + }, []); + + const dismiss = useCallback((id: number) => { + setToasts((prev) => prev.filter((t) => t.id !== id)); + }, []); + + return ( +
+ {toasts.map((t) => ( + + ))} +
+ ); +} + +function ToastItemView({ + item, + onDismiss, +}: { + item: ToastItem; + onDismiss: (id: number) => void; +}) { + useEffect(() => { + const timer = setTimeout(() => onDismiss(item.id), 3000); + return () => clearTimeout(timer); + }, [item.id, onDismiss]); + + return ( +
onDismiss(item.id)}> + {item.message} +
+ ); +} diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts index d187e8d..a9112f6 100644 --- a/frontend/src/hooks/useVisionSession.ts +++ b/frontend/src/hooks/useVisionSession.ts @@ -4,11 +4,14 @@ // 来源:docs/02-系统架构.md 核心 Hook 设计 // ============================================================ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { v4 as uuidv4 } from "uuid"; import { wsClient } from "../lib/websocket"; import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio"; +import { getErrorMessage } from "../lib/errors"; +import { showToast } from "../lib/toast"; import { useCamera } from "../components/CameraManager"; +import { useMicrophone } from "../components/MicManager"; import { useVAD } from "../components/EdgeProcessor"; import { useWebSocketManager } from "../components/WebSocketManager"; import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types"; @@ -19,12 +22,31 @@ export function useVisionSession() { const [isProcessing, setIsProcessing] = useState(false); const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); + const { startMic, stopMic } = useMicrophone(); const { status, connect, disconnect, send } = useWebSocketManager(); + // 用 ref 跟踪 isProcessing,避免 VAD 回调闭包问题 + const isProcessingRef = useRef(false); + useEffect(() => { + isProcessingRef.current = isProcessing; + }, [isProcessing]); + // VAD:语音结束时自动发送 query - const { isSpeaking, start: startVAD, stop: stopVAD } = useVAD({ + const { + isSpeaking, + isReady: isVADReady, + error: vadError, + start: startVAD, + stop: stopVAD, + } = useVAD({ onSpeechEnd: useCallback( (audio: Float32Array) => { + // 处理中忽略,防止重复发送 + if (isProcessingRef.current) { + console.warn("[Session] 正在处理中,忽略语音输入"); + return; + } + const frame = captureFrame(); if (!frame) { console.warn("[Session] 无法捕获图像帧"); @@ -39,7 +61,7 @@ export function useVisionSession() { audio: encodeAudioToBase64(audio), }); - // 添加用户消息(STT 结果到达后会更新文本) + // 添加用户消息(STT 流式结果会逐步更新文本) setMessages((prev) => [ ...prev, { role: "user", content: "(语音识别中...)", timestamp: Date.now() }, @@ -54,43 +76,51 @@ export function useVisionSession() { useEffect(() => { const unsub = wsClient.onMessage((msg: ServerMessage) => { switch (msg.type) { - case "stt_result": - if (msg.is_final) { - setMessages((prev) => { - const updated = [...prev]; - const lastUserIdx = updated.findLastIndex((m) => m.role === "user"); - if (lastUserIdx >= 0) { - updated[lastUserIdx] = { ...updated[lastUserIdx], content: msg.text }; - } - return updated; - }); - } + case "stt_result": { + // 流式更新用户消息文本(包括中间结果和最终结果) + setMessages((prev) => { + const updated = [...prev]; + const lastUserIdx = updated.findLastIndex((m) => m.role === "user"); + if (lastUserIdx >= 0) { + updated[lastUserIdx] = { + ...updated[lastUserIdx], + content: msg.text || "(未识别到语音)", + }; + } + return updated; + }); break; + } case "llm_chunk": setCurrentReply((prev) => prev + msg.delta); break; - case "llm_done": + case "llm_done": { + const done = msg as LLMDoneMessage; setMessages((prev) => [ ...prev, { role: "assistant", - content: (msg as LLMDoneMessage).full_text, + content: done.full_text, timestamp: Date.now(), - tokensUsed: (msg as LLMDoneMessage).tokens_used?.total, + tokensUsed: done.tokens_used?.total, + latencyMs: done.latency_ms, + model: done.model, }, ]); setCurrentReply(""); setIsProcessing(false); break; + } case "tts_audio": - // TODO: 音频流播放 + // TODO: 阶段 5 音频流播放 break; case "error": console.error("[Session] 服务端错误:", msg.code, msg.message); + showToast(getErrorMessage(msg.code), "error"); setIsProcessing(false); break; } @@ -99,31 +129,64 @@ export function useVisionSession() { return unsub; }, []); + // 连接断开时显示提示 + useEffect(() => { + if (status === "disconnected") { + // 只在非主动断开时提示(通过检查是否有活跃会话判断) + // 这里简单处理,由 App 层根据状态显示 + } + }, [status]); + /** 启动会话 */ const startSession = useCallback(async () => { + // 1. 获取摄像头和麦克风 await startCamera(); + const micStream = await startMic(); + if (!micStream) { + showToast("无法获取麦克风权限", "error"); + return; + } + + // 2. 连接 WebSocket connect(); - startVAD(); - }, [startCamera, connect, startVAD]); + + // 3. 启动 VAD(传入麦克风 stream) + await startVAD(micStream); + }, [startCamera, startMic, connect, startVAD]); /** 结束会话 */ - const stopSession = useCallback(() => { - stopVAD(); + const stopSession = useCallback(async () => { + await stopVAD(); + stopMic(); stopCamera(); disconnect(); - }, [stopVAD, stopCamera, disconnect]); + // 清理所有对话状态 + setMessages([]); + setCurrentReply(""); + setIsProcessing(false); + }, [stopVAD, stopMic, stopCamera, disconnect]); /** 打断当前回复 */ const interrupt = useCallback(() => { send({ type: "interrupt" }); + // 将未完成的流式内容保存为最终消息 + if (currentReply) { + setMessages((prev) => [ + ...prev, + { role: "assistant", content: currentReply + "(已打断)", timestamp: Date.now() }, + ]); + } + setCurrentReply(""); setIsProcessing(false); - }, [send]); + }, [send, currentReply]); return { messages, currentReply, isProcessing, isSpeaking, + isVADReady, + vadError, connectionStatus: status, videoRef, stream, diff --git a/frontend/src/lib/errors.ts b/frontend/src/lib/errors.ts new file mode 100644 index 0000000..9aa56e2 --- /dev/null +++ b/frontend/src/lib/errors.ts @@ -0,0 +1,24 @@ +// ============================================================ +// 错误码 → 用户友好文案映射 +// 来源:docs/03-接口文档.md §五 错误码 +// ============================================================ + +import type { ErrorCode } from "../types"; + +const ERROR_MESSAGES: Record = { + INVALID_MESSAGE: "消息格式异常,请重试", + SESSION_NOT_FOUND: "会话已过期,请重新连接", + RATE_LIMITED: "请求太频繁,请稍后再试", + IMAGE_TOO_LARGE: "图像过大,请降低分辨率", + AUDIO_TOO_SHORT: "语音太短,请再说一句", + LLM_TIMEOUT: "AI 响应超时,请重试", + LLM_ERROR: "AI 服务异常,请稍后重试", + STT_ERROR: "语音识别失败,请重试", + TTS_ERROR: "语音合成失败", + INTERNAL_ERROR: "服务内部错误,请重试", +}; + +/** 将错误码转为用户友好文案 */ +export function getErrorMessage(code: string): string { + return ERROR_MESSAGES[code as ErrorCode] ?? `未知错误: ${code}`; +} diff --git a/frontend/src/lib/toast.ts b/frontend/src/lib/toast.ts new file mode 100644 index 0000000..3074e90 --- /dev/null +++ b/frontend/src/lib/toast.ts @@ -0,0 +1,29 @@ +// ============================================================ +// Toast 全局状态管理 +// 与 Toast 组件配合使用 +// ============================================================ + +export type ToastType = "error" | "warning" | "info"; + +let nextId = 0; +let _setToasts: React.Dispatch> | null = null; + +export interface ToastItem { + id: number; + type: ToastType; + message: string; +} + +/** 注册 Toast state setter(由 ToastContainer 组件调用) */ +export function registerToastSetter( + setter: React.Dispatch> | null, +) { + _setToasts = setter; +} + +/** 显示一条 Toast */ +export function showToast(message: string, type: ToastType = "info") { + if (!_setToasts) return; + const id = nextId++; + _setToasts((prev) => [...prev, { id, type, message }]); +} diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 2755ae3..4cad08a 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -25,6 +25,8 @@ export interface ChatMessage { imageUrl?: string; timestamp: number; tokensUsed?: number; + latencyMs?: number; + model?: string; } // ---- WebSocket 通用信封 ----