feat: 初始化前端项目,搭建 React + TypeScript + Vite 工程结构
- 创建 Vite + React 18 + TypeScript 项目(严格模式) - 安装 @ricky0123/vad-web、onnxruntime-web、uuid 依赖 - 定义全部 WebSocket 消息类型和数据模型(对齐接口文档) - 实现 WebSocket 连接管理(心跳保活、指数退避重连) - 实现音频编码工具(PCM↔Base64、DataURL 转换) - 创建组件骨架:CameraManager、MicManager、EdgeProcessor、WebSocketManager、ChatPanel、VideoPreview - 实现核心 useVisionSession Hook 骨架 - 配置 ESLint(_前缀变量忽略规则) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
59
frontend/src/components/CameraManager/index.tsx
Normal file
59
frontend/src/components/CameraManager/index.tsx
Normal file
@@ -0,0 +1,59 @@
|
||||
// ============================================================
|
||||
// CameraManager — 摄像头流采集
|
||||
// 职责:获取用户摄像头 MediaStream,提供给 VideoPreview 和 EdgeProcessor
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
|
||||
export interface CameraManagerHandle {
|
||||
/** 获取当前视频轨道 */
|
||||
stream: MediaStream | null;
|
||||
/** 捕获当前帧为 JPEG DataURL */
|
||||
captureFrame: () => string | null;
|
||||
}
|
||||
|
||||
export function useCamera() {
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const [stream, setStream] = useState<MediaStream | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const startCamera = useCallback(async () => {
|
||||
try {
|
||||
const mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { facingMode: "environment", width: 640, height: 480 },
|
||||
audio: false,
|
||||
});
|
||||
setStream(mediaStream);
|
||||
if (videoRef.current) {
|
||||
videoRef.current.srcObject = mediaStream;
|
||||
}
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "无法访问摄像头";
|
||||
setError(message);
|
||||
console.error("[Camera] 获取摄像头失败:", err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stopCamera = useCallback(() => {
|
||||
stream?.getTracks().forEach((track) => track.stop());
|
||||
setStream(null);
|
||||
}, [stream]);
|
||||
|
||||
/** 从 video 元素捕获当前帧为 JPEG DataURL */
|
||||
const captureFrame = useCallback((): string | null => {
|
||||
const video = videoRef.current;
|
||||
if (!video || video.readyState < 2) return null;
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = video.videoWidth;
|
||||
canvas.height = video.videoHeight;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return null;
|
||||
|
||||
ctx.drawImage(video, 0, 0);
|
||||
return canvas.toDataURL("image/jpeg", 0.7);
|
||||
}, []);
|
||||
|
||||
return { videoRef, stream, error, startCamera, stopCamera, captureFrame };
|
||||
}
|
||||
33
frontend/src/components/ChatPanel/index.tsx
Normal file
33
frontend/src/components/ChatPanel/index.tsx
Normal file
@@ -0,0 +1,33 @@
|
||||
// ============================================================
|
||||
// ChatPanel — 消息展示面板
|
||||
// 职责:渲染对话消息列表(用户提问 + AI 回复)
|
||||
// ============================================================
|
||||
|
||||
import type { ChatMessage } from "../../types";
|
||||
|
||||
interface ChatPanelProps {
|
||||
messages: ChatMessage[];
|
||||
}
|
||||
|
||||
export function ChatPanel({ messages }: ChatPanelProps) {
|
||||
if (messages.length === 0) {
|
||||
return (
|
||||
<div className="chat-panel chat-panel--empty">
|
||||
<p>开始对话:对着摄像头说话即可</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="chat-panel">
|
||||
{messages.map((msg, index) => (
|
||||
<div key={index} className={`chat-message chat-message--${msg.role}`}>
|
||||
<div className="chat-message__role">
|
||||
{msg.role === "user" ? "你" : "AI"}
|
||||
</div>
|
||||
<div className="chat-message__content">{msg.content}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
58
frontend/src/components/EdgeProcessor/index.tsx
Normal file
58
frontend/src/components/EdgeProcessor/index.tsx
Normal file
@@ -0,0 +1,58 @@
|
||||
// ============================================================
|
||||
// EdgeProcessor — 边缘预处理(VAD + 关键帧检测)
|
||||
// 职责:浏览器端语音活动检测、关键帧筛选
|
||||
// 技术:@ricky0123/vad-web(VAD)、ONNX Runtime Web(关键帧检测)
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useState } from "react";
|
||||
|
||||
export interface VADOptions {
|
||||
/** 语音结束回调,携带录音 Float32Array */
|
||||
onSpeechEnd?: (audio: Float32Array) => void;
|
||||
/** 语音开始回调 */
|
||||
onSpeechStart?: () => void;
|
||||
}
|
||||
|
||||
export function useVAD(_options?: VADOptions) {
|
||||
const [isSpeaking] = useState(false);
|
||||
// TODO: 初始化 @ricky0123/vad-web,加载后设为 true
|
||||
const isReady = false;
|
||||
|
||||
// TODO: 实现 VAD 初始化
|
||||
// 1. 加载 @ricky0123/vad-web
|
||||
// 2. 配置 VAD 参数(阈值、最小语音时长等)
|
||||
// 3. 连接麦克风 stream
|
||||
// 4. 在 onSpeechEnd 时收集音频并回调 _options.onSpeechEnd
|
||||
|
||||
const start = useCallback(() => {
|
||||
// TODO: 启动 VAD 监听
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
// TODO: 停止 VAD 监听
|
||||
}, []);
|
||||
|
||||
return { isSpeaking, isReady, start, stop };
|
||||
}
|
||||
|
||||
// ---- 关键帧检测(ONNX Runtime Web)----
|
||||
|
||||
export function useKeyframeDetection() {
|
||||
// TODO: 加载 ONNX 模型后设为 true
|
||||
const isReady = false;
|
||||
|
||||
// TODO: 实现关键帧检测
|
||||
// 1. 加载 ONNX 模型
|
||||
// 2. 对比当前帧与上一帧的像素差异
|
||||
// 3. 超过阈值则判定为关键帧
|
||||
|
||||
const isKeyframe = useCallback(
|
||||
(_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
|
||||
// TODO: 实现像素差异对比
|
||||
return true; // 暂时所有帧都视为关键帧
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
return { isReady, isKeyframe };
|
||||
}
|
||||
51
frontend/src/components/MicManager/index.tsx
Normal file
51
frontend/src/components/MicManager/index.tsx
Normal file
@@ -0,0 +1,51 @@
|
||||
// ============================================================
|
||||
// MicManager — 麦克风音频采集
|
||||
// 职责:获取麦克风 MediaStream,供 VAD 和音频录制使用
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
|
||||
export function useMicrophone() {
|
||||
const [stream, setStream] = useState<MediaStream | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
|
||||
const startMic = useCallback(async () => {
|
||||
try {
|
||||
const mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
sampleRate: 16000,
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
video: false,
|
||||
});
|
||||
setStream(mediaStream);
|
||||
setError(null);
|
||||
return mediaStream;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "无法访问麦克风";
|
||||
setError(message);
|
||||
console.error("[Mic] 获取麦克风失败:", err);
|
||||
return null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stopMic = useCallback(() => {
|
||||
stream?.getTracks().forEach((track) => track.stop());
|
||||
setStream(null);
|
||||
audioContextRef.current?.close();
|
||||
audioContextRef.current = null;
|
||||
}, [stream]);
|
||||
|
||||
/** 获取或创建 AudioContext */
|
||||
const getAudioContext = useCallback((): AudioContext => {
|
||||
if (!audioContextRef.current) {
|
||||
audioContextRef.current = new AudioContext({ sampleRate: 16000 });
|
||||
}
|
||||
return audioContextRef.current;
|
||||
}, []);
|
||||
|
||||
return { stream, error, startMic, stopMic, getAudioContext };
|
||||
}
|
||||
31
frontend/src/components/VideoPreview/index.tsx
Normal file
31
frontend/src/components/VideoPreview/index.tsx
Normal file
@@ -0,0 +1,31 @@
|
||||
// ============================================================
|
||||
// VideoPreview — 摄像头画面预览
|
||||
// 职责:显示实时摄像头画面
|
||||
// ============================================================
|
||||
|
||||
import { forwardRef } from "react";
|
||||
|
||||
interface VideoPreviewProps {
|
||||
isStreaming: boolean;
|
||||
}
|
||||
|
||||
export const VideoPreview = forwardRef<HTMLVideoElement, VideoPreviewProps>(
|
||||
function VideoPreview({ isStreaming }, ref) {
|
||||
return (
|
||||
<div className="video-preview">
|
||||
<video
|
||||
ref={ref}
|
||||
autoPlay
|
||||
playsInline
|
||||
muted
|
||||
className="video-preview__video"
|
||||
/>
|
||||
{!isStreaming && (
|
||||
<div className="video-preview__placeholder">
|
||||
摄像头未开启
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
);
|
||||
32
frontend/src/components/WebSocketManager/index.tsx
Normal file
32
frontend/src/components/WebSocketManager/index.tsx
Normal file
@@ -0,0 +1,32 @@
|
||||
// ============================================================
|
||||
// WebSocketManager — WebSocket 连接生命周期管理
|
||||
// 职责:管理连接状态、消息分发
|
||||
// ============================================================
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import { wsClient } from "../../lib/websocket";
|
||||
import type { ConnectionStatus } from "../../lib/websocket";
|
||||
import type { ServerMessage } from "../../types";
|
||||
|
||||
export function useWebSocketManager() {
|
||||
const [status, setStatus] = useState<ConnectionStatus>(wsClient.status);
|
||||
const [lastMessage, setLastMessage] = useState<ServerMessage | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubStatus = wsClient.onStatusChange(setStatus);
|
||||
const unsubMessage = wsClient.onMessage(setLastMessage);
|
||||
|
||||
return () => {
|
||||
unsubStatus();
|
||||
unsubMessage();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return {
|
||||
status,
|
||||
lastMessage,
|
||||
connect: () => wsClient.connect(),
|
||||
disconnect: () => wsClient.disconnect(),
|
||||
send: wsClient.send.bind(wsClient),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user