feat: 实现摄像头和麦克风设备选择功能
- 新增 useDeviceList hook,枚举音视频输入设备并监听热插拔 - CameraManager/MicManager 的 startCamera/startMic 支持可选 deviceId 参数 - useVisionSession 集成设备选择:授权后自动枚举、切换设备时热重启 - 连接后显示设备下拉选择器,未连接时隐藏(避免未授权时空列表) - SessionConfig 新增 cameraDeviceId/micDeviceId 持久化到 localStorage
This commit is contained in:
@@ -96,6 +96,9 @@ function AppContent() {
|
|||||||
toggleCamera,
|
toggleCamera,
|
||||||
toggleMic,
|
toggleMic,
|
||||||
sendTextMessage,
|
sendTextMessage,
|
||||||
|
cameras,
|
||||||
|
mics,
|
||||||
|
switchDevice,
|
||||||
} = useVisionSession(accessToken, activeSessionId);
|
} = useVisionSession(accessToken, activeSessionId);
|
||||||
|
|
||||||
const isConnected = connectionStatus === "connected";
|
const isConnected = connectionStatus === "connected";
|
||||||
@@ -406,21 +409,6 @@ function AppContent() {
|
|||||||
<button className="btn btn--primary btn--lg" onClick={startSession}>
|
<button className="btn btn--primary btn--lg" onClick={startSession}>
|
||||||
{connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")}
|
{connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")}
|
||||||
</button>
|
</button>
|
||||||
{/* 设备选择器 */}
|
|
||||||
<div className="video-controls__devices">
|
|
||||||
<div className="device-select-wrapper">
|
|
||||||
<label className="device-select-label">📷 {tr("controls.device.camera")}</label>
|
|
||||||
<select className="device-select" defaultValue="default">
|
|
||||||
<option value="default">{tr("controls.device.default")}</option>
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
<div className="device-select-wrapper">
|
|
||||||
<label className="device-select-label">🎤 {tr("controls.device.mic")}</label>
|
|
||||||
<select className="device-select" defaultValue="default">
|
|
||||||
<option value="default">{tr("controls.device.default")}</option>
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</>
|
</>
|
||||||
) : isCameraOn ? (
|
) : isCameraOn ? (
|
||||||
<>
|
<>
|
||||||
@@ -457,6 +445,35 @@ function AppContent() {
|
|||||||
{tr("controls.stopVideo")}
|
{tr("controls.stopVideo")}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
{/* 设备选择器 */}
|
||||||
|
<div className="video-controls__devices">
|
||||||
|
<div className="device-select-wrapper">
|
||||||
|
<label className="device-select-label">📷</label>
|
||||||
|
<select
|
||||||
|
className="device-select"
|
||||||
|
value={config.cameraDeviceId || "default"}
|
||||||
|
onChange={(e) => switchDevice("camera", e.target.value === "default" ? "" : e.target.value)}
|
||||||
|
>
|
||||||
|
<option value="default">{tr("controls.device.default")}</option>
|
||||||
|
{cameras.map((d) => (
|
||||||
|
<option key={d.deviceId} value={d.deviceId}>{d.label}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div className="device-select-wrapper">
|
||||||
|
<label className="device-select-label">🎤</label>
|
||||||
|
<select
|
||||||
|
className="device-select"
|
||||||
|
value={config.micDeviceId || "default"}
|
||||||
|
onChange={(e) => switchDevice("mic", e.target.value === "default" ? "" : e.target.value)}
|
||||||
|
>
|
||||||
|
<option value="default">{tr("controls.device.default")}</option>
|
||||||
|
{mics.map((d) => (
|
||||||
|
<option key={d.deviceId} value={d.deviceId}>{d.label}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</>
|
</>
|
||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
|
|||||||
@@ -19,10 +19,13 @@ export function useCamera() {
|
|||||||
const [stream, setStream] = useState<MediaStream | null>(null);
|
const [stream, setStream] = useState<MediaStream | null>(null);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
const startCamera = useCallback(async () => {
|
const startCamera = useCallback(async (deviceId?: string) => {
|
||||||
try {
|
try {
|
||||||
|
const videoConstraints: MediaTrackConstraints = deviceId
|
||||||
|
? { deviceId: { exact: deviceId }, width: 640, height: 480 }
|
||||||
|
: { facingMode: "environment", width: 640, height: 480 };
|
||||||
const mediaStream = await navigator.mediaDevices.getUserMedia({
|
const mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||||
video: { facingMode: "environment", width: 640, height: 480 },
|
video: videoConstraints,
|
||||||
audio: false,
|
audio: false,
|
||||||
});
|
});
|
||||||
setStream(mediaStream);
|
setStream(mediaStream);
|
||||||
|
|||||||
@@ -12,15 +12,13 @@ export function useMicrophone() {
|
|||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const audioContextRef = useRef<AudioContext | null>(null);
|
const audioContextRef = useRef<AudioContext | null>(null);
|
||||||
|
|
||||||
const startMic = useCallback(async () => {
|
const startMic = useCallback(async (deviceId?: string) => {
|
||||||
try {
|
try {
|
||||||
|
const audioConstraints: MediaTrackConstraints = deviceId
|
||||||
|
? { deviceId: { exact: deviceId }, sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true }
|
||||||
|
: { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true };
|
||||||
const mediaStream = await navigator.mediaDevices.getUserMedia({
|
const mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||||
audio: {
|
audio: audioConstraints,
|
||||||
sampleRate: 16000,
|
|
||||||
channelCount: 1,
|
|
||||||
echoCancellation: true,
|
|
||||||
noiseSuppression: true,
|
|
||||||
},
|
|
||||||
video: false,
|
video: false,
|
||||||
});
|
});
|
||||||
setStream(mediaStream);
|
setStream(mediaStream);
|
||||||
|
|||||||
44
frontend/src/hooks/useDeviceList.ts
Normal file
44
frontend/src/hooks/useDeviceList.ts
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
// ============================================================
|
||||||
|
// useDeviceList — 设备枚举 Hook
|
||||||
|
// 职责:枚举摄像头/麦克风设备列表,监听设备热插拔
|
||||||
|
// 注意:首次枚举需要先有一次成功的 getUserMedia 授权
|
||||||
|
// ============================================================
|
||||||
|
|
||||||
|
import { useCallback, useEffect, useState } from "react";
|
||||||
|
|
||||||
|
export interface DeviceInfo {
|
||||||
|
deviceId: string;
|
||||||
|
label: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useDeviceList() {
|
||||||
|
const [cameras, setCameras] = useState<DeviceInfo[]>([]);
|
||||||
|
const [mics, setMics] = useState<DeviceInfo[]>([]);
|
||||||
|
|
||||||
|
/** 枚举当前可用的音视频输入设备 */
|
||||||
|
const refreshDevices = useCallback(async () => {
|
||||||
|
try {
|
||||||
|
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||||
|
setCameras(
|
||||||
|
devices
|
||||||
|
.filter((d) => d.kind === "videoinput")
|
||||||
|
.map((d) => ({ deviceId: d.deviceId, label: d.label || `摄像头 ${d.deviceId.slice(0, 4)}` }))
|
||||||
|
);
|
||||||
|
setMics(
|
||||||
|
devices
|
||||||
|
.filter((d) => d.kind === "audioinput")
|
||||||
|
.map((d) => ({ deviceId: d.deviceId, label: d.label || `麦克风 ${d.deviceId.slice(0, 4)}` }))
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn("[DeviceList] 枚举设备失败:", err);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// 监听设备热插拔
|
||||||
|
useEffect(() => {
|
||||||
|
navigator.mediaDevices.addEventListener("devicechange", refreshDevices);
|
||||||
|
return () => navigator.mediaDevices.removeEventListener("devicechange", refreshDevices);
|
||||||
|
}, [refreshDevices]);
|
||||||
|
|
||||||
|
return { cameras, mics, refreshDevices };
|
||||||
|
}
|
||||||
@@ -17,6 +17,7 @@ import { useCamera } from "../components/CameraManager";
|
|||||||
import { useMicrophone } from "../components/MicManager";
|
import { useMicrophone } from "../components/MicManager";
|
||||||
import { useVAD, sampleFrame } from "../components/EdgeProcessor";
|
import { useVAD, sampleFrame } from "../components/EdgeProcessor";
|
||||||
import { useWebSocketManager } from "../components/WebSocketManager";
|
import { useWebSocketManager } from "../components/WebSocketManager";
|
||||||
|
import { useDeviceList } from "./useDeviceList";
|
||||||
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
||||||
|
|
||||||
export interface SessionStats {
|
export interface SessionStats {
|
||||||
@@ -55,6 +56,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
|||||||
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
|
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
|
||||||
const { startMic, stopMic } = useMicrophone();
|
const { startMic, stopMic } = useMicrophone();
|
||||||
const { status, connect, disconnect, send } = useWebSocketManager();
|
const { status, connect, disconnect, send } = useWebSocketManager();
|
||||||
|
const { cameras, mics, refreshDevices } = useDeviceList();
|
||||||
|
|
||||||
// 用 ref 跟踪 isProcessing,避免 VAD 回调闭包问题
|
// 用 ref 跟踪 isProcessing,避免 VAD 回调闭包问题
|
||||||
const isProcessingRef = useRef(false);
|
const isProcessingRef = useRef(false);
|
||||||
@@ -277,14 +279,14 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
|||||||
|
|
||||||
// 2. 尝试获取摄像头(可选)
|
// 2. 尝试获取摄像头(可选)
|
||||||
try {
|
try {
|
||||||
await startCamera();
|
await startCamera(config.cameraDeviceId || undefined);
|
||||||
setIsCameraOn(true);
|
setIsCameraOn(true);
|
||||||
} catch {
|
} catch {
|
||||||
console.warn("[Session] 无法获取摄像头权限,将以纯文本模式运行");
|
console.warn("[Session] 无法获取摄像头权限,将以纯文本模式运行");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. 尝试获取麦克风(可选)
|
// 3. 尝试获取麦克风(可选)
|
||||||
const micStream = await startMic();
|
const micStream = await startMic(config.micDeviceId || undefined);
|
||||||
if (micStream) {
|
if (micStream) {
|
||||||
setIsMicOn(true);
|
setIsMicOn(true);
|
||||||
// 4. 启动 VAD(仅在麦克风可用时)
|
// 4. 启动 VAD(仅在麦克风可用时)
|
||||||
@@ -292,7 +294,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
|||||||
} else {
|
} else {
|
||||||
console.warn("[Session] 无法获取麦克风权限,将以文本输入模式运行");
|
console.warn("[Session] 无法获取麦克风权限,将以文本输入模式运行");
|
||||||
}
|
}
|
||||||
}, [startCamera, startMic, connect, startVAD, accessToken]);
|
|
||||||
|
// 5. 授权后刷新设备列表
|
||||||
|
refreshDevices();
|
||||||
|
}, [startCamera, startMic, connect, startVAD, accessToken, config.cameraDeviceId, config.micDeviceId, refreshDevices]);
|
||||||
|
|
||||||
/** 结束会话 */
|
/** 结束会话 */
|
||||||
const stopSession = useCallback(async () => {
|
const stopSession = useCallback(async () => {
|
||||||
@@ -332,10 +337,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
|||||||
stopCamera();
|
stopCamera();
|
||||||
setIsCameraOn(false);
|
setIsCameraOn(false);
|
||||||
} else {
|
} else {
|
||||||
await startCamera();
|
await startCamera(config.cameraDeviceId || undefined);
|
||||||
setIsCameraOn(true);
|
setIsCameraOn(true);
|
||||||
}
|
}
|
||||||
}, [isCameraOn, startCamera, stopCamera]);
|
}, [isCameraOn, startCamera, stopCamera, config.cameraDeviceId]);
|
||||||
|
|
||||||
/** 麦克风开关 */
|
/** 麦克风开关 */
|
||||||
const toggleMic = useCallback(async () => {
|
const toggleMic = useCallback(async () => {
|
||||||
@@ -344,13 +349,13 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
|||||||
stopMic();
|
stopMic();
|
||||||
setIsMicOn(false);
|
setIsMicOn(false);
|
||||||
} else {
|
} else {
|
||||||
const micStream = await startMic();
|
const micStream = await startMic(config.micDeviceId || undefined);
|
||||||
if (micStream) {
|
if (micStream) {
|
||||||
await startVAD(micStream);
|
await startVAD(micStream);
|
||||||
setIsMicOn(true);
|
setIsMicOn(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, [isMicOn, startMic, stopMic, startVAD, stopVAD]);
|
}, [isMicOn, startMic, stopMic, startVAD, stopVAD, config.micDeviceId]);
|
||||||
|
|
||||||
/** 打断当前回复 */
|
/** 打断当前回复 */
|
||||||
const interrupt = useCallback(() => {
|
const interrupt = useCallback(() => {
|
||||||
@@ -370,6 +375,31 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
|||||||
setIsProcessing(false);
|
setIsProcessing(false);
|
||||||
}, [send, currentReply]);
|
}, [send, currentReply]);
|
||||||
|
|
||||||
|
/** 切换设备(摄像头或麦克风) */
|
||||||
|
const switchDevice = useCallback(async (kind: "camera" | "mic", deviceId: string) => {
|
||||||
|
if (kind === "camera") {
|
||||||
|
updateConfig({ cameraDeviceId: deviceId || undefined });
|
||||||
|
if (isCameraOn) {
|
||||||
|
stopCamera();
|
||||||
|
try {
|
||||||
|
await startCamera(deviceId || undefined);
|
||||||
|
} catch {
|
||||||
|
console.warn("[Session] 切换摄像头失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
updateConfig({ micDeviceId: deviceId || undefined });
|
||||||
|
if (isMicOn) {
|
||||||
|
await stopVAD();
|
||||||
|
stopMic();
|
||||||
|
const micStream = await startMic(deviceId || undefined);
|
||||||
|
if (micStream) {
|
||||||
|
await startVAD(micStream);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, [isCameraOn, isMicOn, stopCamera, startCamera, stopMic, startMic, stopVAD, startVAD, updateConfig]);
|
||||||
|
|
||||||
/** 发送文本消息(手动输入) */
|
/** 发送文本消息(手动输入) */
|
||||||
const sendTextMessage = useCallback(
|
const sendTextMessage = useCallback(
|
||||||
(text: string) => {
|
(text: string) => {
|
||||||
@@ -439,5 +469,8 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
|||||||
toggleMic,
|
toggleMic,
|
||||||
sendTextMessage,
|
sendTextMessage,
|
||||||
captureFrame,
|
captureFrame,
|
||||||
|
cameras,
|
||||||
|
mics,
|
||||||
|
switchDevice,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -133,8 +133,6 @@ export const enUS: TranslationMap = {
|
|||||||
|
|
||||||
// Video controls (enhanced)
|
// Video controls (enhanced)
|
||||||
"controls.recognize": "Analyze Scene",
|
"controls.recognize": "Analyze Scene",
|
||||||
"controls.device.camera": "Camera",
|
|
||||||
"controls.device.mic": "Microphone",
|
|
||||||
"controls.device.default": "Default",
|
"controls.device.default": "Default",
|
||||||
|
|
||||||
// Status bar
|
// Status bar
|
||||||
|
|||||||
@@ -133,8 +133,6 @@ export const jaJP: TranslationMap = {
|
|||||||
|
|
||||||
// Video controls (enhanced)
|
// Video controls (enhanced)
|
||||||
"controls.recognize": "シーンを分析",
|
"controls.recognize": "シーンを分析",
|
||||||
"controls.device.camera": "カメラ",
|
|
||||||
"controls.device.mic": "マイク",
|
|
||||||
"controls.device.default": "デフォルト",
|
"controls.device.default": "デフォルト",
|
||||||
|
|
||||||
// Status bar
|
// Status bar
|
||||||
|
|||||||
@@ -133,8 +133,6 @@ export const zhCN: TranslationMap = {
|
|||||||
|
|
||||||
// Video controls (enhanced)
|
// Video controls (enhanced)
|
||||||
"controls.recognize": "识别画面",
|
"controls.recognize": "识别画面",
|
||||||
"controls.device.camera": "摄像头",
|
|
||||||
"controls.device.mic": "麦克风",
|
|
||||||
"controls.device.default": "默认",
|
"controls.device.default": "默认",
|
||||||
|
|
||||||
// Status bar
|
// Status bar
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ export interface SessionConfig {
|
|||||||
detailLevel: "low" | "high";
|
detailLevel: "low" | "high";
|
||||||
language: string;
|
language: string;
|
||||||
scenario: string; // 情景 ID,如 "free_chat"、"interviewer"
|
scenario: string; // 情景 ID,如 "free_chat"、"interviewer"
|
||||||
|
cameraDeviceId?: string; // 摄像头设备 ID,空 = 系统默认
|
||||||
|
micDeviceId?: string; // 麦克风设备 ID,空 = 系统默认
|
||||||
}
|
}
|
||||||
|
|
||||||
export type Theme = "dark" | "light";
|
export type Theme = "dark" | "light";
|
||||||
|
|||||||
Reference in New Issue
Block a user