Compare commits

...

2 Commits

Author SHA1 Message Date
6967dd7b2e feat: 实现摄像头和麦克风设备选择功能
- 新增 useDeviceList hook,枚举音视频输入设备并监听热插拔
- CameraManager/MicManager 的 startCamera/startMic 支持可选 deviceId 参数
- useVisionSession 集成设备选择:授权后自动枚举、切换设备时热重启
- 连接后显示设备下拉选择器,未连接时隐藏(避免未授权时空列表)
- SessionConfig 新增 cameraDeviceId/micDeviceId 持久化到 localStorage
2026-06-21 16:49:48 +08:00
eea5c07eaa refactor: 删除实时分析/按需识别/纯聊天三种模式切换
三种模式仅为前端 UI 区分,后端无感知,实际使用中价值不大:
- 纯聊天模式名不副实(VAD 仍附带摄像头帧)
- 实时分析增加 API 成本且体验不佳
- 按需识别已是默认且最自然的交互方式

删除后保留按需识别行为:语音带图 + 手动识别按钮,UI 更简洁。
2026-06-21 16:24:09 +08:00
11 changed files with 140 additions and 316 deletions

View File

@@ -644,12 +644,6 @@ body {
transition: all var(--transition-fast); transition: all var(--transition-fast);
} }
.ai-vision-indicator--active {
color: var(--color-success);
background: rgba(52, 211, 153, 0.15);
animation: pulse 2s cubic-bezier(0.25, 0.46, 0.45, 0.94) infinite;
}
/* ---- 视频控制栏 ---- */ /* ---- 视频控制栏 ---- */
.video-controls { .video-controls {
@@ -750,16 +744,6 @@ body {
gap: 10px; gap: 10px;
} }
.chat-panel-header__mode {
font-size: 0.68rem;
font-weight: 500;
color: var(--color-success);
background: rgba(52, 211, 153, 0.08);
padding: 3px 10px;
border-radius: 12px;
letter-spacing: 0.02em;
}
/* ---- Scenario Quick Selector (chat header) ---- */ /* ---- Scenario Quick Selector (chat header) ---- */
.scenario-selector { .scenario-selector {
appearance: none; appearance: none;
@@ -1473,38 +1457,6 @@ body {
outline-offset: 1px; outline-offset: 1px;
} }
/* Mode switcher */
.video-controls__mode {
display: flex;
gap: 2px;
background: var(--color-surface-2);
border-radius: var(--radius-sm);
padding: 2px;
justify-content: center;
}
.mode-btn {
background: none;
border: none;
color: var(--color-text-muted);
font-size: 0.7rem;
font-weight: 500;
padding: 5px 12px;
border-radius: 6px;
cursor: pointer;
transition: all var(--transition-fast);
white-space: nowrap;
}
.mode-btn:hover {
color: var(--color-text);
}
.mode-btn--active {
background: var(--color-primary);
color: white;
}
/* ---- Text-only mode (video ended, chat preserved) ---- */ /* ---- Text-only mode (video ended, chat preserved) ---- */
.video-controls__text-only { .video-controls__text-only {

View File

@@ -24,9 +24,6 @@ import type { Theme } from "./types";
import type { UserScenario } from "./lib/api/scenarios"; import type { UserScenario } from "./lib/api/scenarios";
import "./App.css"; import "./App.css";
/** AI 视觉模式 */
type VisionMode = "realtime" | "ondemand" | "chat";
/** 内部组件,确保在 I18nContext.Provider 内部使用 hooks */ /** 内部组件,确保在 I18nContext.Provider 内部使用 hooks */
function AppContent() { function AppContent() {
const { isAuthenticated, isLoading, user, logout, accessToken } = useAuth(); const { isAuthenticated, isLoading, user, logout, accessToken } = useAuth();
@@ -36,7 +33,6 @@ function AppContent() {
const [theme, setTheme] = useState<Theme>(loadTheme); const [theme, setTheme] = useState<Theme>(loadTheme);
const [elapsed, setElapsed] = useState(0); const [elapsed, setElapsed] = useState(0);
const [sidebarOpen, setSidebarOpen] = useState(false); const [sidebarOpen, setSidebarOpen] = useState(false);
const [visionMode, setVisionMode] = useState<VisionMode>("ondemand");
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null); const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
// ---- 自建情景管理 ---- // ---- 自建情景管理 ----
@@ -91,8 +87,6 @@ function AppContent() {
config, config,
updateConfig, updateConfig,
stats, stats,
mode,
isObserving,
startSession, startSession,
stopSession, stopSession,
stopVideo, stopVideo,
@@ -102,6 +96,9 @@ function AppContent() {
toggleCamera, toggleCamera,
toggleMic, toggleMic,
sendTextMessage, sendTextMessage,
cameras,
mics,
switchDevice,
} = useVisionSession(accessToken, activeSessionId); } = useVisionSession(accessToken, activeSessionId);
const isConnected = connectionStatus === "connected"; const isConnected = connectionStatus === "connected";
@@ -365,13 +362,13 @@ function AppContent() {
<div className="detail-badge">HD</div> <div className="detail-badge">HD</div>
)} )}
{/* AI 视觉状态指示 */} {/* AI 视觉状态指示 */}
{isConnected && visionMode !== "chat" && ( {isConnected && (
<div className={`ai-vision-indicator ${isObserving ? "ai-vision-indicator--active" : ""}`}> <div className="ai-vision-indicator">
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"> <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
<path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z" /> <path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z" />
<circle cx="12" cy="12" r="3" /> <circle cx="12" cy="12" r="3" />
</svg> </svg>
<span>{isObserving ? tr("video.observing") : "AI"}</span> <span>AI</span>
</div> </div>
)} )}
{isSpeaking && ( {isSpeaking && (
@@ -412,42 +409,6 @@ function AppContent() {
<button className="btn btn--primary btn--lg" onClick={startSession}> <button className="btn btn--primary btn--lg" onClick={startSession}>
{connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")} {connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")}
</button> </button>
{/* 设备选择器 */}
<div className="video-controls__devices">
<div className="device-select-wrapper">
<label className="device-select-label">📷 {tr("controls.device.camera")}</label>
<select className="device-select" defaultValue="default">
<option value="default">{tr("controls.device.default")}</option>
</select>
</div>
<div className="device-select-wrapper">
<label className="device-select-label">🎤 {tr("controls.device.mic")}</label>
<select className="device-select" defaultValue="default">
<option value="default">{tr("controls.device.default")}</option>
</select>
</div>
</div>
{/* 模式切换器 */}
<div className="video-controls__mode">
<button
className={`mode-btn ${visionMode === "realtime" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("realtime")}
>
{tr("controls.mode.realtime")}
</button>
<button
className={`mode-btn ${visionMode === "ondemand" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("ondemand")}
>
{tr("controls.mode.ondemand")}
</button>
<button
className={`mode-btn ${visionMode === "chat" ? "mode-btn--active" : ""}`}
onClick={() => setVisionMode("chat")}
>
{tr("controls.mode.chat")}
</button>
</div>
</> </>
) : isCameraOn ? ( ) : isCameraOn ? (
<> <>
@@ -467,16 +428,14 @@ function AppContent() {
> >
🎤 🎤
</button> </button>
{/* 识别画面按钮(按需模式下显示) */} {/* 识别画面按钮 */}
{visionMode === "ondemand" && ( <button
<button className="btn--recognize"
className="btn--recognize" onClick={handleRecognize}
onClick={handleRecognize} disabled={isProcessing}
disabled={isProcessing} >
> 🔍 {tr("controls.recognize")}
🔍 {tr("controls.recognize")} </button>
</button>
)}
{isProcessing && ( {isProcessing && (
<button className="btn btn--warning" onClick={interrupt}> <button className="btn btn--warning" onClick={interrupt}>
{tr("controls.interrupt")} {tr("controls.interrupt")}
@@ -486,26 +445,34 @@ function AppContent() {
{tr("controls.stopVideo")} {tr("controls.stopVideo")}
</button> </button>
</div> </div>
{/* 通话态模式切换 */} {/* 设备选择器 */}
<div className="video-controls__mode"> <div className="video-controls__devices">
<button <div className="device-select-wrapper">
className={`mode-btn ${visionMode === "realtime" ? "mode-btn--active" : ""}`} <label className="device-select-label">📷</label>
onClick={() => setVisionMode("realtime")} <select
> className="device-select"
{tr("controls.mode.realtime")} value={config.cameraDeviceId || "default"}
</button> onChange={(e) => switchDevice("camera", e.target.value === "default" ? "" : e.target.value)}
<button >
className={`mode-btn ${visionMode === "ondemand" ? "mode-btn--active" : ""}`} <option value="default">{tr("controls.device.default")}</option>
onClick={() => setVisionMode("ondemand")} {cameras.map((d) => (
> <option key={d.deviceId} value={d.deviceId}>{d.label}</option>
{tr("controls.mode.ondemand")} ))}
</button> </select>
<button </div>
className={`mode-btn ${visionMode === "chat" ? "mode-btn--active" : ""}`} <div className="device-select-wrapper">
onClick={() => setVisionMode("chat")} <label className="device-select-label">🎤</label>
> <select
{tr("controls.mode.chat")} className="device-select"
</button> value={config.micDeviceId || "default"}
onChange={(e) => switchDevice("mic", e.target.value === "default" ? "" : e.target.value)}
>
<option value="default">{tr("controls.device.default")}</option>
{mics.map((d) => (
<option key={d.deviceId} value={d.deviceId}>{d.label}</option>
))}
</select>
</div>
</div> </div>
</> </>
) : ( ) : (
@@ -547,9 +514,6 @@ function AppContent() {
</option> </option>
))} ))}
</select> </select>
{isConnected && mode === "observation" && (
<span className="chat-panel-header__mode">{tr("chat.mode.observation")}</span>
)}
{isConnected && stats.queryCount > 0 && ( {isConnected && stats.queryCount > 0 && (
<span className="chat-panel-header__stats"> <span className="chat-panel-header__stats">
{stats.queryCount} {tr("statusbar.recognitions")} {stats.queryCount} {tr("statusbar.recognitions")}

View File

@@ -19,10 +19,13 @@ export function useCamera() {
const [stream, setStream] = useState<MediaStream | null>(null); const [stream, setStream] = useState<MediaStream | null>(null);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const startCamera = useCallback(async () => { const startCamera = useCallback(async (deviceId?: string) => {
try { try {
const videoConstraints: MediaTrackConstraints = deviceId
? { deviceId: { exact: deviceId }, width: 640, height: 480 }
: { facingMode: "environment", width: 640, height: 480 };
const mediaStream = await navigator.mediaDevices.getUserMedia({ const mediaStream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: "environment", width: 640, height: 480 }, video: videoConstraints,
audio: false, audio: false,
}); });
setStream(mediaStream); setStream(mediaStream);

View File

@@ -12,15 +12,13 @@ export function useMicrophone() {
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const audioContextRef = useRef<AudioContext | null>(null); const audioContextRef = useRef<AudioContext | null>(null);
const startMic = useCallback(async () => { const startMic = useCallback(async (deviceId?: string) => {
try { try {
const audioConstraints: MediaTrackConstraints = deviceId
? { deviceId: { exact: deviceId }, sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true }
: { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true };
const mediaStream = await navigator.mediaDevices.getUserMedia({ const mediaStream = await navigator.mediaDevices.getUserMedia({
audio: { audio: audioConstraints,
sampleRate: 16000,
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
},
video: false, video: false,
}); });
setStream(mediaStream); setStream(mediaStream);

View File

@@ -0,0 +1,44 @@
// ============================================================
// useDeviceList — 设备枚举 Hook
// 职责:枚举摄像头/麦克风设备列表,监听设备热插拔
// 注意:首次枚举需要先有一次成功的 getUserMedia 授权
// ============================================================
import { useCallback, useEffect, useState } from "react";
export interface DeviceInfo {
deviceId: string;
label: string;
}
export function useDeviceList() {
const [cameras, setCameras] = useState<DeviceInfo[]>([]);
const [mics, setMics] = useState<DeviceInfo[]>([]);
/** 枚举当前可用的音视频输入设备 */
const refreshDevices = useCallback(async () => {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
setCameras(
devices
.filter((d) => d.kind === "videoinput")
.map((d) => ({ deviceId: d.deviceId, label: d.label || `摄像头 ${d.deviceId.slice(0, 4)}` }))
);
setMics(
devices
.filter((d) => d.kind === "audioinput")
.map((d) => ({ deviceId: d.deviceId, label: d.label || `麦克风 ${d.deviceId.slice(0, 4)}` }))
);
} catch (err) {
console.warn("[DeviceList] 枚举设备失败:", err);
}
}, []);
// 监听设备热插拔
useEffect(() => {
navigator.mediaDevices.addEventListener("devicechange", refreshDevices);
return () => navigator.mediaDevices.removeEventListener("devicechange", refreshDevices);
}, [refreshDevices]);
return { cameras, mics, refreshDevices };
}

View File

@@ -1,88 +0,0 @@
// ============================================================
// useObservationMode — 观察模式 Hook
// 职责:定时采帧 → 关键帧检测 → 画面变化时触发回调
// 来源docs/05-用户故事.md US-05持续场景监控
// ============================================================
import { useCallback, useEffect, useRef, useState } from "react";
import { sampleFrame, compareFrames } from "../components/EdgeProcessor";
/** 画面变化显著阈值 */
const CHANGE_THRESHOLD = 0.85;
/** 采样间隔ms */
const SAMPLE_INTERVAL = 5000;
export interface ObservationOptions {
/** 画面变化回调,携带当前帧的 DataURL */
onChange?: (frameDataUrl: string) => void;
}
export function useObservationMode(options?: ObservationOptions) {
const [isObserving, setIsObserving] = useState(false);
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
const optionsRef = useRef(options);
useEffect(() => {
optionsRef.current = options;
}, [options]);
/**
* 启动观察模式
* @param video 摄像头 video 元素
* @param captureFrame 从 video 捕获 DataURL 的函数
*/
const startObserving = useCallback(
(video: HTMLVideoElement | null, captureFrame: () => string | null) => {
if (!video) return;
// 立即采一帧作为基准
prevFrameRef.current = sampleFrame(video);
intervalRef.current = setInterval(() => {
const current = sampleFrame(video);
if (!current) return;
if (prevFrameRef.current) {
const { similarity } = compareFrames(prevFrameRef.current, current);
if (similarity < CHANGE_THRESHOLD) {
console.log(
`[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`,
);
const frameDataUrl = captureFrame();
if (frameDataUrl) {
optionsRef.current?.onChange?.(frameDataUrl);
}
}
}
prevFrameRef.current = current;
}, SAMPLE_INTERVAL);
setIsObserving(true);
},
[],
);
/** 停止观察模式 */
const stopObserving = useCallback(() => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
prevFrameRef.current = null;
setIsObserving(false);
}, []);
// 组件卸载时清理
useEffect(() => {
return () => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
}
};
}, []);
return { isObserving, startObserving, stopObserving };
}

View File

@@ -17,11 +17,9 @@ import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager"; import { useMicrophone } from "../components/MicManager";
import { useVAD, sampleFrame } from "../components/EdgeProcessor"; import { useVAD, sampleFrame } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager"; import { useWebSocketManager } from "../components/WebSocketManager";
import { useObservationMode } from "./useObservationMode"; import { useDeviceList } from "./useDeviceList";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types"; import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
export type SessionMode = "dialogue" | "observation";
export interface SessionStats { export interface SessionStats {
queryCount: number; queryCount: number;
totalTokens: number; totalTokens: number;
@@ -35,7 +33,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
const [isAudioPlaying, setIsAudioPlaying] = useState(false); const [isAudioPlaying, setIsAudioPlaying] = useState(false);
const [config, setConfig] = useState<SessionConfig>(loadConfig); const [config, setConfig] = useState<SessionConfig>(loadConfig);
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 }); const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
const [mode, setMode] = useState<SessionMode>("dialogue");
const [isCameraOn, setIsCameraOn] = useState(false); const [isCameraOn, setIsCameraOn] = useState(false);
const [isMicOn, setIsMicOn] = useState(false); const [isMicOn, setIsMicOn] = useState(false);
@@ -59,6 +56,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
const { startMic, stopMic } = useMicrophone(); const { startMic, stopMic } = useMicrophone();
const { status, connect, disconnect, send } = useWebSocketManager(); const { status, connect, disconnect, send } = useWebSocketManager();
const { cameras, mics, refreshDevices } = useDeviceList();
// 用 ref 跟踪 isProcessing避免 VAD 回调闭包问题 // 用 ref 跟踪 isProcessing避免 VAD 回调闭包问题
const isProcessingRef = useRef(false); const isProcessingRef = useRef(false);
@@ -78,49 +76,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
conversationIdRef.current = conversationId; conversationIdRef.current = conversationId;
}, [conversationId]); }, [conversationId]);
// 观察模式:画面变化时自动发送 query
const { isObserving, startObserving, stopObserving } = useObservationMode({
onChange: useCallback(
(frameDataUrl: string) => {
if (isProcessingRef.current) return;
const requestId = uuidv4();
send({
type: "query",
request_id: requestId,
image: dataUrlToBase64(frameDataUrl),
audio: "", // 观察模式无音频
});
setMessages((prev) => [
...prev,
{
id: uuidv4(),
role: "user",
content: t("session.changeDetected"),
timestamp: Date.now(),
},
]);
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
setIsProcessing(true);
},
[send],
),
});
/** 切换对话/观察模式 */
const toggleMode = useCallback(() => {
setMode((prev) => {
const next = prev === "dialogue" ? "observation" : "dialogue";
if (next === "observation") {
startObserving(videoRef.current, captureFrame);
} else {
stopObserving();
}
return next;
});
}, [videoRef, captureFrame, startObserving, stopObserving]);
// WebSocket 连接成功后发送 config + flush 待发消息 // WebSocket 连接成功后发送 config + flush 待发消息
useEffect(() => { useEffect(() => {
if (status === "connected") { if (status === "connected") {
@@ -324,14 +279,14 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
// 2. 尝试获取摄像头(可选) // 2. 尝试获取摄像头(可选)
try { try {
await startCamera(); await startCamera(config.cameraDeviceId || undefined);
setIsCameraOn(true); setIsCameraOn(true);
} catch { } catch {
console.warn("[Session] 无法获取摄像头权限,将以纯文本模式运行"); console.warn("[Session] 无法获取摄像头权限,将以纯文本模式运行");
} }
// 3. 尝试获取麦克风(可选) // 3. 尝试获取麦克风(可选)
const micStream = await startMic(); const micStream = await startMic(config.micDeviceId || undefined);
if (micStream) { if (micStream) {
setIsMicOn(true); setIsMicOn(true);
// 4. 启动 VAD仅在麦克风可用时 // 4. 启动 VAD仅在麦克风可用时
@@ -339,12 +294,13 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
} else { } else {
console.warn("[Session] 无法获取麦克风权限,将以文本输入模式运行"); console.warn("[Session] 无法获取麦克风权限,将以文本输入模式运行");
} }
}, [startCamera, startMic, connect, startVAD, accessToken]);
// 5. 授权后刷新设备列表
refreshDevices();
}, [startCamera, startMic, connect, startVAD, accessToken, config.cameraDeviceId, config.micDeviceId, refreshDevices]);
/** 结束会话 */ /** 结束会话 */
const stopSession = useCallback(async () => { const stopSession = useCallback(async () => {
stopObserving();
setMode("dialogue");
await stopVAD(); await stopVAD();
stopMic(); stopMic();
stopCamera(); stopCamera();
@@ -359,12 +315,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
prevFrameRef.current = null; prevFrameRef.current = null;
setIsCameraOn(false); setIsCameraOn(false);
setIsMicOn(false); setIsMicOn(false);
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]); }, [stopVAD, stopMic, stopCamera, disconnect]);
/** 结束视频,保留聊天和连接 */ /** 结束视频,保留聊天和连接 */
const stopVideo = useCallback(async () => { const stopVideo = useCallback(async () => {
stopObserving();
setMode("dialogue");
await stopVAD(); await stopVAD();
stopMic(); stopMic();
stopCamera(); stopCamera();
@@ -375,7 +329,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
setIsCameraOn(false); setIsCameraOn(false);
setIsMicOn(false); setIsMicOn(false);
// 不断开 WebSocket不清空消息、统计 // 不断开 WebSocket不清空消息、统计
}, [stopObserving, stopVAD, stopMic, stopCamera]); }, [stopVAD, stopMic, stopCamera]);
/** 摄像头开关 */ /** 摄像头开关 */
const toggleCamera = useCallback(async () => { const toggleCamera = useCallback(async () => {
@@ -383,10 +337,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
stopCamera(); stopCamera();
setIsCameraOn(false); setIsCameraOn(false);
} else { } else {
await startCamera(); await startCamera(config.cameraDeviceId || undefined);
setIsCameraOn(true); setIsCameraOn(true);
} }
}, [isCameraOn, startCamera, stopCamera]); }, [isCameraOn, startCamera, stopCamera, config.cameraDeviceId]);
/** 麦克风开关 */ /** 麦克风开关 */
const toggleMic = useCallback(async () => { const toggleMic = useCallback(async () => {
@@ -395,13 +349,13 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
stopMic(); stopMic();
setIsMicOn(false); setIsMicOn(false);
} else { } else {
const micStream = await startMic(); const micStream = await startMic(config.micDeviceId || undefined);
if (micStream) { if (micStream) {
await startVAD(micStream); await startVAD(micStream);
setIsMicOn(true); setIsMicOn(true);
} }
} }
}, [isMicOn, startMic, stopMic, startVAD, stopVAD]); }, [isMicOn, startMic, stopMic, startVAD, stopVAD, config.micDeviceId]);
/** 打断当前回复 */ /** 打断当前回复 */
const interrupt = useCallback(() => { const interrupt = useCallback(() => {
@@ -421,6 +375,31 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
setIsProcessing(false); setIsProcessing(false);
}, [send, currentReply]); }, [send, currentReply]);
/** 切换设备(摄像头或麦克风) */
const switchDevice = useCallback(async (kind: "camera" | "mic", deviceId: string) => {
if (kind === "camera") {
updateConfig({ cameraDeviceId: deviceId || undefined });
if (isCameraOn) {
stopCamera();
try {
await startCamera(deviceId || undefined);
} catch {
console.warn("[Session] 切换摄像头失败");
}
}
} else {
updateConfig({ micDeviceId: deviceId || undefined });
if (isMicOn) {
await stopVAD();
stopMic();
const micStream = await startMic(deviceId || undefined);
if (micStream) {
await startVAD(micStream);
}
}
}
}, [isCameraOn, isMicOn, stopCamera, startCamera, stopMic, startMic, stopVAD, startVAD, updateConfig]);
/** 发送文本消息(手动输入) */ /** 发送文本消息(手动输入) */
const sendTextMessage = useCallback( const sendTextMessage = useCallback(
(text: string) => { (text: string) => {
@@ -480,9 +459,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
config, config,
updateConfig, updateConfig,
stats, stats,
mode,
isObserving,
toggleMode,
startSession, startSession,
stopSession, stopSession,
stopVideo, stopVideo,
@@ -493,5 +469,8 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
toggleMic, toggleMic,
sendTextMessage, sendTextMessage,
captureFrame, captureFrame,
cameras,
mics,
switchDevice,
}; };
} }

View File

@@ -48,7 +48,6 @@ export const enUS: TranslationMap = {
"scenario.interpreter.hint": "AI will translate your speech in real-time (Chinese-English), without explanations", "scenario.interpreter.hint": "AI will translate your speech in real-time (Chinese-English), without explanations",
// Video indicators // Video indicators
"video.observing": "👁️ Observing",
"video.listening": "🎤 Listening...", "video.listening": "🎤 Listening...",
"video.playing": "🔊 Playing...", "video.playing": "🔊 Playing...",
"video.initVad": "Initializing voice detection...", "video.initVad": "Initializing voice detection...",
@@ -67,8 +66,6 @@ export const enUS: TranslationMap = {
"controls.cameraOn": "Turn on camera", "controls.cameraOn": "Turn on camera",
"controls.micOff": "Turn off microphone", "controls.micOff": "Turn off microphone",
"controls.micOn": "Turn on microphone", "controls.micOn": "Turn on microphone",
"controls.observing": "👁️ Observing",
"controls.observation": "👁️ Observe",
"controls.interrupt": "⏹ Interrupt", "controls.interrupt": "⏹ Interrupt",
"controls.stop": "End Session", "controls.stop": "End Session",
"controls.stopVideo": "End Video", "controls.stopVideo": "End Video",
@@ -77,7 +74,6 @@ export const enUS: TranslationMap = {
// Chat panel // Chat panel
"chat.title": "Chat", "chat.title": "Chat",
"chat.mode.observation": "Observing",
"chat.reconnecting": "Connection lost, reconnecting...", "chat.reconnecting": "Connection lost, reconnecting...",
"chat.connecting": "Connecting to server...", "chat.connecting": "Connecting to server...",
"chat.vadInit": "Initializing voice detection...", "chat.vadInit": "Initializing voice detection...",
@@ -91,7 +87,6 @@ export const enUS: TranslationMap = {
"chat.scenarioSwitched": "Switched to {name} mode", "chat.scenarioSwitched": "Switched to {name} mode",
// Session messages // Session messages
"session.changeDetected": "👁️ Scene change detected",
"session.recognizing": "(Recognizing speech...)", "session.recognizing": "(Recognizing speech...)",
"session.sttFailed": "(Speech recognition failed, please try again)", "session.sttFailed": "(Speech recognition failed, please try again)",
"session.noSpeech": "(No speech detected)", "session.noSpeech": "(No speech detected)",
@@ -138,12 +133,7 @@ export const enUS: TranslationMap = {
// Video controls (enhanced) // Video controls (enhanced)
"controls.recognize": "Analyze Scene", "controls.recognize": "Analyze Scene",
"controls.device.camera": "Camera",
"controls.device.mic": "Microphone",
"controls.device.default": "Default", "controls.device.default": "Default",
"controls.mode.realtime": "Realtime",
"controls.mode.ondemand": "On-demand",
"controls.mode.chat": "Chat only",
// Status bar // Status bar
"statusbar.ready": "Ready · Select devices to start", "statusbar.ready": "Ready · Select devices to start",

View File

@@ -48,7 +48,6 @@ export const jaJP: TranslationMap = {
"scenario.interpreter.hint": "AIがあなたの発言をリアルタイムで翻訳します中日相互翻訳、説明なし", "scenario.interpreter.hint": "AIがあなたの発言をリアルタイムで翻訳します中日相互翻訳、説明なし",
// Video indicators // Video indicators
"video.observing": "👁️ 観察中",
"video.listening": "🎤 聞き取り中...", "video.listening": "🎤 聞き取り中...",
"video.playing": "🔊 再生中...", "video.playing": "🔊 再生中...",
"video.initVad": "音声検出を初期化中...", "video.initVad": "音声検出を初期化中...",
@@ -67,8 +66,6 @@ export const jaJP: TranslationMap = {
"controls.cameraOn": "カメラをオン", "controls.cameraOn": "カメラをオン",
"controls.micOff": "マイクをオフ", "controls.micOff": "マイクをオフ",
"controls.micOn": "マイクをオン", "controls.micOn": "マイクをオン",
"controls.observing": "👁️ 観察中",
"controls.observation": "👁️ 観察モード",
"controls.interrupt": "⏹ 中断", "controls.interrupt": "⏹ 中断",
"controls.stop": "対話を終了", "controls.stop": "対話を終了",
"controls.stopVideo": "ビデオ終了", "controls.stopVideo": "ビデオ終了",
@@ -77,7 +74,6 @@ export const jaJP: TranslationMap = {
// Chat panel // Chat panel
"chat.title": "チャット", "chat.title": "チャット",
"chat.mode.observation": "観察モード",
"chat.reconnecting": "接続が切断されました。再接続中...", "chat.reconnecting": "接続が切断されました。再接続中...",
"chat.connecting": "サーバーに接続中...", "chat.connecting": "サーバーに接続中...",
"chat.vadInit": "音声検出を初期化中...", "chat.vadInit": "音声検出を初期化中...",
@@ -91,7 +87,6 @@ export const jaJP: TranslationMap = {
"chat.scenarioSwitched": "{name} モードに切り替えました", "chat.scenarioSwitched": "{name} モードに切り替えました",
// Session messages // Session messages
"session.changeDetected": "👁️ シーン変化を検出",
"session.recognizing": "(音声認識中...", "session.recognizing": "(音声認識中...",
"session.sttFailed": "(音声認識に失敗しました。もう一度お試しください)", "session.sttFailed": "(音声認識に失敗しました。もう一度お試しください)",
"session.noSpeech": "(音声が検出されませんでした)", "session.noSpeech": "(音声が検出されませんでした)",
@@ -138,12 +133,7 @@ export const jaJP: TranslationMap = {
// Video controls (enhanced) // Video controls (enhanced)
"controls.recognize": "シーンを分析", "controls.recognize": "シーンを分析",
"controls.device.camera": "カメラ",
"controls.device.mic": "マイク",
"controls.device.default": "デフォルト", "controls.device.default": "デフォルト",
"controls.mode.realtime": "リアルタイム",
"controls.mode.ondemand": "オンデマンド",
"controls.mode.chat": "チャットのみ",
// Status bar // Status bar
"statusbar.ready": "準備完了 · デバイスを選択して開始", "statusbar.ready": "準備完了 · デバイスを選択して開始",

View File

@@ -48,7 +48,6 @@ export const zhCN: TranslationMap = {
"scenario.interpreter.hint": "AI 会实时翻译你的话(中英互译),无解释评论", "scenario.interpreter.hint": "AI 会实时翻译你的话(中英互译),无解释评论",
// Video indicators // Video indicators
"video.observing": "👁️ 观察中",
"video.listening": "🎤 正在聆听...", "video.listening": "🎤 正在聆听...",
"video.playing": "🔊 正在播放...", "video.playing": "🔊 正在播放...",
"video.initVad": "正在初始化语音检测...", "video.initVad": "正在初始化语音检测...",
@@ -67,8 +66,6 @@ export const zhCN: TranslationMap = {
"controls.cameraOn": "开启摄像头", "controls.cameraOn": "开启摄像头",
"controls.micOff": "关闭麦克风", "controls.micOff": "关闭麦克风",
"controls.micOn": "开启麦克风", "controls.micOn": "开启麦克风",
"controls.observing": "👁️ 观察中",
"controls.observation": "👁️ 观察模式",
"controls.interrupt": "⏹ 打断", "controls.interrupt": "⏹ 打断",
"controls.stop": "结束对话", "controls.stop": "结束对话",
"controls.stopVideo": "结束视频", "controls.stopVideo": "结束视频",
@@ -77,7 +74,6 @@ export const zhCN: TranslationMap = {
// Chat panel // Chat panel
"chat.title": "对话", "chat.title": "对话",
"chat.mode.observation": "观察模式",
"chat.reconnecting": "连接已断开,正在重连...", "chat.reconnecting": "连接已断开,正在重连...",
"chat.connecting": "正在连接服务...", "chat.connecting": "正在连接服务...",
"chat.vadInit": "正在初始化语音检测...", "chat.vadInit": "正在初始化语音检测...",
@@ -91,7 +87,6 @@ export const zhCN: TranslationMap = {
"chat.scenarioSwitched": "已切换到 {name} 模式", "chat.scenarioSwitched": "已切换到 {name} 模式",
// Session messages // Session messages
"session.changeDetected": "👁️ 画面变化检测",
"session.recognizing": "(语音识别中...", "session.recognizing": "(语音识别中...",
"session.sttFailed": "(语音识别失败,请重试)", "session.sttFailed": "(语音识别失败,请重试)",
"session.noSpeech": "(未识别到语音)", "session.noSpeech": "(未识别到语音)",
@@ -138,12 +133,7 @@ export const zhCN: TranslationMap = {
// Video controls (enhanced) // Video controls (enhanced)
"controls.recognize": "识别画面", "controls.recognize": "识别画面",
"controls.device.camera": "摄像头",
"controls.device.mic": "麦克风",
"controls.device.default": "默认", "controls.device.default": "默认",
"controls.mode.realtime": "实时分析",
"controls.mode.ondemand": "按需识别",
"controls.mode.chat": "纯聊天",
// Status bar // Status bar
"statusbar.ready": "就绪 · 选择设备后开始通话", "statusbar.ready": "就绪 · 选择设备后开始通话",

View File

@@ -10,6 +10,8 @@ export interface SessionConfig {
detailLevel: "low" | "high"; detailLevel: "low" | "high";
language: string; language: string;
scenario: string; // 情景 ID如 "free_chat"、"interviewer" scenario: string; // 情景 ID如 "free_chat"、"interviewer"
cameraDeviceId?: string; // 摄像头设备 ID空 = 系统默认
micDeviceId?: string; // 麦克风设备 ID空 = 系统默认
} }
export type Theme = "dark" | "light"; export type Theme = "dark" | "light";