Compare commits
2 Commits
1079e22699
...
6967dd7b2e
| Author | SHA1 | Date | |
|---|---|---|---|
| 6967dd7b2e | |||
| eea5c07eaa |
@@ -644,12 +644,6 @@ body {
|
||||
transition: all var(--transition-fast);
|
||||
}
|
||||
|
||||
.ai-vision-indicator--active {
|
||||
color: var(--color-success);
|
||||
background: rgba(52, 211, 153, 0.15);
|
||||
animation: pulse 2s cubic-bezier(0.25, 0.46, 0.45, 0.94) infinite;
|
||||
}
|
||||
|
||||
/* ---- 视频控制栏 ---- */
|
||||
|
||||
.video-controls {
|
||||
@@ -750,16 +744,6 @@ body {
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.chat-panel-header__mode {
|
||||
font-size: 0.68rem;
|
||||
font-weight: 500;
|
||||
color: var(--color-success);
|
||||
background: rgba(52, 211, 153, 0.08);
|
||||
padding: 3px 10px;
|
||||
border-radius: 12px;
|
||||
letter-spacing: 0.02em;
|
||||
}
|
||||
|
||||
/* ---- Scenario Quick Selector (chat header) ---- */
|
||||
.scenario-selector {
|
||||
appearance: none;
|
||||
@@ -1473,38 +1457,6 @@ body {
|
||||
outline-offset: 1px;
|
||||
}
|
||||
|
||||
/* Mode switcher */
|
||||
.video-controls__mode {
|
||||
display: flex;
|
||||
gap: 2px;
|
||||
background: var(--color-surface-2);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 2px;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.mode-btn {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.7rem;
|
||||
font-weight: 500;
|
||||
padding: 5px 12px;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
transition: all var(--transition-fast);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.mode-btn:hover {
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.mode-btn--active {
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
}
|
||||
|
||||
/* ---- Text-only mode (video ended, chat preserved) ---- */
|
||||
|
||||
.video-controls__text-only {
|
||||
|
||||
@@ -24,9 +24,6 @@ import type { Theme } from "./types";
|
||||
import type { UserScenario } from "./lib/api/scenarios";
|
||||
import "./App.css";
|
||||
|
||||
/** AI 视觉模式 */
|
||||
type VisionMode = "realtime" | "ondemand" | "chat";
|
||||
|
||||
/** 内部组件,确保在 I18nContext.Provider 内部使用 hooks */
|
||||
function AppContent() {
|
||||
const { isAuthenticated, isLoading, user, logout, accessToken } = useAuth();
|
||||
@@ -36,7 +33,6 @@ function AppContent() {
|
||||
const [theme, setTheme] = useState<Theme>(loadTheme);
|
||||
const [elapsed, setElapsed] = useState(0);
|
||||
const [sidebarOpen, setSidebarOpen] = useState(false);
|
||||
const [visionMode, setVisionMode] = useState<VisionMode>("ondemand");
|
||||
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
// ---- 自建情景管理 ----
|
||||
@@ -91,8 +87,6 @@ function AppContent() {
|
||||
config,
|
||||
updateConfig,
|
||||
stats,
|
||||
mode,
|
||||
isObserving,
|
||||
startSession,
|
||||
stopSession,
|
||||
stopVideo,
|
||||
@@ -102,6 +96,9 @@ function AppContent() {
|
||||
toggleCamera,
|
||||
toggleMic,
|
||||
sendTextMessage,
|
||||
cameras,
|
||||
mics,
|
||||
switchDevice,
|
||||
} = useVisionSession(accessToken, activeSessionId);
|
||||
|
||||
const isConnected = connectionStatus === "connected";
|
||||
@@ -365,13 +362,13 @@ function AppContent() {
|
||||
<div className="detail-badge">HD</div>
|
||||
)}
|
||||
{/* AI 视觉状态指示 */}
|
||||
{isConnected && visionMode !== "chat" && (
|
||||
<div className={`ai-vision-indicator ${isObserving ? "ai-vision-indicator--active" : ""}`}>
|
||||
{isConnected && (
|
||||
<div className="ai-vision-indicator">
|
||||
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z" />
|
||||
<circle cx="12" cy="12" r="3" />
|
||||
</svg>
|
||||
<span>{isObserving ? tr("video.observing") : "AI"}</span>
|
||||
<span>AI</span>
|
||||
</div>
|
||||
)}
|
||||
{isSpeaking && (
|
||||
@@ -412,42 +409,6 @@ function AppContent() {
|
||||
<button className="btn btn--primary btn--lg" onClick={startSession}>
|
||||
{connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")}
|
||||
</button>
|
||||
{/* 设备选择器 */}
|
||||
<div className="video-controls__devices">
|
||||
<div className="device-select-wrapper">
|
||||
<label className="device-select-label">📷 {tr("controls.device.camera")}</label>
|
||||
<select className="device-select" defaultValue="default">
|
||||
<option value="default">{tr("controls.device.default")}</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="device-select-wrapper">
|
||||
<label className="device-select-label">🎤 {tr("controls.device.mic")}</label>
|
||||
<select className="device-select" defaultValue="default">
|
||||
<option value="default">{tr("controls.device.default")}</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
{/* 模式切换器 */}
|
||||
<div className="video-controls__mode">
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "realtime" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("realtime")}
|
||||
>
|
||||
{tr("controls.mode.realtime")}
|
||||
</button>
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "ondemand" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("ondemand")}
|
||||
>
|
||||
{tr("controls.mode.ondemand")}
|
||||
</button>
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "chat" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("chat")}
|
||||
>
|
||||
{tr("controls.mode.chat")}
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
) : isCameraOn ? (
|
||||
<>
|
||||
@@ -467,16 +428,14 @@ function AppContent() {
|
||||
>
|
||||
🎤
|
||||
</button>
|
||||
{/* 识别画面按钮(按需模式下显示) */}
|
||||
{visionMode === "ondemand" && (
|
||||
<button
|
||||
className="btn--recognize"
|
||||
onClick={handleRecognize}
|
||||
disabled={isProcessing}
|
||||
>
|
||||
🔍 {tr("controls.recognize")}
|
||||
</button>
|
||||
)}
|
||||
{/* 识别画面按钮 */}
|
||||
<button
|
||||
className="btn--recognize"
|
||||
onClick={handleRecognize}
|
||||
disabled={isProcessing}
|
||||
>
|
||||
🔍 {tr("controls.recognize")}
|
||||
</button>
|
||||
{isProcessing && (
|
||||
<button className="btn btn--warning" onClick={interrupt}>
|
||||
{tr("controls.interrupt")}
|
||||
@@ -486,26 +445,34 @@ function AppContent() {
|
||||
{tr("controls.stopVideo")}
|
||||
</button>
|
||||
</div>
|
||||
{/* 通话态模式切换 */}
|
||||
<div className="video-controls__mode">
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "realtime" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("realtime")}
|
||||
>
|
||||
{tr("controls.mode.realtime")}
|
||||
</button>
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "ondemand" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("ondemand")}
|
||||
>
|
||||
{tr("controls.mode.ondemand")}
|
||||
</button>
|
||||
<button
|
||||
className={`mode-btn ${visionMode === "chat" ? "mode-btn--active" : ""}`}
|
||||
onClick={() => setVisionMode("chat")}
|
||||
>
|
||||
{tr("controls.mode.chat")}
|
||||
</button>
|
||||
{/* 设备选择器 */}
|
||||
<div className="video-controls__devices">
|
||||
<div className="device-select-wrapper">
|
||||
<label className="device-select-label">📷</label>
|
||||
<select
|
||||
className="device-select"
|
||||
value={config.cameraDeviceId || "default"}
|
||||
onChange={(e) => switchDevice("camera", e.target.value === "default" ? "" : e.target.value)}
|
||||
>
|
||||
<option value="default">{tr("controls.device.default")}</option>
|
||||
{cameras.map((d) => (
|
||||
<option key={d.deviceId} value={d.deviceId}>{d.label}</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<div className="device-select-wrapper">
|
||||
<label className="device-select-label">🎤</label>
|
||||
<select
|
||||
className="device-select"
|
||||
value={config.micDeviceId || "default"}
|
||||
onChange={(e) => switchDevice("mic", e.target.value === "default" ? "" : e.target.value)}
|
||||
>
|
||||
<option value="default">{tr("controls.device.default")}</option>
|
||||
{mics.map((d) => (
|
||||
<option key={d.deviceId} value={d.deviceId}>{d.label}</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
@@ -547,9 +514,6 @@ function AppContent() {
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
{isConnected && mode === "observation" && (
|
||||
<span className="chat-panel-header__mode">{tr("chat.mode.observation")}</span>
|
||||
)}
|
||||
{isConnected && stats.queryCount > 0 && (
|
||||
<span className="chat-panel-header__stats">
|
||||
{stats.queryCount} {tr("statusbar.recognitions")}
|
||||
|
||||
@@ -19,10 +19,13 @@ export function useCamera() {
|
||||
const [stream, setStream] = useState<MediaStream | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const startCamera = useCallback(async () => {
|
||||
const startCamera = useCallback(async (deviceId?: string) => {
|
||||
try {
|
||||
const videoConstraints: MediaTrackConstraints = deviceId
|
||||
? { deviceId: { exact: deviceId }, width: 640, height: 480 }
|
||||
: { facingMode: "environment", width: 640, height: 480 };
|
||||
const mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { facingMode: "environment", width: 640, height: 480 },
|
||||
video: videoConstraints,
|
||||
audio: false,
|
||||
});
|
||||
setStream(mediaStream);
|
||||
|
||||
@@ -12,15 +12,13 @@ export function useMicrophone() {
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
|
||||
const startMic = useCallback(async () => {
|
||||
const startMic = useCallback(async (deviceId?: string) => {
|
||||
try {
|
||||
const audioConstraints: MediaTrackConstraints = deviceId
|
||||
? { deviceId: { exact: deviceId }, sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true }
|
||||
: { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true };
|
||||
const mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
sampleRate: 16000,
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
audio: audioConstraints,
|
||||
video: false,
|
||||
});
|
||||
setStream(mediaStream);
|
||||
|
||||
44
frontend/src/hooks/useDeviceList.ts
Normal file
44
frontend/src/hooks/useDeviceList.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
// ============================================================
|
||||
// useDeviceList — 设备枚举 Hook
|
||||
// 职责:枚举摄像头/麦克风设备列表,监听设备热插拔
|
||||
// 注意:首次枚举需要先有一次成功的 getUserMedia 授权
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
|
||||
export interface DeviceInfo {
|
||||
deviceId: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export function useDeviceList() {
|
||||
const [cameras, setCameras] = useState<DeviceInfo[]>([]);
|
||||
const [mics, setMics] = useState<DeviceInfo[]>([]);
|
||||
|
||||
/** 枚举当前可用的音视频输入设备 */
|
||||
const refreshDevices = useCallback(async () => {
|
||||
try {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
setCameras(
|
||||
devices
|
||||
.filter((d) => d.kind === "videoinput")
|
||||
.map((d) => ({ deviceId: d.deviceId, label: d.label || `摄像头 ${d.deviceId.slice(0, 4)}` }))
|
||||
);
|
||||
setMics(
|
||||
devices
|
||||
.filter((d) => d.kind === "audioinput")
|
||||
.map((d) => ({ deviceId: d.deviceId, label: d.label || `麦克风 ${d.deviceId.slice(0, 4)}` }))
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn("[DeviceList] 枚举设备失败:", err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// 监听设备热插拔
|
||||
useEffect(() => {
|
||||
navigator.mediaDevices.addEventListener("devicechange", refreshDevices);
|
||||
return () => navigator.mediaDevices.removeEventListener("devicechange", refreshDevices);
|
||||
}, [refreshDevices]);
|
||||
|
||||
return { cameras, mics, refreshDevices };
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
// ============================================================
|
||||
// useObservationMode — 观察模式 Hook
|
||||
// 职责:定时采帧 → 关键帧检测 → 画面变化时触发回调
|
||||
// 来源:docs/05-用户故事.md US-05(持续场景监控)
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { sampleFrame, compareFrames } from "../components/EdgeProcessor";
|
||||
|
||||
/** 画面变化显著阈值 */
|
||||
const CHANGE_THRESHOLD = 0.85;
|
||||
/** 采样间隔(ms) */
|
||||
const SAMPLE_INTERVAL = 5000;
|
||||
|
||||
export interface ObservationOptions {
|
||||
/** 画面变化回调,携带当前帧的 DataURL */
|
||||
onChange?: (frameDataUrl: string) => void;
|
||||
}
|
||||
|
||||
export function useObservationMode(options?: ObservationOptions) {
|
||||
const [isObserving, setIsObserving] = useState(false);
|
||||
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
|
||||
const optionsRef = useRef(options);
|
||||
|
||||
useEffect(() => {
|
||||
optionsRef.current = options;
|
||||
}, [options]);
|
||||
|
||||
/**
|
||||
* 启动观察模式
|
||||
* @param video 摄像头 video 元素
|
||||
* @param captureFrame 从 video 捕获 DataURL 的函数
|
||||
*/
|
||||
const startObserving = useCallback(
|
||||
(video: HTMLVideoElement | null, captureFrame: () => string | null) => {
|
||||
if (!video) return;
|
||||
|
||||
// 立即采一帧作为基准
|
||||
prevFrameRef.current = sampleFrame(video);
|
||||
|
||||
intervalRef.current = setInterval(() => {
|
||||
const current = sampleFrame(video);
|
||||
if (!current) return;
|
||||
|
||||
if (prevFrameRef.current) {
|
||||
const { similarity } = compareFrames(prevFrameRef.current, current);
|
||||
|
||||
if (similarity < CHANGE_THRESHOLD) {
|
||||
console.log(
|
||||
`[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`,
|
||||
);
|
||||
const frameDataUrl = captureFrame();
|
||||
if (frameDataUrl) {
|
||||
optionsRef.current?.onChange?.(frameDataUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prevFrameRef.current = current;
|
||||
}, SAMPLE_INTERVAL);
|
||||
|
||||
setIsObserving(true);
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
/** 停止观察模式 */
|
||||
const stopObserving = useCallback(() => {
|
||||
if (intervalRef.current) {
|
||||
clearInterval(intervalRef.current);
|
||||
intervalRef.current = null;
|
||||
}
|
||||
prevFrameRef.current = null;
|
||||
setIsObserving(false);
|
||||
}, []);
|
||||
|
||||
// 组件卸载时清理
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (intervalRef.current) {
|
||||
clearInterval(intervalRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
return { isObserving, startObserving, stopObserving };
|
||||
}
|
||||
@@ -17,11 +17,9 @@ import { useCamera } from "../components/CameraManager";
|
||||
import { useMicrophone } from "../components/MicManager";
|
||||
import { useVAD, sampleFrame } from "../components/EdgeProcessor";
|
||||
import { useWebSocketManager } from "../components/WebSocketManager";
|
||||
import { useObservationMode } from "./useObservationMode";
|
||||
import { useDeviceList } from "./useDeviceList";
|
||||
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
||||
|
||||
export type SessionMode = "dialogue" | "observation";
|
||||
|
||||
export interface SessionStats {
|
||||
queryCount: number;
|
||||
totalTokens: number;
|
||||
@@ -35,7 +33,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
|
||||
const [config, setConfig] = useState<SessionConfig>(loadConfig);
|
||||
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
|
||||
const [mode, setMode] = useState<SessionMode>("dialogue");
|
||||
const [isCameraOn, setIsCameraOn] = useState(false);
|
||||
const [isMicOn, setIsMicOn] = useState(false);
|
||||
|
||||
@@ -59,6 +56,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
|
||||
const { startMic, stopMic } = useMicrophone();
|
||||
const { status, connect, disconnect, send } = useWebSocketManager();
|
||||
const { cameras, mics, refreshDevices } = useDeviceList();
|
||||
|
||||
// 用 ref 跟踪 isProcessing,避免 VAD 回调闭包问题
|
||||
const isProcessingRef = useRef(false);
|
||||
@@ -78,49 +76,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
conversationIdRef.current = conversationId;
|
||||
}, [conversationId]);
|
||||
|
||||
// 观察模式:画面变化时自动发送 query
|
||||
const { isObserving, startObserving, stopObserving } = useObservationMode({
|
||||
onChange: useCallback(
|
||||
(frameDataUrl: string) => {
|
||||
if (isProcessingRef.current) return;
|
||||
|
||||
const requestId = uuidv4();
|
||||
send({
|
||||
type: "query",
|
||||
request_id: requestId,
|
||||
image: dataUrlToBase64(frameDataUrl),
|
||||
audio: "", // 观察模式无音频
|
||||
});
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: uuidv4(),
|
||||
role: "user",
|
||||
content: t("session.changeDetected"),
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
]);
|
||||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||||
setIsProcessing(true);
|
||||
},
|
||||
[send],
|
||||
),
|
||||
});
|
||||
|
||||
/** 切换对话/观察模式 */
|
||||
const toggleMode = useCallback(() => {
|
||||
setMode((prev) => {
|
||||
const next = prev === "dialogue" ? "observation" : "dialogue";
|
||||
if (next === "observation") {
|
||||
startObserving(videoRef.current, captureFrame);
|
||||
} else {
|
||||
stopObserving();
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [videoRef, captureFrame, startObserving, stopObserving]);
|
||||
|
||||
// WebSocket 连接成功后发送 config + flush 待发消息
|
||||
useEffect(() => {
|
||||
if (status === "connected") {
|
||||
@@ -324,14 +279,14 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
|
||||
// 2. 尝试获取摄像头(可选)
|
||||
try {
|
||||
await startCamera();
|
||||
await startCamera(config.cameraDeviceId || undefined);
|
||||
setIsCameraOn(true);
|
||||
} catch {
|
||||
console.warn("[Session] 无法获取摄像头权限,将以纯文本模式运行");
|
||||
}
|
||||
|
||||
// 3. 尝试获取麦克风(可选)
|
||||
const micStream = await startMic();
|
||||
const micStream = await startMic(config.micDeviceId || undefined);
|
||||
if (micStream) {
|
||||
setIsMicOn(true);
|
||||
// 4. 启动 VAD(仅在麦克风可用时)
|
||||
@@ -339,12 +294,13 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
} else {
|
||||
console.warn("[Session] 无法获取麦克风权限,将以文本输入模式运行");
|
||||
}
|
||||
}, [startCamera, startMic, connect, startVAD, accessToken]);
|
||||
|
||||
// 5. 授权后刷新设备列表
|
||||
refreshDevices();
|
||||
}, [startCamera, startMic, connect, startVAD, accessToken, config.cameraDeviceId, config.micDeviceId, refreshDevices]);
|
||||
|
||||
/** 结束会话 */
|
||||
const stopSession = useCallback(async () => {
|
||||
stopObserving();
|
||||
setMode("dialogue");
|
||||
await stopVAD();
|
||||
stopMic();
|
||||
stopCamera();
|
||||
@@ -359,12 +315,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
prevFrameRef.current = null;
|
||||
setIsCameraOn(false);
|
||||
setIsMicOn(false);
|
||||
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
|
||||
}, [stopVAD, stopMic, stopCamera, disconnect]);
|
||||
|
||||
/** 结束视频,保留聊天和连接 */
|
||||
const stopVideo = useCallback(async () => {
|
||||
stopObserving();
|
||||
setMode("dialogue");
|
||||
await stopVAD();
|
||||
stopMic();
|
||||
stopCamera();
|
||||
@@ -375,7 +329,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
setIsCameraOn(false);
|
||||
setIsMicOn(false);
|
||||
// 不断开 WebSocket,不清空消息、统计
|
||||
}, [stopObserving, stopVAD, stopMic, stopCamera]);
|
||||
}, [stopVAD, stopMic, stopCamera]);
|
||||
|
||||
/** 摄像头开关 */
|
||||
const toggleCamera = useCallback(async () => {
|
||||
@@ -383,10 +337,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
stopCamera();
|
||||
setIsCameraOn(false);
|
||||
} else {
|
||||
await startCamera();
|
||||
await startCamera(config.cameraDeviceId || undefined);
|
||||
setIsCameraOn(true);
|
||||
}
|
||||
}, [isCameraOn, startCamera, stopCamera]);
|
||||
}, [isCameraOn, startCamera, stopCamera, config.cameraDeviceId]);
|
||||
|
||||
/** 麦克风开关 */
|
||||
const toggleMic = useCallback(async () => {
|
||||
@@ -395,13 +349,13 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
stopMic();
|
||||
setIsMicOn(false);
|
||||
} else {
|
||||
const micStream = await startMic();
|
||||
const micStream = await startMic(config.micDeviceId || undefined);
|
||||
if (micStream) {
|
||||
await startVAD(micStream);
|
||||
setIsMicOn(true);
|
||||
}
|
||||
}
|
||||
}, [isMicOn, startMic, stopMic, startVAD, stopVAD]);
|
||||
}, [isMicOn, startMic, stopMic, startVAD, stopVAD, config.micDeviceId]);
|
||||
|
||||
/** 打断当前回复 */
|
||||
const interrupt = useCallback(() => {
|
||||
@@ -421,6 +375,31 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
setIsProcessing(false);
|
||||
}, [send, currentReply]);
|
||||
|
||||
/** 切换设备(摄像头或麦克风) */
|
||||
const switchDevice = useCallback(async (kind: "camera" | "mic", deviceId: string) => {
|
||||
if (kind === "camera") {
|
||||
updateConfig({ cameraDeviceId: deviceId || undefined });
|
||||
if (isCameraOn) {
|
||||
stopCamera();
|
||||
try {
|
||||
await startCamera(deviceId || undefined);
|
||||
} catch {
|
||||
console.warn("[Session] 切换摄像头失败");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
updateConfig({ micDeviceId: deviceId || undefined });
|
||||
if (isMicOn) {
|
||||
await stopVAD();
|
||||
stopMic();
|
||||
const micStream = await startMic(deviceId || undefined);
|
||||
if (micStream) {
|
||||
await startVAD(micStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [isCameraOn, isMicOn, stopCamera, startCamera, stopMic, startMic, stopVAD, startVAD, updateConfig]);
|
||||
|
||||
/** 发送文本消息(手动输入) */
|
||||
const sendTextMessage = useCallback(
|
||||
(text: string) => {
|
||||
@@ -480,9 +459,6 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
config,
|
||||
updateConfig,
|
||||
stats,
|
||||
mode,
|
||||
isObserving,
|
||||
toggleMode,
|
||||
startSession,
|
||||
stopSession,
|
||||
stopVideo,
|
||||
@@ -493,5 +469,8 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
|
||||
toggleMic,
|
||||
sendTextMessage,
|
||||
captureFrame,
|
||||
cameras,
|
||||
mics,
|
||||
switchDevice,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -48,7 +48,6 @@ export const enUS: TranslationMap = {
|
||||
"scenario.interpreter.hint": "AI will translate your speech in real-time (Chinese-English), without explanations",
|
||||
|
||||
// Video indicators
|
||||
"video.observing": "👁️ Observing",
|
||||
"video.listening": "🎤 Listening...",
|
||||
"video.playing": "🔊 Playing...",
|
||||
"video.initVad": "Initializing voice detection...",
|
||||
@@ -67,8 +66,6 @@ export const enUS: TranslationMap = {
|
||||
"controls.cameraOn": "Turn on camera",
|
||||
"controls.micOff": "Turn off microphone",
|
||||
"controls.micOn": "Turn on microphone",
|
||||
"controls.observing": "👁️ Observing",
|
||||
"controls.observation": "👁️ Observe",
|
||||
"controls.interrupt": "⏹ Interrupt",
|
||||
"controls.stop": "End Session",
|
||||
"controls.stopVideo": "End Video",
|
||||
@@ -77,7 +74,6 @@ export const enUS: TranslationMap = {
|
||||
|
||||
// Chat panel
|
||||
"chat.title": "Chat",
|
||||
"chat.mode.observation": "Observing",
|
||||
"chat.reconnecting": "Connection lost, reconnecting...",
|
||||
"chat.connecting": "Connecting to server...",
|
||||
"chat.vadInit": "Initializing voice detection...",
|
||||
@@ -91,7 +87,6 @@ export const enUS: TranslationMap = {
|
||||
"chat.scenarioSwitched": "Switched to {name} mode",
|
||||
|
||||
// Session messages
|
||||
"session.changeDetected": "👁️ Scene change detected",
|
||||
"session.recognizing": "(Recognizing speech...)",
|
||||
"session.sttFailed": "(Speech recognition failed, please try again)",
|
||||
"session.noSpeech": "(No speech detected)",
|
||||
@@ -138,12 +133,7 @@ export const enUS: TranslationMap = {
|
||||
|
||||
// Video controls (enhanced)
|
||||
"controls.recognize": "Analyze Scene",
|
||||
"controls.device.camera": "Camera",
|
||||
"controls.device.mic": "Microphone",
|
||||
"controls.device.default": "Default",
|
||||
"controls.mode.realtime": "Realtime",
|
||||
"controls.mode.ondemand": "On-demand",
|
||||
"controls.mode.chat": "Chat only",
|
||||
|
||||
// Status bar
|
||||
"statusbar.ready": "Ready · Select devices to start",
|
||||
|
||||
@@ -48,7 +48,6 @@ export const jaJP: TranslationMap = {
|
||||
"scenario.interpreter.hint": "AIがあなたの発言をリアルタイムで翻訳します(中日相互翻訳、説明なし)",
|
||||
|
||||
// Video indicators
|
||||
"video.observing": "👁️ 観察中",
|
||||
"video.listening": "🎤 聞き取り中...",
|
||||
"video.playing": "🔊 再生中...",
|
||||
"video.initVad": "音声検出を初期化中...",
|
||||
@@ -67,8 +66,6 @@ export const jaJP: TranslationMap = {
|
||||
"controls.cameraOn": "カメラをオン",
|
||||
"controls.micOff": "マイクをオフ",
|
||||
"controls.micOn": "マイクをオン",
|
||||
"controls.observing": "👁️ 観察中",
|
||||
"controls.observation": "👁️ 観察モード",
|
||||
"controls.interrupt": "⏹ 中断",
|
||||
"controls.stop": "対話を終了",
|
||||
"controls.stopVideo": "ビデオ終了",
|
||||
@@ -77,7 +74,6 @@ export const jaJP: TranslationMap = {
|
||||
|
||||
// Chat panel
|
||||
"chat.title": "チャット",
|
||||
"chat.mode.observation": "観察モード",
|
||||
"chat.reconnecting": "接続が切断されました。再接続中...",
|
||||
"chat.connecting": "サーバーに接続中...",
|
||||
"chat.vadInit": "音声検出を初期化中...",
|
||||
@@ -91,7 +87,6 @@ export const jaJP: TranslationMap = {
|
||||
"chat.scenarioSwitched": "{name} モードに切り替えました",
|
||||
|
||||
// Session messages
|
||||
"session.changeDetected": "👁️ シーン変化を検出",
|
||||
"session.recognizing": "(音声認識中...)",
|
||||
"session.sttFailed": "(音声認識に失敗しました。もう一度お試しください)",
|
||||
"session.noSpeech": "(音声が検出されませんでした)",
|
||||
@@ -138,12 +133,7 @@ export const jaJP: TranslationMap = {
|
||||
|
||||
// Video controls (enhanced)
|
||||
"controls.recognize": "シーンを分析",
|
||||
"controls.device.camera": "カメラ",
|
||||
"controls.device.mic": "マイク",
|
||||
"controls.device.default": "デフォルト",
|
||||
"controls.mode.realtime": "リアルタイム",
|
||||
"controls.mode.ondemand": "オンデマンド",
|
||||
"controls.mode.chat": "チャットのみ",
|
||||
|
||||
// Status bar
|
||||
"statusbar.ready": "準備完了 · デバイスを選択して開始",
|
||||
|
||||
@@ -48,7 +48,6 @@ export const zhCN: TranslationMap = {
|
||||
"scenario.interpreter.hint": "AI 会实时翻译你的话(中英互译),无解释评论",
|
||||
|
||||
// Video indicators
|
||||
"video.observing": "👁️ 观察中",
|
||||
"video.listening": "🎤 正在聆听...",
|
||||
"video.playing": "🔊 正在播放...",
|
||||
"video.initVad": "正在初始化语音检测...",
|
||||
@@ -67,8 +66,6 @@ export const zhCN: TranslationMap = {
|
||||
"controls.cameraOn": "开启摄像头",
|
||||
"controls.micOff": "关闭麦克风",
|
||||
"controls.micOn": "开启麦克风",
|
||||
"controls.observing": "👁️ 观察中",
|
||||
"controls.observation": "👁️ 观察模式",
|
||||
"controls.interrupt": "⏹ 打断",
|
||||
"controls.stop": "结束对话",
|
||||
"controls.stopVideo": "结束视频",
|
||||
@@ -77,7 +74,6 @@ export const zhCN: TranslationMap = {
|
||||
|
||||
// Chat panel
|
||||
"chat.title": "对话",
|
||||
"chat.mode.observation": "观察模式",
|
||||
"chat.reconnecting": "连接已断开,正在重连...",
|
||||
"chat.connecting": "正在连接服务...",
|
||||
"chat.vadInit": "正在初始化语音检测...",
|
||||
@@ -91,7 +87,6 @@ export const zhCN: TranslationMap = {
|
||||
"chat.scenarioSwitched": "已切换到 {name} 模式",
|
||||
|
||||
// Session messages
|
||||
"session.changeDetected": "👁️ 画面变化检测",
|
||||
"session.recognizing": "(语音识别中...)",
|
||||
"session.sttFailed": "(语音识别失败,请重试)",
|
||||
"session.noSpeech": "(未识别到语音)",
|
||||
@@ -138,12 +133,7 @@ export const zhCN: TranslationMap = {
|
||||
|
||||
// Video controls (enhanced)
|
||||
"controls.recognize": "识别画面",
|
||||
"controls.device.camera": "摄像头",
|
||||
"controls.device.mic": "麦克风",
|
||||
"controls.device.default": "默认",
|
||||
"controls.mode.realtime": "实时分析",
|
||||
"controls.mode.ondemand": "按需识别",
|
||||
"controls.mode.chat": "纯聊天",
|
||||
|
||||
// Status bar
|
||||
"statusbar.ready": "就绪 · 选择设备后开始通话",
|
||||
|
||||
@@ -10,6 +10,8 @@ export interface SessionConfig {
|
||||
detailLevel: "low" | "high";
|
||||
language: string;
|
||||
scenario: string; // 情景 ID,如 "free_chat"、"interviewer"
|
||||
cameraDeviceId?: string; // 摄像头设备 ID,空 = 系统默认
|
||||
micDeviceId?: string; // 麦克风设备 ID,空 = 系统默认
|
||||
}
|
||||
|
||||
export type Theme = "dark" | "light";
|
||||
|
||||
Reference in New Issue
Block a user