From 51ed6aa563a739ae60c7460854c5873497161567 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Sun, 14 Jun 2026 12:52:39 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=96=87?= =?UTF-8?q?=E5=AD=97=E8=BE=93=E5=85=A5=E5=AF=B9=E8=AF=9D=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 后端 WsQuery 添加 text 字段，支持文本输入模式 - Pipeline 支持文本查询时跳过 STT 直接使用输入文本 - 前端 ChatPanel 添加文字输入框，连接状态下可用 - useVisionSession 添加 sendTextMessage 方法 - 更新接口文档，添加文本输入模式说明 --- backend/internal/models/models.go | 1 + backend/internal/orchestrator/pipeline.go | 100 ++++++++++++-------- docs/03-接口文档.md | 28 +++++- frontend/src/App.css | 57 +++++++++++ frontend/src/App.tsx | 2 + frontend/src/components/ChatPanel/index.tsx | 95 +++++++++++++------ frontend/src/hooks/useVisionSession.ts | 39 ++++++++ frontend/src/types/index.ts | 3 +- 8 files changed, 252 insertions(+), 73 deletions(-) diff --git a/backend/internal/models/models.go b/backend/internal/models/models.go index 73d017a..49347ea 100644 --- a/backend/internal/models/models.go +++ b/backend/internal/models/models.go @@ -55,6 +55,7 @@ type WsQuery struct { RequestID string `json:"request_id"` Image string `json:"image"` // base64 Audio string `json:"audio"` // base64 + Text string `json:"text"` // 用户手动输入的文本（有值时跳过 STT） MimeType string `json:"mime_type"` // 默认 "audio/pcm" } diff --git a/backend/internal/orchestrator/pipeline.go b/backend/internal/orchestrator/pipeline.go index 048055c..cb103b8 100644 --- a/backend/internal/orchestrator/pipeline.go +++ b/backend/internal/orchestrator/pipeline.go @@ -62,22 +62,27 @@ func (p *Pipeline) ProcessQuery( log := logger.Log startTime := time.Now() - // 解码音频数据 - audio, err := base64.StdEncoding.DecodeString(req.Audio) - if err != nil { - log.Errorw("音频解码失败", "error", err) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "INVALID_MESSAGE", - Message: "音频数据解码失败", - }) - return err + // 解码音频数据（文本输入模式可跳过） + var audio []byte + if req.Text == "" && req.Audio != "" { + var err error + audio, err = base64.StdEncoding.DecodeString(req.Audio) + if err != nil { + log.Errorw("音频解码失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "INVALID_MESSAGE", + Message: "音频数据解码失败", + }) + return err + } } // 解码图片数据（可选） var image []byte if req.Image != "" { + var err error image, err = base64.StdEncoding.DecodeString(req.Image) if err != nil { log.Errorw("图片解码失败", "error", err) @@ -110,45 +115,64 @@ func (p *Pipeline) ProcessQuery( return err } - // Step 1: STT 语音识别 - log.Infow("开始语音识别", "request_id", req.RequestID) - sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{ - Encoding: "pcm_s16le", - SampleRate: 16000, - Language: sess.Config.Language, - }) - if err != nil { - log.Errorw("语音识别失败", "error", err) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "STT_ERROR", - Message: "语音识别失败", - }) - return err - } + // Step 1: 获取用户文本（语音识别或直接使用输入文本） + var userText string + if req.Text != "" { + // 文本输入模式：跳过 STT，直接使用用户输入的文本 + log.Infow("使用文本输入", "request_id", req.RequestID, "text", req.Text) + userText = req.Text - // 发送 STT 结果 - if err := sender.SendSTTResult(models.WsSTTResult{ - Type: "stt_result", - RequestID: req.RequestID, - Text: sttResult, - IsFinal: true, - }); err != nil { - log.Errorw("发送 STT 结果失败", "error", err) + // 发送 stt_result 以保持前端消息流一致性 + if err := sender.SendSTTResult(models.WsSTTResult{ + Type: "stt_result", + RequestID: req.RequestID, + Text: userText, + IsFinal: true, + }); err != nil { + log.Errorw("发送 STT 结果失败", "error", err) + } + } else { + // 语音模式：执行 STT 语音识别 + log.Infow("开始语音识别", "request_id", req.RequestID) + sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{ + Encoding: "pcm_s16le", + SampleRate: 16000, + Language: sess.Config.Language, + }) + if err != nil { + log.Errorw("语音识别失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "STT_ERROR", + Message: "语音识别失败", + }) + return err + } + userText = sttResult + + // 发送 STT 结果 + if err := sender.SendSTTResult(models.WsSTTResult{ + Type: "stt_result", + RequestID: req.RequestID, + Text: userText, + IsFinal: true, + }); err != nil { + log.Errorw("发送 STT 结果失败", "error", err) + } } // 追加用户消息到历史 p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{ Role: "user", - Content: sttResult, + Content: userText, }) // Step 2+3: LLM 流式推理 + TTS 并行合成 log.Infow("开始 LLM 推理", "request_id", req.RequestID) llmReq := llm.Request{ Image: image, - Text: sttResult, + Text: userText, History: history, Language: sess.Config.Language, } diff --git a/docs/03-接口文档.md b/docs/03-接口文档.md index 152ac89..cacc3fa 100644 --- a/docs/03-接口文档.md +++ b/docs/03-接口文档.md @@ -41,19 +41,22 @@ interface WsMessage { #### `query` — 发起一次视觉对话 -用户说完话后，客户端同时发送当前图像帧和语音片段： +用户说完话后，客户端同时发送当前图像帧和语音片段。也支持文本输入模式（手动输入文字时跳过语音识别）： ```typescript interface QueryMessage { type: "query"; request_id: string; // 客户端生成的 UUID image: string; // Base64 编码的 JPEG 图像（不含 data: 前缀） - audio: string; // Base64 编码的音频片段（PCM 16kHz） + audio: string; // Base64 编码的音频片段（PCM 16kHz），文本输入时为空字符串 + text?: string; // 用户手动输入的文本（有值时跳过 STT，直接使用此文本） mime_type?: string; // 音频格式，默认 "audio/pcm" } ``` > 为什么图像和音频放在同一条消息里？因为 VAD 检测到用户说完话时，需要同时捕获"此刻的画面"和"说的话"，拆成两条消息会增加时序同步的复杂度。 +> +> **文本输入模式**：当用户关闭麦克风后，可通过对话框手动输入文字。此时 `text` 字段携带用户输入，`audio` 为空字符串，服务端跳过 STT 直接使用 `text` 进行 LLM 推理。 #### `config` — 更新会话配置 @@ -211,13 +214,13 @@ interface PongMessage { ### 消息流时序 -一次完整交互： +**语音模式**（麦克风开启）： ``` Client Server | | |-- query {image, audio} ------>| - |<-- stt_result {text} ---------| + |<-- stt_result {text} ---------| (语音识别) | | |<-- llm_chunk {delta: "这"} ---| (LLM 流式输出) |<-- llm_chunk {delta: "是一"} -| @@ -228,6 +231,22 @@ Client Server |<-- tts_audio {is_last: true} -| ``` +**文本输入模式**（麦克风关闭，手动输入文字）： + +``` +Client Server + | | + |-- query {image, text} ------->| (跳过 STT) + |<-- stt_result {text} ---------| (回显用户文本) + | | + |<-- llm_chunk {delta: "好的"} -| (LLM 流式输出) + |<-- llm_chunk {delta: "，我"} -| + |<-- llm_done {full_text} ------| + | | + |<-- tts_audio {audio} ---------| (TTS 音频流) + |<-- tts_audio {is_last: true} -| +``` + --- ## 二、REST API @@ -891,6 +910,7 @@ type QueryRequest struct { RequestID string `json:"request_id"` Image []byte `json:"-"` // Base64 解码后 Audio []byte `json:"-"` // Base64 解码后 + Text string `json:"text"` // 用户手动输入的文本（有值时跳过 STT） MimeType string `json:"mime_type"` } diff --git a/frontend/src/App.css b/frontend/src/App.css index fdfb826..c88346b 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -473,6 +473,13 @@ body { /* ---- Chat Panel (覆盖子组件样式) ---- */ .chat-panel { + flex: 1; + display: flex; + flex-direction: column; + min-height: 0; +} + +.chat-panel__messages { flex: 1; overflow-y: auto; padding: 16px 24px; @@ -560,6 +567,56 @@ body { letter-spacing: 0.01em; } +/* ---- Chat Input ---- */ + +.chat-input { + display: flex; + gap: 8px; + padding: 12px 24px 16px; + border-top: 1px solid var(--color-border); + background: var(--color-surface); +} + +.chat-input__field { + flex: 1; + padding: 10px 14px; + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + background: var(--color-surface-2); + color: var(--color-text); + font-size: 0.82rem; + outline: none; + transition: border-color var(--transition-fast); +} + +.chat-input__field::placeholder { + color: var(--color-text-muted); +} + +.chat-input__field:focus { + border-color: var(--color-primary); +} + +.chat-input__send { + padding: 10px 16px; + border: none; + border-radius: var(--radius-sm); + background: var(--color-primary); + color: white; + font-size: 0.9rem; + cursor: pointer; + transition: opacity var(--transition-fast); +} + +.chat-input__send:hover:not(:disabled) { + opacity: 0.9; +} + +.chat-input__send:disabled { + opacity: 0.4; + cursor: not-allowed; +} + /* ---- Streaming Cursor ---- */ .cursor { diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 0ef33b5..71517d7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -58,6 +58,7 @@ function App() { isMicOn, toggleCamera, toggleMic, + sendTextMessage, } = useVisionSession(); const isConnected = connectionStatus === "connected"; @@ -233,6 +234,7 @@ function App() { messages={messages} currentReply={currentReply} connectionStatus={connectionStatus} + onSendText={sendTextMessage} /> diff --git a/frontend/src/components/ChatPanel/index.tsx b/frontend/src/components/ChatPanel/index.tsx index 0e6ce3a..b405ef5 100644 --- a/frontend/src/components/ChatPanel/index.tsx +++ b/frontend/src/components/ChatPanel/index.tsx @@ -1,9 +1,9 @@ // ============================================================ // ChatPanel — 消息展示面板 -// 职责：渲染对话消息列表、流式光标、元数据、自动滚动 +// 职责：渲染对话消息列表、流式光标、元数据、自动滚动、文本输入 // ============================================================ -import { useEffect, useRef } from "react"; +import { useEffect, useRef, useState } from "react"; import type { ChatMessage } from "../../types"; import type { ConnectionStatus } from "../../lib/websocket"; @@ -11,12 +11,16 @@ interface ChatPanelProps { messages: ChatMessage[]; currentReply?: string; connectionStatus: ConnectionStatus; + onSendText?: (text: string) => void; } -export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPanelProps) { +export function ChatPanel({ messages, currentReply, connectionStatus, onSendText }: ChatPanelProps) { const bottomRef = useRef(null); const containerRef = useRef(null); const isAutoScroll = useRef(true); + const [inputText, setInputText] = useState(""); + + const isConnected = connectionStatus === "connected"; // 用户上滚时暂停自动滚动，滚到底部时恢复 useEffect(() => { @@ -39,9 +43,17 @@ export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPane } }, [messages, currentReply]); + // 提交文本消息 + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (!inputText.trim() || !onSendText) return; + onSendText(inputText); + setInputText(""); + }; + // 空状态 if (messages.length === 0 && !currentReply) { - if (connectionStatus !== "connected") { + if (!isConnected) { return (
💬 @@ -60,35 +72,58 @@ export function ChatPanel({ messages, currentReply, connectionStatus }: ChatPane } return ( -
- {messages.map((msg, index) => ( -
-
- {msg.role === "user" ? "你" : "AI"} -
-
{msg.content}
- {msg.role === "assistant" && msg.tokensUsed !== undefined && ( -
- {msg.tokensUsed} tokens - {msg.latencyMs !== undefined && ` · ${(msg.latencyMs / 1000).toFixed(1)}s`} - {msg.model && ` · ${msg.model}`} +
+
+ {messages.map((msg, index) => ( +
+
+ {msg.role === "user" ? "你" : "AI"}
- )} -
- ))} - - {/* 流式回复（尚未完成） */} - {currentReply && ( -
-
AI
-
- {currentReply} - ▌ +
{msg.content}
+ {msg.role === "assistant" && msg.tokensUsed !== undefined && ( +
+ {msg.tokensUsed} tokens + {msg.latencyMs !== undefined && ` · ${(msg.latencyMs / 1000).toFixed(1)}s`} + {msg.model && ` · ${msg.model}`} +
+ )}
-
- )} + ))} -
+ {/* 流式回复（尚未完成） */} + {currentReply && ( +
+
AI
+
+ {currentReply} + ▌ +
+
+ )} + +
+
+ + {/* 文本输入框 */} + {isConnected && onSendText && ( +
+ setInputText(e.target.value)} + /> + +
+ )}
); } diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts index a2ba760..e85640e 100644 --- a/frontend/src/hooks/useVisionSession.ts +++ b/frontend/src/hooks/useVisionSession.ts @@ -363,6 +363,44 @@ export function useVisionSession() { setIsProcessing(false); }, [send, currentReply]); + /** 发送文本消息（手动输入） */ + const sendTextMessage = useCallback( + (text: string) => { + if (!text.trim() || isProcessingRef.current) return; + + // 停止上一轮的 TTS 播放 + ttsPlayerRef.current?.stop(); + setIsAudioPlaying(false); + + // 捕获当前摄像头画面 + const frame = captureFrame(); + + const requestId = uuidv4(); + send({ + type: "query", + request_id: requestId, + image: frame ? dataUrlToBase64(frame) : "", + audio: "", // 文本输入无音频 + text: text.trim(), + }); + + // 更新请求统计 + setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 })); + + // 添加用户消息 + setMessages((prev) => [ + ...prev, + { role: "user", content: text.trim(), timestamp: Date.now() }, + ]); + + // 记录到对话历史 + historyRef.current.push({ role: "user", content: text.trim() }); + + setIsProcessing(true); + }, + [captureFrame, send], + ); + return { messages, currentReply, @@ -387,5 +425,6 @@ export function useVisionSession() { isMicOn, toggleCamera, toggleMic, + sendTextMessage, }; } diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 0f29ec5..fe21e36 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -46,7 +46,8 @@ export interface QueryMessage { type: "query"; request_id: string; image: string; // Base64 JPEG（不含 data: 前缀） - audio: string; // Base64 PCM 16kHz + audio: string; // Base64 PCM 16kHz（文本输入时为空字符串） + text?: string; // 用户手动输入的文本（有值时跳过 STT） mime_type?: string; // 默认 "audio/pcm" } From 41cacaa74087d96a8d91fe63a78df2adb94d03f6 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Sun, 14 Jun 2026 12:54:58 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=BA=A6=E5=85=8B?= =?UTF-8?q?=E9=A3=8E=E5=85=B3=E9=97=AD=E5=90=8E=E9=87=8D=E6=96=B0=E6=89=93?= =?UTF-8?q?=E5=BC=80=E6=97=A0=E6=B3=95=E4=BD=BF=E7=94=A8=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 关闭麦克风时同时停止 VAD - 打开麦克风时重新初始化 VAD --- frontend/src/hooks/useVisionSession.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts index e85640e..49b5891 100644 --- a/frontend/src/hooks/useVisionSession.ts +++ b/frontend/src/hooks/useVisionSession.ts @@ -336,13 +336,17 @@ export function useVisionSession() { /** 麦克风开关 */ const toggleMic = useCallback(async () => { if (isMicOn) { + await stopVAD(); stopMic(); setIsMicOn(false); } else { const micStream = await startMic(); - setIsMicOn(!!micStream); + if (micStream) { + await startVAD(micStream); + setIsMicOn(true); + } } - }, [isMicOn, startMic, stopMic]); + }, [isMicOn, startMic, stopMic, startVAD, stopVAD]); /** 打断当前回复 */ const interrupt = useCallback(() => { From 312e762aff340e1c62471aa5506874e55e5aabe6 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Sun, 14 Jun 2026 13:02:07 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=96=87=E6=9C=AC?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E6=97=B6=20LLM=20API=20=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除 contentPart.Text 的 omitempty 标签 - 确保 text 字段始终存在于 JSON 请求中 --- backend/internal/ai/llm/openai.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/internal/ai/llm/openai.go b/backend/internal/ai/llm/openai.go index 754176c..e421d7e 100644 --- a/backend/internal/ai/llm/openai.go +++ b/backend/internal/ai/llm/openai.go @@ -61,7 +61,7 @@ type chatMessage struct { type contentPart struct { Type string `json:"type"` - Text string `json:"text,omitempty"` + Text string `json:"text"` ImageURL *imageURL `json:"image_url,omitempty"` } @@ -100,6 +100,9 @@ func (o *OpenAIService) ChatStream(ctx context.Context, req Request) (<-chan Chu if err != nil { return nil, fmt.Errorf("llm: marshal request: %w", err) } + if err != nil { + return nil, fmt.Errorf("llm: marshal request: %w", err) + } // 创建带超时的 context ctx, cancel := context.WithTimeout(ctx, o.timeout)