diff --git a/.env.example b/.env.example index 3be820f..21ade5f 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,7 @@ CAMTALK_AI_TTS_API_KEY= # CAMTALK_AI_LLM_MODEL=gpt-4o # CAMTALK_AI_LLM_ENDPOINT=https://api.openai.com/v1 # CAMTALK_AI_LLM_TIMEOUT=10 -# CAMTALK_AI_STT_ENDPOINT=wss://api.deepgram.com/v1/listen +# CAMTALK_AI_STT_ENDPOINT=https://api.xiaomimimo.com/v1 # CAMTALK_AI_TTS_ENDPOINT=https://api.openai.com/v1 # CAMTALK_AI_TTS_VOICE=alloy # CAMTALK_AI_TTS_SPEED=1.0 diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..9e9d93c --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,63 @@ +name: Deploy + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + # ---- PR 时：验证 Docker 镜像可构建 ---- + verify: + if: github.event_name == 'pull_request' + runs-on: aliyun + steps: + - name: Setup Node.js + run: | + sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories + apk add --no-cache nodejs + + - name: Checkout + uses: "http://8.161.227.145:3000/huanghaosheng/checkout@releases/v4" + + - name: Verify Frontend Build + run: docker build -t camtalk-frontend-test ./frontend + + - name: Verify Backend Build + run: docker build -t camtalk-backend-test ./backend + + - name: Cleanup + run: | + docker rmi camtalk-frontend-test camtalk-backend-test || true + + # ---- push main 时：构建并部署 ---- + deploy: + if: github.event_name == 'push' + runs-on: aliyun + steps: + - name: Setup Node.js + run: | + sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories + apk add --no-cache nodejs + + - name: Checkout + uses: "http://8.161.227.145:3000/huanghaosheng/checkout@releases/v4" + + - name: Build and Deploy + run: | + chmod +x deploy.sh + ./deploy.sh build + ./deploy.sh restart + + - name: Health Check + run: | + for i in $(seq 1 10); do + if curl -sf http://localhost/api/health > /dev/null 2>&1; then + echo "服务启动成功" + exit 0 + fi + echo "等待服务启动... ($i/10)" + sleep 3 + done + echo "服务启动超时" + exit 1 diff --git a/CLAUDE.md b/CLAUDE.md index f66681a..8a8a26a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,24 +12,23 @@ CamTalk 是一款多模态实时 AI 视觉对话助手。用户通过摄像头 三层系统： -1. **浏览器客户端**（React 18 + TypeScript, Vite）—— 媒体采集、边缘预处理（VAD 通过 `@ricky0123/vad-web`、关键帧检测通过 ONNX Runtime Web）、UI 渲染。核心 Hook：`useVisionSession()` -2. **Go 网关**（gorilla/websocket, Redis, Viper, Zap）—— WebSocket 服务器、会话管理、模型路由、AI 编排、速率限制。每个 WebSocket 连接一个 goroutine。 -3. **云端 AI 服务** —— GPT-4o（LLM）、Deepgram（STT）、OpenAI TTS。仅通过 Go 网关访问，浏览器不直连。 +1. **浏览器客户端**（React 18 + TypeScript, Vite）—— 媒体采集、边缘预处理（VAD 通过 `@ricky0123/vad-web`、关键帧检测通过 Canvas 像素比较）、UI 渲染。核心 Hook：`useVisionSession()` +2. **Go 网关**（Gin, gorilla/websocket, Viper, Zap）—— WebSocket 服务器、会话管理、AI 编排。每个 WebSocket 连接一个 goroutine。 +3. **云端 AI 服务** —— 通过 OpenAI 兼容接口可灵活切换。默认：GPT-4o（LLM）、Deepgram（STT）、OpenAI TTS。仅通过 Go 网关访问，浏览器不直连。 **关键模式**：LLM 文本流和 TTS 音频流并行推送给客户端，以最小化感知延迟。 -**存储**：冷热分离 —— Redis 存实时会话状态，PostgreSQL 存对话历史和用量统计（MVP 后引入）。Repository 接口模式（`HistoryRepository`、`UsageRepository`），MVP 用内存实现。 +**存储**：MVP 阶段使用进程内存（`MemoryManager`），Redis 实现已就绪可通过配置切换，PostgreSQL 为规划中。Repository 接口模式（`HistoryRepository`、`UsageRepository`），MVP 用内存实现。 ## 技术栈 | 层级 | 技术 | |------|------| -| 前端 | React 18, TypeScript, Vite, ONNX Runtime Web, @ricky0123/vad-web | -| 后端 | Go, gorilla/websocket, Redis, Viper, Zap | -| LLM | GPT-4o（主）, Claude Sonnet（备） | -| STT | Deepgram（主）, FunASR（自部署备选） | -| TTS | OpenAI TTS（主）, Edge TTS（免费替代） | -| 模型路由 | GPT-4o-mini 用于轻量分类 | +| 前端 | React 18, TypeScript, Vite, @ricky0123/vad-web | +| 后端 | Go, Gin, gorilla/websocket, Viper, Zap | +| LLM | GPT-4o（默认，通过 OpenAI 兼容接口可切换） | +| STT | Deepgram（默认） / MiMo ASR | +| TTS | OpenAI TTS（默认） / MiMo TTS | ## 构建与运行命令 @@ -50,7 +49,7 @@ go test -run TestName ./path # 运行单个测试 go vet ./... # 静态分析 ``` -基础设施：Redis 为会话状态必需。PostgreSQL 为 MVP 可选（内存回退）。 +基础设施：MVP 使用进程内存管理会话状态。Redis 已实现可通过配置切换，PostgreSQL 为规划中。 ## WebSocket 协议 @@ -80,7 +79,7 @@ go vet ./... # 静态分析 |------|------| | `CameraManager` | 摄像头流采集 | | `MicManager` | 麦克风音频采集 | -| `EdgeProcessor` | VAD + 关键帧检测（ONNX Runtime） | +| `EdgeProcessor` | VAD + 关键帧检测（Canvas 像素比较） | | `WebSocketManager` | WebSocket 连接生命周期管理 | | `ChatPanel` | 消息展示 | | `VideoPreview` | 摄像头画面预览 | @@ -89,11 +88,14 @@ go vet ./... # 静态分析 | 模块 | 职责 | |------|------| -| WebSocket Hub | 连接管理、广播/定向推送 | -| Session Manager | 会话状态、对话历史（Redis + TTL） | -| Model Router | 按请求选择 AI 模型（规则引擎 + 成本阈值） | -| AI Orchestrator | 并行/串行 AI 调用编排，context 超时控制 | -| Rate Limiter | 按用户的令牌桶速率限制 | +| WebSocket Handler | 连接管理、单播消息推送 | +| Session Manager | 会话状态、对话历史（Memory/Redis，30 分钟 TTL） | +| AI Orchestrator | STT→LLM→TTS 流式并行管道编排 | +| AI Service Layer | AI 服务抽象层（STT/LLM/TTS 多 provider） | +| REST API | 健康检查、会话管理（Gin 路由） | +| Models | 数据模型定义 | +| Model Router | 按请求选择 AI 模型（规划中） | +| Rate Limiter | 按用户的令牌桶速率限制（规划中） | ## 编码规范 diff --git a/README.md b/README.md index d51a20a..89baf13 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,137 @@ # CamTalk +多模态实时 AI 视觉对话助手。用户通过摄像头和麦克风与 AI 交互，AI 理解视觉场景和语音输入后，以文字和语音形式给出自然回应。 + +## 架构 + +三层系统，前端做轻量预处理，后端做智能编排，云端 AI 服务按需调用： + +``` +浏览器客户端 Go 网关 :8080 云端 AI 服务 +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ 媒体采集 │ │ WebSocket Handler│ │ STT（语音识别） │ +│ VAD 语音检测 │ WebSocket│ Session Manager │ HTTP │ LLM（多模态推理） │ +│ 关键帧检测 │ ◄──────► │ AI Orchestrator │ ◄──────► │ TTS（语音合成） │ +│ UI 渲染 │ │ REST API │ │ │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +**关键模式**：LLM 文本流和 TTS 音频流并行推送，用户先看到文字、紧接着听到语音，感知延迟 < 0.5 秒。 + +## 技术栈 + +| 层级 | 技术 | +|------|------| +| 前端 | React 18, TypeScript, Vite, @ricky0123/vad-web | +| 后端 | Go, Gin, gorilla/websocket, Viper, Zap | +| STT | Deepgram（默认） / MiMo ASR | +| LLM | GPT-4o（默认，通过 OpenAI 兼容接口可切换） | +| TTS | OpenAI TTS（默认） / MiMo TTS | + +## 项目结构 + +``` +CamTalk/ +├── frontend/ # 浏览器客户端 +│ └── src/ +│ ├── components/ # UI 组件 +│ │ ├── CameraManager/ # 摄像头流采集 +│ │ ├── MicManager/ # 麦克风音频采集 +│ │ ├── EdgeProcessor/ # VAD + 关键帧检测 +│ │ ├── WebSocketManager/ # WS 连接管理 +│ │ ├── ChatPanel/ # 消息展示 +│ │ ├── VideoPreview/ # 摄像头画面预览 +│ │ ├── ConfigPanel/ # 配置面板 +│ │ └── Toast/ # 通知提示 +│ ├── hooks/ # 自定义 Hooks +│ │ ├── useVisionSession.ts # 核心会话 Hook +│ │ └── useObservationMode.ts # 观察模式 +│ ├── lib/ # 工具库 +│ │ ├── websocket.ts # WebSocket 连接管理 +│ │ ├── audio.ts # 音频编码 +│ │ ├── ttsPlayer.ts # TTS 播放器 +│ │ └── sampling.ts # 采样策略 +│ └── types/ # TypeScript 类型定义 +├── backend/ # Go 网关 +│ ├── cmd/server/ # 入口 +│ └── internal/ +│ ├── ai/ # AI 服务抽象层 +│ │ ├── llm/ # LLM 服务（OpenAI 兼容） +│ │ ├── stt/ # STT 服务（Deepgram/MiMo） +│ │ └── tts/ # TTS 服务（OpenAI/MiMo） +│ ├── orchestrator/ # AI 编排器（STT→LLM→TTS 管道） +│ ├── session/ # 会话管理（Memory/Redis） +│ ├── ws/ # WebSocket Handler +│ ├── api/ # REST API +│ ├── config/ # 配置管理 +│ ├── models/ # 数据模型 +│ ├── errors/ # 错误码 +│ └── logger/ # 日志 +├── docs/ # 设计文档 +└── CLAUDE.md # Claude Code 指引 +``` + +## 快速开始 + +### 前置条件 + +- Node.js >= 18 +- Go >= 1.24 + +### 前端 + +```bash +cd frontend +npm install +npm run dev # Vite 开发服务器 http://localhost:5173 +``` + +### 后端 + +```bash +cd backend +go mod download +go run ./cmd/server # 启动网关 :8080 +``` + +### 配置 + +后端配置文件位于 `backend/config.yaml`，支持环境变量覆盖（前缀 `CAMTALK_`）。 + +```bash +# 最小启动（需要至少一个 AI 服务的 API Key） +cd backend +CAMTALK_AI_LLM_API_KEY=sk-xxx \ +CAMTALK_AI_STT_API_KEY=xxx \ +go run ./cmd/server +``` + +配置优先级：环境变量 > `config.{env}.yaml` > `config.yaml` > `.env` + +## WebSocket 协议 + +连接地址：`ws://localhost:8080/ws` + +所有消息为 JSON 文本帧，统一信封格式 `{type, request_id?, timestamp?}`。 + +**客户端 → 服务端**：`query`、`config`、`interrupt`、`ping` +**服务端 → 客户端**：`connected`、`stt_result`、`llm_chunk`、`llm_done`、`tts_audio`、`error`、`pong` + +完整协议见 [docs/03-接口文档.md](docs/03-接口文档.md)。 + +## 文档 + +| 文档 | 内容 | +|------|------| +| [01-项目概述](docs/01-项目概述.md) | 项目目标与核心挑战 | +| [02-系统架构](docs/02-系统架构.md) | 三层架构、技术栈、部署方案 | +| [03-接口文档](docs/03-接口文档.md) | WebSocket 协议、REST API、配置管理 | +| [04-技术选型](docs/04-技术选型.md) | AI 服务栈、持久化层、前端边缘处理选型 | +| [05-用户故事](docs/05-用户故事.md) | 用户场景与优先级 | +| [06-语音交互](docs/06-语音交互.md) | VAD → STT → LLM → TTS 全链路 | +| [07-视觉理解](docs/07-视觉理解.md) | 帧采样、关键帧检测、多模态输入 | +| [08-成本控制](docs/08-成本控制.md) | 采样策略、端云协同、模型分级 | + +## License + +[MIT](LICENSE) © XEngineers diff --git a/backend/.gitignore b/backend/.gitignore index 8304260..ac13b7b 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -3,7 +3,6 @@ bin/ # 环境配置 -.env config.dev.yaml config.prod.yaml diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..67bd3f3 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,27 @@ +# ---- 构建阶段 ---- +FROM golang:1.26-alpine AS builder + +WORKDIR /app + +# 先复制依赖清单，利用 Docker 缓存层 +COPY go.mod go.sum ./ +RUN go mod download + +# 复制源码并构建 +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -o /camtalk ./cmd/server + +# ---- 运行阶段 ---- +FROM alpine:3.20 + +RUN apk add --no-cache ca-certificates tzdata + +WORKDIR /app + +# 复制二进制和配置 +COPY --from=builder /camtalk . +COPY config.yaml . + +EXPOSE 8080 + +ENTRYPOINT ["./camtalk"] diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 134b791..829c415 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" "os/signal" + "strings" "syscall" "time" @@ -21,6 +22,10 @@ import ( "github.com/hhs/camtalk/internal/ws" ) +// Version 通过构建时 -ldflags 注入，如： +// go build -ldflags "-X main.Version=v1.0.0" ./cmd/server +var Version string + var startTime = time.Now() func main() { @@ -42,16 +47,47 @@ func main() { // 初始化 Session Manager（MVP 默认内存实现） var sessionMgr session.Manager // TODO: 当 Redis 配置非空时切换为 RedisManager - sessionMgr = session.NewMemoryManager(30*time.Minute, 20) + sessionMgr = session.NewMemoryManager( + time.Duration(cfg.Session.TTL)*time.Minute, + cfg.Session.MaxHistory, + ) defer sessionMgr.(*session.MemoryManager).Stop() // 初始化 AI 服务 - sttService := stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, logger.Log) - llmService := llm.NewOpenAIService(cfg.AI.LLM.APIKey, cfg.AI.LLM.Model, cfg.AI.LLM.Endpoint, cfg.AI.LLM.Timeout, logger.Log) - ttsService := tts.NewOpenAIService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Speed, cfg.AI.TTS.Timeout, logger.Log) + logger.Log.Infow("initializing AI services", + "stt.provider", cfg.AI.STT.Provider, + "stt.model", cfg.AI.STT.Model, + "llm.provider", cfg.AI.LLM.Provider, + "llm.model", cfg.AI.LLM.Model, + "tts.provider", cfg.AI.TTS.Provider, + "tts.model", cfg.AI.TTS.Model, + "tts.voice", cfg.AI.TTS.Voice, + ) + + var sttService stt.Service + switch strings.ToLower(cfg.AI.STT.Provider) { + case "mimo", "xiaomi": + sttService = stt.NewMiMoService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, cfg.AI.STT.Timeout, logger.Log) + logger.Log.Infow("STT service initialized", "provider", "mimo", "model", cfg.AI.STT.Model, "endpoint", cfg.AI.STT.Endpoint) + default: + sttService = stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, cfg.AI.STT.Timeout, logger.Log) + logger.Log.Infow("STT service initialized", "provider", "deepgram", "model", cfg.AI.STT.Model) + } + llmService := llm.NewOpenAIService(cfg.AI.LLM.APIKey, cfg.AI.LLM.Model, cfg.AI.LLM.Endpoint, cfg.AI.LLM.Timeout, cfg.AI.LLM.HTTPClientTimeout, logger.Log) + logger.Log.Infow("LLM service initialized", "provider", cfg.AI.LLM.Provider, "model", cfg.AI.LLM.Model, "endpoint", cfg.AI.LLM.Endpoint, "timeout", cfg.AI.LLM.Timeout) + + var ttsService tts.Service + switch strings.ToLower(cfg.AI.TTS.Provider) { + case "mimo", "xiaomi": + ttsService = tts.NewMiMoService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Timeout, cfg.AI.TTS.HTTPClientTimeout, logger.Log) + logger.Log.Infow("TTS service initialized", "provider", "mimo", "model", cfg.AI.TTS.Model, "voice", cfg.AI.TTS.Voice, "endpoint", cfg.AI.TTS.Endpoint) + default: + ttsService = tts.NewOpenAIService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Speed, cfg.AI.TTS.Timeout, cfg.AI.TTS.HTTPClientTimeout, logger.Log) + logger.Log.Infow("TTS service initialized", "provider", "openai", "model", cfg.AI.TTS.Model, "voice", cfg.AI.TTS.Voice, "speed", cfg.AI.TTS.Speed) + } // 初始化 Orchestrator - orch := orchestrator.New(sttService, llmService, ttsService, sessionMgr, cfg.AI.LLM.Model) + orch := orchestrator.New(sttService, llmService, ttsService, sessionMgr, cfg) // Gin 模式 if cfg.App.Env == "prod" { @@ -64,7 +100,7 @@ func main() { // REST API apiGroup := r.Group("/api") { - apiGroup.GET("/health", healthHandler(sessionMgr)) + apiGroup.GET("/health", healthHandler(sessionMgr, cfg)) } // Session REST 端点 @@ -72,7 +108,7 @@ func main() { sessionHandler.RegisterRoutes(apiGroup) // WebSocket - r.GET("/ws", ws.ServeWS(sessionMgr, orch)) + r.GET("/ws", ws.ServeWS(sessionMgr, orch, cfg)) // HTTP Server srv := &http.Server{ @@ -96,7 +132,7 @@ func main() { <-ctx.Done() logger.Log.Info("shutting down...") - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.Server.ShutdownTimeout)*time.Second) defer cancel() if err := srv.Shutdown(shutdownCtx); err != nil { @@ -106,11 +142,15 @@ func main() { } // healthHandler 健康检查。 -func healthHandler(sessionMgr session.Manager) gin.HandlerFunc { +func healthHandler(sessionMgr session.Manager, cfg *config.Config) gin.HandlerFunc { return func(c *gin.Context) { + version := Version + if version == "" { + version = cfg.App.Version + } c.JSON(200, gin.H{ "status": "ok", - "version": "0.1.0", + "version": version, "uptime_seconds": int(time.Since(startTime).Seconds()), "active_sessions": sessionMgr.ActiveCount(), }) diff --git a/backend/config.yaml b/backend/config.yaml index 988f336..6e427d1 100644 --- a/backend/config.yaml +++ b/backend/config.yaml @@ -15,20 +15,23 @@ redis: ai: stt: - provider: Xiaomi MiMo - model: mimo-v2.5 - endpoint: "https://token-plan-cn.xiaomimimo.com/v1" + provider: mimo + model: mimo-v2.5-asr + endpoint: "https://api.xiaomimimo.com/v1" + api_key: "sk-c3jhv58rr5djhxw398w2rrij5tfpnpdgxqq1bojagshzviah" llm: provider: dashscope model: qwen3-vl-plus endpoint: "https://dashscope.aliyuncs.com/compatible-mode/v1" + api_key: "sk-ws-H.REHELLY.C4s3.MEUCIQCRee37XWEKp2szaxVLFDtR1rxNNsf372zMvCR0Xl6UvQIgZgvhRTvaa1FmhbCQJgaHu4Jny29AQkn01-3hX9CWBOg" timeout: 30 tts: - provider: Xiaomi MiMo - model: mimo-v2.5 - voice: alloy + provider: mimo + model: mimo-v2.5-tts + voice: mimo_default speed: 1.0 endpoint: "https://token-plan-cn.xiaomimimo.com/v1" + api_key: "tp-c9e7scwfx94qvqyhpnahnw8uaiya01za2qzvg4xe24rp3xiv" timeout: 5 storage: diff --git a/backend/internal/ai/llm/openai.go b/backend/internal/ai/llm/openai.go index bbc9444..e421d7e 100644 --- a/backend/internal/ai/llm/openai.go +++ b/backend/internal/ai/llm/openai.go @@ -26,24 +26,23 @@ type OpenAIService struct { } // NewOpenAIService 创建 OpenAI LLM 服务。 -func NewOpenAIService(apiKey, model, endpoint string, timeoutSec int, logger *zap.SugaredLogger) *OpenAIService { - if model == "" { - model = "gpt-4o" - } - if endpoint == "" { - endpoint = "https://api.openai.com/v1" - } +// model、endpoint 由 config 层保证非空。 +func NewOpenAIService(apiKey, model, endpoint string, timeoutSec, httpClientTimeoutSec int, logger *zap.SugaredLogger) *OpenAIService { timeout := time.Duration(timeoutSec) * time.Second if timeout <= 0 { timeout = 10 * time.Second } + httpClientTimeout := time.Duration(httpClientTimeoutSec) * time.Second + if httpClientTimeout <= 0 { + httpClientTimeout = 60 * time.Second + } return &OpenAIService{ apiKey: apiKey, model: model, endpoint: endpoint, timeout: timeout, logger: logger, - client: &http.Client{Timeout: 60 * time.Second}, // HTTP client timeout > LLM timeout + client: &http.Client{Timeout: httpClientTimeout}, } } @@ -62,7 +61,7 @@ type chatMessage struct { type contentPart struct { Type string `json:"type"` - Text string `json:"text,omitempty"` + Text string `json:"text"` ImageURL *imageURL `json:"image_url,omitempty"` } @@ -101,6 +100,9 @@ func (o *OpenAIService) ChatStream(ctx context.Context, req Request) (<-chan Chu if err != nil { return nil, fmt.Errorf("llm: marshal request: %w", err) } + if err != nil { + return nil, fmt.Errorf("llm: marshal request: %w", err) + } // 创建带超时的 context ctx, cancel := context.WithTimeout(ctx, o.timeout) diff --git a/backend/internal/ai/llm/openai_test.go b/backend/internal/ai/llm/openai_test.go index 9f2eee2..240e523 100644 --- a/backend/internal/ai/llm/openai_test.go +++ b/backend/internal/ai/llm/openai_test.go @@ -53,7 +53,7 @@ func TestOpenAIService_ChatStream_Success(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) ch, err := svc.ChatStream(context.Background(), Request{ Text: "这是什么？", @@ -99,7 +99,7 @@ func TestOpenAIService_ChatStream_WithImage(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) ch, err := svc.ChatStream(context.Background(), Request{ Image: []byte("fake-jpeg-data"), @@ -123,7 +123,7 @@ func TestOpenAIService_ChatStream_WithHistory(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) ch, err := svc.ChatStream(context.Background(), Request{ Text: "继续", @@ -149,7 +149,7 @@ func TestOpenAIService_ChatStream_APIError(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("bad-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + svc := NewOpenAIService("bad-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) _, err := svc.ChatStream(context.Background(), Request{ Text: "test", @@ -172,7 +172,7 @@ func TestOpenAIService_ChatStream_Timeout(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 1, zap.NewNop().Sugar()) // 1s timeout + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 1, 60, zap.NewNop().Sugar()) // 1s timeout ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -204,7 +204,7 @@ func TestOpenAIService_ChatStream_UsageInResponse(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) ch, err := svc.ChatStream(context.Background(), Request{Text: "test"}) if err != nil { diff --git a/backend/internal/ai/stt/deepgram.go b/backend/internal/ai/stt/deepgram.go index 54da674..8d19246 100644 --- a/backend/internal/ai/stt/deepgram.go +++ b/backend/internal/ai/stt/deepgram.go @@ -18,21 +18,22 @@ type DeepgramService struct { apiKey string model string endpoint string + timeout time.Duration logger *zap.SugaredLogger } // NewDeepgramService 创建 Deepgram STT 服务。 -func NewDeepgramService(apiKey, model, endpoint string, logger *zap.SugaredLogger) *DeepgramService { - if model == "" { - model = "nova-2" - } - if endpoint == "" { - endpoint = "wss://api.deepgram.com/v1/listen" +// model、endpoint 由 config 层保证非空，timeoutSec 为 0 时默认 5 秒。 +func NewDeepgramService(apiKey, model, endpoint string, timeoutSec int, logger *zap.SugaredLogger) *DeepgramService { + timeout := time.Duration(timeoutSec) * time.Second + if timeout <= 0 { + timeout = 5 * time.Second } return &DeepgramService{ apiKey: apiKey, model: model, endpoint: endpoint, + timeout: timeout, logger: logger, } } @@ -57,8 +58,8 @@ func (d *DeepgramService) Recognize(ctx context.Context, audio []byte, opts Opti // 构建 WebSocket URL，附带查询参数 wsURL := d.buildURL(opts) - // 5 秒总超时 - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + // 总超时 + ctx, cancel := context.WithTimeout(ctx, d.timeout) defer cancel() // 建立 WebSocket 连接 diff --git a/backend/internal/ai/stt/deepgram_test.go b/backend/internal/ai/stt/deepgram_test.go deleted file mode 100644 index 2d00e53..0000000 --- a/backend/internal/ai/stt/deepgram_test.go +++ /dev/null @@ -1,191 +0,0 @@ -package stt - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/gorilla/websocket" - "go.uber.org/zap" -) - -var upgrader = websocket.Upgrader{ - CheckOrigin: func(r *http.Request) bool { return true }, -} - -// newMockDeepgram 创建模拟 Deepgram WebSocket 服务。 -// 返回 httptest.Server 和对应的 ws:// URL。 -func newMockDeepgram(t *testing.T, handler func(conn *websocket.Conn)) *httptest.Server { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - conn, err := upgrader.Upgrade(w, r, nil) - if err != nil { - t.Logf("upgrade error: %v", err) - return - } - defer conn.Close() - handler(conn) - })) - return srv -} - -// wsToWss 将 http:// 转换为 ws://。 -func wsToWss(httpURL string) string { - return "ws" + strings.TrimPrefix(httpURL, "http") -} - -func TestDeepgramService_Recognize_Success(t *testing.T) { - srv := newMockDeepgram(t, func(conn *websocket.Conn) { - // 读取音频数据 - _, _, err := conn.ReadMessage() - if err != nil { - t.Errorf("read audio: %v", err) - return - } - - // 发送中间结果（非 final） - intermediate := deepgramResponse{ - IsFinal: false, - } - intermediate.Channel.Alternatives = []struct { - Transcript string `json:"transcript"` - Confidence float64 `json:"confidence"` - }{{Transcript: "你好", Confidence: 0.9}} - data, _ := json.Marshal(intermediate) - _ = conn.WriteMessage(websocket.TextMessage, data) - - // 发送最终结果 - final := deepgramResponse{ - IsFinal: true, - } - final.Channel.Alternatives = []struct { - Transcript string `json:"transcript"` - Confidence float64 `json:"confidence"` - }{{Transcript: "你好世界", Confidence: 0.95}} - data, _ = json.Marshal(final) - _ = conn.WriteMessage(websocket.TextMessage, data) - - // 等待客户端关闭 - _, _, _ = conn.ReadMessage() - }) - defer srv.Close() - - svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar()) - - text, err := svc.Recognize(context.Background(), []byte("fake-pcm-audio"), Options{ - Encoding: "pcm_s16le", - SampleRate: 16000, - Language: "zh-CN", - }) - if err != nil { - t.Fatalf("Recognize() error: %v", err) - } - if text != "你好世界" { - t.Errorf("Recognize() = %q, want %q", text, "你好世界") - } -} - -func TestDeepgramService_Recognize_EmptyAudio(t *testing.T) { - svc := NewDeepgramService("test-key", "", "ws://localhost", zap.NewNop().Sugar()) - _, err := svc.Recognize(context.Background(), nil, Options{}) - if err == nil { - t.Fatal("Recognize() with empty audio should return error") - } -} - -func TestDeepgramService_Recognize_ConnectError(t *testing.T) { - svc := NewDeepgramService("test-key", "", "ws://localhost:1", zap.NewNop().Sugar()) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - _, err := svc.Recognize(ctx, []byte("audio"), Options{}) - if err == nil { - t.Fatal("Recognize() with bad endpoint should return error") - } -} - -func TestDeepgramService_Recognize_Timeout(t *testing.T) { - // 模拟一个永不响应的服务端 - srv := newMockDeepgram(t, func(conn *websocket.Conn) { - // 读取音频但不发送任何结果，让客户端超时 - _, _, _ = conn.ReadMessage() - time.Sleep(10 * time.Second) - }) - defer srv.Close() - - svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar()) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - _, err := svc.Recognize(ctx, []byte("audio"), Options{}) - if err == nil { - t.Fatal("Recognize() should timeout") - } -} - -func TestDeepgramService_Recognize_MultipleFinals(t *testing.T) { - srv := newMockDeepgram(t, func(conn *websocket.Conn) { - _, _, _ = conn.ReadMessage() - - // 发送多个 final 结果（多句话场景） - for _, text := range []string{"你好", "世界"} { - resp := deepgramResponse{IsFinal: true} - resp.Channel.Alternatives = []struct { - Transcript string `json:"transcript"` - Confidence float64 `json:"confidence"` - }{{Transcript: text, Confidence: 0.9}} - data, _ := json.Marshal(resp) - _ = conn.WriteMessage(websocket.TextMessage, data) - } - - _, _, _ = conn.ReadMessage() - }) - defer srv.Close() - - svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar()) - - text, err := svc.Recognize(context.Background(), []byte("audio"), Options{}) - if err != nil { - t.Fatalf("Recognize() error: %v", err) - } - if text != "你好世界" { - t.Errorf("Recognize() = %q, want %q", text, "你好世界") - } -} - -func TestDeepgramService_buildURL(t *testing.T) { - svc := NewDeepgramService("key", "", "wss://api.deepgram.com/v1/listen", zap.NewNop().Sugar()) - - tests := []struct { - name string - opts Options - want []string // URL 中应包含的参数 - }{ - { - name: "defaults", - opts: Options{}, - want: []string{"encoding=pcm_s16le", "sample_rate=16000", "language=zh-CN"}, - }, - { - name: "custom", - opts: Options{Encoding: "wav", SampleRate: 44100, Language: "en"}, - want: []string{"encoding=wav", "sample_rate=44100", "language=en"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - u := svc.buildURL(tt.opts) - for _, param := range tt.want { - if !strings.Contains(u, param) { - t.Errorf("buildURL() = %q, should contain %q", u, param) - } - } - }) - } -} diff --git a/backend/internal/ai/stt/mimo.go b/backend/internal/ai/stt/mimo.go new file mode 100644 index 0000000..68e0c0d --- /dev/null +++ b/backend/internal/ai/stt/mimo.go @@ -0,0 +1,233 @@ +package stt + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "go.uber.org/zap" +) + +// MiMoService 基于 Xiaomi MiMo ASR HTTP API 的语音识别实现。 +// 接口兼容 OpenAI chat/completions 格式，音频仅支持 mp3/wav。 +type MiMoService struct { + apiKey string + model string + endpoint string + timeout time.Duration + logger *zap.SugaredLogger +} + +// NewMiMoService 创建 MiMo STT 服务。 +// model、endpoint 由 config 层保证非空，timeoutSec 为 0 时默认 10 秒。 +func NewMiMoService(apiKey, model, endpoint string, timeoutSec int, logger *zap.SugaredLogger) *MiMoService { + timeout := time.Duration(timeoutSec) * time.Second + if timeout <= 0 { + timeout = 10 * time.Second + } + return &MiMoService{ + apiKey: apiKey, + model: model, + endpoint: endpoint, + timeout: timeout, + logger: logger, + } +} + +// mimoRequest MiMo ASR 请求体。 +type mimoRequest struct { + Model string `json:"model"` + Messages []mimoMessage `json:"messages"` + ASROptions *mimoASROptions `json:"asr_options,omitempty"` +} + +type mimoMessage struct { + Role string `json:"role"` + Content []mimoContent `json:"content"` +} + +type mimoContent struct { + Type string `json:"type"` + InputAudio *mimoAudioIn `json:"input_audio,omitempty"` +} + +type mimoAudioIn struct { + Data string `json:"data"` // data URL: data:{mime};base64,{data} +} + +type mimoASROptions struct { + Language string `json:"language"` +} + +// mimoResponse MiMo ASR 非流式响应。 +type mimoResponse struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` +} + +// Recognize 实现 stt.Service。将音频发送到 MiMo ASR API，返回识别文本。 +func (m *MiMoService) Recognize(ctx context.Context, audio []byte, opts Options) (string, error) { + if len(audio) == 0 { + return "", fmt.Errorf("stt: empty audio") + } + + // MiMo 仅支持 mp3/wav，若输入为原始 PCM 则封装为 WAV + audioData := audio + mimeType := "audio/wav" + if !isWAV(audio) && !isMP3(audio) { + wav, err := pcmToWAV(audio, opts.SampleRate, 1) + if err != nil { + return "", fmt.Errorf("stt: pcm to wav: %w", err) + } + audioData = wav + } else if isMP3(audio) { + mimeType = "audio/mpeg" + } + + b64 := base64.StdEncoding.EncodeToString(audioData) + dataURL := fmt.Sprintf("data:%s;base64,%s", mimeType, b64) + + // 映射语言代码 + language := mapLanguage(opts.Language) + + reqBody := mimoRequest{ + Model: m.model, + Messages: []mimoMessage{ + { + Role: "user", + Content: []mimoContent{ + { + Type: "input_audio", + InputAudio: &mimoAudioIn{ + Data: dataURL, + }, + }, + }, + }, + }, + } + if language != "" { + reqBody.ASROptions = &mimoASROptions{Language: language} + } + + body, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("stt: marshal request: %w", err) + } + + url := strings.TrimRight(m.endpoint, "/") + "/chat/completions" + + ctx, cancel := context.WithTimeout(ctx, m.timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return "", fmt.Errorf("stt: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+m.apiKey) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("stt: request mimo: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("stt: read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("stt: mimo returned %d: %s", resp.StatusCode, string(respBody)) + } + + var mResp mimoResponse + if err := json.Unmarshal(respBody, &mResp); err != nil { + return "", fmt.Errorf("stt: unmarshal response: %w", err) + } + + if len(mResp.Choices) == 0 { + return "", fmt.Errorf("stt: mimo returned empty choices") + } + + text := strings.TrimSpace(mResp.Choices[0].Message.Content) + return text, nil +} + +// mapLanguage 将标准语言代码映射为 MiMo 支持的值（auto/zh/en）。 +func mapLanguage(lang string) string { + switch { + case lang == "": + return "auto" + case strings.HasPrefix(lang, "zh"): + return "zh" + case strings.HasPrefix(lang, "en"): + return "en" + default: + return "auto" + } +} + +// isWAV 检查数据是否为 WAV 格式（RIFF 头）。 +func isWAV(data []byte) bool { + return len(data) > 4 && string(data[:4]) == "RIFF" +} + +// isMP3 检查数据是否为 MP3 格式（ID3 标签或帧同步字）。 +func isMP3(data []byte) bool { + if len(data) > 3 && string(data[:3]) == "ID3" { + return true + } + // 帧同步字：0xFF 0xFB/0xF3/0xF2 + return len(data) > 2 && data[0] == 0xFF && (data[1]&0xE0) == 0xE0 +} + +// pcmToWAV 将原始 PCM 数据封装为 WAV 文件。 +func pcmToWAV(pcm []byte, sampleRate, channels int) ([]byte, error) { + if sampleRate == 0 { + sampleRate = 16000 + } + if channels == 0 { + channels = 1 + } + + bitsPerSample := 16 + byteRate := sampleRate * channels * bitsPerSample / 8 + blockAlign := channels * bitsPerSample / 8 + dataSize := len(pcm) + + var buf bytes.Buffer + + // RIFF header + buf.WriteString("RIFF") + binary.Write(&buf, binary.LittleEndian, uint32(36+dataSize)) + buf.WriteString("WAVE") + + // fmt 子块 + buf.WriteString("fmt ") + binary.Write(&buf, binary.LittleEndian, uint32(16)) // 子块大小 + binary.Write(&buf, binary.LittleEndian, uint16(1)) // PCM 格式 + binary.Write(&buf, binary.LittleEndian, uint16(channels)) // 通道数 + binary.Write(&buf, binary.LittleEndian, uint32(sampleRate)) // 采样率 + binary.Write(&buf, binary.LittleEndian, uint32(byteRate)) // 字节率 + binary.Write(&buf, binary.LittleEndian, uint16(blockAlign)) // 块对齐 + binary.Write(&buf, binary.LittleEndian, uint16(bitsPerSample)) // 每样本位数 + + // data 子块 + buf.WriteString("data") + binary.Write(&buf, binary.LittleEndian, uint32(dataSize)) + buf.Write(pcm) + + return buf.Bytes(), nil +} diff --git a/backend/internal/ai/stt/mimo_test.go b/backend/internal/ai/stt/mimo_test.go new file mode 100644 index 0000000..78ab038 --- /dev/null +++ b/backend/internal/ai/stt/mimo_test.go @@ -0,0 +1,255 @@ +package stt + +import ( + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "go.uber.org/zap" +) + +func newTestMiMoService(handler http.HandlerFunc) (*MiMoService, *httptest.Server) { + srv := httptest.NewServer(handler) + s := NewMiMoService("test-key", "mimo-v2.5-asr", srv.URL, 0, zap.NewNop().Sugar()) + return s, srv +} + +func TestMiMoService_Recognize_Success(t *testing.T) { + s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) { + // 验证请求 + if r.Header.Get("Authorization") != "Bearer test-key" { + t.Errorf("expected Authorization Bearer test-key, got %s", r.Header.Get("Authorization")) + } + if r.URL.Path != "/chat/completions" { + t.Errorf("expected path /chat/completions, got %s", r.URL.Path) + } + + var req mimoRequest + body, _ := io.ReadAll(r.Body) + if err := json.Unmarshal(body, &req); err != nil { + t.Fatalf("unmarshal request: %v", err) + } + if req.Model != "mimo-v2.5-asr" { + t.Errorf("expected model mimo-v2.5-asr, got %s", req.Model) + } + if len(req.Messages) == 0 || req.Messages[0].Role != "user" { + t.Error("expected user message") + } + if req.ASROptions == nil || req.ASROptions.Language != "zh" { + t.Errorf("expected language zh, got %v", req.ASROptions) + } + + resp := mimoResponse{ + Choices: []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + }{ + {Message: struct { + Content string `json:"content"` + }{Content: "你好世界"}}, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + }) + defer srv.Close() + + // 发送一个简单的有效 WAV（44 字节头 + 少量 PCM） + wav := makeValidWAV([]byte{0x00, 0x00, 0x00, 0x00}) + text, err := s.Recognize(context.Background(), wav, Options{Language: "zh-CN"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if text != "你好世界" { + t.Errorf("expected '你好世界', got '%s'", text) + } +} + +func TestMiMoService_Recognize_EmptyAudio(t *testing.T) { + s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) {}) + defer srv.Close() + + _, err := s.Recognize(context.Background(), nil, Options{}) + if err == nil { + t.Fatal("expected error for empty audio") + } +} + +func TestMiMoService_Recognize_ServerError(t *testing.T) { + s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("internal error")) + }) + defer srv.Close() + + wav := makeValidWAV([]byte{0x00, 0x00}) + _, err := s.Recognize(context.Background(), wav, Options{}) + if err == nil { + t.Fatal("expected error for 500 response") + } +} + +func TestMiMoService_Recognize_EmptyChoices(t *testing.T) { + s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) { + resp := mimoResponse{Choices: nil} + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + }) + defer srv.Close() + + wav := makeValidWAV([]byte{0x00, 0x00}) + _, err := s.Recognize(context.Background(), wav, Options{}) + if err == nil { + t.Fatal("expected error for empty choices") + } +} + +func TestMiMoService_Recognize_PCMAutoWrap(t *testing.T) { + // 测试原始 PCM 数据自动封装为 WAV + s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) { + var req mimoRequest + body, _ := io.ReadAll(r.Body) + if err := json.Unmarshal(body, &req); err != nil { + t.Fatalf("unmarshal request: %v", err) + } + + // 验证 data URL 格式 + if len(req.Messages) == 0 || len(req.Messages[0].Content) == 0 { + t.Fatal("empty message content") + } + dataURL := req.Messages[0].Content[0].InputAudio.Data + if len(dataURL) < 22 || dataURL[:14] != "data:audio/wav" { + t.Errorf("expected wav data URL, got prefix: %s", dataURL[:min(len(dataURL), 30)]) + } + + // 验证 base64 可解码 + b64Part := dataURL[22:] // skip "data:audio/wav;base64," + decoded, err := base64.StdEncoding.DecodeString(b64Part) + if err != nil { + t.Fatalf("base64 decode failed: %v", err) + } + // 应该是有效 WAV（RIFF 头） + if len(decoded) < 44 || string(decoded[:4]) != "RIFF" { + t.Error("decoded data is not a valid WAV") + } + + resp := mimoResponse{ + Choices: []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + }{ + {Message: struct { + Content string `json:"content"` + }{Content: "test"}}, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + }) + defer srv.Close() + + // 发送原始 PCM（非 WAV/MP3） + pcm := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07} + text, err := s.Recognize(context.Background(), pcm, Options{SampleRate: 16000}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if text != "test" { + t.Errorf("expected 'test', got '%s'", text) + } +} + +func TestMapLanguage(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"", "auto"}, + {"zh-CN", "zh"}, + {"zh", "zh"}, + {"en-US", "en"}, + {"en", "en"}, + {"ja", "auto"}, + } + for _, tt := range tests { + got := mapLanguage(tt.input) + if got != tt.want { + t.Errorf("mapLanguage(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestIsWAV(t *testing.T) { + if !isWAV([]byte("RIFF....")) { + t.Error("expected true for RIFF header") + } + if isWAV([]byte("ID3...")) { + t.Error("expected false for ID3 header") + } + if isWAV([]byte{0x00}) { + t.Error("expected false for short data") + } +} + +func TestIsMP3(t *testing.T) { + if !isMP3([]byte("ID3\x03")) { + t.Error("expected true for ID3 header") + } + if !isMP3([]byte{0xFF, 0xFB, 0x00}) { + t.Error("expected true for MP3 sync word") + } + if isMP3([]byte("RIFF")) { + t.Error("expected false for RIFF header") + } +} + +func makeValidWAV(pcm []byte) []byte { + // 构造一个最小有效 WAV + wav := make([]byte, 44+len(pcm)) + copy(wav[:4], "RIFF") + // little-endian size = 36 + len(pcm) + size := uint32(36 + len(pcm)) + wav[4] = byte(size) + wav[5] = byte(size >> 8) + wav[6] = byte(size >> 16) + wav[7] = byte(size >> 24) + copy(wav[8:12], "WAVE") + copy(wav[12:16], "fmt ") + // fmt chunk size = 16 + wav[16] = 16 + // PCM format = 1 + wav[20] = 1 + // channels = 1 + wav[22] = 1 + // sample rate = 16000 + wav[24] = 0x80 + wav[25] = 0x3E + // byte rate = 32000 + wav[28] = 0x00 + wav[29] = 0x7D + // block align = 2 + wav[32] = 2 + // bits per sample = 16 + wav[34] = 16 + copy(wav[36:40], "data") + dSize := uint32(len(pcm)) + wav[40] = byte(dSize) + wav[41] = byte(dSize >> 8) + wav[42] = byte(dSize >> 16) + wav[43] = byte(dSize >> 24) + copy(wav[44:], pcm) + return wav +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/backend/internal/ai/tts/mimo.go b/backend/internal/ai/tts/mimo.go new file mode 100644 index 0000000..717b9f7 --- /dev/null +++ b/backend/internal/ai/tts/mimo.go @@ -0,0 +1,208 @@ +package tts + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "go.uber.org/zap" +) + +// MiMoService 基于 Xiaomi MiMo TTS API 的语音合成实现。 +// 接口兼容 OpenAI chat/completions 格式，通过 messages 传递待合成文本与风格指令。 +type MiMoService struct { + apiKey string + model string + voice string + endpoint string + timeout time.Duration + logger *zap.SugaredLogger + client *http.Client +} + +// NewMiMoService 创建 MiMo TTS 服务。 +// model、voice、endpoint 由 config 层保证非空。 +func NewMiMoService(apiKey, model, voice, endpoint string, timeoutSec, httpClientTimeoutSec int, logger *zap.SugaredLogger) *MiMoService { + timeout := time.Duration(timeoutSec) * time.Second + if timeout <= 0 { + timeout = 5 * time.Second + } + httpClientTimeout := time.Duration(httpClientTimeoutSec) * time.Second + if httpClientTimeout <= 0 { + httpClientTimeout = 30 * time.Second + } + return &MiMoService{ + apiKey: apiKey, + model: model, + voice: voice, + endpoint: endpoint, + timeout: timeout, + logger: logger, + client: &http.Client{Timeout: httpClientTimeout}, + } +} + +// mimoTTSRequest MiMo TTS API 请求体。 +type mimoTTSRequest struct { + Model string `json:"model"` + Messages []mimoTTSMessage `json:"messages"` + Audio mimoTTSAudio `json:"audio"` + Stream bool `json:"stream"` +} + +// mimoTTSMessage MiMo TTS 消息。 +type mimoTTSMessage struct { + Role string `json:"role"` // "user"（风格指令）| "assistant"（待合成文本） + Content string `json:"content"` +} + +// mimoTTSAudio MiMo TTS 音频配置。 +type mimoTTSAudio struct { + Format string `json:"format"` // "mp3" | "wav" | "pcm16" + Voice string `json:"voice"` // 预置音色 ID +} + +// mimoTTSResponse MiMo TTS 非流式响应。 +type mimoTTSResponse struct { + Choices []struct { + Message struct { + Audio struct { + Data string `json:"data"` // base64 编码的音频数据 + } `json:"audio"` + } `json:"message"` + } `json:"choices"` +} + +// mimoTTSStreamResponse MiMo TTS 流式响应。 +type mimoTTSStreamResponse struct { + Choices []struct { + Delta struct { + Audio struct { + Data string `json:"data"` // base64 编码的音频数据片段 + } `json:"audio"` + } `json:"delta"` + } `json:"choices"` +} + +// SynthesizeStream 实现 tts.Service。从 textStream 读取句子，逐句调用 MiMo TTS API。 +func (m *MiMoService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error) { + voice := opts.Voice + if voice == "" { + voice = m.voice + } + + ch := make(chan Chunk, 4) + go func() { + defer close(ch) + + for text := range textStream { + if text == "" { + continue + } + + audio, err := m.synthesize(ctx, text, voice) + if err != nil { + m.logger.Warnw("mimo tts: synthesize failed", "error", err, "text", text) + // 静默跳过，不中断整个流 + continue + } + + select { + case ch <- Chunk{Audio: audio, IsLast: true, Final: false}: + case <-ctx.Done(): + return + } + } + + // textStream 关闭，发送 Final 标记 + select { + case ch <- Chunk{Audio: nil, IsLast: false, Final: true}: + case <-ctx.Done(): + } + }() + + return ch, nil +} + +// synthesize 调用 MiMo TTS API 合成单个句子。 +// 使用非流式调用，返回完整音频数据（base64 解码后）。 +func (m *MiMoService) synthesize(ctx context.Context, text, voice string) ([]byte, error) { + // 单句超时 + ctx, cancel := context.WithTimeout(ctx, m.timeout) + defer cancel() + + // 构建 MiMo TTS 请求：文本放在 assistant 消息中 + reqBody := mimoTTSRequest{ + Model: m.model, + Messages: []mimoTTSMessage{ + { + Role: "assistant", + Content: text, + }, + }, + Audio: mimoTTSAudio{ + Format: "mp3", + Voice: voice, + }, + Stream: false, + } + + payload, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("mimo tts: marshal request: %w", err) + } + + url := strings.TrimRight(m.endpoint, "/") + "/chat/completions" + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return nil, fmt.Errorf("mimo tts: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("api-key", m.apiKey) + + resp, err := m.client.Do(req) + if err != nil { + return nil, fmt.Errorf("mimo tts: send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("mimo tts: api error (status %d): %s", resp.StatusCode, string(errBody)) + } + + // 非流式响应：解析 JSON，提取 base64 音频数据 + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("mimo tts: read response: %w", err) + } + + var ttsResp mimoTTSResponse + if err := json.Unmarshal(respBody, &ttsResp); err != nil { + return nil, fmt.Errorf("mimo tts: unmarshal response: %w", err) + } + + if len(ttsResp.Choices) == 0 { + return nil, fmt.Errorf("mimo tts: empty choices in response") + } + + audioData := ttsResp.Choices[0].Message.Audio.Data + if audioData == "" { + return nil, fmt.Errorf("mimo tts: empty audio data in response") + } + + // base64 解码音频数据 + audio, err := base64.StdEncoding.DecodeString(audioData) + if err != nil { + return nil, fmt.Errorf("mimo tts: decode audio base64: %w", err) + } + + return audio, nil +} diff --git a/backend/internal/ai/tts/mimo_test.go b/backend/internal/ai/tts/mimo_test.go new file mode 100644 index 0000000..35d2641 --- /dev/null +++ b/backend/internal/ai/tts/mimo_test.go @@ -0,0 +1,414 @@ +package tts + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "go.uber.org/zap" +) + +// mockMiMoTTSServer 创建模拟 MiMo TTS API 的 HTTP 服务器。 +func mockMiMoTTSServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + return httptest.NewServer(handler) +} + +// buildMiMoTTSResponse 构造 MiMo TTS 非流式响应 JSON。 +func buildMiMoTTSResponse(audioData string) []byte { + resp := mimoTTSResponse{ + Choices: []struct { + Message struct { + Audio struct { + Data string `json:"data"` + } `json:"audio"` + } `json:"message"` + }{ + { + Message: struct { + Audio struct { + Data string `json:"data"` + } `json:"audio"` + }{ + Audio: struct { + Data string `json:"data"` + }{Data: audioData}, + }, + }, + }, + } + data, _ := json.Marshal(resp) + return data +} + +func TestMiMoService_SynthesizeStream_Success(t *testing.T) { + var callCount int32 + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&callCount, 1) + + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + if !strings.Contains(r.URL.Path, "/chat/completions") { + t.Errorf("path = %s, should contain /chat/completions", r.URL.Path) + } + + // 验证 api-key 认证头 + apiKey := r.Header.Get("api-key") + if apiKey != "test-key" { + t.Errorf("api-key = %q, want %q", apiKey, "test-key") + } + + // 验证请求体 + body, _ := io.ReadAll(r.Body) + var req mimoTTSRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Errorf("unmarshal request: %v", err) + } + if req.Model != "mimo-v2.5-tts" { + t.Errorf("model = %q, want %q", req.Model, "mimo-v2.5-tts") + } + if len(req.Messages) != 1 || req.Messages[0].Role != "assistant" { + t.Errorf("expected 1 assistant message, got %d messages", len(req.Messages)) + } + if req.Audio.Voice != "冰糖" { + t.Errorf("voice = %q, want %q", req.Audio.Voice, "冰糖") + } + if req.Audio.Format != "mp3" { + t.Errorf("format = %q, want %q", req.Audio.Format, "mp3") + } + + // 返回假音频数据（base64 编码） + audioB64 := base64.StdEncoding.EncodeToString([]byte("fake-mp3-data")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 5, 30, zap.NewNop().Sugar()) + + textStream := sendSentences("你好", "世界", "！") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{ + Voice: "冰糖", OutputFmt: "mp3", SampleRate: 24000, + }) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 应该有 3 个音频 chunk + 1 个 Final 标记 + if len(chunks) != 4 { + t.Fatalf("got %d chunks, want 4", len(chunks)) + } + + // 验证前 3 个有音频数据，IsLast 为 true（每句结束） + for i := 0; i < 3; i++ { + if string(chunks[i].Audio) != "fake-mp3-data" { + t.Errorf("chunk[%d].Audio = %q, want %q", i, string(chunks[i].Audio), "fake-mp3-data") + } + if !chunks[i].IsLast { + t.Errorf("chunk[%d].IsLast should be true (sentence end)", i) + } + if chunks[i].Final { + t.Errorf("chunk[%d].Final should be false", i) + } + } + + // 验证最后一个是 Final（整轮结束） + if !chunks[3].Final { + t.Error("last chunk should be Final") + } + if chunks[3].Audio != nil { + t.Error("last chunk Audio should be nil") + } + + // 验证调用了 3 次 API（3 个句子） + if atomic.LoadInt32(&callCount) != 3 { + t.Errorf("API called %d times, want 3", callCount) + } +} + +func TestMiMoService_SynthesizeStream_APIError(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "internal error") + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 5, 30, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + // 应该只有一个 Final chunk（音频被跳过） + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1 (Final only)", len(chunks)) + } + if !chunks[0].Final { + t.Error("chunk should be Final") + } +} + +func TestMiMoService_SynthesizeStream_Timeout(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(3 * time.Second) + audioB64 := base64.StdEncoding.EncodeToString([]byte("late-mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + // 1 秒超时 + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 1, 30, zap.NewNop().Sugar()) + + textStream := sendSentences("很长的句子") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ch, err := svc.SynthesizeStream(ctx, textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 超时后音频被跳过，只有 Final + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1", len(chunks)) + } + if !chunks[0].Final { + t.Error("chunk should be Final") + } +} + +func TestMiMoService_SynthesizeStream_EmptyText(t *testing.T) { + var callCount int32 + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&callCount, 1) + audioB64 := base64.StdEncoding.EncodeToString([]byte("mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 5, 30, zap.NewNop().Sugar()) + + // 空句子应该被跳过 + textStream := sendSentences("", "你好", "") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 只有 "你好" 应该被合成 + if atomic.LoadInt32(&callCount) != 1 { + t.Errorf("API called %d times, want 1", callCount) + } + + // 1 个音频（IsLast: true）+ 1 个 Final + if len(chunks) != 2 { + t.Fatalf("got %d chunks, want 2", len(chunks)) + } +} + +func TestMiMoService_SynthesizeStream_ContextCancelled(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + audioB64 := base64.StdEncoding.EncodeToString([]byte("mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 5, 30, zap.NewNop().Sugar()) + + textStream := make(chan string, 3) + textStream <- "第一句" + textStream <- "第二句" + textStream <- "第三句" + close(textStream) + + ctx, cancel := context.WithCancel(context.Background()) + // 立即取消 + cancel() + + ch, err := svc.SynthesizeStream(ctx, textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + // 消费 channel，应该很快结束 + var count int + for range ch { + count++ + } + // 可能收到 0 个或 1 个 chunk，取决于时序 + t.Logf("received %d chunks after context cancel", count) +} + +func TestMiMoService_SynthesizeStream_PartialFailure(t *testing.T) { + var callCount int32 + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&callCount, 1) + if n == 2 { + // 第二个句子失败 + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "error") + return + } + audioB64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("mp3-%d", n))) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 5, 30, zap.NewNop().Sugar()) + + textStream := sendSentences("第一句", "第二句", "第三句") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 2 个成功音频（IsLast: true）+ 1 个 Final（第二句被跳过） + if len(chunks) != 3 { + t.Fatalf("got %d chunks, want 3", len(chunks)) + } + if !chunks[0].IsLast { + t.Error("first audio chunk should be IsLast") + } + if !chunks[1].IsLast { + t.Error("second audio chunk should be IsLast") + } + if !chunks[len(chunks)-1].Final { + t.Error("last chunk should be Final") + } +} + +func TestMiMoService_SynthesizeStream_CustomVoice(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var req mimoTTSRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Errorf("unmarshal request: %v", err) + } + if req.Audio.Voice != "茉莉" { + t.Errorf("voice = %q, want %q", req.Audio.Voice, "茉莉") + } + audioB64 := base64.StdEncoding.EncodeToString([]byte("mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 5, 30, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{Voice: "茉莉"}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + for range ch { + } +} + +func TestMiMoService_SynthesizeStream_DefaultVoice(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var req mimoTTSRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Errorf("unmarshal request: %v", err) + } + // 未指定 voice 时应使用默认 "冰糖" + if req.Audio.Voice != "冰糖" { + t.Errorf("voice = %q, want %q (default)", req.Audio.Voice, "冰糖") + } + audioB64 := base64.StdEncoding.EncodeToString([]byte("mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + // 不指定 voice + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 5, 30, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + for range ch { + } +} + +func TestMiMoService_SynthesizeStream_EmptyAudioData(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + // 返回空音频数据 + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse("")) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "mimo-v2.5-tts", "冰糖", srv.URL, 5, 30, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 空音频数据导致错误，句子被跳过，只有 Final + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1", len(chunks)) + } + if !chunks[0].Final { + t.Error("chunk should be Final") + } +} diff --git a/backend/internal/ai/tts/openai.go b/backend/internal/ai/tts/openai.go index 6e0123e..ba7f788 100644 --- a/backend/internal/ai/tts/openai.go +++ b/backend/internal/ai/tts/openai.go @@ -25,23 +25,19 @@ type OpenAIService struct { } // NewOpenAIService 创建 OpenAI TTS 服务。 -func NewOpenAIService(apiKey, model, voice, endpoint string, speed float64, timeoutSec int, logger *zap.SugaredLogger) *OpenAIService { - if model == "" { - model = "tts-1" - } - if voice == "" { - voice = "alloy" - } +// model、voice、endpoint 由 config 层保证非空。 +func NewOpenAIService(apiKey, model, voice, endpoint string, speed float64, timeoutSec, httpClientTimeoutSec int, logger *zap.SugaredLogger) *OpenAIService { if speed <= 0 { speed = 1.0 } - if endpoint == "" { - endpoint = "https://api.openai.com/v1" - } timeout := time.Duration(timeoutSec) * time.Second if timeout <= 0 { timeout = 5 * time.Second } + httpClientTimeout := time.Duration(httpClientTimeoutSec) * time.Second + if httpClientTimeout <= 0 { + httpClientTimeout = 30 * time.Second + } return &OpenAIService{ apiKey: apiKey, model: model, @@ -50,7 +46,7 @@ func NewOpenAIService(apiKey, model, voice, endpoint string, speed float64, time endpoint: endpoint, timeout: timeout, logger: logger, - client: &http.Client{Timeout: 30 * time.Second}, + client: &http.Client{Timeout: httpClientTimeout}, } } @@ -91,15 +87,15 @@ func (o *OpenAIService) SynthesizeStream(ctx context.Context, textStream <-chan } select { - case ch <- Chunk{Audio: audio, IsLast: false}: + case ch <- Chunk{Audio: audio, IsLast: true, Final: false}: case <-ctx.Done(): return } } - // textStream 关闭，发送 IsLast 标记 + // textStream 关闭，发送 Final 标记 select { - case ch <- Chunk{Audio: nil, IsLast: true}: + case ch <- Chunk{Audio: nil, IsLast: false, Final: true}: case <-ctx.Done(): } }() diff --git a/backend/internal/ai/tts/openai_test.go b/backend/internal/ai/tts/openai_test.go index 36afd31..2d7b8ad 100644 --- a/backend/internal/ai/tts/openai_test.go +++ b/backend/internal/ai/tts/openai_test.go @@ -58,7 +58,7 @@ func TestOpenAIService_SynthesizeStream_Success(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "tts-1", "alloy", srv.URL, 1.0, 5, 30, zap.NewNop().Sugar()) textStream := sendSentences("你好", "世界", "！") @@ -74,24 +74,27 @@ func TestOpenAIService_SynthesizeStream_Success(t *testing.T) { chunks = append(chunks, c) } - // 应该有 3 个音频 chunk + 1 个 IsLast 标记 + // 应该有 3 个音频 chunk + 1 个 Final 标记 if len(chunks) != 4 { t.Fatalf("got %d chunks, want 4", len(chunks)) } - // 验证前 3 个有音频数据 + // 验证前 3 个有音频数据，IsLast 为 true（每句结束） for i := 0; i < 3; i++ { if string(chunks[i].Audio) != "fake-mp3-data" { t.Errorf("chunk[%d].Audio = %q, want %q", i, string(chunks[i].Audio), "fake-mp3-data") } - if chunks[i].IsLast { - t.Errorf("chunk[%d].IsLast should be false", i) + if !chunks[i].IsLast { + t.Errorf("chunk[%d].IsLast should be true (sentence end)", i) + } + if chunks[i].Final { + t.Errorf("chunk[%d].Final should be false", i) } } - // 验证最后一个是 IsLast - if !chunks[3].IsLast { - t.Error("last chunk should be IsLast") + // 验证最后一个是 Final（整轮结束） + if !chunks[3].Final { + t.Error("last chunk should be Final") } if chunks[3].Audio != nil { t.Error("last chunk Audio should be nil") @@ -110,7 +113,7 @@ func TestOpenAIService_SynthesizeStream_APIError(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "tts-1", "alloy", srv.URL, 1.0, 5, 30, zap.NewNop().Sugar()) textStream := sendSentences("你好") @@ -119,17 +122,17 @@ func TestOpenAIService_SynthesizeStream_APIError(t *testing.T) { t.Fatalf("SynthesizeStream() error: %v", err) } - // 应该只有一个 IsLast chunk（音频被跳过） + // 应该只有一个 Final chunk（音频被跳过） var chunks []Chunk for c := range ch { chunks = append(chunks, c) } if len(chunks) != 1 { - t.Fatalf("got %d chunks, want 1 (IsLast only)", len(chunks)) + t.Fatalf("got %d chunks, want 1 (Final only)", len(chunks)) } - if !chunks[0].IsLast { - t.Error("chunk should be IsLast") + if !chunks[0].Final { + t.Error("chunk should be Final") } } @@ -142,7 +145,7 @@ func TestOpenAIService_SynthesizeStream_Timeout(t *testing.T) { defer srv.Close() // 1 秒超时 - svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 1, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "tts-1", "alloy", srv.URL, 1.0, 1, 30, zap.NewNop().Sugar()) textStream := sendSentences("很长的句子") @@ -159,12 +162,12 @@ func TestOpenAIService_SynthesizeStream_Timeout(t *testing.T) { chunks = append(chunks, c) } - // 超时后音频被跳过，只有 IsLast + // 超时后音频被跳过，只有 Final if len(chunks) != 1 { t.Fatalf("got %d chunks, want 1", len(chunks)) } - if !chunks[0].IsLast { - t.Error("chunk should be IsLast") + if !chunks[0].Final { + t.Error("chunk should be Final") } } @@ -177,7 +180,7 @@ func TestOpenAIService_SynthesizeStream_EmptyText(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "tts-1", "alloy", srv.URL, 1.0, 5, 30, zap.NewNop().Sugar()) // 空句子应该被跳过 textStream := sendSentences("", "你好", "") @@ -197,7 +200,7 @@ func TestOpenAIService_SynthesizeStream_EmptyText(t *testing.T) { t.Errorf("API called %d times, want 1", callCount) } - // 1 个音频 + 1 个 IsLast + // 1 个音频（IsLast: true）+ 1 个 Final if len(chunks) != 2 { t.Fatalf("got %d chunks, want 2", len(chunks)) } @@ -210,7 +213,7 @@ func TestOpenAIService_SynthesizeStream_ContextCancelled(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "tts-1", "alloy", srv.URL, 1.0, 5, 30, zap.NewNop().Sugar()) // 发送多个句子，但在第一个后取消 textStream := make(chan string, 3) @@ -252,7 +255,7 @@ func TestOpenAIService_SynthesizeStream_PartialFailure(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "tts-1", "alloy", srv.URL, 1.0, 5, 30, zap.NewNop().Sugar()) textStream := sendSentences("第一句", "第二句", "第三句") @@ -266,12 +269,18 @@ func TestOpenAIService_SynthesizeStream_PartialFailure(t *testing.T) { chunks = append(chunks, c) } - // 2 个成功音频 + 1 个 IsLast（第二句被跳过） + // 2 个成功音频（IsLast: true）+ 1 个 Final（第二句被跳过） if len(chunks) != 3 { t.Fatalf("got %d chunks, want 3", len(chunks)) } - if !chunks[len(chunks)-1].IsLast { - t.Error("last chunk should be IsLast") + if !chunks[0].IsLast { + t.Error("first audio chunk should be IsLast") + } + if !chunks[1].IsLast { + t.Error("second audio chunk should be IsLast") + } + if !chunks[len(chunks)-1].Final { + t.Error("last chunk should be Final") } } @@ -286,7 +295,7 @@ func TestOpenAIService_SynthesizeStream_CustomVoice(t *testing.T) { }) defer srv.Close() - svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + svc := NewOpenAIService("test-key", "tts-1", "alloy", srv.URL, 1.0, 5, 30, zap.NewNop().Sugar()) textStream := sendSentences("你好") diff --git a/backend/internal/ai/tts/tts.go b/backend/internal/ai/tts/tts.go index 9b08d0e..e478ff7 100644 --- a/backend/internal/ai/tts/tts.go +++ b/backend/internal/ai/tts/tts.go @@ -21,5 +21,6 @@ type Options struct { // Chunk 一个音频片段。 type Chunk struct { Audio []byte // MP3 音频数据（未 Base64 编码） - IsLast bool // 是否为最后一片 + IsLast bool // 当前句子是否为最后一片（每句结束时为 true） + Final bool // 整轮 TTS 是否结束（所有句子合成完毕后为 true，此时 Audio 为 nil） } diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 1a5cac8..aff3a8e 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -11,12 +11,19 @@ import ( // Config 应用配置。 type Config struct { - App AppConfig `mapstructure:"app"` - Server ServerConfig `mapstructure:"server"` - Redis RedisConfig `mapstructure:"redis"` - AI AIConfig `mapstructure:"ai"` - Storage StorageConfig `mapstructure:"storage"` - Log LogConfig `mapstructure:"log"` + App AppConfig `mapstructure:"app"` + Server ServerConfig `mapstructure:"server"` + Session SessionConfig `mapstructure:"session"` + Redis RedisConfig `mapstructure:"redis"` + AI AIConfig `mapstructure:"ai"` + Storage StorageConfig `mapstructure:"storage"` + Log LogConfig `mapstructure:"log"` +} + +// SessionConfig 会话管理配置。 +type SessionConfig struct { + TTL int `mapstructure:"ttl"` // 会话过期时间（分钟） + MaxHistory int `mapstructure:"max_history"` // 对话历史上限（条） } type AppConfig struct { @@ -25,10 +32,14 @@ type AppConfig struct { } type ServerConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - ReadTimeout int `mapstructure:"read_timeout"` - WriteTimeout int `mapstructure:"write_timeout"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + ReadTimeout int `mapstructure:"read_timeout"` + WriteTimeout int `mapstructure:"write_timeout"` + HeartbeatInterval int `mapstructure:"heartbeat_interval"` // 心跳检查间隔（秒） + HeartbeatTimeout int `mapstructure:"heartbeat_timeout"` // 心跳超时（秒） + ShutdownTimeout int `mapstructure:"shutdown_timeout"` // 优雅关闭超时（秒） + AllowedOrigins []string `mapstructure:"allowed_origins"` // CORS 允许的来源，空表示允许所有 } // Addr 返回 host:port 地址。 @@ -49,28 +60,34 @@ type AIConfig struct { } type STTConfig struct { - Provider string `mapstructure:"provider"` - APIKey string `mapstructure:"api_key"` - Model string `mapstructure:"model"` - Endpoint string `mapstructure:"endpoint"` + Provider string `mapstructure:"provider"` + APIKey string `mapstructure:"api_key"` + Model string `mapstructure:"model"` + Endpoint string `mapstructure:"endpoint"` + Timeout int `mapstructure:"timeout"` // STT 超时（秒） + HTTPClientTimeout int `mapstructure:"http_client_timeout"` // HTTP 客户端超时（秒） } type LLMConfig struct { - Provider string `mapstructure:"provider"` - APIKey string `mapstructure:"api_key"` - Model string `mapstructure:"model"` - Endpoint string `mapstructure:"endpoint"` - Timeout int `mapstructure:"timeout"` + Provider string `mapstructure:"provider"` + APIKey string `mapstructure:"api_key"` + Model string `mapstructure:"model"` + Endpoint string `mapstructure:"endpoint"` + Timeout int `mapstructure:"timeout"` + HTTPClientTimeout int `mapstructure:"http_client_timeout"` // HTTP 客户端超时（秒） } type TTSConfig struct { - Provider string `mapstructure:"provider"` - APIKey string `mapstructure:"api_key"` - Model string `mapstructure:"model"` - Voice string `mapstructure:"voice"` - Speed float64 `mapstructure:"speed"` - Endpoint string `mapstructure:"endpoint"` - Timeout int `mapstructure:"timeout"` + Provider string `mapstructure:"provider"` + APIKey string `mapstructure:"api_key"` + Model string `mapstructure:"model"` + Voice string `mapstructure:"voice"` + Speed float64 `mapstructure:"speed"` + Endpoint string `mapstructure:"endpoint"` + Timeout int `mapstructure:"timeout"` + HTTPClientTimeout int `mapstructure:"http_client_timeout"` // HTTP 客户端超时（秒） + OutputFormat string `mapstructure:"output_format"` // 输出格式：mp3/wav + SampleRate int `mapstructure:"sample_rate"` // 输出采样率 } type StorageConfig struct { @@ -91,28 +108,42 @@ func Load() (*Config, error) { v.AddConfigPath(".") v.AddConfigPath("./config") v.AddConfigPath("./backend") + v.AddConfigPath("..") // 兼容从 backend/cmd/ 启动 + v.AddConfigPath("../..") // 兼容从 backend/cmd/server/ 启动 // 默认值 v.SetDefault("app.env", "dev") + v.SetDefault("app.version", "dev") v.SetDefault("server.host", "0.0.0.0") v.SetDefault("server.port", 8080) v.SetDefault("server.read_timeout", 30) v.SetDefault("server.write_timeout", 30) + v.SetDefault("server.heartbeat_interval", 30) + v.SetDefault("server.heartbeat_timeout", 60) + v.SetDefault("server.shutdown_timeout", 10) + v.SetDefault("session.ttl", 30) + v.SetDefault("session.max_history", 20) v.SetDefault("redis.addr", "localhost:6379") v.SetDefault("redis.db", 0) v.SetDefault("ai.stt.provider", "deepgram") v.SetDefault("ai.stt.model", "nova-2") v.SetDefault("ai.stt.endpoint", "wss://api.deepgram.com/v1/listen") + v.SetDefault("ai.stt.timeout", 5) + v.SetDefault("ai.stt.http_client_timeout", 30) v.SetDefault("ai.llm.provider", "openai") v.SetDefault("ai.llm.model", "gpt-4o") v.SetDefault("ai.llm.endpoint", "https://api.openai.com/v1") v.SetDefault("ai.llm.timeout", 10) + v.SetDefault("ai.llm.http_client_timeout", 60) v.SetDefault("ai.tts.provider", "openai") v.SetDefault("ai.tts.model", "tts-1") - v.SetDefault("ai.tts.voice", "alloy") + v.SetDefault("ai.tts.voice", "mimo_default") v.SetDefault("ai.tts.speed", 1.0) v.SetDefault("ai.tts.endpoint", "https://api.openai.com/v1") v.SetDefault("ai.tts.timeout", 5) + v.SetDefault("ai.tts.http_client_timeout", 30) + v.SetDefault("ai.tts.output_format", "mp3") + v.SetDefault("ai.tts.sample_rate", 24000) v.SetDefault("storage.driver", "memory") v.SetDefault("log.level", "info") v.SetDefault("log.format", "console") @@ -134,6 +165,7 @@ func Load() (*Config, error) { // 按优先级尝试：当前目录、上级目录（兼容从 backend/ 或项目根目录启动） _ = godotenv.Load() _ = godotenv.Load("../.env") + _ = godotenv.Load("../../.env") // 兼容从 backend/cmd/server/ 启动 // 环境变量覆盖 v.SetEnvPrefix("CAMTALK") diff --git a/backend/internal/models/models.go b/backend/internal/models/models.go index 73d017a..b07924c 100644 --- a/backend/internal/models/models.go +++ b/backend/internal/models/models.go @@ -55,6 +55,7 @@ type WsQuery struct { RequestID string `json:"request_id"` Image string `json:"image"` // base64 Audio string `json:"audio"` // base64 + Text string `json:"text"` // 用户手动输入的文本（有值时跳过 STT） MimeType string `json:"mime_type"` // 默认 "audio/pcm" } @@ -111,7 +112,8 @@ type WsTTSAudio struct { RequestID string `json:"request_id"` Audio string `json:"audio"` // base64 MimeType string `json:"mime_type"` // "audio/mp3" 或 "audio/pcm" - IsLast bool `json:"is_last"` + IsLast bool `json:"is_last"` // 当前句子的音频是否完整（每句结束时为 true） + Final bool `json:"final"` // 整轮 TTS 是否结束（所有句子合成完毕后为 true） } // WsError 服务端 error 消息。 diff --git a/backend/internal/orchestrator/pipeline.go b/backend/internal/orchestrator/pipeline.go index 0186108..914b70e 100644 --- a/backend/internal/orchestrator/pipeline.go +++ b/backend/internal/orchestrator/pipeline.go @@ -11,6 +11,7 @@ import ( "github.com/hhs/camtalk/internal/ai/llm" "github.com/hhs/camtalk/internal/ai/stt" "github.com/hhs/camtalk/internal/ai/tts" + "github.com/hhs/camtalk/internal/config" "github.com/hhs/camtalk/internal/logger" "github.com/hhs/camtalk/internal/models" "github.com/hhs/camtalk/internal/session" @@ -18,11 +19,15 @@ import ( // Pipeline 实现 Orchestrator 接口，管理 STT → LLM → TTS 流式管道。 type Pipeline struct { - sttService stt.Service - llmService llm.Service - ttsService tts.Service - sessionMgr session.Manager - model string // LLM 模型名，用于 llm_done 上报 + sttService stt.Service + llmService llm.Service + ttsService tts.Service + sessionMgr session.Manager + model string // LLM 模型名，用于 llm_done 上报 + ttsVoice string // TTS 音色 + ttsSpeed float64 // TTS 语速 + ttsOutputFmt string // TTS 输出格式 + ttsSampleRate int // TTS 输出采样率 } // New 创建 Pipeline 实例。 @@ -31,14 +36,18 @@ func New( llmService llm.Service, ttsService tts.Service, sessionMgr session.Manager, - model string, + cfg *config.Config, ) *Pipeline { return &Pipeline{ - sttService: sttService, - llmService: llmService, - ttsService: ttsService, - sessionMgr: sessionMgr, - model: model, + sttService: sttService, + llmService: llmService, + ttsService: ttsService, + sessionMgr: sessionMgr, + model: cfg.AI.LLM.Model, + ttsVoice: cfg.AI.TTS.Voice, + ttsSpeed: cfg.AI.TTS.Speed, + ttsOutputFmt: cfg.AI.TTS.OutputFormat, + ttsSampleRate: cfg.AI.TTS.SampleRate, } } @@ -53,22 +62,27 @@ func (p *Pipeline) ProcessQuery( log := logger.Log startTime := time.Now() - // 解码音频数据 - audio, err := base64.StdEncoding.DecodeString(req.Audio) - if err != nil { - log.Errorw("音频解码失败", "error", err) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "INVALID_MESSAGE", - Message: "音频数据解码失败", - }) - return err + // 解码音频数据（文本输入模式可跳过） + var audio []byte + if req.Text == "" && req.Audio != "" { + var err error + audio, err = base64.StdEncoding.DecodeString(req.Audio) + if err != nil { + log.Errorw("音频解码失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "INVALID_MESSAGE", + Message: "音频数据解码失败", + }) + return err + } } // 解码图片数据（可选） var image []byte if req.Image != "" { + var err error image, err = base64.StdEncoding.DecodeString(req.Image) if err != nil { log.Errorw("图片解码失败", "error", err) @@ -101,45 +115,64 @@ func (p *Pipeline) ProcessQuery( return err } - // Step 1: STT 语音识别 - log.Infow("开始语音识别", "request_id", req.RequestID) - sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{ - Encoding: "pcm_s16le", - SampleRate: 16000, - Language: sess.Config.Language, - }) - if err != nil { - log.Errorw("语音识别失败", "error", err) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "STT_ERROR", - Message: "语音识别失败", - }) - return err - } + // Step 1: 获取用户文本（语音识别或直接使用输入文本） + var userText string + if req.Text != "" { + // 文本输入模式：跳过 STT，直接使用用户输入的文本 + log.Infow("使用文本输入", "request_id", req.RequestID, "text", req.Text) + userText = req.Text - // 发送 STT 结果 - if err := sender.SendSTTResult(models.WsSTTResult{ - Type: "stt_result", - RequestID: req.RequestID, - Text: sttResult, - IsFinal: true, - }); err != nil { - log.Errorw("发送 STT 结果失败", "error", err) + // 发送 stt_result 以保持前端消息流一致性 + if err := sender.SendSTTResult(models.WsSTTResult{ + Type: "stt_result", + RequestID: req.RequestID, + Text: userText, + IsFinal: true, + }); err != nil { + log.Errorw("发送 STT 结果失败", "error", err) + } + } else { + // 语音模式：执行 STT 语音识别 + log.Infow("开始语音识别", "request_id", req.RequestID) + sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{ + Encoding: "pcm_s16le", + SampleRate: 16000, + Language: sess.Config.Language, + }) + if err != nil { + log.Errorw("语音识别失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "STT_ERROR", + Message: "语音识别失败", + }) + return err + } + userText = sttResult + + // 发送 STT 结果 + if err := sender.SendSTTResult(models.WsSTTResult{ + Type: "stt_result", + RequestID: req.RequestID, + Text: userText, + IsFinal: true, + }); err != nil { + log.Errorw("发送 STT 结果失败", "error", err) + } } // 追加用户消息到历史 p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{ Role: "user", - Content: sttResult, + Content: userText, }) // Step 2+3: LLM 流式推理 + TTS 并行合成 log.Infow("开始 LLM 推理", "request_id", req.RequestID) llmReq := llm.Request{ Image: image, - Text: sttResult, + Text: userText, History: history, Language: sess.Config.Language, } @@ -166,11 +199,12 @@ func (p *Pipeline) ProcessQuery( var ttsErr error // goroutine 1: 消费 LLM token + 句子切分 + var tokenUsage *llm.TokenUsage wg.Add(1) go func() { defer wg.Done() defer close(sentenceCh) - fullText = p.consumeLLMStream(ctx, llmStream, req.RequestID, sender, splitter) + fullText, tokenUsage = p.consumeLLMStream(ctx, llmStream, req.RequestID, sender, splitter) }() // goroutine 2: TTS 合成（如果启用） @@ -205,13 +239,25 @@ func (p *Pipeline) ProcessQuery( // 发送 llm_done latency := time.Since(startTime).Milliseconds() - if err := sender.SendLLMDone(models.WsLLMDone{ + done := models.WsLLMDone{ Type: "llm_done", RequestID: req.RequestID, FullText: fullText, Model: p.model, LatencyMs: latency, - }); err != nil { + } + if tokenUsage != nil { + done.TokensUsed = struct { + Prompt int `json:"prompt"` + Completion int `json:"completion"` + Total int `json:"total"` + }{ + Prompt: tokenUsage.Prompt, + Completion: tokenUsage.Completion, + Total: tokenUsage.Total, + } + } + if err := sender.SendLLMDone(done); err != nil { log.Errorw("发送 llm_done 失败", "error", err) } @@ -225,33 +271,36 @@ func (p *Pipeline) ProcessQuery( } // consumeLLMStream 消费 LLM 流式输出，发送 llm_chunk 并进行句子切分。 +// 返回完整文本和 token 用量。 func (p *Pipeline) consumeLLMStream( ctx context.Context, stream <-chan llm.Chunk, requestID string, sender Sender, splitter *Splitter, -) string { +) (string, *llm.TokenUsage) { log := logger.Log var fullText strings.Builder + var tokenUsage *llm.TokenUsage for chunk := range stream { // 检查上下文是否已取消 select { case <-ctx.Done(): log.Infow("LLM 流被中断", "request_id", requestID) - return fullText.String() + return fullText.String(), tokenUsage default: } if chunk.Done { - // 流结束 + // 流结束，记录 token 用量 if chunk.TokensUsed != nil { + tokenUsage = chunk.TokensUsed log.Infow("LLM 用量统计", "request_id", requestID, - "prompt_tokens", chunk.TokensUsed.Prompt, - "completion_tokens", chunk.TokensUsed.Completion, - "total_tokens", chunk.TokensUsed.Total, + "prompt_tokens", tokenUsage.Prompt, + "completion_tokens", tokenUsage.Completion, + "total_tokens", tokenUsage.Total, ) } break @@ -277,7 +326,7 @@ func (p *Pipeline) consumeLLMStream( // 刷新切分器中的剩余文本 splitter.Flush() - return fullText.String() + return fullText.String(), tokenUsage } // synthesizeTTS 从句子 channel 读取文本，进行 TTS 合成并发送音频。 @@ -290,10 +339,10 @@ func (p *Pipeline) synthesizeTTS( log := logger.Log ttsStream, err := p.ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{ - Voice: "alloy", - Speed: 1.0, - OutputFmt: "mp3", - SampleRate: 24000, + Voice: p.ttsVoice, + Speed: p.ttsSpeed, + OutputFmt: p.ttsOutputFmt, + SampleRate: p.ttsSampleRate, }) if err != nil { log.Errorw("TTS 合成启动失败", "error", err) @@ -319,6 +368,7 @@ func (p *Pipeline) synthesizeTTS( Audio: audioBase64, MimeType: "audio/mp3", IsLast: chunk.IsLast, + Final: chunk.Final, }); err != nil { log.Errorw("发送 tts_audio 失败", "error", err) } diff --git a/backend/internal/orchestrator/pipeline_test.go b/backend/internal/orchestrator/pipeline_test.go index 4a3f2a1..ad9b847 100644 --- a/backend/internal/orchestrator/pipeline_test.go +++ b/backend/internal/orchestrator/pipeline_test.go @@ -13,6 +13,7 @@ import ( "github.com/hhs/camtalk/internal/ai/llm" "github.com/hhs/camtalk/internal/ai/stt" "github.com/hhs/camtalk/internal/ai/tts" + "github.com/hhs/camtalk/internal/config" "github.com/hhs/camtalk/internal/logger" "github.com/hhs/camtalk/internal/models" ) @@ -254,7 +255,12 @@ func TestProcessQuery_Success(t *testing.T) { mockSender.On("SendTTSAudio", mock.Anything).Return(nil) // 创建 Pipeline - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ + AI: config.AIConfig{ + LLM: config.LLMConfig{Model: "gpt-4o"}, + TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, + }, + }) // 执行 ctx := context.Background() @@ -305,7 +311,12 @@ func TestProcessQuery_STTError(t *testing.T) { mockSender.On("SendError", mock.Anything).Return(nil) - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ + AI: config.AIConfig{ + LLM: config.LLMConfig{Model: "gpt-4o"}, + TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, + }, + }) ctx := context.Background() err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) @@ -357,7 +368,12 @@ func TestProcessQuery_LLMError(t *testing.T) { mockSender.On("SendError", mock.Anything).Return(nil) - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ + AI: config.AIConfig{ + LLM: config.LLMConfig{Model: "gpt-4o"}, + TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, + }, + }) ctx := context.Background() err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) @@ -415,7 +431,12 @@ func TestProcessQuery_TTSError(t *testing.T) { mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything). Return(nil, errors.New("TTS service unavailable")) - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ + AI: config.AIConfig{ + LLM: config.LLMConfig{Model: "gpt-4o"}, + TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, + }, + }) ctx := context.Background() err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) @@ -485,7 +506,12 @@ func TestProcessQuery_ContextCancelled(t *testing.T) { }() mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return((<-chan tts.Chunk)(ttsCh), nil) - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ + AI: config.AIConfig{ + LLM: config.LLMConfig{Model: "gpt-4o"}, + TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, + }, + }) // 创建可取消的上下文 ctx, cancel := context.WithCancel(context.Background()) @@ -545,7 +571,12 @@ func TestProcessQuery_DisabledTTS(t *testing.T) { mockSender.On("SendLLMChunk", mock.Anything).Return(nil) mockSender.On("SendLLMDone", mock.Anything).Return(nil) - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ + AI: config.AIConfig{ + LLM: config.LLMConfig{Model: "gpt-4o"}, + TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, + }, + }) ctx := context.Background() err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) @@ -617,7 +648,12 @@ func TestProcessQuery_InvalidAudio(t *testing.T) { mockSender.On("SendError", mock.Anything).Return(nil) - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ + AI: config.AIConfig{ + LLM: config.LLMConfig{Model: "gpt-4o"}, + TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, + }, + }) ctx := context.Background() err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) @@ -650,7 +686,12 @@ func TestProcessQuery_SessionNotFound(t *testing.T) { mockSender.On("SendError", mock.Anything).Return(nil) - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ + AI: config.AIConfig{ + LLM: config.LLMConfig{Model: "gpt-4o"}, + TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, + }, + }) ctx := context.Background() err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) diff --git a/backend/internal/ws/handler.go b/backend/internal/ws/handler.go index eb84af9..5aa2acd 100644 --- a/backend/internal/ws/handler.go +++ b/backend/internal/ws/handler.go @@ -10,6 +10,7 @@ import ( "github.com/gin-gonic/gin" "github.com/gorilla/websocket" + "github.com/hhs/camtalk/internal/config" "github.com/hhs/camtalk/internal/errors" "github.com/hhs/camtalk/internal/logger" "github.com/hhs/camtalk/internal/models" @@ -17,8 +18,23 @@ import ( "github.com/hhs/camtalk/internal/session" ) -var upgrader = websocket.Upgrader{ - CheckOrigin: func(r *http.Request) bool { return true }, // 开发阶段允许所有来源 +// newUpgrader 根据配置创建 WebSocket upgrader。 +func newUpgrader(cfg *config.Config) websocket.Upgrader { + allowedOrigins := cfg.Server.AllowedOrigins + return websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + if len(allowedOrigins) == 0 { + return true // 未配置则允许所有来源（开发模式） + } + origin := r.Header.Get("Origin") + for _, o := range allowedOrigins { + if o == origin || o == "*" { + return true + } + } + return false + }, + } } // Client 代表一个 WebSocket 客户端连接。 @@ -75,13 +91,21 @@ func (w *WSClient) SendError(err models.WsError) error { } // ServeWS 处理 WebSocket 升级请求。 -func ServeWS(sessionMgr session.Manager, orch orchestrator.Orchestrator) gin.HandlerFunc { +func ServeWS(sessionMgr session.Manager, orch orchestrator.Orchestrator, cfg *config.Config) gin.HandlerFunc { + upgrader := newUpgrader(cfg) + heartbeatInterval := time.Duration(cfg.Server.HeartbeatInterval) * time.Second + heartbeatTimeout := time.Duration(cfg.Server.HeartbeatTimeout) * time.Second + version := cfg.App.Version + + maxHistory := cfg.Session.MaxHistory + return func(c *gin.Context) { - serveWS(c, sessionMgr, orch) + serveWS(c, sessionMgr, orch, upgrader, heartbeatInterval, heartbeatTimeout, version, maxHistory) } } -func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orchestrator) { +func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orchestrator, + upgrader websocket.Upgrader, heartbeatInterval, heartbeatTimeout time.Duration, version string, maxHistory int) { conn, err := upgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { logger.Log.Errorw("websocket upgrade failed", "error", err) @@ -108,7 +132,7 @@ func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orche _ = client.SendJSON(models.WsConnected{ Type: "connected", SessionID: sessionID, - ServerVersion: "0.1.0", + ServerVersion: version, }) logger.Log.Infow("client connected", "session", sessionID) @@ -122,12 +146,12 @@ func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orche // 启动心跳检查 goroutine done := make(chan struct{}) go func() { - ticker := time.NewTicker(30 * time.Second) + ticker := time.NewTicker(heartbeatInterval) defer ticker.Stop() for { select { case <-ticker.C: - if time.Since(lastPong) > 60*time.Second { + if time.Since(lastPong) > heartbeatTimeout { logger.Log.Warnw("heartbeat timeout", "session", sessionID) conn.Close() return @@ -159,6 +183,7 @@ func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orche switch envelope.Type { case "ping": + lastPong = time.Now() // 刷新心跳计时器 _ = client.SendJSON(models.WsPong{Type: "pong"}) case "query": @@ -180,7 +205,7 @@ func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orche } // 获取对话历史 - history, _ := client.sessionMgr.GetHistory(context.Background(), sessionID, 20) + history, _ := client.sessionMgr.GetHistory(context.Background(), sessionID, maxHistory) // 创建可取消的 context ctx, cancel := context.WithCancel(context.Background()) diff --git a/backend/internal/ws/handler_test.go b/backend/internal/ws/handler_test.go index 97cf697..2872d5c 100644 --- a/backend/internal/ws/handler_test.go +++ b/backend/internal/ws/handler_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "context" + "github.com/hhs/camtalk/internal/config" "github.com/hhs/camtalk/internal/logger" "github.com/hhs/camtalk/internal/models" "github.com/hhs/camtalk/internal/orchestrator" @@ -138,7 +139,12 @@ func setupTestServer(t *testing.T, orch orchestrator.Orchestrator) (*httptest.Se t.Cleanup(func() { sessionMgr.Stop() }) r := gin.New() - r.GET("/ws", ServeWS(sessionMgr, orch)) + cfg := &config.Config{ + App: config.AppConfig{Version: "test"}, + Server: config.ServerConfig{HeartbeatInterval: 30, HeartbeatTimeout: 60}, + Session: config.SessionConfig{MaxHistory: 20}, + } + r.GET("/ws", ServeWS(sessionMgr, orch, cfg)) srv := httptest.NewServer(r) @@ -181,7 +187,7 @@ func TestWS_Connected(t *testing.T) { msg := readJSON(t, conn) assert.Equal(t, "connected", msg["type"]) assert.NotEmpty(t, msg["session_id"]) - assert.Equal(t, "0.1.0", msg["server_version"]) + assert.Equal(t, "test", msg["server_version"]) } // TestWS_PingPong 验证 ping/pong 心跳。 diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..ebf1858 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +PROJECT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$PROJECT_DIR" + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +info() { echo -e "${GREEN}[INFO]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } + +# 确保 .env 存在 +ensure_env() { + if [ ! -f .env ]; then + if [ -f .env.example ]; then + warn ".env 文件不存在，从 .env.example 复制模板" + cp .env.example .env + warn "请编辑 .env 填入实际的 API Key，然后重新运行" + exit 1 + else + error ".env.example 也不存在，请手动创建 .env" + exit 1 + fi + fi +} + +cmd_build() { + info "构建 Docker 镜像..." + docker compose build + info "构建完成" +} + +cmd_up() { + ensure_env + info "启动服务..." + docker compose up -d + info "服务已启动" + info "前端: http://localhost" + info "健康检查: http://localhost/api/health" +} + +cmd_down() { + info "停止服务..." + docker compose down + info "服务已停止" +} + +cmd_restart() { + info "重启服务..." + cmd_down + cmd_up +} + +cmd_logs() { + docker compose logs -f "${@}" +} + +cmd_status() { + docker compose ps +} + +usage() { + cat < + +命令: + build 构建 Docker 镜像 + up 启动服务（后台运行） + down 停止服务 + restart 重启服务 + logs 查看日志（可加服务名，如: $0 logs backend） + status 查看服务状态 +EOF +} + +case "${1:-}" in + build) cmd_build ;; + up) cmd_up ;; + down) cmd_down ;; + restart) cmd_restart ;; + logs) shift; cmd_logs "$@" ;; + status) cmd_status ;; + *) usage; exit 1 ;; +esac diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d331f9d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +services: + frontend: + build: + context: ./frontend + dockerfile: Dockerfile + container_name: camtalk-frontend + ports: + - "80:80" + depends_on: + - backend + networks: + - camtalk-net + restart: unless-stopped + + backend: + build: + context: ./backend + dockerfile: Dockerfile + container_name: camtalk-backend + env_file: + - .env + environment: + - APP_ENV=production + networks: + - camtalk-net + restart: unless-stopped + +networks: + camtalk-net: + driver: bridge diff --git a/docs/02-系统架构.md b/docs/02-系统架构.md index 41f2a33..0a2b316 100644 --- a/docs/02-系统架构.md +++ b/docs/02-系统架构.md @@ -9,7 +9,7 @@ | 层级 | 职责 | 关键约束 | |------|------|---------| | **客户端（浏览器）** | 媒体采集、边缘预处理、UI 渲染 | 浏览器资源有限，模型需轻量 | -| **Go 网关** | 会话管理、模型路由、AI 服务编排 | 高并发、低延迟、状态管理 | +| **Go 网关** | 会话管理、AI 服务编排、流式管道 | 高并发、低延迟、状态管理 | | **AI 服务** | LLM 推理、语音识别、语音合成 | 按量计费，需控制调用频率 | > 为什么要单独加一层 Go 网关，而不是让前端直连 AI API？1）API Key 安全性；2）统一的速率限制和成本管控；3）多模型路由逻辑集中在一处便于维护。 @@ -22,7 +22,7 @@ |------|------|---------| | 框架 | React 18 + TypeScript | 组件化开发，类型安全，生态成熟 | | 构建 | Vite | 开发热更新快，构建产物小 | -| 实时通信 | WebSocket（原生 API） | 浏览器原生支持，无需额外依赖 | +| 实时通信 | WebSocket（原生 API） + 自封装连接管理 | 浏览器原生支持，封装心跳/重连/消息分发 | | 边缘推理 | ONNX Runtime Web | 浏览器端跑轻量模型（VAD、关键帧检测） | | 语音检测 | @ricky0123/vad-web | 基于 WebRTC VAD，纯前端零延迟 | | 媒体采集 | MediaDevices API | 浏览器原生摄像头/麦克风访问 | @@ -32,22 +32,22 @@ | 技术 | 选型 | 选择理由 | |------|------|---------| | 语言 | Go | 高并发 goroutine 模型，适合长连接管理 | +| HTTP 框架 | Gin | 高性能 HTTP 路由，中间件生态成熟 | | WebSocket | gorilla/websocket | Go 生态最成熟的 WebSocket 库 | -| 会话存储 | Redis | 高速 KV 存储，适合会话状态和上下文缓存 | -| 持久化存储 | PostgreSQL | 对话历史、用量统计、用户偏好（MVP 阶段可选） | -| 配置管理 | Viper | 支持 YAML + 环境变量覆盖，详见 `03-接口文档.md` 第六章 | +| 会话存储 | Redis（规划中） / Memory（MVP 默认） | 高速 KV 存储，MVP 阶段使用进程内存，可通过配置切换到 Redis | +| 持久化存储 | PostgreSQL（规划中） | 对话历史、用量统计、用户偏好（MVP 阶段未实现） | +| 配置管理 | Viper + godotenv | 支持 YAML + .env + 环境变量覆盖，详见 `03-接口文档.md` 第六章 | | 日志 | Zap | 高性能结构化日志 | ### AI 服务 | 能力 | 主选方案 | 备选方案 | 选型考量 | |------|---------|---------|---------| -| 多模态 LLM | GPT-4o | Claude Sonnet | 视觉理解能力强，API 成熟 | -| 语音识别 STT | Deepgram | FunASR 自部署 | 流式识别延迟低（<500ms） | -| 语音合成 TTS | OpenAI TTS | Edge TTS（免费） | 音质自然，支持流式 | -| 轻量分类 | GPT-4o-mini | Haiku | 模型路由时的复杂度判断 | +| 多模态 LLM | GPT-4o（默认） | 通义千问等 OpenAI 兼容模型 | 通过 OpenAI 兼容接口，可灵活切换 | +| 语音识别 STT | Deepgram（默认） | MiMo ASR（小米） | 支持多 provider 切换 | +| 语音合成 TTS | OpenAI TTS（默认） | MiMo TTS（小米） | 支持多 provider 切换 | -> 不必绑定单一厂商。Go 网关的模型路由层统一封装不同 AI 服务的调用接口，按场景动态切换。 +> 不必绑定单一厂商。Go 网关的 AI 服务层统一封装不同服务商的调用接口，通过配置切换 provider。 ## 核心交互流程 @@ -78,52 +78,35 @@ Browser Go Gateway STT LLM TTS | 模块 | 职责 | 关键实现 | |------|------|---------| -| WebSocket Hub | 管理所有客户端连接，广播/定向推送 | goroutine per connection | -| Session Manager | 维护用户会话状态、对话历史 | Redis Hash + List，30 分钟 TTL（详见 `03-接口文档.md` 第五章） | -| Model Router | 根据请求类型选择 AI 模型 | 规则引擎 + 成本阈值 | -| AI Orchestrator | 编排多路 AI 调用（并行/串行） | context 取消 + 超时控制 | -| Rate Limiter | 防止单用户过度消耗 API 额度 | 令牌桶算法 | +| WebSocket Handler | 管理客户端连接生命周期，单播消息推送 | goroutine per connection | +| Session Manager | 维护用户会话状态、对话历史 | Memory（MVP 默认）/ Redis（可切换），30 分钟 TTL（详见 `03-接口文档.md` 第五章） | +| AI Orchestrator | 编排 STT→LLM→TTS 流式并行管道 | context 取消 + 超时控制 + 句子切分 | +| AI Service Layer | AI 服务抽象层（STT/LLM/TTS） | 多 provider 支持（Deepgram/MiMo/OpenAI 等） | +| REST API | 健康检查、会话管理端点 | Gin 路由 | +| Error Handler | 统一错误码定义与发送 | 错误码枚举 | +| Logger | 日志初始化封装 | Zap 结构化日志 | +| Models | 数据模型定义 | WebSocket 消息、会话、配置等 | +| Model Router | 根据请求类型选择 AI 模型（规划中） | 规则引擎 + 成本阈值 | +| Rate Limiter | 防止单用户过度消耗 API 额度（规划中） | 令牌桶算法 | -AI Orchestrator 核心代码（句子级流式并行）： +AI Orchestrator 核心接口（`internal/orchestrator/orchestrator.go`）： ```go -func (o *Orchestrator) ProcessQuery(ctx context.Context, client MessageSender, req *QueryRequest) { - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - // Step 1: STT — 识别用户语音（串行） - text, err := o.stt.Recognize(ctx, req.Audio, STTOptions{...}) - if err != nil { - client.SendError(req.RequestID, "STT_ERROR", err.Error()) - return - } - client.SendSTTResult(req.RequestID, text, true) - - // Step 2: LLM 流式输出 + 句子切分（并行） - llmStream, _ := o.llm.ChatStream(ctx, LLMRequest{Image: req.Image, Text: text, ...}) - sentenceCh := make(chan string, 4) - go func() { - defer close(sentenceCh) - var buf strings.Builder - for chunk := range llmStream { - client.SendLLMChunk(req.RequestID, chunk.Delta) // 逐 token 推送文字 - buf.WriteString(chunk.Delta) - if isSentenceEnd(chunk.Delta) { // 按 。！？\n 切分 - sentenceCh <- buf.String() - buf.Reset() - } - } - if buf.Len() > 0 { sentenceCh <- buf.String() } - }() - - // Step 3: TTS 并行消费句子流 - ttsStream, _ := o.tts.SynthesizeStream(ctx, sentenceCh, TTSOptions{...}) - for chunk := range ttsStream { - client.SendTTSAudio(req.RequestID, chunk.Audio, chunk.IsLast) - } +// Orchestrator AI 编排器接口。 +type Orchestrator interface { + ProcessQuery(ctx context.Context, sessionID string, req models.WsQuery, + history []models.Message, sender Sender) error } ``` +Pipeline 实现（`internal/orchestrator/pipeline.go`）流程： +1. Base64 解码音频/图片 +2. 调用 `stt.Recognize()` → 发送 `stt_result` +3. 调用 `llm.ChatStream()` 获取流式输出，goroutine 消费 token → 发送 `llm_chunk` + 句子切分 +4. 另一 goroutine 从句子 channel 读取 → 调用 `tts.SynthesizeStream()` → 发送 `tts_audio` +5. 流结束 → 发送 `llm_done` +6. TTS 失败静默跳过，STT/LLM 失败发送对应 error 消息 + > **关键优化**：LLM 文本流和 TTS 音频流**并行推送**——客户端先逐 token 展示文字，同时 TTS 逐句子合成并推送音频，用户感知延迟大幅降低。详细的 AI 服务层接口和编排策略见 `03-接口文档.md` 第三、四章。 ## 前端组件 @@ -132,17 +115,19 @@ func (o *Orchestrator) ProcessQuery(ctx context.Context, client MessageSender, r |------|------| | CameraManager | 摄像头流采集 | | MicManager | 麦克风音频采集 | -| EdgeProcessor | VAD + 关键帧检测（ONNX Runtime） | +| EdgeProcessor | VAD + 关键帧检测（Canvas 像素比较） | | WebSocketManager | WS 连接生命周期管理 | | ChatPanel | 消息展示 | | VideoPreview | 摄像头画面预览 | +| ConfigPanel | 右侧抽屉式配置面板（主题、TTS 开关、detail level、语言） | +| Toast | 轻量通知提示（3 秒自动消失） | 核心 Hook：`useVisionSession()` 封装一次完整的视觉对话会话（摄像头、VAD、WebSocket、消息状态）。 ```typescript function useVisionSession() { const [messages, setMessages] = useState([]); - const wsRef = useWebSocket("ws://localhost:8080/ws"); + const wsRef = useWebSocket(`${window.location.protocol === "https:" ? "wss:" : "ws:"}//${window.location.host}/ws`); const videoRef = useRef(null); const { captureFrame } = useCamera(videoRef); @@ -173,7 +158,7 @@ function useVisionSession() { | 阶段 | 存储方案 | 持久化内容 | 理由 | |------|---------|-----------|------| -| MVP | Redis only | 无 | 快速验证核心功能，重启丢数据可接受 | +| MVP | Memory（进程内） | 无 | 快速验证核心功能，重启丢数据可接受。Redis 实现已就绪，可通过 `storage.driver` 配置切换 | | 上线 | Redis + PostgreSQL | 对话历史、用户偏好、用量统计 | 用户需要查看历史，运营需要成本数据 | | 规模化 | Redis + PG + 对象存储 | 图像帧、音频片段归档 | 大文件不适合存关系库 | @@ -262,24 +247,8 @@ server { > WebSocket 是长连接，Nginx 必须配置 `Upgrade` 和 `Connection` 头。`proxy_read_timeout` 需要覆盖心跳间隔（客户端 30s ping），否则 Nginx 会主动断开空闲连接。 -### 开发环境（Vite proxy） +### 开发环境 -开发时前端（Vite :5173）和后端（Gin :8080）不同端口，用 Vite 内置代理解决跨域： +开发时前端（Vite :5173）和后端（Gin :8080）不同端口。前端 WebSocket 地址基于 `window.location.host` 动态构建，通过 Vite `server.proxy` 转发到后端，无需硬编码端口。 -```typescript -// frontend/vite.config.ts -export default defineConfig({ - plugins: [react()], - server: { - proxy: { - "/api": "http://localhost:8080", - "/ws": { - target: "ws://localhost:8080", - ws: true, - }, - }, - }, -}); -``` - -前端代码中 WebSocket 地址改为相对路径 `ws://localhost:5173/ws`，Vite 自动代理到后端。部署时 Nginx 同理，前端无需区分开发/生产地址。 +`vite.config.ts` 中配置了 `/ws`（WebSocket）和 `/api`（REST）的代理，目标为 `http://localhost:8080`。 diff --git a/docs/03-接口文档.md b/docs/03-接口文档.md index f1b7e1c..f26a79a 100644 --- a/docs/03-接口文档.md +++ b/docs/03-接口文档.md @@ -41,19 +41,22 @@ interface WsMessage { #### `query` — 发起一次视觉对话 -用户说完话后，客户端同时发送当前图像帧和语音片段： +用户说完话后，客户端同时发送当前图像帧和语音片段。也支持文本输入模式（手动输入文字时跳过语音识别）： ```typescript interface QueryMessage { type: "query"; request_id: string; // 客户端生成的 UUID image: string; // Base64 编码的 JPEG 图像（不含 data: 前缀） - audio: string; // Base64 编码的音频片段（PCM 16kHz） + audio: string; // Base64 编码的音频片段（PCM 16kHz），文本输入时为空字符串 + text?: string; // 用户手动输入的文本（有值时跳过 STT，直接使用此文本） mime_type?: string; // 音频格式，默认 "audio/pcm" } ``` > 为什么图像和音频放在同一条消息里？因为 VAD 检测到用户说完话时，需要同时捕获"此刻的画面"和"说的话"，拆成两条消息会增加时序同步的复杂度。 +> +> **文本输入模式**：当用户关闭麦克风后，可通过对话框手动输入文字。此时 `text` 字段携带用户输入，`audio` 为空字符串，服务端跳过 STT 直接使用 `text` 进行 LLM 推理。 #### `config` — 更新会话配置 @@ -73,7 +76,7 @@ interface ConfigMessage { ```typescript interface InterruptMessage { type: "interrupt"; - request_id?: string; // 可选，指定打断哪次请求 + request_id?: string; // 可选，当前实现不使用此字段，服务端始终取消当前活跃请求 } ``` @@ -143,16 +146,22 @@ interface TTSAudioMessage { type: "tts_audio"; request_id: string; audio: string; // Base64 编码的音频片段 - mime_type: string; // "audio/mpeg" - is_last: boolean; // 是否为最后一片 + mime_type: string; // "audio/mp3" + is_last: boolean; // 当前句子的音频是否完整（每句结束时为 true） + final: boolean; // 整轮 TTS 是否结束（所有句子合成完毕后为 true） } ``` +**字段语义**： + +- `is_last`: 每个句子合成完毕后为 `true`，前端收到此信号即可将该句子加入播放队列。每句 TTS 音频由一次独立的 API 调用生成，对应一个 `tts_audio` 消息。 +- `final`: 所有句子合成完毕后为 `true`（此时 `audio` 为空字符串），用于前端判断本轮 TTS 已全部到齐。 + **音频格式规范**（前端播放依赖此约定）： | 属性 | 值 | 说明 | |------|------|------| -| 编码 | `audio/mpeg`（MP3） | 浏览器 `