From e967d89e7eb1df2ec97b2eb3e906cfc008551976 Mon Sep 17 00:00:00 2001 From: hhs <386998068@qq.com> Date: Sun, 14 Jun 2026 10:05:56 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20MiMo=20TTS=20?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E5=90=88=E6=88=90=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 实现基于 Xiaomi MiMo TTS API 的语音合成 provider，使用冰糖音色。 - 新增 MiMoService 实现 tts.Service 接口 - 使用 chat/completions 格式，与 MiMo STT 保持一致 - 在 main.go 添加 TTS provider 切换逻辑（mimo/xiaomi） - 配置文件默认音色设为冰糖 - 包含完整测试覆盖（9 个测试用例） --- backend/cmd/server/main.go | 8 +- backend/config.yaml | 4 +- backend/internal/ai/tts/mimo.go | 212 ++++++++++++++ backend/internal/ai/tts/mimo_test.go | 405 +++++++++++++++++++++++++++ 4 files changed, 626 insertions(+), 3 deletions(-) create mode 100644 backend/internal/ai/tts/mimo.go create mode 100644 backend/internal/ai/tts/mimo_test.go diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 5840f87..a84b701 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -55,7 +55,13 @@ func main() { sttService = stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, logger.Log) } llmService := llm.NewOpenAIService(cfg.AI.LLM.APIKey, cfg.AI.LLM.Model, cfg.AI.LLM.Endpoint, cfg.AI.LLM.Timeout, logger.Log) - ttsService := tts.NewOpenAIService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Speed, cfg.AI.TTS.Timeout, logger.Log) + var ttsService tts.Service + switch strings.ToLower(cfg.AI.TTS.Provider) { + case "mimo", "xiaomi": + ttsService = tts.NewMiMoService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Timeout, logger.Log) + default: + ttsService = tts.NewOpenAIService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Speed, cfg.AI.TTS.Timeout, logger.Log) + } // 初始化 Orchestrator orch := orchestrator.New(sttService, llmService, ttsService, sessionMgr, cfg.AI.LLM.Model) diff --git a/backend/config.yaml b/backend/config.yaml index a998a33..d61dc4b 100644 --- a/backend/config.yaml +++ b/backend/config.yaml @@ -26,9 +26,9 @@ ai: api_key: "sk-ws-H.REHELLY.C4s3.MEUCIQCRee37XWEKp2szaxVLFDtR1rxNNsf372zMvCR0Xl6UvQIgZgvhRTvaa1FmhbCQJgaHu4Jny29AQkn01-3hX9CWBOg" timeout: 30 tts: - provider: Xiaomi MiMo + provider: mimo model: mimo-v2.5-tts - voice: alloy + voice: 冰糖 speed: 1.0 endpoint: "https://token-plan-cn.xiaomimimo.com/v1" api_key: "tp-c9e7scwfx94qvqyhpnahnw8uaiya01za2qzvg4xe24rp3xiv" diff --git a/backend/internal/ai/tts/mimo.go b/backend/internal/ai/tts/mimo.go new file mode 100644 index 0000000..208d301 --- /dev/null +++ b/backend/internal/ai/tts/mimo.go @@ -0,0 +1,212 @@ +package tts + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "go.uber.org/zap" +) + +// MiMoService 基于 Xiaomi MiMo TTS API 的语音合成实现。 +// 接口兼容 OpenAI chat/completions 格式，通过 messages 传递待合成文本与风格指令。 +type MiMoService struct { + apiKey string + model string + voice string + endpoint string + timeout time.Duration + logger *zap.SugaredLogger + client *http.Client +} + +// NewMiMoService 创建 MiMo TTS 服务。 +func NewMiMoService(apiKey, model, voice, endpoint string, timeoutSec int, logger *zap.SugaredLogger) *MiMoService { + if model == "" { + model = "mimo-v2.5-tts" + } + if voice == "" { + voice = "冰糖" + } + if endpoint == "" { + endpoint = "https://api.xiaomimimo.com/v1" + } + timeout := time.Duration(timeoutSec) * time.Second + if timeout <= 0 { + timeout = 5 * time.Second + } + return &MiMoService{ + apiKey: apiKey, + model: model, + voice: voice, + endpoint: endpoint, + timeout: timeout, + logger: logger, + client: &http.Client{Timeout: 30 * time.Second}, + } +} + +// mimoTTSRequest MiMo TTS API 请求体。 +type mimoTTSRequest struct { + Model string `json:"model"` + Messages []mimoTTSMessage `json:"messages"` + Audio mimoTTSAudio `json:"audio"` + Stream bool `json:"stream"` +} + +// mimoTTSMessage MiMo TTS 消息。 +type mimoTTSMessage struct { + Role string `json:"role"` // "user"（风格指令）| "assistant"（待合成文本） + Content string `json:"content"` +} + +// mimoTTSAudio MiMo TTS 音频配置。 +type mimoTTSAudio struct { + Format string `json:"format"` // "mp3" | "wav" | "pcm16" + Voice string `json:"voice"` // 预置音色 ID +} + +// mimoTTSResponse MiMo TTS 非流式响应。 +type mimoTTSResponse struct { + Choices []struct { + Message struct { + Audio struct { + Data string `json:"data"` // base64 编码的音频数据 + } `json:"audio"` + } `json:"message"` + } `json:"choices"` +} + +// mimoTTSStreamResponse MiMo TTS 流式响应。 +type mimoTTSStreamResponse struct { + Choices []struct { + Delta struct { + Audio struct { + Data string `json:"data"` // base64 编码的音频数据片段 + } `json:"audio"` + } `json:"delta"` + } `json:"choices"` +} + +// SynthesizeStream 实现 tts.Service。从 textStream 读取句子，逐句调用 MiMo TTS API。 +func (m *MiMoService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error) { + voice := opts.Voice + if voice == "" { + voice = m.voice + } + + ch := make(chan Chunk, 4) + go func() { + defer close(ch) + + for text := range textStream { + if text == "" { + continue + } + + audio, err := m.synthesize(ctx, text, voice) + if err != nil { + m.logger.Warnw("mimo tts: synthesize failed", "error", err, "text", text) + // 静默跳过，不中断整个流 + continue + } + + select { + case ch <- Chunk{Audio: audio, IsLast: false}: + case <-ctx.Done(): + return + } + } + + // textStream 关闭，发送 IsLast 标记 + select { + case ch <- Chunk{Audio: nil, IsLast: true}: + case <-ctx.Done(): + } + }() + + return ch, nil +} + +// synthesize 调用 MiMo TTS API 合成单个句子。 +// 使用非流式调用，返回完整音频数据（base64 解码后）。 +func (m *MiMoService) synthesize(ctx context.Context, text, voice string) ([]byte, error) { + // 单句超时 + ctx, cancel := context.WithTimeout(ctx, m.timeout) + defer cancel() + + // 构建 MiMo TTS 请求：文本放在 assistant 消息中 + reqBody := mimoTTSRequest{ + Model: m.model, + Messages: []mimoTTSMessage{ + { + Role: "assistant", + Content: text, + }, + }, + Audio: mimoTTSAudio{ + Format: "mp3", + Voice: voice, + }, + Stream: false, + } + + payload, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("mimo tts: marshal request: %w", err) + } + + url := strings.TrimRight(m.endpoint, "/") + "/chat/completions" + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return nil, fmt.Errorf("mimo tts: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("api-key", m.apiKey) + + resp, err := m.client.Do(req) + if err != nil { + return nil, fmt.Errorf("mimo tts: send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("mimo tts: api error (status %d): %s", resp.StatusCode, string(errBody)) + } + + // 非流式响应：解析 JSON，提取 base64 音频数据 + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("mimo tts: read response: %w", err) + } + + var ttsResp mimoTTSResponse + if err := json.Unmarshal(respBody, &ttsResp); err != nil { + return nil, fmt.Errorf("mimo tts: unmarshal response: %w", err) + } + + if len(ttsResp.Choices) == 0 { + return nil, fmt.Errorf("mimo tts: empty choices in response") + } + + audioData := ttsResp.Choices[0].Message.Audio.Data + if audioData == "" { + return nil, fmt.Errorf("mimo tts: empty audio data in response") + } + + // base64 解码音频数据 + audio, err := base64.StdEncoding.DecodeString(audioData) + if err != nil { + return nil, fmt.Errorf("mimo tts: decode audio base64: %w", err) + } + + return audio, nil +} diff --git a/backend/internal/ai/tts/mimo_test.go b/backend/internal/ai/tts/mimo_test.go new file mode 100644 index 0000000..0e419bf --- /dev/null +++ b/backend/internal/ai/tts/mimo_test.go @@ -0,0 +1,405 @@ +package tts + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "go.uber.org/zap" +) + +// mockMiMoTTSServer 创建模拟 MiMo TTS API 的 HTTP 服务器。 +func mockMiMoTTSServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + return httptest.NewServer(handler) +} + +// buildMiMoTTSResponse 构造 MiMo TTS 非流式响应 JSON。 +func buildMiMoTTSResponse(audioData string) []byte { + resp := mimoTTSResponse{ + Choices: []struct { + Message struct { + Audio struct { + Data string `json:"data"` + } `json:"audio"` + } `json:"message"` + }{ + { + Message: struct { + Audio struct { + Data string `json:"data"` + } `json:"audio"` + }{ + Audio: struct { + Data string `json:"data"` + }{Data: audioData}, + }, + }, + }, + } + data, _ := json.Marshal(resp) + return data +} + +func TestMiMoService_SynthesizeStream_Success(t *testing.T) { + var callCount int32 + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&callCount, 1) + + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + if !strings.Contains(r.URL.Path, "/chat/completions") { + t.Errorf("path = %s, should contain /chat/completions", r.URL.Path) + } + + // 验证 api-key 认证头 + apiKey := r.Header.Get("api-key") + if apiKey != "test-key" { + t.Errorf("api-key = %q, want %q", apiKey, "test-key") + } + + // 验证请求体 + body, _ := io.ReadAll(r.Body) + var req mimoTTSRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Errorf("unmarshal request: %v", err) + } + if req.Model != "mimo-v2.5-tts" { + t.Errorf("model = %q, want %q", req.Model, "mimo-v2.5-tts") + } + if len(req.Messages) != 1 || req.Messages[0].Role != "assistant" { + t.Errorf("expected 1 assistant message, got %d messages", len(req.Messages)) + } + if req.Audio.Voice != "冰糖" { + t.Errorf("voice = %q, want %q", req.Audio.Voice, "冰糖") + } + if req.Audio.Format != "mp3" { + t.Errorf("format = %q, want %q", req.Audio.Format, "mp3") + } + + // 返回假音频数据（base64 编码） + audioB64 := base64.StdEncoding.EncodeToString([]byte("fake-mp3-data")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "", "冰糖", srv.URL, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("你好", "世界", "！") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{ + Voice: "冰糖", OutputFmt: "mp3", SampleRate: 24000, + }) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 应该有 3 个音频 chunk + 1 个 IsLast 标记 + if len(chunks) != 4 { + t.Fatalf("got %d chunks, want 4", len(chunks)) + } + + // 验证前 3 个有音频数据 + for i := 0; i < 3; i++ { + if string(chunks[i].Audio) != "fake-mp3-data" { + t.Errorf("chunk[%d].Audio = %q, want %q", i, string(chunks[i].Audio), "fake-mp3-data") + } + if chunks[i].IsLast { + t.Errorf("chunk[%d].IsLast should be false", i) + } + } + + // 验证最后一个是 IsLast + if !chunks[3].IsLast { + t.Error("last chunk should be IsLast") + } + if chunks[3].Audio != nil { + t.Error("last chunk Audio should be nil") + } + + // 验证调用了 3 次 API（3 个句子） + if atomic.LoadInt32(&callCount) != 3 { + t.Errorf("API called %d times, want 3", callCount) + } +} + +func TestMiMoService_SynthesizeStream_APIError(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "internal error") + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "", "冰糖", srv.URL, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + // 应该只有一个 IsLast chunk（音频被跳过） + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1 (IsLast only)", len(chunks)) + } + if !chunks[0].IsLast { + t.Error("chunk should be IsLast") + } +} + +func TestMiMoService_SynthesizeStream_Timeout(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(3 * time.Second) + audioB64 := base64.StdEncoding.EncodeToString([]byte("late-mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + // 1 秒超时 + svc := NewMiMoService("test-key", "", "冰糖", srv.URL, 1, zap.NewNop().Sugar()) + + textStream := sendSentences("很长的句子") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ch, err := svc.SynthesizeStream(ctx, textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 超时后音频被跳过，只有 IsLast + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1", len(chunks)) + } + if !chunks[0].IsLast { + t.Error("chunk should be IsLast") + } +} + +func TestMiMoService_SynthesizeStream_EmptyText(t *testing.T) { + var callCount int32 + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&callCount, 1) + audioB64 := base64.StdEncoding.EncodeToString([]byte("mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "", "冰糖", srv.URL, 5, zap.NewNop().Sugar()) + + // 空句子应该被跳过 + textStream := sendSentences("", "你好", "") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 只有 "你好" 应该被合成 + if atomic.LoadInt32(&callCount) != 1 { + t.Errorf("API called %d times, want 1", callCount) + } + + // 1 个音频 + 1 个 IsLast + if len(chunks) != 2 { + t.Fatalf("got %d chunks, want 2", len(chunks)) + } +} + +func TestMiMoService_SynthesizeStream_ContextCancelled(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + audioB64 := base64.StdEncoding.EncodeToString([]byte("mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "", "冰糖", srv.URL, 5, zap.NewNop().Sugar()) + + textStream := make(chan string, 3) + textStream <- "第一句" + textStream <- "第二句" + textStream <- "第三句" + close(textStream) + + ctx, cancel := context.WithCancel(context.Background()) + // 立即取消 + cancel() + + ch, err := svc.SynthesizeStream(ctx, textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + // 消费 channel，应该很快结束 + var count int + for range ch { + count++ + } + // 可能收到 0 个或 1 个 chunk，取决于时序 + t.Logf("received %d chunks after context cancel", count) +} + +func TestMiMoService_SynthesizeStream_PartialFailure(t *testing.T) { + var callCount int32 + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&callCount, 1) + if n == 2 { + // 第二个句子失败 + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "error") + return + } + audioB64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("mp3-%d", n))) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "", "冰糖", srv.URL, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("第一句", "第二句", "第三句") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 2 个成功音频 + 1 个 IsLast（第二句被跳过） + if len(chunks) != 3 { + t.Fatalf("got %d chunks, want 3", len(chunks)) + } + if !chunks[len(chunks)-1].IsLast { + t.Error("last chunk should be IsLast") + } +} + +func TestMiMoService_SynthesizeStream_CustomVoice(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var req mimoTTSRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Errorf("unmarshal request: %v", err) + } + if req.Audio.Voice != "茉莉" { + t.Errorf("voice = %q, want %q", req.Audio.Voice, "茉莉") + } + audioB64 := base64.StdEncoding.EncodeToString([]byte("mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "", "冰糖", srv.URL, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{Voice: "茉莉"}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + for range ch { + } +} + +func TestMiMoService_SynthesizeStream_DefaultVoice(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var req mimoTTSRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Errorf("unmarshal request: %v", err) + } + // 未指定 voice 时应使用默认 "冰糖" + if req.Audio.Voice != "冰糖" { + t.Errorf("voice = %q, want %q (default)", req.Audio.Voice, "冰糖") + } + audioB64 := base64.StdEncoding.EncodeToString([]byte("mp3")) + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse(audioB64)) + }) + defer srv.Close() + + // 不指定 voice + svc := NewMiMoService("test-key", "", "", srv.URL, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + for range ch { + } +} + +func TestMiMoService_SynthesizeStream_EmptyAudioData(t *testing.T) { + srv := mockMiMoTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + // 返回空音频数据 + w.Header().Set("Content-Type", "application/json") + w.Write(buildMiMoTTSResponse("")) + }) + defer srv.Close() + + svc := NewMiMoService("test-key", "", "冰糖", srv.URL, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 空音频数据导致错误，句子被跳过，只有 IsLast + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1", len(chunks)) + } + if !chunks[0].IsLast { + t.Error("chunk should be IsLast") + } +} -- 2.49.1