部署 #117
21
.env.example
Normal file
21
.env.example
Normal file
@@ -0,0 +1,21 @@
|
||||
# CamTalk 环境变量模板
|
||||
# 复制为 .env 并填入实际值:cp .env.example .env
|
||||
# .env 已在 .gitignore 中,不会提交到版本控制
|
||||
|
||||
# ---- AI 服务 API Key ----
|
||||
CAMTALK_AI_LLM_API_KEY=sk-xxx
|
||||
CAMTALK_AI_STT_API_KEY=
|
||||
CAMTALK_AI_TTS_API_KEY=
|
||||
|
||||
# ---- 可选覆盖(默认值见 config.yaml)----
|
||||
# CAMTALK_AI_LLM_MODEL=gpt-4o
|
||||
# CAMTALK_AI_LLM_ENDPOINT=https://api.openai.com/v1
|
||||
# CAMTALK_AI_LLM_TIMEOUT=10
|
||||
# CAMTALK_AI_STT_ENDPOINT=wss://api.deepgram.com/v1/listen
|
||||
# CAMTALK_AI_TTS_ENDPOINT=https://api.openai.com/v1
|
||||
# CAMTALK_AI_TTS_VOICE=alloy
|
||||
# CAMTALK_AI_TTS_SPEED=1.0
|
||||
# CAMTALK_AI_TTS_TIMEOUT=5
|
||||
|
||||
# ---- 应用 ----
|
||||
# APP_ENV=dev
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -4,6 +4,7 @@ frontend/dist/
|
||||
|
||||
# ---- 后端 ----
|
||||
backend/bin/
|
||||
backend/server
|
||||
|
||||
# ---- 环境变量 ----
|
||||
.env
|
||||
@@ -18,5 +19,11 @@ Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# ---- Playwright MCP ----
|
||||
.playwright-mcp/
|
||||
|
||||
# ---- 截图 ----
|
||||
*.png
|
||||
|
||||
# ---- Obsidian ----
|
||||
.obsidian/
|
||||
|
||||
17
backend/.gitignore
vendored
Normal file
17
backend/.gitignore
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
# 编译产物
|
||||
/server
|
||||
bin/
|
||||
|
||||
# 环境配置
|
||||
.env
|
||||
config.dev.yaml
|
||||
config.prod.yaml
|
||||
|
||||
# 临时文件
|
||||
tmp/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
@@ -1,40 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/hhs/camtalk/internal/api"
|
||||
"github.com/hhs/camtalk/internal/ai/llm"
|
||||
"github.com/hhs/camtalk/internal/ai/stt"
|
||||
"github.com/hhs/camtalk/internal/ai/tts"
|
||||
"github.com/hhs/camtalk/internal/config"
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/orchestrator"
|
||||
"github.com/hhs/camtalk/internal/session"
|
||||
"github.com/hhs/camtalk/internal/ws"
|
||||
)
|
||||
|
||||
var startTime = time.Now()
|
||||
|
||||
func main() {
|
||||
r := gin.Default()
|
||||
// 加载配置
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
panic("failed to load config: " + err.Error())
|
||||
}
|
||||
|
||||
// 初始化日志
|
||||
logger.Init(cfg.Log.Level, cfg.Log.Format)
|
||||
defer logger.Sync()
|
||||
|
||||
logger.Log.Infow("config loaded",
|
||||
"env", cfg.App.Env,
|
||||
"addr", cfg.Server.Addr(),
|
||||
)
|
||||
|
||||
// 初始化 Session Manager(MVP 默认内存实现)
|
||||
var sessionMgr session.Manager
|
||||
// TODO: 当 Redis 配置非空时切换为 RedisManager
|
||||
sessionMgr = session.NewMemoryManager(30*time.Minute, 20)
|
||||
defer sessionMgr.(*session.MemoryManager).Stop()
|
||||
|
||||
// 初始化 AI 服务
|
||||
sttService := stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, logger.Log)
|
||||
llmService := llm.NewOpenAIService(cfg.AI.LLM.APIKey, cfg.AI.LLM.Model, cfg.AI.LLM.Endpoint, cfg.AI.LLM.Timeout, logger.Log)
|
||||
ttsService := tts.NewOpenAIService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Speed, cfg.AI.TTS.Timeout, logger.Log)
|
||||
|
||||
// 初始化 Orchestrator
|
||||
orch := orchestrator.New(sttService, llmService, ttsService, sessionMgr, cfg.AI.LLM.Model)
|
||||
|
||||
// Gin 模式
|
||||
if cfg.App.Env == "prod" {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
|
||||
r := gin.New()
|
||||
r.Use(gin.Recovery())
|
||||
|
||||
// REST API
|
||||
api := r.Group("/api")
|
||||
apiGroup := r.Group("/api")
|
||||
{
|
||||
api.GET("/health", healthHandler)
|
||||
apiGroup.GET("/health", healthHandler(sessionMgr))
|
||||
}
|
||||
|
||||
// Session REST 端点
|
||||
sessionHandler := api.NewSessionHandler(sessionMgr)
|
||||
sessionHandler.RegisterRoutes(apiGroup)
|
||||
|
||||
// WebSocket
|
||||
r.GET("/ws", ws.ServeWS)
|
||||
r.GET("/ws", ws.ServeWS(sessionMgr, orch))
|
||||
|
||||
log.Println("CamTalk gateway starting on :8080")
|
||||
if err := r.Run(":8080"); err != nil {
|
||||
log.Fatalf("failed to start server: %v", err)
|
||||
// HTTP Server
|
||||
srv := &http.Server{
|
||||
Addr: cfg.Server.Addr(),
|
||||
Handler: r,
|
||||
ReadTimeout: time.Duration(cfg.Server.ReadTimeout) * time.Second,
|
||||
WriteTimeout: time.Duration(cfg.Server.WriteTimeout) * time.Second,
|
||||
}
|
||||
|
||||
// Graceful shutdown
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
go func() {
|
||||
logger.Log.Infow("server starting", "addr", srv.Addr)
|
||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
logger.Log.Fatalw("listen failed", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
<-ctx.Done()
|
||||
logger.Log.Info("shutting down...")
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||
logger.Log.Errorw("shutdown error", "error", err)
|
||||
}
|
||||
logger.Log.Info("server stopped")
|
||||
}
|
||||
|
||||
// healthHandler 健康检查。
|
||||
func healthHandler(c *gin.Context) {
|
||||
c.JSON(200, gin.H{
|
||||
"status": "ok",
|
||||
"version": "0.1.0",
|
||||
"uptime": time.Since(startTime).String(),
|
||||
"active_sessions": 0, // TODO: 接入 Session Manager
|
||||
})
|
||||
func healthHandler(sessionMgr session.Manager) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{
|
||||
"status": "ok",
|
||||
"version": "0.1.0",
|
||||
"uptime_seconds": int(time.Since(startTime).Seconds()),
|
||||
"active_sessions": sessionMgr.ActiveCount(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
39
backend/config.yaml
Normal file
39
backend/config.yaml
Normal file
@@ -0,0 +1,39 @@
|
||||
# config.yaml — 默认配置
|
||||
app:
|
||||
env: dev
|
||||
|
||||
server:
|
||||
host: "0.0.0.0"
|
||||
port: 8080
|
||||
read_timeout: 30
|
||||
write_timeout: 30
|
||||
|
||||
redis:
|
||||
addr: "localhost:6379"
|
||||
password: ""
|
||||
db: 0
|
||||
|
||||
ai:
|
||||
stt:
|
||||
provider: Xiaomi MiMo
|
||||
model: mimo-v2.5
|
||||
endpoint: "https://token-plan-cn.xiaomimimo.com/v1"
|
||||
llm:
|
||||
provider: dashscope
|
||||
model: qwen3-vl-plus
|
||||
endpoint: "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
timeout: 30
|
||||
tts:
|
||||
provider: Xiaomi MiMo
|
||||
model: mimo-v2.5
|
||||
voice: alloy
|
||||
speed: 1.0
|
||||
endpoint: "https://token-plan-cn.xiaomimimo.com/v1"
|
||||
timeout: 5
|
||||
|
||||
storage:
|
||||
driver: memory
|
||||
|
||||
log:
|
||||
level: info
|
||||
format: console
|
||||
@@ -1,38 +1,58 @@
|
||||
module github.com/hhs/camtalk
|
||||
|
||||
go 1.23
|
||||
go 1.24
|
||||
|
||||
require (
|
||||
github.com/gin-gonic/gin v1.10.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/redis/go-redis/v9 v9.20.1
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
go.uber.org/zap v1.28.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/bytedance/sonic v1.11.6 // indirect
|
||||
github.com/bytedance/sonic/loader v0.1.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.4 // indirect
|
||||
github.com/cloudwego/iasm v0.2.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.20.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/joho/godotenv v1.5.1 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/arch v0.8.0 // indirect
|
||||
golang.org/x/crypto v0.23.0 // indirect
|
||||
golang.org/x/net v0.25.0 // indirect
|
||||
golang.org/x/sys v0.20.0 // indirect
|
||||
golang.org/x/text v0.15.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
google.golang.org/protobuf v1.34.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
|
||||
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
|
||||
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
|
||||
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
|
||||
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
|
||||
@@ -9,6 +15,10 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
@@ -23,21 +33,29 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
|
||||
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
@@ -47,26 +65,56 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.20.1 h1:sfCU6A8P3dXbKyWes02uxA2baehGux9dZHfEKtsTB1w=
|
||||
github.com/redis/go-redis/v9 v9.20.1/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
|
||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
|
||||
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
|
||||
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
|
||||
@@ -74,18 +122,16 @@ golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
||||
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
|
||||
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
37
backend/internal/ai/llm/llm.go
Normal file
37
backend/internal/ai/llm/llm.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
// Service 多模态大模型服务契约。
|
||||
type Service interface {
|
||||
// ChatStream 流式推理,返回增量文本的 channel。
|
||||
// 调用方必须消费 channel 直到 Done=true,否则需 cancel ctx 以释放连接。
|
||||
ChatStream(ctx context.Context, req Request) (<-chan Chunk, error)
|
||||
}
|
||||
|
||||
// Request 推理请求。
|
||||
type Request struct {
|
||||
Image []byte // JPEG 图片(已从 Base64 解码)
|
||||
Text string // 用户语音识别后的文本
|
||||
History []models.Message // 最近 N 轮对话历史
|
||||
Language string // 语言,如 "zh-CN"
|
||||
}
|
||||
|
||||
// Chunk 流式推理的一个增量片段。
|
||||
type Chunk struct {
|
||||
Delta string // 增量文本
|
||||
Done bool // 是否结束
|
||||
TokensUsed *TokenUsage // 仅 Done=true 时有值
|
||||
Model string // 实际使用的模型名
|
||||
}
|
||||
|
||||
// TokenUsage 用量统计。
|
||||
type TokenUsage struct {
|
||||
Prompt int
|
||||
Completion int
|
||||
Total int
|
||||
}
|
||||
237
backend/internal/ai/llm/openai.go
Normal file
237
backend/internal/ai/llm/openai.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// OpenAIService 基于 OpenAI Chat Completions API 的 LLM 实现。
|
||||
type OpenAIService struct {
|
||||
apiKey string
|
||||
model string
|
||||
endpoint string
|
||||
timeout time.Duration
|
||||
logger *zap.SugaredLogger
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewOpenAIService 创建 OpenAI LLM 服务。
|
||||
func NewOpenAIService(apiKey, model, endpoint string, timeoutSec int, logger *zap.SugaredLogger) *OpenAIService {
|
||||
if model == "" {
|
||||
model = "gpt-4o"
|
||||
}
|
||||
if endpoint == "" {
|
||||
endpoint = "https://api.openai.com/v1"
|
||||
}
|
||||
timeout := time.Duration(timeoutSec) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
return &OpenAIService{
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
endpoint: endpoint,
|
||||
timeout: timeout,
|
||||
logger: logger,
|
||||
client: &http.Client{Timeout: 60 * time.Second}, // HTTP client timeout > LLM timeout
|
||||
}
|
||||
}
|
||||
|
||||
// --- OpenAI API 请求/响应结构 ---
|
||||
|
||||
type chatRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []chatMessage `json:"messages"`
|
||||
Stream bool `json:"stream"`
|
||||
}
|
||||
|
||||
type chatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content []contentPart `json:"content"`
|
||||
}
|
||||
|
||||
type contentPart struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ImageURL *imageURL `json:"image_url,omitempty"`
|
||||
}
|
||||
|
||||
type imageURL struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// streamDelta SSE 流式响应的单个 delta。
|
||||
type streamDelta struct {
|
||||
Choices []struct {
|
||||
Delta struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"delta"`
|
||||
FinishReason *string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage *struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// ChatStream 实现 llm.Service。调用 OpenAI Chat Completions API 流式推理。
|
||||
func (o *OpenAIService) ChatStream(ctx context.Context, req Request) (<-chan Chunk, error) {
|
||||
// 构建请求
|
||||
messages := o.buildMessages(req)
|
||||
|
||||
body := chatRequest{
|
||||
Model: o.model,
|
||||
Messages: messages,
|
||||
Stream: true,
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llm: marshal request: %w", err)
|
||||
}
|
||||
|
||||
// 创建带超时的 context
|
||||
ctx, cancel := context.WithTimeout(ctx, o.timeout)
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, o.endpoint+"/chat/completions", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("llm: create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Authorization", "Bearer "+o.apiKey)
|
||||
|
||||
resp, err := o.client.Do(httpReq)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("llm: send request: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
cancel()
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("llm: api error (status %d): %s", resp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
|
||||
// 启动 goroutine 解析 SSE 流
|
||||
ch := make(chan Chunk, 64)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
defer cancel()
|
||||
defer resp.Body.Close()
|
||||
|
||||
o.parseSSEStream(resp.Body, ch)
|
||||
}()
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// parseSSEStream 解析 SSE 流,将 delta 发送到 channel。
|
||||
func (o *OpenAIService) parseSSEStream(body io.Reader, ch chan<- Chunk) {
|
||||
scanner := bufio.NewScanner(body)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 256*1024)
|
||||
|
||||
var fullText strings.Builder
|
||||
var lastModel string
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
// SSE 格式:data: {...}
|
||||
if !strings.HasPrefix(line, "data: ") {
|
||||
continue
|
||||
}
|
||||
data := strings.TrimPrefix(line, "data: ")
|
||||
if data == "[DONE]" {
|
||||
// 流结束,发送最终 chunk
|
||||
ch <- Chunk{Delta: "", Done: true, Model: lastModel}
|
||||
return
|
||||
}
|
||||
|
||||
var delta streamDelta
|
||||
if err := json.Unmarshal([]byte(data), &delta); err != nil {
|
||||
o.logger.Warnw("llm: unmarshal delta failed", "error", err, "data", data)
|
||||
continue
|
||||
}
|
||||
|
||||
if delta.Model != "" {
|
||||
lastModel = delta.Model
|
||||
}
|
||||
|
||||
// 提取增量文本
|
||||
if len(delta.Choices) > 0 {
|
||||
content := delta.Choices[0].Delta.Content
|
||||
if content != "" {
|
||||
fullText.WriteString(content)
|
||||
ch <- Chunk{Delta: content, Done: false, Model: lastModel}
|
||||
}
|
||||
|
||||
// 某些模型在最后一个 choice 中携带 usage
|
||||
if delta.Choices[0].FinishReason != nil && delta.Usage != nil {
|
||||
ch <- Chunk{
|
||||
Delta: "",
|
||||
Done: true,
|
||||
Model: lastModel,
|
||||
TokensUsed: &TokenUsage{
|
||||
Prompt: delta.Usage.PromptTokens,
|
||||
Completion: delta.Usage.CompletionTokens,
|
||||
Total: delta.Usage.TotalTokens,
|
||||
},
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// scanner 结束但没收到 [DONE]
|
||||
if err := scanner.Err(); err != nil {
|
||||
o.logger.Warnw("llm: scan error", "error", err)
|
||||
}
|
||||
ch <- Chunk{Delta: "", Done: true, Model: lastModel}
|
||||
}
|
||||
|
||||
// buildMessages 构建 OpenAI Chat API 的 messages 数组。
|
||||
func (o *OpenAIService) buildMessages(req Request) []chatMessage {
|
||||
var messages []chatMessage
|
||||
|
||||
// System prompt
|
||||
messages = append(messages, chatMessage{
|
||||
Role: "system",
|
||||
Content: []contentPart{{Type: "text", Text: BuildSystemPrompt(req.Language, "")}},
|
||||
})
|
||||
|
||||
// 历史消息
|
||||
for _, msg := range req.History {
|
||||
messages = append(messages, chatMessage{
|
||||
Role: msg.Role,
|
||||
Content: []contentPart{{Type: "text", Text: msg.Content}},
|
||||
})
|
||||
}
|
||||
|
||||
// 当前用户消息(图像 + 文本)
|
||||
var parts []contentPart
|
||||
if len(req.Image) > 0 {
|
||||
b64 := base64.StdEncoding.EncodeToString(req.Image)
|
||||
parts = append(parts, contentPart{
|
||||
Type: "image_url",
|
||||
ImageURL: &imageURL{URL: "data:image/jpeg;base64," + b64},
|
||||
})
|
||||
}
|
||||
parts = append(parts, contentPart{Type: "text", Text: req.Text})
|
||||
messages = append(messages, chatMessage{Role: "user", Content: parts})
|
||||
|
||||
return messages
|
||||
}
|
||||
251
backend/internal/ai/llm/openai_test.go
Normal file
251
backend/internal/ai/llm/openai_test.go
Normal file
@@ -0,0 +1,251 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
// mockLLMServer 创建模拟 OpenAI SSE 流式响应的 HTTP 服务器。
|
||||
func mockLLMServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(handler)
|
||||
}
|
||||
|
||||
func TestOpenAIService_ChatStream_Success(t *testing.T) {
|
||||
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
// 验证请求
|
||||
if r.Method != http.MethodPost {
|
||||
t.Errorf("method = %s, want POST", r.Method)
|
||||
}
|
||||
if !strings.Contains(r.URL.Path, "/chat/completions") {
|
||||
t.Errorf("path = %s, should contain /chat/completions", r.URL.Path)
|
||||
}
|
||||
auth := r.Header.Get("Authorization")
|
||||
if auth != "Bearer test-key" {
|
||||
t.Errorf("Authorization = %q, want %q", auth, "Bearer test-key")
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
t.Fatal("ResponseWriter does not support Flusher")
|
||||
}
|
||||
|
||||
// 发送几个 delta
|
||||
deltas := []string{"你好", "世界", "!"}
|
||||
for _, d := range deltas {
|
||||
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"%s\"}}],\"model\":\"gpt-4o\"}\n\n", d)
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// 发送 [DONE]
|
||||
fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
flusher.Flush()
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
|
||||
|
||||
ch, err := svc.ChatStream(context.Background(), Request{
|
||||
Text: "这是什么?",
|
||||
Language: "zh-CN",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ChatStream() error: %v", err)
|
||||
}
|
||||
|
||||
var chunks []Chunk
|
||||
for c := range ch {
|
||||
chunks = append(chunks, c)
|
||||
}
|
||||
|
||||
// 应该有 3 个文本 chunk + 1 个 Done chunk
|
||||
if len(chunks) != 4 {
|
||||
t.Fatalf("got %d chunks, want 4", len(chunks))
|
||||
}
|
||||
|
||||
// 验证文本内容
|
||||
if chunks[0].Delta != "你好" {
|
||||
t.Errorf("chunk[0].Delta = %q, want %q", chunks[0].Delta, "你好")
|
||||
}
|
||||
if chunks[1].Delta != "世界" {
|
||||
t.Errorf("chunk[1].Delta = %q, want %q", chunks[1].Delta, "世界")
|
||||
}
|
||||
|
||||
// 验证最后一个 chunk 是 Done
|
||||
last := chunks[len(chunks)-1]
|
||||
if !last.Done {
|
||||
t.Error("last chunk should be Done")
|
||||
}
|
||||
if last.Model != "gpt-4o" {
|
||||
t.Errorf("last chunk Model = %q, want %q", last.Model, "gpt-4o")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_ChatStream_WithImage(t *testing.T) {
|
||||
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}],\"model\":\"gpt-4o\"}\n\n")
|
||||
fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
|
||||
|
||||
ch, err := svc.ChatStream(context.Background(), Request{
|
||||
Image: []byte("fake-jpeg-data"),
|
||||
Text: "描述图片",
|
||||
Language: "zh-CN",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ChatStream() error: %v", err)
|
||||
}
|
||||
|
||||
// 消费 channel
|
||||
for range ch {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_ChatStream_WithHistory(t *testing.T) {
|
||||
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}],\"model\":\"gpt-4o\"}\n\n")
|
||||
fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
|
||||
|
||||
ch, err := svc.ChatStream(context.Background(), Request{
|
||||
Text: "继续",
|
||||
Language: "zh-CN",
|
||||
History: []models.Message{
|
||||
{Role: "user", Content: "你好"},
|
||||
{Role: "assistant", Content: "你好!有什么可以帮助你的吗?"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ChatStream() error: %v", err)
|
||||
}
|
||||
|
||||
for range ch {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_ChatStream_APIError(t *testing.T) {
|
||||
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
fmt.Fprintf(w, `{"error":{"message":"Invalid API key"}}`)
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("bad-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
|
||||
|
||||
_, err := svc.ChatStream(context.Background(), Request{
|
||||
Text: "test",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ChatStream() should return error for 401")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "401") {
|
||||
t.Errorf("error should mention 401, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_ChatStream_Timeout(t *testing.T) {
|
||||
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
// 模拟慢响应
|
||||
time.Sleep(5 * time.Second)
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"late\"}}]}\n\n")
|
||||
fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 1, zap.NewNop().Sugar()) // 1s timeout
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ch, err := svc.ChatStream(ctx, Request{Text: "test"})
|
||||
if err != nil {
|
||||
// 超时可能在建立连接时或读取时发生
|
||||
return
|
||||
}
|
||||
|
||||
// 如果连接成功,消费 channel 应该超时
|
||||
var gotContent bool
|
||||
for c := range ch {
|
||||
if c.Delta != "" {
|
||||
gotContent = true
|
||||
}
|
||||
}
|
||||
if gotContent {
|
||||
t.Error("should not receive content before timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_ChatStream_UsageInResponse(t *testing.T) {
|
||||
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
// 带 usage 的最后一个 chunk
|
||||
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"},\"finish_reason\":\"stop\"}],\"model\":\"gpt-4o\",\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":5,\"total_tokens\":15}}\n\n")
|
||||
fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
|
||||
|
||||
ch, err := svc.ChatStream(context.Background(), Request{Text: "test"})
|
||||
if err != nil {
|
||||
t.Fatalf("ChatStream() error: %v", err)
|
||||
}
|
||||
|
||||
var last Chunk
|
||||
for c := range ch {
|
||||
last = c
|
||||
}
|
||||
|
||||
if !last.Done {
|
||||
t.Error("last chunk should be Done")
|
||||
}
|
||||
if last.TokensUsed == nil {
|
||||
t.Fatal("last chunk should have TokensUsed")
|
||||
}
|
||||
if last.TokensUsed.Total != 15 {
|
||||
t.Errorf("TokensUsed.Total = %d, want 15", last.TokensUsed.Total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSystemPrompt(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
language string
|
||||
detailLevel string
|
||||
wantContain string
|
||||
}{
|
||||
{"chinese default", "zh-CN", "", "视觉助手"},
|
||||
{"chinese high", "zh-CN", "high", "更详细"},
|
||||
{"english default", "en", "", "visual assistant"},
|
||||
{"english high", "en", "high", "detailed"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := BuildSystemPrompt(tt.language, tt.detailLevel)
|
||||
if !strings.Contains(got, tt.wantContain) {
|
||||
t.Errorf("BuildSystemPrompt(%q, %q) should contain %q", tt.language, tt.detailLevel, tt.wantContain)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
25
backend/internal/ai/llm/prompt.go
Normal file
25
backend/internal/ai/llm/prompt.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package llm
|
||||
|
||||
import "strings"
|
||||
|
||||
// BuildSystemPrompt 根据语言和细节级别构建系统提示词。
|
||||
func BuildSystemPrompt(language, detailLevel string) string {
|
||||
isChinese := strings.HasPrefix(language, "zh")
|
||||
|
||||
var prompt strings.Builder
|
||||
if isChinese {
|
||||
prompt.WriteString("你是一个视觉助手。用户通过摄像头看到一个场景,并用语音向你提问。请用简洁自然的中文回答。如果涉及视觉描述,先说\"我看到……\"。回答控制在3-5句话以内,除非用户要求详细说明。")
|
||||
} else {
|
||||
prompt.WriteString("You are a visual assistant. The user sees a scene through their camera and asks questions by voice. Answer concisely and naturally. If describing visual content, start with 'I see...'. Keep answers to 3-5 sentences unless the user asks for detail.")
|
||||
}
|
||||
|
||||
if detailLevel == "high" {
|
||||
if isChinese {
|
||||
prompt.WriteString("请提供更详细的视觉描述,包括颜色、位置、数量等细节。")
|
||||
} else {
|
||||
prompt.WriteString(" Provide detailed visual descriptions including colors, positions, quantities, and other details.")
|
||||
}
|
||||
}
|
||||
|
||||
return prompt.String()
|
||||
}
|
||||
141
backend/internal/ai/stt/deepgram.go
Normal file
141
backend/internal/ai/stt/deepgram.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package stt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// DeepgramService 基于 Deepgram WebSocket API 的语音识别实现。
|
||||
type DeepgramService struct {
|
||||
apiKey string
|
||||
model string
|
||||
endpoint string
|
||||
logger *zap.SugaredLogger
|
||||
}
|
||||
|
||||
// NewDeepgramService 创建 Deepgram STT 服务。
|
||||
func NewDeepgramService(apiKey, model, endpoint string, logger *zap.SugaredLogger) *DeepgramService {
|
||||
if model == "" {
|
||||
model = "nova-2"
|
||||
}
|
||||
if endpoint == "" {
|
||||
endpoint = "wss://api.deepgram.com/v1/listen"
|
||||
}
|
||||
return &DeepgramService{
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
endpoint: endpoint,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// deepgramResponse Deepgram WebSocket 响应。
|
||||
type deepgramResponse struct {
|
||||
Channel struct {
|
||||
Alternatives []struct {
|
||||
Transcript string `json:"transcript"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
} `json:"alternatives"`
|
||||
} `json:"channel"`
|
||||
IsFinal bool `json:"is_final"`
|
||||
}
|
||||
|
||||
// Recognize 实现 stt.Service。通过 WebSocket 发送音频到 Deepgram,返回最终识别文本。
|
||||
func (d *DeepgramService) Recognize(ctx context.Context, audio []byte, opts Options) (string, error) {
|
||||
if len(audio) == 0 {
|
||||
return "", fmt.Errorf("stt: empty audio")
|
||||
}
|
||||
|
||||
// 构建 WebSocket URL,附带查询参数
|
||||
wsURL := d.buildURL(opts)
|
||||
|
||||
// 5 秒总超时
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// 建立 WebSocket 连接
|
||||
conn, _, err := websocket.DefaultDialer.DialContext(ctx, wsURL, http.Header{
|
||||
"Authorization": []string{"Token " + d.apiKey},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stt: connect deepgram: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// 发送音频数据(一次性)
|
||||
if err := conn.WriteMessage(websocket.BinaryMessage, audio); err != nil {
|
||||
return "", fmt.Errorf("stt: send audio: %w", err)
|
||||
}
|
||||
|
||||
// 发送 Close 消息通知服务端音频已发送完毕
|
||||
closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")
|
||||
_ = conn.WriteMessage(websocket.CloseMessage, closeMsg)
|
||||
|
||||
// 读取识别结果
|
||||
var transcript strings.Builder
|
||||
for {
|
||||
_, message, err := conn.ReadMessage()
|
||||
if err != nil {
|
||||
// Close 帧是正常的结束信号
|
||||
if websocket.IsCloseError(err, websocket.CloseNormalClosure) {
|
||||
break
|
||||
}
|
||||
if websocket.IsUnexpectedCloseError(err, websocket.CloseNormalClosure) {
|
||||
break
|
||||
}
|
||||
return "", fmt.Errorf("stt: read response: %w", err)
|
||||
}
|
||||
|
||||
var resp deepgramResponse
|
||||
if err := json.Unmarshal(message, &resp); err != nil {
|
||||
d.logger.Warnw("stt: unmarshal response failed", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 只累积 final 结果,跳过中间结果
|
||||
if resp.IsFinal && len(resp.Channel.Alternatives) > 0 {
|
||||
text := strings.TrimSpace(resp.Channel.Alternatives[0].Transcript)
|
||||
if text != "" {
|
||||
transcript.WriteString(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(transcript.String()), nil
|
||||
}
|
||||
|
||||
// buildURL 构建 Deepgram WebSocket URL,包含音频格式参数。
|
||||
func (d *DeepgramService) buildURL(opts Options) string {
|
||||
u, _ := url.Parse(d.endpoint)
|
||||
|
||||
encoding := opts.Encoding
|
||||
if encoding == "" {
|
||||
encoding = "pcm_s16le"
|
||||
}
|
||||
sampleRate := opts.SampleRate
|
||||
if sampleRate == 0 {
|
||||
sampleRate = 16000
|
||||
}
|
||||
language := opts.Language
|
||||
if language == "" {
|
||||
language = "zh-CN"
|
||||
}
|
||||
|
||||
q := u.Query()
|
||||
q.Set("encoding", encoding)
|
||||
q.Set("sample_rate", fmt.Sprintf("%d", sampleRate))
|
||||
q.Set("language", language)
|
||||
q.Set("model", d.model)
|
||||
q.Set("punctuate", "true")
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
return u.String()
|
||||
}
|
||||
191
backend/internal/ai/stt/deepgram_test.go
Normal file
191
backend/internal/ai/stt/deepgram_test.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package stt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var upgrader = websocket.Upgrader{
|
||||
CheckOrigin: func(r *http.Request) bool { return true },
|
||||
}
|
||||
|
||||
// newMockDeepgram 创建模拟 Deepgram WebSocket 服务。
|
||||
// 返回 httptest.Server 和对应的 ws:// URL。
|
||||
func newMockDeepgram(t *testing.T, handler func(conn *websocket.Conn)) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
t.Logf("upgrade error: %v", err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
handler(conn)
|
||||
}))
|
||||
return srv
|
||||
}
|
||||
|
||||
// wsToWss 将 http:// 转换为 ws://。
|
||||
func wsToWss(httpURL string) string {
|
||||
return "ws" + strings.TrimPrefix(httpURL, "http")
|
||||
}
|
||||
|
||||
func TestDeepgramService_Recognize_Success(t *testing.T) {
|
||||
srv := newMockDeepgram(t, func(conn *websocket.Conn) {
|
||||
// 读取音频数据
|
||||
_, _, err := conn.ReadMessage()
|
||||
if err != nil {
|
||||
t.Errorf("read audio: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 发送中间结果(非 final)
|
||||
intermediate := deepgramResponse{
|
||||
IsFinal: false,
|
||||
}
|
||||
intermediate.Channel.Alternatives = []struct {
|
||||
Transcript string `json:"transcript"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}{{Transcript: "你好", Confidence: 0.9}}
|
||||
data, _ := json.Marshal(intermediate)
|
||||
_ = conn.WriteMessage(websocket.TextMessage, data)
|
||||
|
||||
// 发送最终结果
|
||||
final := deepgramResponse{
|
||||
IsFinal: true,
|
||||
}
|
||||
final.Channel.Alternatives = []struct {
|
||||
Transcript string `json:"transcript"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}{{Transcript: "你好世界", Confidence: 0.95}}
|
||||
data, _ = json.Marshal(final)
|
||||
_ = conn.WriteMessage(websocket.TextMessage, data)
|
||||
|
||||
// 等待客户端关闭
|
||||
_, _, _ = conn.ReadMessage()
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar())
|
||||
|
||||
text, err := svc.Recognize(context.Background(), []byte("fake-pcm-audio"), Options{
|
||||
Encoding: "pcm_s16le",
|
||||
SampleRate: 16000,
|
||||
Language: "zh-CN",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Recognize() error: %v", err)
|
||||
}
|
||||
if text != "你好世界" {
|
||||
t.Errorf("Recognize() = %q, want %q", text, "你好世界")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramService_Recognize_EmptyAudio(t *testing.T) {
|
||||
svc := NewDeepgramService("test-key", "", "ws://localhost", zap.NewNop().Sugar())
|
||||
_, err := svc.Recognize(context.Background(), nil, Options{})
|
||||
if err == nil {
|
||||
t.Fatal("Recognize() with empty audio should return error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramService_Recognize_ConnectError(t *testing.T) {
|
||||
svc := NewDeepgramService("test-key", "", "ws://localhost:1", zap.NewNop().Sugar())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := svc.Recognize(ctx, []byte("audio"), Options{})
|
||||
if err == nil {
|
||||
t.Fatal("Recognize() with bad endpoint should return error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramService_Recognize_Timeout(t *testing.T) {
|
||||
// 模拟一个永不响应的服务端
|
||||
srv := newMockDeepgram(t, func(conn *websocket.Conn) {
|
||||
// 读取音频但不发送任何结果,让客户端超时
|
||||
_, _, _ = conn.ReadMessage()
|
||||
time.Sleep(10 * time.Second)
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar())
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := svc.Recognize(ctx, []byte("audio"), Options{})
|
||||
if err == nil {
|
||||
t.Fatal("Recognize() should timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramService_Recognize_MultipleFinals(t *testing.T) {
|
||||
srv := newMockDeepgram(t, func(conn *websocket.Conn) {
|
||||
_, _, _ = conn.ReadMessage()
|
||||
|
||||
// 发送多个 final 结果(多句话场景)
|
||||
for _, text := range []string{"你好", "世界"} {
|
||||
resp := deepgramResponse{IsFinal: true}
|
||||
resp.Channel.Alternatives = []struct {
|
||||
Transcript string `json:"transcript"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}{{Transcript: text, Confidence: 0.9}}
|
||||
data, _ := json.Marshal(resp)
|
||||
_ = conn.WriteMessage(websocket.TextMessage, data)
|
||||
}
|
||||
|
||||
_, _, _ = conn.ReadMessage()
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar())
|
||||
|
||||
text, err := svc.Recognize(context.Background(), []byte("audio"), Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Recognize() error: %v", err)
|
||||
}
|
||||
if text != "你好世界" {
|
||||
t.Errorf("Recognize() = %q, want %q", text, "你好世界")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepgramService_buildURL(t *testing.T) {
|
||||
svc := NewDeepgramService("key", "", "wss://api.deepgram.com/v1/listen", zap.NewNop().Sugar())
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
opts Options
|
||||
want []string // URL 中应包含的参数
|
||||
}{
|
||||
{
|
||||
name: "defaults",
|
||||
opts: Options{},
|
||||
want: []string{"encoding=pcm_s16le", "sample_rate=16000", "language=zh-CN"},
|
||||
},
|
||||
{
|
||||
name: "custom",
|
||||
opts: Options{Encoding: "wav", SampleRate: 44100, Language: "en"},
|
||||
want: []string{"encoding=wav", "sample_rate=44100", "language=en"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
u := svc.buildURL(tt.opts)
|
||||
for _, param := range tt.want {
|
||||
if !strings.Contains(u, param) {
|
||||
t.Errorf("buildURL() = %q, should contain %q", u, param)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
16
backend/internal/ai/stt/stt.go
Normal file
16
backend/internal/ai/stt/stt.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package stt
|
||||
|
||||
import "context"
|
||||
|
||||
// Service 语音识别服务契约。
|
||||
type Service interface {
|
||||
// Recognize 识别一段完整音频,返回最终文本。
|
||||
Recognize(ctx context.Context, audio []byte, opts Options) (string, error)
|
||||
}
|
||||
|
||||
// Options 语音识别参数。
|
||||
type Options struct {
|
||||
Encoding string // 音频编码,如 "pcm_s16le"
|
||||
SampleRate int // 采样率,如 16000
|
||||
Language string // 语言,如 "zh-CN"
|
||||
}
|
||||
154
backend/internal/ai/tts/openai.go
Normal file
154
backend/internal/ai/tts/openai.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package tts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// OpenAIService 基于 OpenAI TTS API 的语音合成实现。
|
||||
type OpenAIService struct {
|
||||
apiKey string
|
||||
model string
|
||||
voice string
|
||||
speed float64
|
||||
endpoint string
|
||||
timeout time.Duration
|
||||
logger *zap.SugaredLogger
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewOpenAIService 创建 OpenAI TTS 服务。
|
||||
func NewOpenAIService(apiKey, model, voice, endpoint string, speed float64, timeoutSec int, logger *zap.SugaredLogger) *OpenAIService {
|
||||
if model == "" {
|
||||
model = "tts-1"
|
||||
}
|
||||
if voice == "" {
|
||||
voice = "alloy"
|
||||
}
|
||||
if speed <= 0 {
|
||||
speed = 1.0
|
||||
}
|
||||
if endpoint == "" {
|
||||
endpoint = "https://api.openai.com/v1"
|
||||
}
|
||||
timeout := time.Duration(timeoutSec) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
return &OpenAIService{
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
voice: voice,
|
||||
speed: speed,
|
||||
endpoint: endpoint,
|
||||
timeout: timeout,
|
||||
logger: logger,
|
||||
client: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// ttsRequest OpenAI TTS API 请求。
|
||||
type ttsRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input string `json:"input"`
|
||||
Voice string `json:"voice"`
|
||||
ResponseFormat string `json:"response_format"`
|
||||
Speed float64 `json:"speed"`
|
||||
}
|
||||
|
||||
// SynthesizeStream 实现 tts.Service。从 textStream 读取句子,逐句调用 OpenAI TTS API。
|
||||
func (o *OpenAIService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error) {
|
||||
voice := opts.Voice
|
||||
if voice == "" {
|
||||
voice = o.voice
|
||||
}
|
||||
speed := opts.Speed
|
||||
if speed <= 0 {
|
||||
speed = o.speed
|
||||
}
|
||||
|
||||
ch := make(chan Chunk, 4)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
for text := range textStream {
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
audio, err := o.synthesize(ctx, text, voice, speed)
|
||||
if err != nil {
|
||||
o.logger.Warnw("tts: synthesize failed", "error", err, "text", text)
|
||||
// 静默跳过,不中断整个流
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- Chunk{Audio: audio, IsLast: false}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// textStream 关闭,发送 IsLast 标记
|
||||
select {
|
||||
case ch <- Chunk{Audio: nil, IsLast: true}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// synthesize 调用 OpenAI TTS API 合成单个句子。
|
||||
func (o *OpenAIService) synthesize(ctx context.Context, text, voice string, speed float64) ([]byte, error) {
|
||||
// 单句超时
|
||||
ctx, cancel := context.WithTimeout(ctx, o.timeout)
|
||||
defer cancel()
|
||||
|
||||
body := ttsRequest{
|
||||
Model: o.model,
|
||||
Input: text,
|
||||
Voice: voice,
|
||||
ResponseFormat: "mp3",
|
||||
Speed: speed,
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tts: marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, o.endpoint+"/audio/speech", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tts: create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+o.apiKey)
|
||||
|
||||
resp, err := o.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tts: send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
errBody, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("tts: api error (status %d): %s", resp.StatusCode, string(errBody))
|
||||
}
|
||||
|
||||
// 读取整个 MP3 响应
|
||||
audio, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tts: read response: %w", err)
|
||||
}
|
||||
|
||||
return audio, nil
|
||||
}
|
||||
300
backend/internal/ai/tts/openai_test.go
Normal file
300
backend/internal/ai/tts/openai_test.go
Normal file
@@ -0,0 +1,300 @@
|
||||
package tts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// mockTTSServer 创建模拟 OpenAI TTS API 的 HTTP 服务器。
|
||||
func mockTTSServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(handler)
|
||||
}
|
||||
|
||||
// sendSentences 向 channel 发送句子并关闭。
|
||||
func sendSentences(sentences ...string) <-chan string {
|
||||
ch := make(chan string, len(sentences))
|
||||
for _, s := range sentences {
|
||||
ch <- s
|
||||
}
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
func TestOpenAIService_SynthesizeStream_Success(t *testing.T) {
|
||||
var callCount int32
|
||||
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&callCount, 1)
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
t.Errorf("method = %s, want POST", r.Method)
|
||||
}
|
||||
if !strings.Contains(r.URL.Path, "/audio/speech") {
|
||||
t.Errorf("path = %s, should contain /audio/speech", r.URL.Path)
|
||||
}
|
||||
auth := r.Header.Get("Authorization")
|
||||
if auth != "Bearer test-key" {
|
||||
t.Errorf("Authorization = %q, want %q", auth, "Bearer test-key")
|
||||
}
|
||||
|
||||
// 验证请求体
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if !strings.Contains(string(body), "tts-1") {
|
||||
t.Errorf("request body should contain model tts-1")
|
||||
}
|
||||
|
||||
// 返回假 MP3 数据
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
fmt.Fprintf(w, "fake-mp3-data")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
|
||||
|
||||
textStream := sendSentences("你好", "世界", "!")
|
||||
|
||||
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{
|
||||
Voice: "alloy", Speed: 1.0, OutputFmt: "mp3", SampleRate: 24000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SynthesizeStream() error: %v", err)
|
||||
}
|
||||
|
||||
var chunks []Chunk
|
||||
for c := range ch {
|
||||
chunks = append(chunks, c)
|
||||
}
|
||||
|
||||
// 应该有 3 个音频 chunk + 1 个 IsLast 标记
|
||||
if len(chunks) != 4 {
|
||||
t.Fatalf("got %d chunks, want 4", len(chunks))
|
||||
}
|
||||
|
||||
// 验证前 3 个有音频数据
|
||||
for i := 0; i < 3; i++ {
|
||||
if string(chunks[i].Audio) != "fake-mp3-data" {
|
||||
t.Errorf("chunk[%d].Audio = %q, want %q", i, string(chunks[i].Audio), "fake-mp3-data")
|
||||
}
|
||||
if chunks[i].IsLast {
|
||||
t.Errorf("chunk[%d].IsLast should be false", i)
|
||||
}
|
||||
}
|
||||
|
||||
// 验证最后一个是 IsLast
|
||||
if !chunks[3].IsLast {
|
||||
t.Error("last chunk should be IsLast")
|
||||
}
|
||||
if chunks[3].Audio != nil {
|
||||
t.Error("last chunk Audio should be nil")
|
||||
}
|
||||
|
||||
// 验证调用了 3 次 API(3 个句子)
|
||||
if atomic.LoadInt32(&callCount) != 3 {
|
||||
t.Errorf("API called %d times, want 3", callCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_SynthesizeStream_APIError(t *testing.T) {
|
||||
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
fmt.Fprintf(w, "internal error")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
|
||||
|
||||
textStream := sendSentences("你好")
|
||||
|
||||
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("SynthesizeStream() error: %v", err)
|
||||
}
|
||||
|
||||
// 应该只有一个 IsLast chunk(音频被跳过)
|
||||
var chunks []Chunk
|
||||
for c := range ch {
|
||||
chunks = append(chunks, c)
|
||||
}
|
||||
|
||||
if len(chunks) != 1 {
|
||||
t.Fatalf("got %d chunks, want 1 (IsLast only)", len(chunks))
|
||||
}
|
||||
if !chunks[0].IsLast {
|
||||
t.Error("chunk should be IsLast")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_SynthesizeStream_Timeout(t *testing.T) {
|
||||
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(3 * time.Second)
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
fmt.Fprintf(w, "late-mp3")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
// 1 秒超时
|
||||
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 1, zap.NewNop().Sugar())
|
||||
|
||||
textStream := sendSentences("很长的句子")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ch, err := svc.SynthesizeStream(ctx, textStream, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("SynthesizeStream() error: %v", err)
|
||||
}
|
||||
|
||||
var chunks []Chunk
|
||||
for c := range ch {
|
||||
chunks = append(chunks, c)
|
||||
}
|
||||
|
||||
// 超时后音频被跳过,只有 IsLast
|
||||
if len(chunks) != 1 {
|
||||
t.Fatalf("got %d chunks, want 1", len(chunks))
|
||||
}
|
||||
if !chunks[0].IsLast {
|
||||
t.Error("chunk should be IsLast")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_SynthesizeStream_EmptyText(t *testing.T) {
|
||||
var callCount int32
|
||||
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&callCount, 1)
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
fmt.Fprintf(w, "mp3")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
|
||||
|
||||
// 空句子应该被跳过
|
||||
textStream := sendSentences("", "你好", "")
|
||||
|
||||
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("SynthesizeStream() error: %v", err)
|
||||
}
|
||||
|
||||
var chunks []Chunk
|
||||
for c := range ch {
|
||||
chunks = append(chunks, c)
|
||||
}
|
||||
|
||||
// 只有 "你好" 应该被合成
|
||||
if atomic.LoadInt32(&callCount) != 1 {
|
||||
t.Errorf("API called %d times, want 1", callCount)
|
||||
}
|
||||
|
||||
// 1 个音频 + 1 个 IsLast
|
||||
if len(chunks) != 2 {
|
||||
t.Fatalf("got %d chunks, want 2", len(chunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_SynthesizeStream_ContextCancelled(t *testing.T) {
|
||||
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
fmt.Fprintf(w, "mp3")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
|
||||
|
||||
// 发送多个句子,但在第一个后取消
|
||||
textStream := make(chan string, 3)
|
||||
textStream <- "第一句"
|
||||
textStream <- "第二句"
|
||||
textStream <- "第三句"
|
||||
close(textStream)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
// 立即取消
|
||||
cancel()
|
||||
|
||||
ch, err := svc.SynthesizeStream(ctx, textStream, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("SynthesizeStream() error: %v", err)
|
||||
}
|
||||
|
||||
// 消费 channel,应该很快结束
|
||||
var count int
|
||||
for range ch {
|
||||
count++
|
||||
}
|
||||
// 可能收到 0 个或 1 个 chunk,取决于时序
|
||||
t.Logf("received %d chunks after context cancel", count)
|
||||
}
|
||||
|
||||
func TestOpenAIService_SynthesizeStream_PartialFailure(t *testing.T) {
|
||||
var callCount int32
|
||||
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&callCount, 1)
|
||||
if n == 2 {
|
||||
// 第二个句子失败
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
fmt.Fprintf(w, "error")
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
fmt.Fprintf(w, "mp3-%d", n)
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
|
||||
|
||||
textStream := sendSentences("第一句", "第二句", "第三句")
|
||||
|
||||
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("SynthesizeStream() error: %v", err)
|
||||
}
|
||||
|
||||
var chunks []Chunk
|
||||
for c := range ch {
|
||||
chunks = append(chunks, c)
|
||||
}
|
||||
|
||||
// 2 个成功音频 + 1 个 IsLast(第二句被跳过)
|
||||
if len(chunks) != 3 {
|
||||
t.Fatalf("got %d chunks, want 3", len(chunks))
|
||||
}
|
||||
if !chunks[len(chunks)-1].IsLast {
|
||||
t.Error("last chunk should be IsLast")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIService_SynthesizeStream_CustomVoice(t *testing.T) {
|
||||
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if !strings.Contains(string(body), "nova") {
|
||||
t.Errorf("request body should contain voice 'nova', got: %s", string(body))
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
fmt.Fprintf(w, "mp3")
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
|
||||
|
||||
textStream := sendSentences("你好")
|
||||
|
||||
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{Voice: "nova"})
|
||||
if err != nil {
|
||||
t.Fatalf("SynthesizeStream() error: %v", err)
|
||||
}
|
||||
|
||||
for range ch {
|
||||
}
|
||||
}
|
||||
25
backend/internal/ai/tts/tts.go
Normal file
25
backend/internal/ai/tts/tts.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package tts
|
||||
|
||||
import "context"
|
||||
|
||||
// Service 语音合成服务契约。
|
||||
type Service interface {
|
||||
// SynthesizeStream 流式合成。
|
||||
// textStream 接收句子级文本(由 Orchestrator 的句子切分器产出),
|
||||
// 返回的 channel 持续输出 MP3 音频 chunk。
|
||||
SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error)
|
||||
}
|
||||
|
||||
// Options 合成参数。
|
||||
type Options struct {
|
||||
Voice string // "alloy" | "nova" | "shimmer" 等
|
||||
Speed float64 // 1.0 为正常语速
|
||||
OutputFmt string // "mp3" — 固定使用 MP3
|
||||
SampleRate int // 24000
|
||||
}
|
||||
|
||||
// Chunk 一个音频片段。
|
||||
type Chunk struct {
|
||||
Audio []byte // MP3 音频数据(未 Base64 编码)
|
||||
IsLast bool // 是否为最后一片
|
||||
}
|
||||
91
backend/internal/api/session.go
Normal file
91
backend/internal/api/session.go
Normal file
@@ -0,0 +1,91 @@
|
||||
// Package api 提供 REST API 处理函数。
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
"github.com/hhs/camtalk/internal/session"
|
||||
)
|
||||
|
||||
// SessionHandler 提供会话相关的 REST 端点。
|
||||
type SessionHandler struct {
|
||||
sessionMgr session.Manager
|
||||
}
|
||||
|
||||
// NewSessionHandler 创建 SessionHandler。
|
||||
func NewSessionHandler(sessionMgr session.Manager) *SessionHandler {
|
||||
return &SessionHandler{sessionMgr: sessionMgr}
|
||||
}
|
||||
|
||||
// CreateSessionRequest POST /api/sessions 请求体(所有字段可选)。
|
||||
type CreateSessionRequest struct {
|
||||
Config *models.SessionConfig `json:"config,omitempty"`
|
||||
}
|
||||
|
||||
// CreateSession POST /api/sessions — 创建新会话。
|
||||
func (h *SessionHandler) CreateSession(c *gin.Context) {
|
||||
var req CreateSessionRequest
|
||||
// 请求体可选,解析失败不报错(使用默认配置)
|
||||
_ = c.ShouldBindJSON(&req)
|
||||
|
||||
cfg := models.DefaultConfig()
|
||||
if req.Config != nil {
|
||||
cfg = *req.Config
|
||||
}
|
||||
|
||||
sessionID, err := h.sessionMgr.Create(c.Request.Context(), cfg)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"code": "INTERNAL_ERROR",
|
||||
"message": "failed to create session",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// 获取创建后的会话以返回 created_at
|
||||
sess, err := h.sessionMgr.Get(c.Request.Context(), sessionID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"code": "INTERNAL_ERROR",
|
||||
"message": "failed to retrieve created session",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusCreated, gin.H{
|
||||
"session_id": sess.ID,
|
||||
"created_at": sess.CreatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
// DestroySession DELETE /api/sessions/:id — 销毁会话。
|
||||
func (h *SessionHandler) DestroySession(c *gin.Context) {
|
||||
sessionID := c.Param("id")
|
||||
|
||||
err := h.sessionMgr.Destroy(c.Request.Context(), sessionID)
|
||||
if err != nil {
|
||||
if err == session.ErrSessionNotFound {
|
||||
c.JSON(http.StatusNotFound, gin.H{
|
||||
"code": "SESSION_NOT_FOUND",
|
||||
"message": "session not found or already expired",
|
||||
})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"code": "INTERNAL_ERROR",
|
||||
"message": "failed to destroy session",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.Status(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// RegisterRoutes 注册会话相关路由到给定的路由组。
|
||||
func (h *SessionHandler) RegisterRoutes(rg *gin.RouterGroup) {
|
||||
rg.POST("/sessions", h.CreateSession)
|
||||
rg.DELETE("/sessions/:id", h.DestroySession)
|
||||
}
|
||||
160
backend/internal/config/config.go
Normal file
160
backend/internal/config/config.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// Config 应用配置。
|
||||
type Config struct {
|
||||
App AppConfig `mapstructure:"app"`
|
||||
Server ServerConfig `mapstructure:"server"`
|
||||
Redis RedisConfig `mapstructure:"redis"`
|
||||
AI AIConfig `mapstructure:"ai"`
|
||||
Storage StorageConfig `mapstructure:"storage"`
|
||||
Log LogConfig `mapstructure:"log"`
|
||||
}
|
||||
|
||||
type AppConfig struct {
|
||||
Env string `mapstructure:"env"`
|
||||
Version string `mapstructure:"version"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
Host string `mapstructure:"host"`
|
||||
Port int `mapstructure:"port"`
|
||||
ReadTimeout int `mapstructure:"read_timeout"`
|
||||
WriteTimeout int `mapstructure:"write_timeout"`
|
||||
}
|
||||
|
||||
// Addr 返回 host:port 地址。
|
||||
func (s ServerConfig) Addr() string {
|
||||
return fmt.Sprintf("%s:%d", s.Host, s.Port)
|
||||
}
|
||||
|
||||
type RedisConfig struct {
|
||||
Addr string `mapstructure:"addr"`
|
||||
Password string `mapstructure:"password"`
|
||||
DB int `mapstructure:"db"`
|
||||
}
|
||||
|
||||
type AIConfig struct {
|
||||
STT STTConfig `mapstructure:"stt"`
|
||||
LLM LLMConfig `mapstructure:"llm"`
|
||||
TTS TTSConfig `mapstructure:"tts"`
|
||||
}
|
||||
|
||||
type STTConfig struct {
|
||||
Provider string `mapstructure:"provider"`
|
||||
APIKey string `mapstructure:"api_key"`
|
||||
Model string `mapstructure:"model"`
|
||||
Endpoint string `mapstructure:"endpoint"`
|
||||
}
|
||||
|
||||
type LLMConfig struct {
|
||||
Provider string `mapstructure:"provider"`
|
||||
APIKey string `mapstructure:"api_key"`
|
||||
Model string `mapstructure:"model"`
|
||||
Endpoint string `mapstructure:"endpoint"`
|
||||
Timeout int `mapstructure:"timeout"`
|
||||
}
|
||||
|
||||
type TTSConfig struct {
|
||||
Provider string `mapstructure:"provider"`
|
||||
APIKey string `mapstructure:"api_key"`
|
||||
Model string `mapstructure:"model"`
|
||||
Voice string `mapstructure:"voice"`
|
||||
Speed float64 `mapstructure:"speed"`
|
||||
Endpoint string `mapstructure:"endpoint"`
|
||||
Timeout int `mapstructure:"timeout"`
|
||||
}
|
||||
|
||||
type StorageConfig struct {
|
||||
Driver string `mapstructure:"driver"`
|
||||
DSN string `mapstructure:"dsn"`
|
||||
}
|
||||
|
||||
type LogConfig struct {
|
||||
Level string `mapstructure:"level"`
|
||||
Format string `mapstructure:"format"`
|
||||
}
|
||||
|
||||
// Load 加载配置。优先级:环境变量 > config.{env}.yaml > config.yaml。
|
||||
func Load() (*Config, error) {
|
||||
v := viper.New()
|
||||
v.SetConfigName("config")
|
||||
v.SetConfigType("yaml")
|
||||
v.AddConfigPath(".")
|
||||
v.AddConfigPath("./config")
|
||||
v.AddConfigPath("./backend")
|
||||
|
||||
// 默认值
|
||||
v.SetDefault("app.env", "dev")
|
||||
v.SetDefault("server.host", "0.0.0.0")
|
||||
v.SetDefault("server.port", 8080)
|
||||
v.SetDefault("server.read_timeout", 30)
|
||||
v.SetDefault("server.write_timeout", 30)
|
||||
v.SetDefault("redis.addr", "localhost:6379")
|
||||
v.SetDefault("redis.db", 0)
|
||||
v.SetDefault("ai.stt.provider", "deepgram")
|
||||
v.SetDefault("ai.stt.model", "nova-2")
|
||||
v.SetDefault("ai.stt.endpoint", "wss://api.deepgram.com/v1/listen")
|
||||
v.SetDefault("ai.llm.provider", "openai")
|
||||
v.SetDefault("ai.llm.model", "gpt-4o")
|
||||
v.SetDefault("ai.llm.endpoint", "https://api.openai.com/v1")
|
||||
v.SetDefault("ai.llm.timeout", 10)
|
||||
v.SetDefault("ai.tts.provider", "openai")
|
||||
v.SetDefault("ai.tts.model", "tts-1")
|
||||
v.SetDefault("ai.tts.voice", "alloy")
|
||||
v.SetDefault("ai.tts.speed", 1.0)
|
||||
v.SetDefault("ai.tts.endpoint", "https://api.openai.com/v1")
|
||||
v.SetDefault("ai.tts.timeout", 5)
|
||||
v.SetDefault("storage.driver", "memory")
|
||||
v.SetDefault("log.level", "info")
|
||||
v.SetDefault("log.format", "console")
|
||||
|
||||
// 读取基础配置文件
|
||||
_ = v.ReadInConfig() // 文件不存在不报错
|
||||
|
||||
// 根据 APP_ENV 覆盖
|
||||
env := os.Getenv("APP_ENV")
|
||||
if env == "" {
|
||||
env = v.GetString("app.env")
|
||||
}
|
||||
if env != "" {
|
||||
v.SetConfigName("config." + env)
|
||||
_ = v.MergeInConfig()
|
||||
}
|
||||
|
||||
// 加载 .env 文件(不覆盖已有环境变量)
|
||||
// 按优先级尝试:当前目录、上级目录(兼容从 backend/ 或项目根目录启动)
|
||||
_ = godotenv.Load()
|
||||
_ = godotenv.Load("../.env")
|
||||
|
||||
// 环境变量覆盖
|
||||
v.SetEnvPrefix("CAMTALK")
|
||||
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
||||
v.AutomaticEnv()
|
||||
|
||||
var cfg Config
|
||||
if err := v.Unmarshal(&cfg); err != nil {
|
||||
return nil, fmt.Errorf("config unmarshal: %w", err)
|
||||
}
|
||||
|
||||
// 填充默认值
|
||||
if cfg.Server.Host == "" {
|
||||
cfg.Server.Host = "0.0.0.0"
|
||||
}
|
||||
if cfg.Server.Port == 0 {
|
||||
cfg.Server.Port = 8080
|
||||
}
|
||||
if cfg.App.Env == "" {
|
||||
cfg.App.Env = "dev"
|
||||
}
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
33
backend/internal/errors/codes.go
Normal file
33
backend/internal/errors/codes.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package errors
|
||||
|
||||
import "github.com/hhs/camtalk/internal/models"
|
||||
|
||||
// 错误码常量,与 docs/03-接口文档.md 保持一致。
|
||||
const (
|
||||
CodeInvalidMessage = "INVALID_MESSAGE"
|
||||
CodeSessionNotFound = "SESSION_NOT_FOUND"
|
||||
CodeRateLimited = "RATE_LIMITED"
|
||||
CodeImageTooLarge = "IMAGE_TOO_LARGE"
|
||||
CodeAudioTooShort = "AUDIO_TOO_SHORT"
|
||||
CodeLLMTimeout = "LLM_TIMEOUT"
|
||||
CodeLLMError = "LLM_ERROR"
|
||||
CodeSTTError = "STT_ERROR"
|
||||
CodeTTSError = "TTS_ERROR"
|
||||
CodeInternalError = "INTERNAL_ERROR"
|
||||
)
|
||||
|
||||
// Sender 定义发送 WS 错误消息的接口,便于测试 mock。
|
||||
type Sender interface {
|
||||
SendError(code, requestID, message string)
|
||||
}
|
||||
|
||||
// SendWSError 向客户端发送 error 消息。
|
||||
// sender 是一个具有 sendJSON 方法的对象,这里用接口抽象。
|
||||
func SendWSError(sender interface{ SendJSON(v any) error }, code, requestID string, err error) {
|
||||
_ = sender.SendJSON(models.WsError{
|
||||
Type: "error",
|
||||
Code: code,
|
||||
RequestID: requestID,
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
57
backend/internal/logger/logger.go
Normal file
57
backend/internal/logger/logger.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package logger
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
// Log 是全局 SugaredLogger,由 Init 初始化。
|
||||
var Log *zap.SugaredLogger
|
||||
|
||||
// Init 初始化全局日志器。
|
||||
// level: "debug", "info", "warn", "error"
|
||||
// format: "json" 或 "console"
|
||||
func Init(level, format string) {
|
||||
var lvl zapcore.Level
|
||||
switch level {
|
||||
case "debug":
|
||||
lvl = zapcore.DebugLevel
|
||||
case "warn":
|
||||
lvl = zapcore.WarnLevel
|
||||
case "error":
|
||||
lvl = zapcore.ErrorLevel
|
||||
default:
|
||||
lvl = zapcore.InfoLevel
|
||||
}
|
||||
|
||||
encoderCfg := zap.NewProductionEncoderConfig()
|
||||
encoderCfg.TimeKey = "ts"
|
||||
encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||
|
||||
var core zapcore.Core
|
||||
if format == "console" {
|
||||
core = zapcore.NewCore(
|
||||
zapcore.NewConsoleEncoder(encoderCfg),
|
||||
zapcore.AddSync(os.Stdout),
|
||||
lvl,
|
||||
)
|
||||
} else {
|
||||
core = zapcore.NewCore(
|
||||
zapcore.NewJSONEncoder(encoderCfg),
|
||||
zapcore.AddSync(os.Stdout),
|
||||
lvl,
|
||||
)
|
||||
}
|
||||
|
||||
logger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel))
|
||||
Log = logger.Sugar()
|
||||
}
|
||||
|
||||
// Sync 刷新缓冲区,退出前调用。
|
||||
func Sync() {
|
||||
if Log != nil {
|
||||
_ = Log.Sync()
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,26 @@ func DefaultConfig() SessionConfig {
|
||||
return SessionConfig{TTSEnabled: true, DetailLevel: "low", Language: "zh-CN"}
|
||||
}
|
||||
|
||||
// SessionConfigPatch 会话配置增量更新(指针字段表示"未传则不更新")。
|
||||
type SessionConfigPatch struct {
|
||||
TTSEnabled *bool `json:"tts_enabled,omitempty"`
|
||||
DetailLevel *string `json:"detail_level,omitempty"`
|
||||
Language *string `json:"language,omitempty"`
|
||||
}
|
||||
|
||||
// Apply 将 patch 中的非 nil 字段覆盖到 cfg。
|
||||
func (p SessionConfigPatch) Apply(cfg *SessionConfig) {
|
||||
if p.TTSEnabled != nil {
|
||||
cfg.TTSEnabled = *p.TTSEnabled
|
||||
}
|
||||
if p.DetailLevel != nil {
|
||||
cfg.DetailLevel = *p.DetailLevel
|
||||
}
|
||||
if p.Language != nil {
|
||||
cfg.Language = *p.Language
|
||||
}
|
||||
}
|
||||
|
||||
// Message 对话消息。
|
||||
type Message struct {
|
||||
Role string `json:"role"` // "user" | "assistant"
|
||||
|
||||
33
backend/internal/orchestrator/orchestrator.go
Normal file
33
backend/internal/orchestrator/orchestrator.go
Normal file
@@ -0,0 +1,33 @@
|
||||
// Package orchestrator 实现 STT → LLM → TTS 流式并行管道。
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
// Orchestrator AI 编排器接口。
|
||||
// 接收查询并执行完整的 STT → LLM → TTS 管道。
|
||||
type Orchestrator interface {
|
||||
// ProcessQuery 处理一次用户查询。
|
||||
// ctx 用于整体超时和中断控制。
|
||||
// sessionID 用于会话管理和历史获取。
|
||||
// req 包含图像和音频数据。
|
||||
// history 是最近的对话历史。
|
||||
// sender 用于向客户端推送消息。
|
||||
ProcessQuery(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
req models.WsQuery,
|
||||
history []models.Message,
|
||||
sender Sender,
|
||||
) error
|
||||
}
|
||||
|
||||
// QueryRequest 查询请求(内部使用)。
|
||||
type QueryRequest struct {
|
||||
Image []byte // JPEG 图片(已从 Base64 解码)
|
||||
Audio []byte // 音频数据(已从 Base64 解码)
|
||||
Language string // 语言,如 "zh-CN"
|
||||
}
|
||||
328
backend/internal/orchestrator/pipeline.go
Normal file
328
backend/internal/orchestrator/pipeline.go
Normal file
@@ -0,0 +1,328 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/hhs/camtalk/internal/ai/llm"
|
||||
"github.com/hhs/camtalk/internal/ai/stt"
|
||||
"github.com/hhs/camtalk/internal/ai/tts"
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
"github.com/hhs/camtalk/internal/session"
|
||||
)
|
||||
|
||||
// Pipeline 实现 Orchestrator 接口,管理 STT → LLM → TTS 流式管道。
|
||||
type Pipeline struct {
|
||||
sttService stt.Service
|
||||
llmService llm.Service
|
||||
ttsService tts.Service
|
||||
sessionMgr session.Manager
|
||||
model string // LLM 模型名,用于 llm_done 上报
|
||||
}
|
||||
|
||||
// New 创建 Pipeline 实例。
|
||||
func New(
|
||||
sttService stt.Service,
|
||||
llmService llm.Service,
|
||||
ttsService tts.Service,
|
||||
sessionMgr session.Manager,
|
||||
model string,
|
||||
) *Pipeline {
|
||||
return &Pipeline{
|
||||
sttService: sttService,
|
||||
llmService: llmService,
|
||||
ttsService: ttsService,
|
||||
sessionMgr: sessionMgr,
|
||||
model: model,
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessQuery 实现 Orchestrator 接口。
|
||||
func (p *Pipeline) ProcessQuery(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
req models.WsQuery,
|
||||
history []models.Message,
|
||||
sender Sender,
|
||||
) error {
|
||||
log := logger.Log
|
||||
startTime := time.Now()
|
||||
|
||||
// 解码音频数据
|
||||
audio, err := base64.StdEncoding.DecodeString(req.Audio)
|
||||
if err != nil {
|
||||
log.Errorw("音频解码失败", "error", err)
|
||||
sender.SendError(models.WsError{
|
||||
Type: "error",
|
||||
RequestID: req.RequestID,
|
||||
Code: "INVALID_MESSAGE",
|
||||
Message: "音频数据解码失败",
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// 解码图片数据(可选)
|
||||
var image []byte
|
||||
if req.Image != "" {
|
||||
image, err = base64.StdEncoding.DecodeString(req.Image)
|
||||
if err != nil {
|
||||
log.Errorw("图片解码失败", "error", err)
|
||||
sender.SendError(models.WsError{
|
||||
Type: "error",
|
||||
RequestID: req.RequestID,
|
||||
Code: "INVALID_MESSAGE",
|
||||
Message: "图片数据解码失败",
|
||||
})
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 设置活跃请求
|
||||
if err := p.sessionMgr.SetActiveRequest(ctx, sessionID, req.RequestID); err != nil {
|
||||
log.Errorw("设置活跃请求失败", "error", err)
|
||||
}
|
||||
defer p.sessionMgr.ClearActiveRequest(ctx, sessionID)
|
||||
|
||||
// 获取会话配置
|
||||
sess, err := p.sessionMgr.Get(ctx, sessionID)
|
||||
if err != nil {
|
||||
log.Errorw("获取会话失败", "error", err)
|
||||
sender.SendError(models.WsError{
|
||||
Type: "error",
|
||||
RequestID: req.RequestID,
|
||||
Code: "SESSION_NOT_FOUND",
|
||||
Message: "会话不存在",
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// Step 1: STT 语音识别
|
||||
log.Infow("开始语音识别", "request_id", req.RequestID)
|
||||
sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{
|
||||
Encoding: "pcm_s16le",
|
||||
SampleRate: 16000,
|
||||
Language: sess.Config.Language,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorw("语音识别失败", "error", err)
|
||||
sender.SendError(models.WsError{
|
||||
Type: "error",
|
||||
RequestID: req.RequestID,
|
||||
Code: "STT_ERROR",
|
||||
Message: "语音识别失败",
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// 发送 STT 结果
|
||||
if err := sender.SendSTTResult(models.WsSTTResult{
|
||||
Type: "stt_result",
|
||||
RequestID: req.RequestID,
|
||||
Text: sttResult,
|
||||
IsFinal: true,
|
||||
}); err != nil {
|
||||
log.Errorw("发送 STT 结果失败", "error", err)
|
||||
}
|
||||
|
||||
// 追加用户消息到历史
|
||||
p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{
|
||||
Role: "user",
|
||||
Content: sttResult,
|
||||
})
|
||||
|
||||
// Step 2+3: LLM 流式推理 + TTS 并行合成
|
||||
log.Infow("开始 LLM 推理", "request_id", req.RequestID)
|
||||
llmReq := llm.Request{
|
||||
Image: image,
|
||||
Text: sttResult,
|
||||
History: history,
|
||||
Language: sess.Config.Language,
|
||||
}
|
||||
|
||||
llmStream, err := p.llmService.ChatStream(ctx, llmReq)
|
||||
if err != nil {
|
||||
log.Errorw("LLM 流式推理启动失败", "error", err)
|
||||
sender.SendError(models.WsError{
|
||||
Type: "error",
|
||||
RequestID: req.RequestID,
|
||||
Code: "LLM_ERROR",
|
||||
Message: "LLM 推理失败",
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// 创建句子切分器
|
||||
sentenceCh := make(chan string, 4)
|
||||
splitter := NewSplitter(sentenceCh)
|
||||
|
||||
// 并行:LLM 消费 + TTS 合成
|
||||
var wg sync.WaitGroup
|
||||
var fullText string
|
||||
var ttsErr error
|
||||
|
||||
// goroutine 1: 消费 LLM token + 句子切分
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer close(sentenceCh)
|
||||
fullText = p.consumeLLMStream(ctx, llmStream, req.RequestID, sender, splitter)
|
||||
}()
|
||||
|
||||
// goroutine 2: TTS 合成(如果启用)
|
||||
if sess.Config.TTSEnabled {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
log.Infow("开始 TTS 合成", "request_id", req.RequestID)
|
||||
ttsErr = p.synthesizeTTS(ctx, sentenceCh, req.RequestID, sender)
|
||||
}()
|
||||
} else {
|
||||
// 如果 TTS 未启用,需要消费 sentenceCh 防止阻塞
|
||||
go func() {
|
||||
for range sentenceCh {
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// 等待所有 goroutine 完成
|
||||
wg.Wait()
|
||||
|
||||
// TTS 失败静默跳过
|
||||
if ttsErr != nil {
|
||||
log.Warnw("TTS 合成失败(已跳过)", "error", ttsErr)
|
||||
}
|
||||
|
||||
// 追加助手消息到历史
|
||||
p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{
|
||||
Role: "assistant",
|
||||
Content: fullText,
|
||||
})
|
||||
|
||||
// 发送 llm_done
|
||||
latency := time.Since(startTime).Milliseconds()
|
||||
if err := sender.SendLLMDone(models.WsLLMDone{
|
||||
Type: "llm_done",
|
||||
RequestID: req.RequestID,
|
||||
FullText: fullText,
|
||||
Model: p.model,
|
||||
LatencyMs: latency,
|
||||
}); err != nil {
|
||||
log.Errorw("发送 llm_done 失败", "error", err)
|
||||
}
|
||||
|
||||
log.Infow("查询处理完成",
|
||||
"request_id", req.RequestID,
|
||||
"latency_ms", latency,
|
||||
"text_length", utf8.RuneCountInString(fullText),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// consumeLLMStream 消费 LLM 流式输出,发送 llm_chunk 并进行句子切分。
|
||||
func (p *Pipeline) consumeLLMStream(
|
||||
ctx context.Context,
|
||||
stream <-chan llm.Chunk,
|
||||
requestID string,
|
||||
sender Sender,
|
||||
splitter *Splitter,
|
||||
) string {
|
||||
log := logger.Log
|
||||
var fullText strings.Builder
|
||||
|
||||
for chunk := range stream {
|
||||
// 检查上下文是否已取消
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Infow("LLM 流被中断", "request_id", requestID)
|
||||
return fullText.String()
|
||||
default:
|
||||
}
|
||||
|
||||
if chunk.Done {
|
||||
// 流结束
|
||||
if chunk.TokensUsed != nil {
|
||||
log.Infow("LLM 用量统计",
|
||||
"request_id", requestID,
|
||||
"prompt_tokens", chunk.TokensUsed.Prompt,
|
||||
"completion_tokens", chunk.TokensUsed.Completion,
|
||||
"total_tokens", chunk.TokensUsed.Total,
|
||||
)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// 累积全文
|
||||
fullText.WriteString(chunk.Delta)
|
||||
|
||||
// 发送 llm_chunk
|
||||
if err := sender.SendLLMChunk(models.WsLLMChunk{
|
||||
Type: "llm_chunk",
|
||||
RequestID: requestID,
|
||||
Delta: chunk.Delta,
|
||||
Role: "assistant",
|
||||
}); err != nil {
|
||||
log.Errorw("发送 llm_chunk 失败", "error", err)
|
||||
}
|
||||
|
||||
// 句子切分
|
||||
splitter.Feed(chunk.Delta)
|
||||
}
|
||||
|
||||
// 刷新切分器中的剩余文本
|
||||
splitter.Flush()
|
||||
|
||||
return fullText.String()
|
||||
}
|
||||
|
||||
// synthesizeTTS 从句子 channel 读取文本,进行 TTS 合成并发送音频。
|
||||
func (p *Pipeline) synthesizeTTS(
|
||||
ctx context.Context,
|
||||
sentenceCh <-chan string,
|
||||
requestID string,
|
||||
sender Sender,
|
||||
) error {
|
||||
log := logger.Log
|
||||
|
||||
ttsStream, err := p.ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{
|
||||
Voice: "alloy",
|
||||
Speed: 1.0,
|
||||
OutputFmt: "mp3",
|
||||
SampleRate: 24000,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorw("TTS 合成启动失败", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// 消费 TTS 音频流
|
||||
for chunk := range ttsStream {
|
||||
// 检查上下文是否已取消
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Infow("TTS 流被中断", "request_id", requestID)
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Base64 编码音频数据
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(chunk.Audio)
|
||||
|
||||
if err := sender.SendTTSAudio(models.WsTTSAudio{
|
||||
Type: "tts_audio",
|
||||
RequestID: requestID,
|
||||
Audio: audioBase64,
|
||||
MimeType: "audio/mp3",
|
||||
IsLast: chunk.IsLast,
|
||||
}); err != nil {
|
||||
log.Errorw("发送 tts_audio 失败", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
661
backend/internal/orchestrator/pipeline_test.go
Normal file
661
backend/internal/orchestrator/pipeline_test.go
Normal file
@@ -0,0 +1,661 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
|
||||
"github.com/hhs/camtalk/internal/ai/llm"
|
||||
"github.com/hhs/camtalk/internal/ai/stt"
|
||||
"github.com/hhs/camtalk/internal/ai/tts"
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
logger.Init("debug", "console")
|
||||
}
|
||||
|
||||
// MockSTTService mock STT 服务
|
||||
type MockSTTService struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *MockSTTService) Recognize(ctx context.Context, audio []byte, opts stt.Options) (string, error) {
|
||||
args := m.Called(ctx, audio, opts)
|
||||
return args.String(0), args.Error(1)
|
||||
}
|
||||
|
||||
// MockLLMService mock LLM 服务
|
||||
type MockLLMService struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *MockLLMService) ChatStream(ctx context.Context, req llm.Request) (<-chan llm.Chunk, error) {
|
||||
args := m.Called(ctx, req)
|
||||
if args.Get(0) == nil {
|
||||
return nil, args.Error(1)
|
||||
}
|
||||
return args.Get(0).(<-chan llm.Chunk), args.Error(1)
|
||||
}
|
||||
|
||||
// MockTTSService mock TTS 服务
|
||||
type MockTTSService struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *MockTTSService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts tts.Options) (<-chan tts.Chunk, error) {
|
||||
args := m.Called(ctx, textStream, opts)
|
||||
if args.Get(0) == nil {
|
||||
return nil, args.Error(1)
|
||||
}
|
||||
return args.Get(0).(<-chan tts.Chunk), args.Error(1)
|
||||
}
|
||||
|
||||
// MockSessionManager mock 会话管理器
|
||||
type MockSessionManager struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) Create(ctx context.Context, config models.SessionConfig) (string, error) {
|
||||
args := m.Called(ctx, config)
|
||||
return args.String(0), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) Get(ctx context.Context, sessionID string) (*models.Session, error) {
|
||||
args := m.Called(ctx, sessionID)
|
||||
if args.Get(0) == nil {
|
||||
return nil, args.Error(1)
|
||||
}
|
||||
return args.Get(0).(*models.Session), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error {
|
||||
args := m.Called(ctx, sessionID, patch)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error) {
|
||||
args := m.Called(ctx, sessionID, limit)
|
||||
return args.Get(0).([]models.Message), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) AppendMessage(ctx context.Context, sessionID string, msg models.Message) error {
|
||||
args := m.Called(ctx, sessionID, msg)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) SetActiveRequest(ctx context.Context, sessionID string, requestID string) error {
|
||||
args := m.Called(ctx, sessionID, requestID)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) GetActiveRequestID(ctx context.Context, sessionID string) (string, error) {
|
||||
args := m.Called(ctx, sessionID)
|
||||
return args.String(0), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) ClearActiveRequest(ctx context.Context, sessionID string) error {
|
||||
args := m.Called(ctx, sessionID)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) Touch(ctx context.Context, sessionID string) error {
|
||||
args := m.Called(ctx, sessionID)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) Destroy(ctx context.Context, sessionID string) error {
|
||||
args := m.Called(ctx, sessionID)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSessionManager) ActiveCount() int {
|
||||
args := m.Called()
|
||||
return args.Int(0)
|
||||
}
|
||||
|
||||
// MockSender mock WebSocket 发送器
|
||||
type MockSender struct {
|
||||
mock.Mock
|
||||
STTResults []models.WsSTTResult
|
||||
LLMChunks []models.WsLLMChunk
|
||||
LLMDones []models.WsLLMDone
|
||||
TTSAudios []models.WsTTSAudio
|
||||
Errors []models.WsError
|
||||
}
|
||||
|
||||
func NewMockSender() *MockSender {
|
||||
return &MockSender{
|
||||
STTResults: make([]models.WsSTTResult, 0),
|
||||
LLMChunks: make([]models.WsLLMChunk, 0),
|
||||
LLMDones: make([]models.WsLLMDone, 0),
|
||||
TTSAudios: make([]models.WsTTSAudio, 0),
|
||||
Errors: make([]models.WsError, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MockSender) SendSTTResult(result models.WsSTTResult) error {
|
||||
m.STTResults = append(m.STTResults, result)
|
||||
args := m.Called(result)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSender) SendLLMChunk(chunk models.WsLLMChunk) error {
|
||||
m.LLMChunks = append(m.LLMChunks, chunk)
|
||||
args := m.Called(chunk)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSender) SendLLMDone(done models.WsLLMDone) error {
|
||||
m.LLMDones = append(m.LLMDones, done)
|
||||
args := m.Called(done)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSender) SendTTSAudio(audio models.WsTTSAudio) error {
|
||||
m.TTSAudios = append(m.TTSAudios, audio)
|
||||
args := m.Called(audio)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *MockSender) SendError(err models.WsError) error {
|
||||
m.Errors = append(m.Errors, err)
|
||||
args := m.Called(err)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
// 辅助函数:创建 LLM 流式响应
|
||||
func createLLMStream(chunks []llm.Chunk) <-chan llm.Chunk {
|
||||
ch := make(chan llm.Chunk, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
ch <- chunk
|
||||
}
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// 辅助函数:创建 TTS 流式响应
|
||||
func createTTSStream(chunks []tts.Chunk) <-chan tts.Chunk {
|
||||
ch := make(chan tts.Chunk, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
ch <- chunk
|
||||
}
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// TestProcessQuery_Success 测试完整流程
|
||||
func TestProcessQuery_Success(t *testing.T) {
|
||||
// 准备测试数据
|
||||
audioData := []byte("test audio")
|
||||
imageData := []byte("test image")
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
imageBase64 := base64.StdEncoding.EncodeToString(imageData)
|
||||
|
||||
req := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-123",
|
||||
Image: imageBase64,
|
||||
Audio: audioBase64,
|
||||
}
|
||||
|
||||
session := &models.Session{
|
||||
ID: "session-123",
|
||||
Config: models.SessionConfig{
|
||||
TTSEnabled: true,
|
||||
DetailLevel: "low",
|
||||
Language: "zh-CN",
|
||||
},
|
||||
}
|
||||
|
||||
// 创建 mock
|
||||
mockSTT := new(MockSTTService)
|
||||
mockLLM := new(MockLLMService)
|
||||
mockTTS := new(MockTTSService)
|
||||
mockSession := new(MockSessionManager)
|
||||
mockSender := NewMockSender()
|
||||
|
||||
// 设置 mock 期望
|
||||
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
|
||||
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
|
||||
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
|
||||
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
|
||||
|
||||
mockSTT.On("Recognize", mock.Anything, audioData, stt.Options{
|
||||
Encoding: "pcm_s16le",
|
||||
SampleRate: 16000,
|
||||
Language: "zh-CN",
|
||||
}).Return("你好,世界", nil)
|
||||
|
||||
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
|
||||
|
||||
llmChunks := []llm.Chunk{
|
||||
{Delta: "你好"},
|
||||
{Delta: ",世界!"},
|
||||
{Done: true, TokensUsed: &llm.TokenUsage{Prompt: 10, Completion: 5, Total: 15}},
|
||||
}
|
||||
mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil)
|
||||
|
||||
mockSender.On("SendLLMChunk", mock.Anything).Return(nil)
|
||||
mockSender.On("SendLLMDone", mock.Anything).Return(nil)
|
||||
|
||||
ttsChunks := []tts.Chunk{
|
||||
{Audio: []byte("audio1"), IsLast: false},
|
||||
{Audio: []byte("audio2"), IsLast: true},
|
||||
}
|
||||
mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return(createTTSStream(ttsChunks), nil)
|
||||
|
||||
mockSender.On("SendTTSAudio", mock.Anything).Return(nil)
|
||||
|
||||
// 创建 Pipeline
|
||||
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
|
||||
|
||||
// 执行
|
||||
ctx := context.Background()
|
||||
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
|
||||
|
||||
// 验证
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, mockSender.STTResults, 1)
|
||||
assert.Equal(t, "你好,世界", mockSender.STTResults[0].Text)
|
||||
assert.Len(t, mockSender.LLMChunks, 2)
|
||||
assert.Len(t, mockSender.LLMDones, 1)
|
||||
assert.Len(t, mockSender.TTSAudios, 2)
|
||||
|
||||
mockSTT.AssertExpectations(t)
|
||||
mockLLM.AssertExpectations(t)
|
||||
mockTTS.AssertExpectations(t)
|
||||
mockSession.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// TestProcessQuery_STTError 测试 STT 失败降级
|
||||
func TestProcessQuery_STTError(t *testing.T) {
|
||||
audioData := []byte("test audio")
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
|
||||
req := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-123",
|
||||
Audio: audioBase64,
|
||||
}
|
||||
|
||||
mockSTT := new(MockSTTService)
|
||||
mockLLM := new(MockLLMService)
|
||||
mockTTS := new(MockTTSService)
|
||||
mockSession := new(MockSessionManager)
|
||||
mockSender := NewMockSender()
|
||||
|
||||
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
|
||||
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
|
||||
mockSession.On("Get", mock.Anything, "session-123").Return(&models.Session{
|
||||
ID: "session-123",
|
||||
Config: models.SessionConfig{
|
||||
Language: "zh-CN",
|
||||
},
|
||||
}, nil)
|
||||
|
||||
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).
|
||||
Return("", errors.New("STT service unavailable"))
|
||||
|
||||
mockSender.On("SendError", mock.Anything).Return(nil)
|
||||
|
||||
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
|
||||
|
||||
ctx := context.Background()
|
||||
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Len(t, mockSender.Errors, 1)
|
||||
assert.Equal(t, "STT_ERROR", mockSender.Errors[0].Code)
|
||||
|
||||
mockSTT.AssertExpectations(t)
|
||||
mockLLM.AssertNotCalled(t, "ChatStream")
|
||||
mockTTS.AssertNotCalled(t, "SynthesizeStream")
|
||||
}
|
||||
|
||||
// TestProcessQuery_LLMError 测试 LLM 失败降级
|
||||
func TestProcessQuery_LLMError(t *testing.T) {
|
||||
audioData := []byte("test audio")
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
|
||||
req := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-123",
|
||||
Audio: audioBase64,
|
||||
}
|
||||
|
||||
session := &models.Session{
|
||||
ID: "session-123",
|
||||
Config: models.SessionConfig{
|
||||
TTSEnabled: true,
|
||||
Language: "zh-CN",
|
||||
},
|
||||
}
|
||||
|
||||
mockSTT := new(MockSTTService)
|
||||
mockLLM := new(MockLLMService)
|
||||
mockTTS := new(MockTTSService)
|
||||
mockSession := new(MockSessionManager)
|
||||
mockSender := NewMockSender()
|
||||
|
||||
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
|
||||
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
|
||||
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
|
||||
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
|
||||
|
||||
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil)
|
||||
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
|
||||
|
||||
mockLLM.On("ChatStream", mock.Anything, mock.Anything).
|
||||
Return(nil, errors.New("LLM service unavailable"))
|
||||
|
||||
mockSender.On("SendError", mock.Anything).Return(nil)
|
||||
|
||||
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
|
||||
|
||||
ctx := context.Background()
|
||||
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Len(t, mockSender.Errors, 1)
|
||||
assert.Equal(t, "LLM_ERROR", mockSender.Errors[0].Code)
|
||||
|
||||
mockSTT.AssertExpectations(t)
|
||||
mockLLM.AssertExpectations(t)
|
||||
mockTTS.AssertNotCalled(t, "SynthesizeStream")
|
||||
}
|
||||
|
||||
// TestProcessQuery_TTSError 测试 TTS 失败静默跳过
|
||||
func TestProcessQuery_TTSError(t *testing.T) {
|
||||
audioData := []byte("test audio")
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
|
||||
req := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-123",
|
||||
Audio: audioBase64,
|
||||
}
|
||||
|
||||
session := &models.Session{
|
||||
ID: "session-123",
|
||||
Config: models.SessionConfig{
|
||||
TTSEnabled: true,
|
||||
Language: "zh-CN",
|
||||
},
|
||||
}
|
||||
|
||||
mockSTT := new(MockSTTService)
|
||||
mockLLM := new(MockLLMService)
|
||||
mockTTS := new(MockTTSService)
|
||||
mockSession := new(MockSessionManager)
|
||||
mockSender := NewMockSender()
|
||||
|
||||
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
|
||||
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
|
||||
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
|
||||
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
|
||||
|
||||
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil)
|
||||
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
|
||||
|
||||
llmChunks := []llm.Chunk{
|
||||
{Delta: "你好"},
|
||||
{Done: true},
|
||||
}
|
||||
mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil)
|
||||
mockSender.On("SendLLMChunk", mock.Anything).Return(nil)
|
||||
mockSender.On("SendLLMDone", mock.Anything).Return(nil)
|
||||
|
||||
mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).
|
||||
Return(nil, errors.New("TTS service unavailable"))
|
||||
|
||||
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
|
||||
|
||||
ctx := context.Background()
|
||||
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
|
||||
|
||||
// TTS 失败应该静默跳过,不返回错误
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, mockSender.LLMDones, 1)
|
||||
assert.Len(t, mockSender.TTSAudios, 0)
|
||||
|
||||
mockSTT.AssertExpectations(t)
|
||||
mockLLM.AssertExpectations(t)
|
||||
mockTTS.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// TestProcessQuery_ContextCancelled 测试上下文取消(Interrupt)
|
||||
func TestProcessQuery_ContextCancelled(t *testing.T) {
|
||||
audioData := []byte("test audio")
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
|
||||
req := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-123",
|
||||
Audio: audioBase64,
|
||||
}
|
||||
|
||||
session := &models.Session{
|
||||
ID: "session-123",
|
||||
Config: models.SessionConfig{
|
||||
TTSEnabled: true,
|
||||
Language: "zh-CN",
|
||||
},
|
||||
}
|
||||
|
||||
mockSTT := new(MockSTTService)
|
||||
mockLLM := new(MockLLMService)
|
||||
mockTTS := new(MockTTSService)
|
||||
mockSession := new(MockSessionManager)
|
||||
mockSender := NewMockSender()
|
||||
|
||||
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
|
||||
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
|
||||
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
|
||||
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
|
||||
|
||||
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil)
|
||||
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
|
||||
|
||||
// 创建一个会延迟的 LLM 流,以便我们可以取消上下文
|
||||
llmCh := make(chan llm.Chunk)
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
llmCh <- llm.Chunk{Delta: "你"}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
llmCh <- llm.Chunk{Delta: "好"}
|
||||
close(llmCh)
|
||||
}()
|
||||
|
||||
mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return((<-chan llm.Chunk)(llmCh), nil)
|
||||
mockSender.On("SendLLMChunk", mock.Anything).Return(nil)
|
||||
mockSender.On("SendLLMDone", mock.Anything).Return(nil)
|
||||
|
||||
// 创建一个会延迟的 TTS 流
|
||||
ttsCh := make(chan tts.Chunk)
|
||||
go func() {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
close(ttsCh)
|
||||
}()
|
||||
mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return((<-chan tts.Chunk)(ttsCh), nil)
|
||||
|
||||
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
|
||||
|
||||
// 创建可取消的上下文
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
// 在 50ms 后取消
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
|
||||
|
||||
// 上下文取消后,流程应该正常完成(中断流但不返回错误)
|
||||
assert.NoError(t, err)
|
||||
|
||||
mockSTT.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// TestProcessQuery_DisabledTTS 测试 TTS 未启用的情况
|
||||
func TestProcessQuery_DisabledTTS(t *testing.T) {
|
||||
audioData := []byte("test audio")
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
|
||||
req := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-123",
|
||||
Audio: audioBase64,
|
||||
}
|
||||
|
||||
session := &models.Session{
|
||||
ID: "session-123",
|
||||
Config: models.SessionConfig{
|
||||
TTSEnabled: false, // TTS 未启用
|
||||
Language: "zh-CN",
|
||||
},
|
||||
}
|
||||
|
||||
mockSTT := new(MockSTTService)
|
||||
mockLLM := new(MockLLMService)
|
||||
mockTTS := new(MockTTSService)
|
||||
mockSession := new(MockSessionManager)
|
||||
mockSender := NewMockSender()
|
||||
|
||||
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
|
||||
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
|
||||
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
|
||||
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
|
||||
|
||||
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil)
|
||||
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
|
||||
|
||||
llmChunks := []llm.Chunk{
|
||||
{Delta: "你好"},
|
||||
{Done: true},
|
||||
}
|
||||
mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil)
|
||||
mockSender.On("SendLLMChunk", mock.Anything).Return(nil)
|
||||
mockSender.On("SendLLMDone", mock.Anything).Return(nil)
|
||||
|
||||
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
|
||||
|
||||
ctx := context.Background()
|
||||
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, mockSender.LLMDones, 1)
|
||||
assert.Len(t, mockSender.TTSAudios, 0)
|
||||
|
||||
// TTS 不应该被调用
|
||||
mockTTS.AssertNotCalled(t, "SynthesizeStream")
|
||||
}
|
||||
|
||||
// TestSplitter 测试句子切分器
|
||||
func TestSplitter(t *testing.T) {
|
||||
ch := make(chan string, 10)
|
||||
splitter := NewSplitter(ch)
|
||||
|
||||
// 输入包含多个句子的文本
|
||||
splitter.Feed("你好。")
|
||||
splitter.Feed("世界!")
|
||||
splitter.Feed("这是")
|
||||
splitter.Feed("一个测试。")
|
||||
splitter.Flush()
|
||||
|
||||
// 应该有 3 个句子
|
||||
assert.Equal(t, 3, len(ch))
|
||||
assert.Equal(t, "你好。", <-ch)
|
||||
assert.Equal(t, "世界!", <-ch)
|
||||
assert.Equal(t, "这是一个测试。", <-ch)
|
||||
}
|
||||
|
||||
// TestSplitter_NoDelimiter 测试没有分隔符的情况
|
||||
func TestSplitter_NoDelimiter(t *testing.T) {
|
||||
ch := make(chan string, 10)
|
||||
splitter := NewSplitter(ch)
|
||||
|
||||
splitter.Feed("没有分隔符的文本")
|
||||
splitter.Flush()
|
||||
|
||||
// 应该有 1 个句子(Flush 会发送剩余内容)
|
||||
assert.Equal(t, 1, len(ch))
|
||||
assert.Equal(t, "没有分隔符的文本", <-ch)
|
||||
}
|
||||
|
||||
// TestSplitter_Empty 测试空输入
|
||||
func TestSplitter_Empty(t *testing.T) {
|
||||
ch := make(chan string, 10)
|
||||
splitter := NewSplitter(ch)
|
||||
|
||||
splitter.Flush()
|
||||
|
||||
// 应该没有句子
|
||||
assert.Equal(t, 0, len(ch))
|
||||
}
|
||||
|
||||
// TestProcessQuery_InvalidAudio 测试无效音频数据
|
||||
func TestProcessQuery_InvalidAudio(t *testing.T) {
|
||||
req := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-123",
|
||||
Audio: "invalid-base64!!!",
|
||||
}
|
||||
|
||||
mockSTT := new(MockSTTService)
|
||||
mockLLM := new(MockLLMService)
|
||||
mockTTS := new(MockTTSService)
|
||||
mockSession := new(MockSessionManager)
|
||||
mockSender := NewMockSender()
|
||||
|
||||
mockSender.On("SendError", mock.Anything).Return(nil)
|
||||
|
||||
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
|
||||
|
||||
ctx := context.Background()
|
||||
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Len(t, mockSender.Errors, 1)
|
||||
assert.Equal(t, "INVALID_MESSAGE", mockSender.Errors[0].Code)
|
||||
}
|
||||
|
||||
// TestProcessQuery_SessionNotFound 测试会话不存在
|
||||
func TestProcessQuery_SessionNotFound(t *testing.T) {
|
||||
audioData := []byte("test audio")
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
|
||||
req := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-123",
|
||||
Audio: audioBase64,
|
||||
}
|
||||
|
||||
mockSTT := new(MockSTTService)
|
||||
mockLLM := new(MockLLMService)
|
||||
mockTTS := new(MockTTSService)
|
||||
mockSession := new(MockSessionManager)
|
||||
mockSender := NewMockSender()
|
||||
|
||||
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
|
||||
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
|
||||
mockSession.On("Get", mock.Anything, "session-123").Return(nil, errors.New("session not found"))
|
||||
|
||||
mockSender.On("SendError", mock.Anything).Return(nil)
|
||||
|
||||
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
|
||||
|
||||
ctx := context.Background()
|
||||
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Len(t, mockSender.Errors, 1)
|
||||
assert.Equal(t, "SESSION_NOT_FOUND", mockSender.Errors[0].Code)
|
||||
}
|
||||
22
backend/internal/orchestrator/sender.go
Normal file
22
backend/internal/orchestrator/sender.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package orchestrator
|
||||
|
||||
import "github.com/hhs/camtalk/internal/models"
|
||||
|
||||
// Sender 抽象 WebSocket 消息推送能力。
|
||||
// 便于测试时 mock,避免依赖真实 WebSocket 连接。
|
||||
type Sender interface {
|
||||
// SendSTTResult 发送语音识别结果。
|
||||
SendSTTResult(result models.WsSTTResult) error
|
||||
|
||||
// SendLLMChunk 发送 LLM 流式文本增量。
|
||||
SendLLMChunk(chunk models.WsLLMChunk) error
|
||||
|
||||
// SendLLMDone 发送 LLM 流结束信号。
|
||||
SendLLMDone(done models.WsLLMDone) error
|
||||
|
||||
// SendTTSAudio 发送 TTS 音频数据。
|
||||
SendTTSAudio(audio models.WsTTSAudio) error
|
||||
|
||||
// SendError 发送错误消息。
|
||||
SendError(err models.WsError) error
|
||||
}
|
||||
55
backend/internal/orchestrator/splitter.go
Normal file
55
backend/internal/orchestrator/splitter.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package orchestrator
|
||||
|
||||
import "strings"
|
||||
|
||||
// sentenceDelimiters 句子分隔符集合。
|
||||
var sentenceDelimiters = map[rune]bool{
|
||||
'。': true,
|
||||
'!': true,
|
||||
'?': true,
|
||||
'\n': true,
|
||||
'.': true,
|
||||
'!': true,
|
||||
'?': true,
|
||||
}
|
||||
|
||||
// Splitter 句子切分器。
|
||||
// 将流式文本按句子边界切分,发送到 channel 供 TTS 合成。
|
||||
type Splitter struct {
|
||||
ch chan<- string
|
||||
buffer strings.Builder
|
||||
}
|
||||
|
||||
// NewSplitter 创建句子切分器。
|
||||
// ch 用于接收切分后的句子文本。
|
||||
func NewSplitter(ch chan<- string) *Splitter {
|
||||
return &Splitter{
|
||||
ch: ch,
|
||||
}
|
||||
}
|
||||
|
||||
// Feed 输入增量文本,遇到句子分隔符时发送完整句子。
|
||||
func (s *Splitter) Feed(delta string) {
|
||||
for _, r := range delta {
|
||||
s.buffer.WriteRune(r)
|
||||
if sentenceDelimiters[r] {
|
||||
s.flushBuffer()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush 刷新缓冲区中的剩余文本(即使没有句子分隔符)。
|
||||
func (s *Splitter) Flush() {
|
||||
if s.buffer.Len() > 0 {
|
||||
s.flushBuffer()
|
||||
}
|
||||
}
|
||||
|
||||
// flushBuffer 将缓冲区内容发送到 channel 并清空。
|
||||
func (s *Splitter) flushBuffer() {
|
||||
text := strings.TrimSpace(s.buffer.String())
|
||||
if text != "" {
|
||||
s.ch <- text
|
||||
}
|
||||
s.buffer.Reset()
|
||||
}
|
||||
49
backend/internal/session/manager.go
Normal file
49
backend/internal/session/manager.go
Normal file
@@ -0,0 +1,49 @@
|
||||
// Package session 提供会话生命周期管理能力。
|
||||
package session
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
// ErrSessionNotFound 会话不存在或已过期。
|
||||
var ErrSessionNotFound = errors.New("session not found")
|
||||
|
||||
// Manager 会话管理器接口。
|
||||
// WebSocket Handler 通过此接口操作会话,不直接接触存储层。
|
||||
type Manager interface {
|
||||
// Create 创建新会话,返回 session ID。
|
||||
Create(ctx context.Context, config models.SessionConfig) (string, error)
|
||||
|
||||
// Get 获取会话(含 config)。不存在返回 ErrSessionNotFound。
|
||||
Get(ctx context.Context, sessionID string) (*models.Session, error)
|
||||
|
||||
// UpdateConfig 更新会话配置(config 消息触发)。
|
||||
UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error
|
||||
|
||||
// GetHistory 获取最近 N 轮对话历史(供 Orchestrator 构建 LLM 上下文)。
|
||||
GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error)
|
||||
|
||||
// AppendMessage 追加一条对话消息,同时刷新 TTL。
|
||||
AppendMessage(ctx context.Context, sessionID string, msg models.Message) error
|
||||
|
||||
// SetActiveRequest 标记当前正在处理的请求 ID(interrupt 用)。
|
||||
SetActiveRequest(ctx context.Context, sessionID string, requestID string) error
|
||||
|
||||
// GetActiveRequestID 获取当前活跃请求 ID。
|
||||
GetActiveRequestID(ctx context.Context, sessionID string) (string, error)
|
||||
|
||||
// ClearActiveRequest 清除活跃请求标记(请求完成或中断后)。
|
||||
ClearActiveRequest(ctx context.Context, sessionID string) error
|
||||
|
||||
// Touch 刷新 TTL(心跳时调用)。
|
||||
Touch(ctx context.Context, sessionID string) error
|
||||
|
||||
// Destroy 显式销毁会话(REST API DELETE 或连接断开清理)。
|
||||
Destroy(ctx context.Context, sessionID string) error
|
||||
|
||||
// ActiveCount 返回当前活跃会话数(健康检查用)。
|
||||
ActiveCount() int
|
||||
}
|
||||
275
backend/internal/session/memory.go
Normal file
275
backend/internal/session/memory.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTTL = 30 * time.Minute
|
||||
defaultHistorySize = 20
|
||||
)
|
||||
|
||||
// sessionEntry 内部会话条目。
|
||||
type sessionEntry struct {
|
||||
session models.Session
|
||||
history []models.Message
|
||||
activeReqID string
|
||||
lastActive time.Time
|
||||
}
|
||||
|
||||
// MemoryManager 基于内存的 SessionManager 实现。
|
||||
// 适用于 MVP 和无 Redis 的开发环境。
|
||||
type MemoryManager struct {
|
||||
mu sync.RWMutex
|
||||
sessions map[string]*sessionEntry
|
||||
ttl time.Duration
|
||||
maxHistory int
|
||||
stopCleaner chan struct{}
|
||||
}
|
||||
|
||||
// NewMemoryManager 创建内存版 SessionManager。
|
||||
// ttl 为会话过期时间,maxHistory 为对话历史上限(0 表示使用默认值 20)。
|
||||
func NewMemoryManager(ttl time.Duration, maxHistory int) *MemoryManager {
|
||||
if ttl <= 0 {
|
||||
ttl = defaultTTL
|
||||
}
|
||||
if maxHistory <= 0 {
|
||||
maxHistory = defaultHistorySize
|
||||
}
|
||||
|
||||
m := &MemoryManager{
|
||||
sessions: make(map[string]*sessionEntry),
|
||||
ttl: ttl,
|
||||
maxHistory: maxHistory,
|
||||
stopCleaner: make(chan struct{}),
|
||||
}
|
||||
|
||||
// 启动后台清理 goroutine,每分钟清除过期会话。
|
||||
go m.cleanLoop()
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// cleanLoop 后台定期清理过期会话。
|
||||
func (m *MemoryManager) cleanLoop() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
m.cleanExpired()
|
||||
case <-m.stopCleaner:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanExpired 清除所有过期会话。
|
||||
func (m *MemoryManager) cleanExpired() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
for id, entry := range m.sessions {
|
||||
if now.Sub(entry.lastActive) > m.ttl {
|
||||
delete(m.sessions, id)
|
||||
logger.Log.Debugw("session expired (cleaner)", "session", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop 停止后台清理 goroutine。应用退出前调用。
|
||||
func (m *MemoryManager) Stop() {
|
||||
close(m.stopCleaner)
|
||||
}
|
||||
|
||||
// isExpired 检查会话是否过期(调用方需持锁或在已知 entry 存在时调用)。
|
||||
func (m *MemoryManager) isExpired(entry *sessionEntry) bool {
|
||||
return time.Since(entry.lastActive) > m.ttl
|
||||
}
|
||||
|
||||
// Create 创建新会话。
|
||||
func (m *MemoryManager) Create(_ context.Context, config models.SessionConfig) (string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
id := uuid.New().String()
|
||||
now := time.Now()
|
||||
m.sessions[id] = &sessionEntry{
|
||||
session: models.Session{
|
||||
ID: id,
|
||||
CreatedAt: now,
|
||||
Config: config,
|
||||
},
|
||||
history: make([]models.Message, 0),
|
||||
lastActive: now,
|
||||
}
|
||||
|
||||
logger.Log.Debugw("session created", "session", id)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Get 获取会话。
|
||||
func (m *MemoryManager) Get(_ context.Context, sessionID string) (*models.Session, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
entry, ok := m.sessions[sessionID]
|
||||
if !ok || m.isExpired(entry) {
|
||||
return nil, ErrSessionNotFound
|
||||
}
|
||||
|
||||
sess := entry.session // 复制一份返回
|
||||
return &sess, nil
|
||||
}
|
||||
|
||||
// UpdateConfig 更新会话配置。
|
||||
func (m *MemoryManager) UpdateConfig(_ context.Context, sessionID string, patch models.SessionConfigPatch) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
entry, ok := m.sessions[sessionID]
|
||||
if !ok || m.isExpired(entry) {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
patch.Apply(&entry.session.Config)
|
||||
entry.lastActive = time.Now()
|
||||
|
||||
logger.Log.Debugw("session config updated", "session", sessionID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetHistory 获取最近 N 轮对话历史。
|
||||
func (m *MemoryManager) GetHistory(_ context.Context, sessionID string, limit int) ([]models.Message, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
entry, ok := m.sessions[sessionID]
|
||||
if !ok || m.isExpired(entry) {
|
||||
return nil, ErrSessionNotFound
|
||||
}
|
||||
|
||||
if limit <= 0 || limit > len(entry.history) {
|
||||
limit = len(entry.history)
|
||||
}
|
||||
|
||||
// 返回最近 limit 条的副本
|
||||
result := make([]models.Message, limit)
|
||||
copy(result, entry.history[len(entry.history)-limit:])
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// AppendMessage 追加一条对话消息,同时刷新 TTL。
|
||||
func (m *MemoryManager) AppendMessage(_ context.Context, sessionID string, msg models.Message) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
entry, ok := m.sessions[sessionID]
|
||||
if !ok || m.isExpired(entry) {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
entry.history = append(entry.history, msg)
|
||||
|
||||
// 超过上限时裁剪,保留最新的 maxHistory 条
|
||||
if len(entry.history) > m.maxHistory {
|
||||
entry.history = entry.history[len(entry.history)-m.maxHistory:]
|
||||
}
|
||||
|
||||
entry.lastActive = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetActiveRequest 标记当前正在处理的请求 ID。
|
||||
func (m *MemoryManager) SetActiveRequest(_ context.Context, sessionID string, requestID string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
entry, ok := m.sessions[sessionID]
|
||||
if !ok || m.isExpired(entry) {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
entry.activeReqID = requestID
|
||||
entry.lastActive = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetActiveRequestID 获取当前活跃请求 ID。
|
||||
func (m *MemoryManager) GetActiveRequestID(_ context.Context, sessionID string) (string, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
entry, ok := m.sessions[sessionID]
|
||||
if !ok || m.isExpired(entry) {
|
||||
return "", ErrSessionNotFound
|
||||
}
|
||||
|
||||
return entry.activeReqID, nil
|
||||
}
|
||||
|
||||
// ClearActiveRequest 清除活跃请求标记。
|
||||
func (m *MemoryManager) ClearActiveRequest(_ context.Context, sessionID string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
entry, ok := m.sessions[sessionID]
|
||||
if !ok || m.isExpired(entry) {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
entry.activeReqID = ""
|
||||
entry.lastActive = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Touch 刷新 TTL。
|
||||
func (m *MemoryManager) Touch(_ context.Context, sessionID string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
entry, ok := m.sessions[sessionID]
|
||||
if !ok || m.isExpired(entry) {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
entry.lastActive = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Destroy 显式销毁会话。
|
||||
func (m *MemoryManager) Destroy(_ context.Context, sessionID string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if _, ok := m.sessions[sessionID]; !ok {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
delete(m.sessions, sessionID)
|
||||
logger.Log.Debugw("session destroyed", "session", sessionID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ActiveCount 返回当前活跃会话数。
|
||||
func (m *MemoryManager) ActiveCount() int {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
now := time.Now()
|
||||
count := 0
|
||||
for _, entry := range m.sessions {
|
||||
if now.Sub(entry.lastActive) <= m.ttl {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
286
backend/internal/session/memory_test.go
Normal file
286
backend/internal/session/memory_test.go
Normal file
@@ -0,0 +1,286 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
logger.Init("debug", "console")
|
||||
}
|
||||
|
||||
func TestCreateAndGet(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
config := models.DefaultConfig()
|
||||
id, err := m.Create(ctx, config)
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
if id == "" {
|
||||
t.Fatal("Create returned empty ID")
|
||||
}
|
||||
|
||||
sess, err := m.Get(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if sess.ID != id {
|
||||
t.Errorf("ID = %q, want %q", sess.ID, id)
|
||||
}
|
||||
if sess.Config.Language != "zh-CN" {
|
||||
t.Errorf("Language = %q, want %q", sess.Config.Language, "zh-CN")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNotFound(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := m.Get(ctx, "nonexistent")
|
||||
if err != ErrSessionNotFound {
|
||||
t.Errorf("Get nonexistent: err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpire(t *testing.T) {
|
||||
// 使用极短 TTL 测试过期
|
||||
m := NewMemoryManager(50*time.Millisecond, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
id, _ := m.Create(ctx, models.DefaultConfig())
|
||||
|
||||
// 未过期时应能获取
|
||||
_, err := m.Get(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("Get before expire: %v", err)
|
||||
}
|
||||
|
||||
// 等待过期
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
|
||||
_, err = m.Get(ctx, id)
|
||||
if err != ErrSessionNotFound {
|
||||
t.Errorf("Get after expire: err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDestroy(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
id, _ := m.Create(ctx, models.DefaultConfig())
|
||||
|
||||
if err := m.Destroy(ctx, id); err != nil {
|
||||
t.Fatalf("Destroy: %v", err)
|
||||
}
|
||||
|
||||
_, err := m.Get(ctx, id)
|
||||
if err != ErrSessionNotFound {
|
||||
t.Errorf("Get after Destroy: err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDestroyNotFound(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
err := m.Destroy(ctx, "nonexistent")
|
||||
if err != ErrSessionNotFound {
|
||||
t.Errorf("Destroy nonexistent: err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendMessageAndGetHistory(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
id, _ := m.Create(ctx, models.DefaultConfig())
|
||||
|
||||
msgs := []models.Message{
|
||||
{Role: "user", Content: "你好"},
|
||||
{Role: "assistant", Content: "你好!有什么可以帮你的吗?"},
|
||||
{Role: "user", Content: "这是什么?"},
|
||||
{Role: "assistant", Content: "这是一朵花。"},
|
||||
}
|
||||
|
||||
for _, msg := range msgs {
|
||||
if err := m.AppendMessage(ctx, id, msg); err != nil {
|
||||
t.Fatalf("AppendMessage: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
history, err := m.GetHistory(ctx, id, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetHistory: %v", err)
|
||||
}
|
||||
if len(history) != 4 {
|
||||
t.Fatalf("GetHistory len = %d, want 4", len(history))
|
||||
}
|
||||
if history[0].Content != "你好" {
|
||||
t.Errorf("history[0] = %q, want %q", history[0].Content, "你好")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetHistoryLimit(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
id, _ := m.Create(ctx, models.DefaultConfig())
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
m.AppendMessage(ctx, id, models.Message{Role: "user", Content: "msg"})
|
||||
}
|
||||
|
||||
history, err := m.GetHistory(ctx, id, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("GetHistory: %v", err)
|
||||
}
|
||||
if len(history) != 3 {
|
||||
t.Fatalf("GetHistory limit=3: len = %d, want 3", len(history))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistoryLimit(t *testing.T) {
|
||||
const maxHistory = 5
|
||||
m := NewMemoryManager(30*time.Minute, maxHistory)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
id, _ := m.Create(ctx, models.DefaultConfig())
|
||||
|
||||
// 插入超过上限的消息
|
||||
for i := 0; i < 10; i++ {
|
||||
m.AppendMessage(ctx, id, models.Message{Role: "user", Content: "msg"})
|
||||
}
|
||||
|
||||
history, err := m.GetHistory(ctx, id, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetHistory: %v", err)
|
||||
}
|
||||
if len(history) != maxHistory {
|
||||
t.Fatalf("GetHistory after overflow: len = %d, want %d", len(history), maxHistory)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateConfig(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
id, _ := m.Create(ctx, models.DefaultConfig())
|
||||
|
||||
ttsEnabled := false
|
||||
detailLevel := "high"
|
||||
patch := models.SessionConfigPatch{
|
||||
TTSEnabled: &ttsEnabled,
|
||||
DetailLevel: &detailLevel,
|
||||
}
|
||||
|
||||
if err := m.UpdateConfig(ctx, id, patch); err != nil {
|
||||
t.Fatalf("UpdateConfig: %v", err)
|
||||
}
|
||||
|
||||
sess, _ := m.Get(ctx, id)
|
||||
if sess.Config.TTSEnabled != false {
|
||||
t.Errorf("TTSEnabled = %v, want false", sess.Config.TTSEnabled)
|
||||
}
|
||||
if sess.Config.DetailLevel != "high" {
|
||||
t.Errorf("DetailLevel = %q, want %q", sess.Config.DetailLevel, "high")
|
||||
}
|
||||
// Language 未传,应保持原值
|
||||
if sess.Config.Language != "zh-CN" {
|
||||
t.Errorf("Language = %q, want %q", sess.Config.Language, "zh-CN")
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveRequest(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
id, _ := m.Create(ctx, models.DefaultConfig())
|
||||
|
||||
// 初始应为空
|
||||
reqID, err := m.GetActiveRequestID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetActiveRequestID: %v", err)
|
||||
}
|
||||
if reqID != "" {
|
||||
t.Errorf("initial active request = %q, want empty", reqID)
|
||||
}
|
||||
|
||||
// 设置
|
||||
if err := m.SetActiveRequest(ctx, id, "req-123"); err != nil {
|
||||
t.Fatalf("SetActiveRequest: %v", err)
|
||||
}
|
||||
reqID, _ = m.GetActiveRequestID(ctx, id)
|
||||
if reqID != "req-123" {
|
||||
t.Errorf("active request = %q, want %q", reqID, "req-123")
|
||||
}
|
||||
|
||||
// 清除
|
||||
if err := m.ClearActiveRequest(ctx, id); err != nil {
|
||||
t.Fatalf("ClearActiveRequest: %v", err)
|
||||
}
|
||||
reqID, _ = m.GetActiveRequestID(ctx, id)
|
||||
if reqID != "" {
|
||||
t.Errorf("active request after clear = %q, want empty", reqID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTouchRefreshesTTL(t *testing.T) {
|
||||
m := NewMemoryManager(100*time.Millisecond, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
id, _ := m.Create(ctx, models.DefaultConfig())
|
||||
|
||||
// 50ms 后 Touch,应重置 TTL
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if err := m.Touch(ctx, id); err != nil {
|
||||
t.Fatalf("Touch: %v", err)
|
||||
}
|
||||
|
||||
// 再等 70ms(距创建 120ms,但距 Touch 只有 70ms),不应过期
|
||||
time.Sleep(70 * time.Millisecond)
|
||||
_, err := m.Get(ctx, id)
|
||||
if err != nil {
|
||||
t.Errorf("Get after Touch: %v, want nil (should not expire yet)", err)
|
||||
}
|
||||
|
||||
// 再等 50ms(距 Touch 120ms),应过期
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
_, err = m.Get(ctx, id)
|
||||
if err != ErrSessionNotFound {
|
||||
t.Errorf("Get after TTL: err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveCount(t *testing.T) {
|
||||
m := NewMemoryManager(30*time.Minute, 20)
|
||||
defer m.Stop()
|
||||
ctx := context.Background()
|
||||
|
||||
if m.ActiveCount() != 0 {
|
||||
t.Errorf("initial ActiveCount = %d, want 0", m.ActiveCount())
|
||||
}
|
||||
|
||||
m.Create(ctx, models.DefaultConfig())
|
||||
m.Create(ctx, models.DefaultConfig())
|
||||
if m.ActiveCount() != 2 {
|
||||
t.Errorf("ActiveCount = %d, want 2", m.ActiveCount())
|
||||
}
|
||||
}
|
||||
324
backend/internal/session/redis.go
Normal file
324
backend/internal/session/redis.go
Normal file
@@ -0,0 +1,324 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
// RedisManager 基于 Redis 的 SessionManager 实现。
|
||||
// 数据结构:
|
||||
// - session:{id}:meta → Hash(会话元数据)
|
||||
// - session:{id}:history → List(对话历史)
|
||||
type RedisManager struct {
|
||||
rdb *redis.Client
|
||||
ttl time.Duration
|
||||
maxHistory int
|
||||
}
|
||||
|
||||
// NewRedisManager 创建 Redis 版 SessionManager。
|
||||
func NewRedisManager(rdb *redis.Client, ttl time.Duration, maxHistory int) *RedisManager {
|
||||
if ttl <= 0 {
|
||||
ttl = defaultTTL
|
||||
}
|
||||
if maxHistory <= 0 {
|
||||
maxHistory = defaultHistorySize
|
||||
}
|
||||
return &RedisManager{rdb: rdb, ttl: ttl, maxHistory: maxHistory}
|
||||
}
|
||||
|
||||
func metaKey(id string) string { return fmt.Sprintf("session:%s:meta", id) }
|
||||
func histKey(id string) string { return fmt.Sprintf("session:%s:history", id) }
|
||||
|
||||
// Create 创建新会话。
|
||||
func (m *RedisManager) Create(ctx context.Context, config models.SessionConfig) (string, error) {
|
||||
id := uuidNew()
|
||||
now := time.Now().UTC()
|
||||
|
||||
pipe := m.rdb.Pipeline()
|
||||
|
||||
// 写入 meta Hash
|
||||
pipe.HSet(ctx, metaKey(id), map[string]interface{}{
|
||||
"session_id": id,
|
||||
"config.tts_enabled": strconv.FormatBool(config.TTSEnabled),
|
||||
"config.detail_level": config.DetailLevel,
|
||||
"config.language": config.Language,
|
||||
"created_at": now.Format(time.RFC3339),
|
||||
"last_active": now.Format(time.RFC3339),
|
||||
"active_request_id": "",
|
||||
})
|
||||
pipe.Expire(ctx, metaKey(id), m.ttl)
|
||||
|
||||
// 初始化空 history List
|
||||
pipe.RPush(ctx, histKey(id), placeholderHistoryMark)
|
||||
pipe.Expire(ctx, histKey(id), m.ttl)
|
||||
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return "", fmt.Errorf("redis create session: %w", err)
|
||||
}
|
||||
|
||||
logger.Log.Debugw("redis session created", "session", id)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// placeholderHistoryMark 占位符,避免 Redis 对空 key 的特殊行为。
|
||||
const placeholderHistoryMark = "__placeholder__"
|
||||
|
||||
// Get 获取会话。
|
||||
func (m *RedisManager) Get(ctx context.Context, sessionID string) (*models.Session, error) {
|
||||
vals, err := m.rdb.HGetAll(ctx, metaKey(sessionID)).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("redis get session: %w", err)
|
||||
}
|
||||
if len(vals) == 0 {
|
||||
return nil, ErrSessionNotFound
|
||||
}
|
||||
|
||||
sess := &models.Session{
|
||||
ID: vals["session_id"],
|
||||
}
|
||||
sess.CreatedAt, _ = time.Parse(time.RFC3339, vals["created_at"])
|
||||
sess.Config.TTSEnabled, _ = strconv.ParseBool(vals["config.tts_enabled"])
|
||||
sess.Config.DetailLevel = vals["config.detail_level"]
|
||||
sess.Config.Language = vals["config.language"]
|
||||
|
||||
return sess, nil
|
||||
}
|
||||
|
||||
// UpdateConfig 更新会话配置。
|
||||
func (m *RedisManager) UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error {
|
||||
// 先检查会话是否存在
|
||||
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis check session: %w", err)
|
||||
}
|
||||
if exists == 0 {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
fields := map[string]interface{}{
|
||||
"last_active": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if patch.TTSEnabled != nil {
|
||||
fields["config.tts_enabled"] = strconv.FormatBool(*patch.TTSEnabled)
|
||||
}
|
||||
if patch.DetailLevel != nil {
|
||||
fields["config.detail_level"] = *patch.DetailLevel
|
||||
}
|
||||
if patch.Language != nil {
|
||||
fields["config.language"] = *patch.Language
|
||||
}
|
||||
|
||||
if err := m.rdb.HSet(ctx, metaKey(sessionID), fields).Err(); err != nil {
|
||||
return fmt.Errorf("redis update config: %w", err)
|
||||
}
|
||||
|
||||
// 刷新 TTL
|
||||
m.rdb.Expire(ctx, metaKey(sessionID), m.ttl)
|
||||
logger.Log.Debugw("redis session config updated", "session", sessionID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetHistory 获取最近 N 轮对话历史。
|
||||
func (m *RedisManager) GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error) {
|
||||
// 检查会话是否存在
|
||||
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("redis check session: %w", err)
|
||||
}
|
||||
if exists == 0 {
|
||||
return nil, ErrSessionNotFound
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = m.maxHistory
|
||||
}
|
||||
|
||||
// LRANGE 0 {limit-1},最新在前(LPUSH),需要反转为时间顺序
|
||||
raws, err := m.rdb.LRange(ctx, histKey(sessionID), 0, int64(limit)).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("redis get history: %w", err)
|
||||
}
|
||||
|
||||
var msgs []models.Message
|
||||
for _, raw := range raws {
|
||||
if raw == placeholderHistoryMark {
|
||||
continue
|
||||
}
|
||||
var msg models.Message
|
||||
if err := json.Unmarshal([]byte(raw), &msg); err != nil {
|
||||
logger.Log.Warnw("invalid history entry", "session", sessionID, "raw", raw)
|
||||
continue
|
||||
}
|
||||
msgs = append(msgs, msg)
|
||||
}
|
||||
|
||||
// 反转为时间顺序(LPUSH 最新在前 → 需要最旧在前)
|
||||
for i, j := 0, len(msgs)-1; i < j; i, j = i+1, j-1 {
|
||||
msgs[i], msgs[j] = msgs[j], msgs[i]
|
||||
}
|
||||
|
||||
return msgs, nil
|
||||
}
|
||||
|
||||
// AppendMessage 追加一条对话消息,同时刷新 TTL。
|
||||
func (m *RedisManager) AppendMessage(ctx context.Context, sessionID string, msg models.Message) error {
|
||||
// 检查会话是否存在
|
||||
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis check session: %w", err)
|
||||
}
|
||||
if exists == 0 {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
data, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal message: %w", err)
|
||||
}
|
||||
|
||||
pipe := m.rdb.Pipeline()
|
||||
// LPUSH 新消息到左头(最新在前)
|
||||
pipe.LPush(ctx, histKey(sessionID), string(data))
|
||||
// LTRIM 保留最近 maxHistory 条(+1 是因为有占位符)
|
||||
pipe.LTrim(ctx, histKey(sessionID), 0, int64(m.maxHistory))
|
||||
// 刷新 TTL
|
||||
pipe.Expire(ctx, histKey(sessionID), m.ttl)
|
||||
pipe.Expire(ctx, metaKey(sessionID), m.ttl)
|
||||
// 更新 last_active
|
||||
pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339))
|
||||
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return fmt.Errorf("redis append message: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetActiveRequest 标记当前正在处理的请求 ID。
|
||||
func (m *RedisManager) SetActiveRequest(ctx context.Context, sessionID string, requestID string) error {
|
||||
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis check session: %w", err)
|
||||
}
|
||||
if exists == 0 {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
pipe := m.rdb.Pipeline()
|
||||
pipe.HSet(ctx, metaKey(sessionID), "active_request_id", requestID)
|
||||
pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339))
|
||||
pipe.Expire(ctx, metaKey(sessionID), m.ttl)
|
||||
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return fmt.Errorf("redis set active request: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetActiveRequestID 获取当前活跃请求 ID。
|
||||
func (m *RedisManager) GetActiveRequestID(ctx context.Context, sessionID string) (string, error) {
|
||||
val, err := m.rdb.HGet(ctx, metaKey(sessionID), "active_request_id").Result()
|
||||
if err == redis.Nil {
|
||||
return "", ErrSessionNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("redis get active request: %w", err)
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// ClearActiveRequest 清除活跃请求标记。
|
||||
func (m *RedisManager) ClearActiveRequest(ctx context.Context, sessionID string) error {
|
||||
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis check session: %w", err)
|
||||
}
|
||||
if exists == 0 {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
pipe := m.rdb.Pipeline()
|
||||
pipe.HSet(ctx, metaKey(sessionID), "active_request_id", "")
|
||||
pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339))
|
||||
pipe.Expire(ctx, metaKey(sessionID), m.ttl)
|
||||
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return fmt.Errorf("redis clear active request: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Touch 刷新 TTL。
|
||||
func (m *RedisManager) Touch(ctx context.Context, sessionID string) error {
|
||||
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis check session: %w", err)
|
||||
}
|
||||
if exists == 0 {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
pipe := m.rdb.Pipeline()
|
||||
pipe.Expire(ctx, metaKey(sessionID), m.ttl)
|
||||
pipe.Expire(ctx, histKey(sessionID), m.ttl)
|
||||
pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339))
|
||||
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return fmt.Errorf("redis touch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Destroy 显式销毁会话。
|
||||
func (m *RedisManager) Destroy(ctx context.Context, sessionID string) error {
|
||||
deleted, err := m.rdb.Del(ctx, metaKey(sessionID), histKey(sessionID)).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis destroy session: %w", err)
|
||||
}
|
||||
if deleted == 0 {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
|
||||
logger.Log.Debugw("redis session destroyed", "session", sessionID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ActiveCount 返回当前活跃会话数。
|
||||
// Redis 实现通过 SCAN 遍历 meta key,适用于中等规模。
|
||||
// 大规模部署建议维护独立的活跃会话集合。
|
||||
func (m *RedisManager) ActiveCount() int {
|
||||
ctx := context.Background()
|
||||
count := 0
|
||||
var cursor uint64
|
||||
for {
|
||||
keys, nextCursor, err := m.rdb.Scan(ctx, cursor, "session:*:meta", 100).Result()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, key := range keys {
|
||||
exists, _ := m.rdb.Exists(ctx, key).Result()
|
||||
if exists > 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
cursor = nextCursor
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// uuidNew 生成 UUID,便于测试时 mock。
|
||||
var uuidNew = func() string {
|
||||
return uuid.New().String()
|
||||
}
|
||||
@@ -1,17 +1,20 @@
|
||||
package ws
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/gorilla/websocket"
|
||||
|
||||
"github.com/hhs/camtalk/internal/errors"
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
"github.com/hhs/camtalk/internal/orchestrator"
|
||||
"github.com/hhs/camtalk/internal/session"
|
||||
)
|
||||
|
||||
var upgrader = websocket.Upgrader{
|
||||
@@ -20,36 +23,94 @@ var upgrader = websocket.Upgrader{
|
||||
|
||||
// Client 代表一个 WebSocket 客户端连接。
|
||||
type Client struct {
|
||||
conn *websocket.Conn
|
||||
sessionID string
|
||||
mu sync.Mutex
|
||||
conn *websocket.Conn
|
||||
sessionID string
|
||||
sessionMgr session.Manager
|
||||
orchestrator orchestrator.Orchestrator
|
||||
cancelFuncs map[string]context.CancelFunc // requestID → cancel func
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (c *Client) sendJSON(v any) error {
|
||||
// SendJSON 向客户端发送 JSON 消息(公开以便 errors 包调用)。
|
||||
func (c *Client) SendJSON(v any) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.conn.WriteJSON(v)
|
||||
}
|
||||
|
||||
// WSClient 实现 orchestrator.Sender 接口,将消息推送到 WebSocket 连接。
|
||||
type WSClient struct {
|
||||
client *Client
|
||||
requestID string
|
||||
}
|
||||
|
||||
// SendSTTResult 发送语音识别结果。
|
||||
func (w *WSClient) SendSTTResult(result models.WsSTTResult) error {
|
||||
result.RequestID = w.requestID
|
||||
return w.client.SendJSON(result)
|
||||
}
|
||||
|
||||
// SendLLMChunk 发送 LLM 流式文本增量。
|
||||
func (w *WSClient) SendLLMChunk(chunk models.WsLLMChunk) error {
|
||||
chunk.RequestID = w.requestID
|
||||
return w.client.SendJSON(chunk)
|
||||
}
|
||||
|
||||
// SendLLMDone 发送 LLM 流结束信号。
|
||||
func (w *WSClient) SendLLMDone(done models.WsLLMDone) error {
|
||||
done.RequestID = w.requestID
|
||||
return w.client.SendJSON(done)
|
||||
}
|
||||
|
||||
// SendTTSAudio 发送 TTS 音频数据。
|
||||
func (w *WSClient) SendTTSAudio(audio models.WsTTSAudio) error {
|
||||
audio.RequestID = w.requestID
|
||||
return w.client.SendJSON(audio)
|
||||
}
|
||||
|
||||
// SendError 发送错误消息。
|
||||
func (w *WSClient) SendError(err models.WsError) error {
|
||||
err.RequestID = w.requestID
|
||||
return w.client.SendJSON(err)
|
||||
}
|
||||
|
||||
// ServeWS 处理 WebSocket 升级请求。
|
||||
func ServeWS(c *gin.Context) {
|
||||
func ServeWS(sessionMgr session.Manager, orch orchestrator.Orchestrator) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
serveWS(c, sessionMgr, orch)
|
||||
}
|
||||
}
|
||||
|
||||
func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orchestrator) {
|
||||
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
|
||||
if err != nil {
|
||||
log.Printf("websocket upgrade failed: %v", err)
|
||||
logger.Log.Errorw("websocket upgrade failed", "error", err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
sessionID := uuid.New().String()
|
||||
client := &Client{conn: conn, sessionID: sessionID}
|
||||
// 创建会话
|
||||
sessionID, err := sessionMgr.Create(context.Background(), models.DefaultConfig())
|
||||
if err != nil {
|
||||
logger.Log.Errorw("create session failed", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
client := &Client{
|
||||
conn: conn,
|
||||
sessionID: sessionID,
|
||||
sessionMgr: sessionMgr,
|
||||
orchestrator: orch,
|
||||
cancelFuncs: make(map[string]context.CancelFunc),
|
||||
}
|
||||
|
||||
// 发送 connected 消息
|
||||
_ = client.sendJSON(models.WsConnected{
|
||||
_ = client.SendJSON(models.WsConnected{
|
||||
Type: "connected",
|
||||
SessionID: sessionID,
|
||||
ServerVersion: "0.1.0",
|
||||
})
|
||||
log.Printf("client connected: session=%s", sessionID)
|
||||
logger.Log.Infow("client connected", "session", sessionID)
|
||||
|
||||
// 心跳检测
|
||||
lastPong := time.Now()
|
||||
@@ -67,7 +128,7 @@ func ServeWS(c *gin.Context) {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if time.Since(lastPong) > 60*time.Second {
|
||||
log.Printf("heartbeat timeout: session=%s", sessionID)
|
||||
logger.Log.Warnw("heartbeat timeout", "session", sessionID)
|
||||
conn.Close()
|
||||
return
|
||||
}
|
||||
@@ -82,7 +143,7 @@ func ServeWS(c *gin.Context) {
|
||||
_, message, err := conn.ReadMessage()
|
||||
if err != nil {
|
||||
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) {
|
||||
log.Printf("ws read error: %v", err)
|
||||
logger.Log.Warnw("ws read error", "error", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -92,51 +153,96 @@ func ServeWS(c *gin.Context) {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
if err := json.Unmarshal(message, &envelope); err != nil {
|
||||
_ = client.sendJSON(models.WsError{
|
||||
Type: "error",
|
||||
Code: "INVALID_MESSAGE",
|
||||
Message: "invalid JSON",
|
||||
})
|
||||
errors.SendWSError(client, errors.CodeInvalidMessage, "", err)
|
||||
continue
|
||||
}
|
||||
|
||||
switch envelope.Type {
|
||||
case "ping":
|
||||
_ = client.sendJSON(models.WsPong{Type: "pong"})
|
||||
_ = client.SendJSON(models.WsPong{Type: "pong"})
|
||||
|
||||
case "query":
|
||||
var msg models.WsQuery
|
||||
if err := json.Unmarshal(message, &msg); err != nil {
|
||||
_ = client.sendJSON(models.WsError{
|
||||
Type: "error",
|
||||
Code: "INVALID_MESSAGE",
|
||||
Message: "invalid query message",
|
||||
RequestID: msg.RequestID,
|
||||
})
|
||||
errors.SendWSError(client, errors.CodeInvalidMessage, msg.RequestID, err)
|
||||
continue
|
||||
}
|
||||
log.Printf("query received: session=%s request=%s", sessionID, msg.RequestID)
|
||||
// TODO: 调用 AI 编排流程(STT → LLM → TTS)
|
||||
logger.Log.Infow("query received", "session", sessionID, "request", msg.RequestID)
|
||||
|
||||
// 刷新会话 TTL
|
||||
if err := client.sessionMgr.Touch(context.Background(), sessionID); err != nil {
|
||||
logger.Log.Warnw("touch session failed", "session", sessionID, "error", err)
|
||||
}
|
||||
|
||||
// 标记活跃请求
|
||||
if err := client.sessionMgr.SetActiveRequest(context.Background(), sessionID, msg.RequestID); err != nil {
|
||||
logger.Log.Warnw("set active request failed", "session", sessionID, "error", err)
|
||||
}
|
||||
|
||||
// 获取对话历史
|
||||
history, _ := client.sessionMgr.GetHistory(context.Background(), sessionID, 20)
|
||||
|
||||
// 创建可取消的 context
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
client.mu.Lock()
|
||||
client.cancelFuncs[msg.RequestID] = cancel
|
||||
client.mu.Unlock()
|
||||
|
||||
// 创建 sender
|
||||
sender := &WSClient{client: client, requestID: msg.RequestID}
|
||||
|
||||
// 启动 orchestrator 处理 goroutine
|
||||
go func() {
|
||||
defer func() {
|
||||
// 清理 cancel func
|
||||
client.mu.Lock()
|
||||
delete(client.cancelFuncs, msg.RequestID)
|
||||
client.mu.Unlock()
|
||||
cancel()
|
||||
// 清除活跃请求
|
||||
_ = client.sessionMgr.ClearActiveRequest(context.Background(), sessionID)
|
||||
}()
|
||||
|
||||
if err := client.orchestrator.ProcessQuery(ctx, sessionID, msg, history, sender); err != nil {
|
||||
logger.Log.Errorw("process query failed", "session", sessionID, "request", msg.RequestID, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
case "config":
|
||||
var msg models.WsConfig
|
||||
if err := json.Unmarshal(message, &msg); err != nil {
|
||||
_ = client.sendJSON(models.WsError{
|
||||
Type: "error",
|
||||
Code: "INVALID_MESSAGE",
|
||||
Message: "invalid config message",
|
||||
})
|
||||
errors.SendWSError(client, errors.CodeInvalidMessage, "", err)
|
||||
continue
|
||||
}
|
||||
log.Printf("config update: session=%s", sessionID)
|
||||
// TODO: 更新会话配置
|
||||
|
||||
patch := models.SessionConfigPatch{
|
||||
TTSEnabled: msg.Payload.TTSEnabled,
|
||||
DetailLevel: msg.Payload.DetailLevel,
|
||||
Language: msg.Payload.Language,
|
||||
}
|
||||
if err := client.sessionMgr.UpdateConfig(context.Background(), sessionID, patch); err != nil {
|
||||
errors.SendWSError(client, errors.CodeInternalError, "", err)
|
||||
continue
|
||||
}
|
||||
logger.Log.Infow("config updated", "session", sessionID)
|
||||
|
||||
case "interrupt":
|
||||
log.Printf("interrupt received: session=%s", sessionID)
|
||||
// TODO: 中断当前 AI 响应
|
||||
logger.Log.Infow("interrupt received", "session", sessionID)
|
||||
|
||||
// 获取活跃请求 ID 并取消
|
||||
reqID, _ := client.sessionMgr.GetActiveRequestID(context.Background(), sessionID)
|
||||
if reqID != "" {
|
||||
client.mu.Lock()
|
||||
if cancel, ok := client.cancelFuncs[reqID]; ok {
|
||||
cancel()
|
||||
delete(client.cancelFuncs, reqID)
|
||||
}
|
||||
client.mu.Unlock()
|
||||
_ = client.sessionMgr.ClearActiveRequest(context.Background(), sessionID)
|
||||
}
|
||||
|
||||
default:
|
||||
_ = client.sendJSON(models.WsError{
|
||||
_ = client.SendJSON(models.WsError{
|
||||
Type: "error",
|
||||
Code: "INVALID_MESSAGE",
|
||||
Message: "unknown message type: " + envelope.Type,
|
||||
@@ -145,5 +251,16 @@ func ServeWS(c *gin.Context) {
|
||||
}
|
||||
|
||||
close(done)
|
||||
log.Printf("client disconnected: session=%s", sessionID)
|
||||
|
||||
// 取消所有活跃请求
|
||||
client.mu.Lock()
|
||||
for reqID, cancel := range client.cancelFuncs {
|
||||
logger.Log.Infow("canceling active request on disconnect", "session", sessionID, "request", reqID)
|
||||
cancel()
|
||||
}
|
||||
client.cancelFuncs = make(map[string]context.CancelFunc)
|
||||
client.mu.Unlock()
|
||||
|
||||
// 断开连接时不销毁会话,让其自然过期(支持重连恢复)
|
||||
logger.Log.Infow("client disconnected", "session", sessionID)
|
||||
}
|
||||
|
||||
563
backend/internal/ws/handler_test.go
Normal file
563
backend/internal/ws/handler_test.go
Normal file
@@ -0,0 +1,563 @@
|
||||
package ws
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"context"
|
||||
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
"github.com/hhs/camtalk/internal/orchestrator"
|
||||
"github.com/hhs/camtalk/internal/session"
|
||||
)
|
||||
|
||||
func init() {
|
||||
logger.Init("debug", "console")
|
||||
gin.SetMode(gin.TestMode)
|
||||
}
|
||||
|
||||
// --- Mock Orchestrator ---
|
||||
|
||||
// MockOrchestrator 实现 orchestrator.Orchestrator 接口,
|
||||
// 模拟完整的 STT → LLM → TTS 管道,通过 sender 推送消息。
|
||||
type MockOrchestrator struct {
|
||||
// STTResult 模拟的语音识别结果
|
||||
STTResult string
|
||||
// LLMDeltas 模拟的 LLM 流式输出
|
||||
LLMDeltas []string
|
||||
// TTSAudios 模拟的 TTS 音频数据(每项一个 base64 编码的 MP3 片段)
|
||||
TTSAudios []string
|
||||
// Err 如果非 nil,ProcessQuery 直接返回此错误
|
||||
Err error
|
||||
// Delay 每个消息之间的延迟(用于 interrupt 测试)
|
||||
Delay time.Duration
|
||||
}
|
||||
|
||||
func (m *MockOrchestrator) ProcessQuery(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
req models.WsQuery,
|
||||
history []models.Message,
|
||||
sender orchestrator.Sender,
|
||||
) error {
|
||||
if m.Err != nil {
|
||||
sender.SendError(models.WsError{
|
||||
Type: "error",
|
||||
RequestID: req.RequestID,
|
||||
Code: "INTERNAL_ERROR",
|
||||
Message: m.Err.Error(),
|
||||
})
|
||||
return m.Err
|
||||
}
|
||||
|
||||
// Step 1: 发送 STT 结果
|
||||
if m.STTResult != "" {
|
||||
_ = sender.SendSTTResult(models.WsSTTResult{
|
||||
Type: "stt_result",
|
||||
RequestID: req.RequestID,
|
||||
Text: m.STTResult,
|
||||
IsFinal: true,
|
||||
})
|
||||
}
|
||||
if m.Delay > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil // 中断视为正常完成
|
||||
case <-time.After(m.Delay):
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: 发送 LLM chunks
|
||||
var fullText strings.Builder
|
||||
for _, delta := range m.LLMDeltas {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil // 中断视为正常完成
|
||||
default:
|
||||
}
|
||||
fullText.WriteString(delta)
|
||||
_ = sender.SendLLMChunk(models.WsLLMChunk{
|
||||
Type: "llm_chunk",
|
||||
RequestID: req.RequestID,
|
||||
Delta: delta,
|
||||
Role: "assistant",
|
||||
})
|
||||
if m.Delay > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil // 中断视为正常完成
|
||||
case <-time.After(m.Delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: 发送 TTS 音频
|
||||
for i, audio := range m.TTSAudios {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil // 中断视为正常完成
|
||||
default:
|
||||
}
|
||||
isLast := i == len(m.TTSAudios)-1
|
||||
_ = sender.SendTTSAudio(models.WsTTSAudio{
|
||||
Type: "tts_audio",
|
||||
RequestID: req.RequestID,
|
||||
Audio: audio,
|
||||
MimeType: "audio/mp3",
|
||||
IsLast: isLast,
|
||||
})
|
||||
}
|
||||
|
||||
// Step 4: 发送 llm_done
|
||||
_ = sender.SendLLMDone(models.WsLLMDone{
|
||||
Type: "llm_done",
|
||||
RequestID: req.RequestID,
|
||||
FullText: fullText.String(),
|
||||
Model: "gpt-4o",
|
||||
LatencyMs: 100,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- 测试辅助函数 ---
|
||||
|
||||
// setupTestServer 创建测试用 Gin 服务器和 WebSocket URL。
|
||||
func setupTestServer(t *testing.T, orch orchestrator.Orchestrator) (*httptest.Server, string) {
|
||||
t.Helper()
|
||||
|
||||
sessionMgr := session.NewMemoryManager(5*time.Minute, 20)
|
||||
t.Cleanup(func() { sessionMgr.Stop() })
|
||||
|
||||
r := gin.New()
|
||||
r.GET("/ws", ServeWS(sessionMgr, orch))
|
||||
|
||||
srv := httptest.NewServer(r)
|
||||
|
||||
// 构造 WebSocket URL
|
||||
wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + "/ws"
|
||||
|
||||
return srv, wsURL
|
||||
}
|
||||
|
||||
// connectWS 建立 WebSocket 连接并返回 conn。
|
||||
func connectWS(t *testing.T, wsURL string) *websocket.Conn {
|
||||
t.Helper()
|
||||
|
||||
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
|
||||
require.NoError(t, err, "WebSocket 连接失败")
|
||||
t.Cleanup(func() { conn.Close() })
|
||||
return conn
|
||||
}
|
||||
|
||||
// readJSON 从 WebSocket 读取一条 JSON 消息。
|
||||
func readJSON(t *testing.T, conn *websocket.Conn) map[string]any {
|
||||
t.Helper()
|
||||
|
||||
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
var msg map[string]any
|
||||
err := conn.ReadJSON(&msg)
|
||||
require.NoError(t, err, "读取 WebSocket 消息失败")
|
||||
return msg
|
||||
}
|
||||
|
||||
// --- 测试用例 ---
|
||||
|
||||
// TestWS_Connected 验证连接建立后收到 connected 消息。
|
||||
func TestWS_Connected(t *testing.T) {
|
||||
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
|
||||
msg := readJSON(t, conn)
|
||||
assert.Equal(t, "connected", msg["type"])
|
||||
assert.NotEmpty(t, msg["session_id"])
|
||||
assert.Equal(t, "0.1.0", msg["server_version"])
|
||||
}
|
||||
|
||||
// TestWS_PingPong 验证 ping/pong 心跳。
|
||||
func TestWS_PingPong(t *testing.T) {
|
||||
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
|
||||
// 读取 connected 消息
|
||||
_ = readJSON(t, conn)
|
||||
|
||||
// 发送 ping
|
||||
err := conn.WriteJSON(map[string]string{"type": "ping"})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 读取 pong
|
||||
msg := readJSON(t, conn)
|
||||
assert.Equal(t, "pong", msg["type"])
|
||||
}
|
||||
|
||||
// TestWS_QueryFullFlow 验证完整的 query → stt_result → llm_chunk → tts_audio → llm_done 流程。
|
||||
func TestWS_QueryFullFlow(t *testing.T) {
|
||||
audioB64 := base64.StdEncoding.EncodeToString([]byte("fake-audio-data"))
|
||||
imageB64 := base64.StdEncoding.EncodeToString([]byte("fake-image-data"))
|
||||
|
||||
mock := &MockOrchestrator{
|
||||
STTResult: "你好,世界",
|
||||
LLMDeltas: []string{"你好", ",世界!"},
|
||||
TTSAudios: []string{base64.StdEncoding.EncodeToString([]byte("mp3-data-1")), base64.StdEncoding.EncodeToString([]byte("mp3-data-2"))},
|
||||
}
|
||||
|
||||
srv, wsURL := setupTestServer(t, mock)
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
|
||||
// 1. 读取 connected
|
||||
connected := readJSON(t, conn)
|
||||
assert.Equal(t, "connected", connected["type"])
|
||||
sessionID := connected["session_id"].(string)
|
||||
assert.NotEmpty(t, sessionID)
|
||||
|
||||
// 2. 发送 query
|
||||
queryMsg := models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-test-001",
|
||||
Image: imageB64,
|
||||
Audio: audioB64,
|
||||
MimeType: "audio/pcm",
|
||||
}
|
||||
err := conn.WriteJSON(queryMsg)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 3. 读取 stt_result
|
||||
sttResult := readJSON(t, conn)
|
||||
assert.Equal(t, "stt_result", sttResult["type"])
|
||||
assert.Equal(t, "req-test-001", sttResult["request_id"])
|
||||
assert.Equal(t, "你好,世界", sttResult["text"])
|
||||
assert.Equal(t, true, sttResult["is_final"])
|
||||
|
||||
// 4. 读取 llm_chunk 消息
|
||||
chunk1 := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_chunk", chunk1["type"])
|
||||
assert.Equal(t, "req-test-001", chunk1["request_id"])
|
||||
assert.Equal(t, "你好", chunk1["delta"])
|
||||
assert.Equal(t, "assistant", chunk1["role"])
|
||||
|
||||
chunk2 := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_chunk", chunk2["type"])
|
||||
assert.Equal(t, ",世界!", chunk2["delta"])
|
||||
|
||||
// 5. 读取 tts_audio 消息
|
||||
tts1 := readJSON(t, conn)
|
||||
assert.Equal(t, "tts_audio", tts1["type"])
|
||||
assert.Equal(t, "req-test-001", tts1["request_id"])
|
||||
assert.NotEmpty(t, tts1["audio"])
|
||||
assert.Equal(t, "audio/mp3", tts1["mime_type"])
|
||||
assert.Equal(t, false, tts1["is_last"])
|
||||
|
||||
tts2 := readJSON(t, conn)
|
||||
assert.Equal(t, "tts_audio", tts2["type"])
|
||||
assert.Equal(t, true, tts2["is_last"])
|
||||
|
||||
// 6. 读取 llm_done
|
||||
llmDone := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_done", llmDone["type"])
|
||||
assert.Equal(t, "req-test-001", llmDone["request_id"])
|
||||
assert.Equal(t, "你好,世界!", llmDone["full_text"])
|
||||
assert.Equal(t, "gpt-4o", llmDone["model"])
|
||||
assert.NotNil(t, llmDone["latency_ms"])
|
||||
}
|
||||
|
||||
// TestWS_QuerySTTOnly 验证只有 STT 结果、无 LLM 输出的场景。
|
||||
func TestWS_QuerySTTOnly(t *testing.T) {
|
||||
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
|
||||
|
||||
mock := &MockOrchestrator{
|
||||
STTResult: "测试语音",
|
||||
// LLMDeltas 为空 → 不发送 llm_chunk
|
||||
// TTSAudios 为空 → 不发送 tts_audio
|
||||
}
|
||||
|
||||
srv, wsURL := setupTestServer(t, mock)
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
_ = readJSON(t, conn) // connected
|
||||
|
||||
err := conn.WriteJSON(models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-stt-only",
|
||||
Audio: audioB64,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 应收到 stt_result
|
||||
stt := readJSON(t, conn)
|
||||
assert.Equal(t, "stt_result", stt["type"])
|
||||
assert.Equal(t, "测试语音", stt["text"])
|
||||
|
||||
// 应收到 llm_done(即使没有 chunk)
|
||||
done := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_done", done["type"])
|
||||
assert.Equal(t, "", done["full_text"])
|
||||
}
|
||||
|
||||
// TestWS_UnknownMessageType 验证未知消息类型返回 error。
|
||||
func TestWS_UnknownMessageType(t *testing.T) {
|
||||
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
_ = readJSON(t, conn) // connected
|
||||
|
||||
err := conn.WriteJSON(map[string]string{"type": "unknown_type"})
|
||||
require.NoError(t, err)
|
||||
|
||||
errMsg := readJSON(t, conn)
|
||||
assert.Equal(t, "error", errMsg["type"])
|
||||
assert.Equal(t, "INVALID_MESSAGE", errMsg["code"])
|
||||
assert.Contains(t, errMsg["message"], "unknown message type")
|
||||
}
|
||||
|
||||
// TestWS_InvalidJSON 验证无效 JSON 返回 error。
|
||||
func TestWS_InvalidJSON(t *testing.T) {
|
||||
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
_ = readJSON(t, conn) // connected
|
||||
|
||||
err := conn.WriteMessage(websocket.TextMessage, []byte("not-json"))
|
||||
require.NoError(t, err)
|
||||
|
||||
errMsg := readJSON(t, conn)
|
||||
assert.Equal(t, "error", errMsg["type"])
|
||||
assert.Equal(t, "INVALID_MESSAGE", errMsg["code"])
|
||||
}
|
||||
|
||||
// TestWS_MultipleQueries 验证同一连接上可以发送多次 query。
|
||||
func TestWS_MultipleQueries(t *testing.T) {
|
||||
mock := &MockOrchestrator{
|
||||
STTResult: "识别结果",
|
||||
LLMDeltas: []string{"回复"},
|
||||
}
|
||||
|
||||
srv, wsURL := setupTestServer(t, mock)
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
_ = readJSON(t, conn) // connected
|
||||
|
||||
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
reqID := "req-multi-" + string(rune('0'+i))
|
||||
err := conn.WriteJSON(models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: reqID,
|
||||
Audio: audioB64,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 每次应收到完整的响应序列
|
||||
stt := readJSON(t, conn)
|
||||
assert.Equal(t, "stt_result", stt["type"], "第 %d 次 query", i+1)
|
||||
|
||||
chunk := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_chunk", chunk["type"], "第 %d 次 query", i+1)
|
||||
|
||||
done := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_done", done["type"], "第 %d 次 query", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWS_Interrupt 验证 interrupt 取消正在进行的请求。
|
||||
func TestWS_Interrupt(t *testing.T) {
|
||||
// 使用较长延迟模拟慢请求
|
||||
mock := &MockOrchestrator{
|
||||
STTResult: "识别文本",
|
||||
LLMDeltas: []string{"第一句", "第二句", "第三句", "第四句", "第五句"},
|
||||
Delay: 200 * time.Millisecond,
|
||||
}
|
||||
|
||||
srv, wsURL := setupTestServer(t, mock)
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
_ = readJSON(t, conn) // connected
|
||||
|
||||
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
|
||||
|
||||
// 发送 query
|
||||
err := conn.WriteJSON(models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-interrupt",
|
||||
Audio: audioB64,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 收到 stt_result
|
||||
stt := readJSON(t, conn)
|
||||
assert.Equal(t, "stt_result", stt["type"])
|
||||
|
||||
// 收到第一个 llm_chunk
|
||||
chunk1 := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_chunk", chunk1["type"])
|
||||
|
||||
// 发送 interrupt
|
||||
err = conn.WriteJSON(map[string]string{"type": "interrupt"})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 等待 interrupt 生效
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// 验证连接仍然存活(可以发 ping 收 pong)
|
||||
require.NoError(t, conn.WriteJSON(map[string]string{"type": "ping"}))
|
||||
|
||||
var pong map[string]any
|
||||
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
require.NoError(t, conn.ReadJSON(&pong), "interrupt 后连接应仍存活")
|
||||
assert.Equal(t, "pong", pong["type"])
|
||||
}
|
||||
|
||||
// TestWS_DisconnectCleanup 验证断开连接时清理资源。
|
||||
func TestWS_DisconnectCleanup(t *testing.T) {
|
||||
// 使用较长延迟模拟慢请求
|
||||
mock := &MockOrchestrator{
|
||||
STTResult: "识别文本",
|
||||
LLMDeltas: []string{"长回复第一部分", "长回复第二部分"},
|
||||
Delay: 500 * time.Millisecond,
|
||||
}
|
||||
|
||||
srv, wsURL := setupTestServer(t, mock)
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
_ = readJSON(t, conn) // connected
|
||||
|
||||
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
|
||||
|
||||
// 发送 query
|
||||
err := conn.WriteJSON(models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-disconnect",
|
||||
Audio: audioB64,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 收到 stt_result
|
||||
stt := readJSON(t, conn)
|
||||
assert.Equal(t, "stt_result", stt["type"])
|
||||
|
||||
// 关闭连接(模拟客户端断开)
|
||||
conn.Close()
|
||||
|
||||
// 等待一小段时间让服务器处理断开
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
// 如果没有 panic 或 goroutine 泄漏,测试通过
|
||||
// (Go test 的 -race 检测器会捕获数据竞争)
|
||||
}
|
||||
|
||||
// TestWS_SessionCreated 验证每次连接都创建新会话。
|
||||
func TestWS_SessionCreated(t *testing.T) {
|
||||
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
|
||||
defer srv.Close()
|
||||
|
||||
// 第一次连接
|
||||
conn1 := connectWS(t, wsURL)
|
||||
msg1 := readJSON(t, conn1)
|
||||
sid1 := msg1["session_id"].(string)
|
||||
conn1.Close()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// 第二次连接
|
||||
conn2 := connectWS(t, wsURL)
|
||||
sid2 := readJSON(t, conn2)["session_id"].(string)
|
||||
|
||||
assert.NotEmpty(t, sid1)
|
||||
assert.NotEmpty(t, sid2)
|
||||
assert.NotEqual(t, sid1, sid2, "两次连接应创建不同的会话")
|
||||
}
|
||||
|
||||
// TestWS_QueryWithoutImage 验证不带图片的 query。
|
||||
func TestWS_QueryWithoutImage(t *testing.T) {
|
||||
mock := &MockOrchestrator{
|
||||
STTResult: "纯语音输入",
|
||||
LLMDeltas: []string{"收到"},
|
||||
}
|
||||
|
||||
srv, wsURL := setupTestServer(t, mock)
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
_ = readJSON(t, conn) // connected
|
||||
|
||||
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
|
||||
|
||||
err := conn.WriteJSON(models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-no-image",
|
||||
Audio: audioB64,
|
||||
// Image 为空
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
stt := readJSON(t, conn)
|
||||
assert.Equal(t, "stt_result", stt["type"])
|
||||
assert.Equal(t, "纯语音输入", stt["text"])
|
||||
|
||||
chunk := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_chunk", chunk["type"])
|
||||
|
||||
done := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_done", done["type"])
|
||||
}
|
||||
|
||||
// TestWS_QueryWithTTSDisabled 验证 TTS 未启用时不应收到 tts_audio。
|
||||
func TestWS_QueryWithTTSDisabled(t *testing.T) {
|
||||
// MockOrchestrator 的 TTSAudios 为空 → 不发送 tts_audio
|
||||
mock := &MockOrchestrator{
|
||||
STTResult: "语音",
|
||||
LLMDeltas: []string{"回复"},
|
||||
// TTSAudios 留空
|
||||
}
|
||||
|
||||
srv, wsURL := setupTestServer(t, mock)
|
||||
defer srv.Close()
|
||||
|
||||
conn := connectWS(t, wsURL)
|
||||
_ = readJSON(t, conn) // connected
|
||||
|
||||
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
|
||||
|
||||
err := conn.WriteJSON(models.WsQuery{
|
||||
Type: "query",
|
||||
RequestID: "req-no-tts",
|
||||
Audio: audioB64,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
stt := readJSON(t, conn)
|
||||
assert.Equal(t, "stt_result", stt["type"])
|
||||
|
||||
chunk := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_chunk", chunk["type"])
|
||||
|
||||
done := readJSON(t, conn)
|
||||
assert.Equal(t, "llm_done", done["type"])
|
||||
|
||||
// 不应有 tts_audio 消息;设置短超时验证
|
||||
conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond))
|
||||
var extra map[string]any
|
||||
err = conn.ReadJSON(&extra)
|
||||
assert.Error(t, err, "不应有额外消息")
|
||||
}
|
||||
@@ -213,11 +213,73 @@ CREATE TABLE usage_daily (
|
||||
## 部署架构
|
||||
|
||||
```
|
||||
CDN(静态资源) ← 用户浏览器
|
||||
Nginx 负载均衡(sticky session for WebSocket)
|
||||
├── Gateway-1 ──→ Redis
|
||||
├── Gateway-2 ──→ Redis
|
||||
└── Gateway-N ──→ AI Services(外部 API)
|
||||
用户浏览器
|
||||
↓
|
||||
Nginx(同源反代 + 负载均衡)
|
||||
├── / → 前端静态资源(CDN 或本地 dist)
|
||||
├── /api/* → Go Gateway(REST API)
|
||||
└── /ws → Go Gateway(WebSocket)
|
||||
├── Gateway-1 ──→ Redis
|
||||
├── Gateway-2 ──→ Redis
|
||||
└── Gateway-N ──→ AI Services(外部 API)
|
||||
```
|
||||
|
||||
WebSocket 是长连接,Nginx 需要配置 `proxy_set_header Upgrade` 和 sticky session,确保同一用户的请求始终路由到同一个 Gateway 实例。
|
||||
**跨域策略**:Nginx 将前端和后端统一到同一域名下,浏览器无跨域问题。
|
||||
|
||||
### Nginx 配置
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name camtalk.example.com;
|
||||
|
||||
# 前端静态资源
|
||||
location / {
|
||||
root /var/www/camtalk/dist;
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# REST API 反代
|
||||
location /api/ {
|
||||
proxy_pass http://127.0.0.1:8080;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
}
|
||||
|
||||
# WebSocket 反代
|
||||
location /ws {
|
||||
proxy_pass http://127.0.0.1:8080;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_read_timeout 86400s; # 长连接超时 24h
|
||||
proxy_send_timeout 86400s;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> WebSocket 是长连接,Nginx 必须配置 `Upgrade` 和 `Connection` 头。`proxy_read_timeout` 需要覆盖心跳间隔(客户端 30s ping),否则 Nginx 会主动断开空闲连接。
|
||||
|
||||
### 开发环境(Vite proxy)
|
||||
|
||||
开发时前端(Vite :5173)和后端(Gin :8080)不同端口,用 Vite 内置代理解决跨域:
|
||||
|
||||
```typescript
|
||||
// frontend/vite.config.ts
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
proxy: {
|
||||
"/api": "http://localhost:8080",
|
||||
"/ws": {
|
||||
target: "ws://localhost:8080",
|
||||
ws: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
前端代码中 WebSocket 地址改为相对路径 `ws://localhost:5173/ws`,Vite 自动代理到后端。部署时 Nginx 同理,前端无需区分开发/生产地址。
|
||||
|
||||
201
docs/03-接口文档.md
201
docs/03-接口文档.md
@@ -256,7 +256,6 @@ POST /api/sessions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"user_id": "optional-user-id",
|
||||
"config": {
|
||||
"tts_enabled": true,
|
||||
"detail_level": "low",
|
||||
@@ -301,28 +300,18 @@ Go 网关内部与外部 AI 服务(Deepgram STT、GPT-4o、OpenAI TTS)的调
|
||||
语音识别:接收前端采集的音频,返回识别文本。
|
||||
|
||||
```go
|
||||
// STTService 语音识别服务契约。
|
||||
type STTService interface {
|
||||
// Service 语音识别服务契约。
|
||||
type Service interface {
|
||||
// Recognize 识别一段完整音频,返回最终文本。
|
||||
Recognize(ctx context.Context, audio []byte, opts STTOptions) (string, error)
|
||||
|
||||
// RecognizeStream 流式识别(边说边识别,可选实现)。
|
||||
// audioStream 持续接收音频片段,返回的 channel 持续输出中间结果。
|
||||
RecognizeStream(ctx context.Context, audioStream <-chan []byte, opts STTOptions) (<-chan STTPartial, error)
|
||||
Recognize(ctx context.Context, audio []byte, opts Options) (string, error)
|
||||
}
|
||||
|
||||
// STTOptions 语音识别参数。
|
||||
type STTOptions struct {
|
||||
// Options 语音识别参数。
|
||||
type Options struct {
|
||||
Encoding string // "pcm_s16le" — 前端 VAD 输出格式
|
||||
SampleRate int // 16000 — 前端麦克风采样率
|
||||
Language string // "zh-CN"
|
||||
}
|
||||
|
||||
// STTPartial 流式识别的中间/最终结果。
|
||||
type STTPartial struct {
|
||||
Text string
|
||||
IsFinal bool
|
||||
}
|
||||
```
|
||||
|
||||
**Deepgram 接入约定**:
|
||||
@@ -336,23 +325,23 @@ type STTPartial struct {
|
||||
多模态推理:接收图像 + 文本 + 对话历史,流式返回回复。
|
||||
|
||||
```go
|
||||
// LLMService 多模态大模型服务契约。
|
||||
type LLMService interface {
|
||||
// Service 多模态大模型服务契约。
|
||||
type Service interface {
|
||||
// ChatStream 流式推理,返回增量文本的 channel。
|
||||
// 调用方必须消费 channel 直到 Done=true,否则需 cancel ctx 以释放连接。
|
||||
ChatStream(ctx context.Context, req LLMRequest) (<-chan LLMChunk, error)
|
||||
ChatStream(ctx context.Context, req Request) (<-chan Chunk, error)
|
||||
}
|
||||
|
||||
// LLMRequest 推理请求。
|
||||
type LLMRequest struct {
|
||||
Image []byte // JPEG 图片(已从 Base64 解码)
|
||||
Text string // 用户语音识别后的文本
|
||||
History []Message // 最近 N 轮对话历史
|
||||
Language string // "zh-CN"
|
||||
// Request 推理请求。
|
||||
type Request struct {
|
||||
Image []byte // JPEG 图片(已从 Base64 解码)
|
||||
Text string // 用户语音识别后的文本
|
||||
History []models.Message // 最近 N 轮对话历史
|
||||
Language string // "zh-CN"
|
||||
}
|
||||
|
||||
// LLMChunk 流式推理的一个增量片段。
|
||||
type LLMChunk struct {
|
||||
// Chunk 流式推理的一个增量片段。
|
||||
type Chunk struct {
|
||||
Delta string // 增量文本
|
||||
Done bool // 是否结束
|
||||
TokensUsed *TokenUsage // 仅 Done=true 时有值
|
||||
@@ -388,24 +377,24 @@ user: [图片 + 用户语音文本]
|
||||
语音合成:接收文本流,输出音频 chunk 流。
|
||||
|
||||
```go
|
||||
// TTSService 语音合成服务契约。
|
||||
type TTSService interface {
|
||||
// Service 语音合成服务契约。
|
||||
type Service interface {
|
||||
// SynthesizeStream 流式合成。
|
||||
// textStream 接收句子级文本(由 Orchestrator 的句子切分器产出),
|
||||
// 返回的 channel 持续输出 MP3 音频 chunk。
|
||||
SynthesizeStream(ctx context.Context, textStream <-chan string, opts TTSOptions) (<-chan TTSChunk, error)
|
||||
SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error)
|
||||
}
|
||||
|
||||
// TTSOptions 合成参数。
|
||||
type TTSOptions struct {
|
||||
// Options 合成参数。
|
||||
type Options struct {
|
||||
Voice string // "alloy" | "nova" | "shimmer" | ...
|
||||
Speed float64 // 1.0 为正常语速
|
||||
OutputFmt string // "mp3" — 固定使用 MP3,浏览器原生支持
|
||||
SampleRate int // 24000
|
||||
}
|
||||
|
||||
// TTSChunk 一个音频片段。
|
||||
type TTSChunk struct {
|
||||
// Chunk 一个音频片段。
|
||||
type Chunk struct {
|
||||
Audio []byte // MP3 音频数据(未 Base64 编码,由发送层编码)
|
||||
IsLast bool // 是否为最后一片
|
||||
}
|
||||
@@ -446,73 +435,32 @@ LLM 流式输出: "这" "是一" "朵红色" "的花。" "它看起" "来很美
|
||||
### Orchestrator 接口
|
||||
|
||||
```go
|
||||
// Orchestrator AI 编排器,协调 STT → LLM → TTS 全链路。
|
||||
type Orchestrator struct {
|
||||
stt STTService
|
||||
llm LLMService
|
||||
tts TTSService
|
||||
// Orchestrator AI 编排器接口。
|
||||
type Orchestrator interface {
|
||||
// ProcessQuery 处理一次完整的视觉对话请求。
|
||||
// 通过 sender 向前端实时推送 stt_result、llm_chunk、llm_done、tts_audio 消息。
|
||||
ProcessQuery(ctx context.Context, sessionID string, req models.WsQuery,
|
||||
history []models.Message, sender Sender) error
|
||||
}
|
||||
|
||||
// ProcessQuery 处理一次完整的视觉对话请求。
|
||||
// 通过 client 向前端实时推送 stt_result、llm_chunk、llm_done、tts_audio 消息。
|
||||
func (o *Orchestrator) ProcessQuery(ctx context.Context, client MessageSender, req *QueryRequest) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Step 1: STT — 识别用户语音
|
||||
text, err := o.stt.Recognize(ctx, req.Audio, STTOptions{
|
||||
Encoding: "pcm_s16le", SampleRate: 16000, Language: "zh-CN",
|
||||
})
|
||||
if err != nil {
|
||||
client.SendError(req.RequestID, "STT_ERROR", err.Error())
|
||||
return
|
||||
}
|
||||
client.SendSTTResult(req.RequestID, text, true)
|
||||
|
||||
// Step 2: LLM 流式输出 + 句子切分
|
||||
llmStream, _ := o.llm.ChatStream(ctx, LLMRequest{
|
||||
Image: req.Image, Text: text, Language: "zh-CN",
|
||||
})
|
||||
|
||||
sentenceCh := make(chan string, 4)
|
||||
go func() {
|
||||
defer close(sentenceCh)
|
||||
var buf strings.Builder
|
||||
var fullText strings.Builder
|
||||
for chunk := range llmStream {
|
||||
// 即时推送文字给客户端(逐 token 显示)
|
||||
client.SendLLMChunk(req.RequestID, chunk.Delta)
|
||||
fullText.WriteString(chunk.Delta)
|
||||
buf.WriteString(chunk.Delta)
|
||||
// 遇到句子边界就吐出
|
||||
if isSentenceEnd(chunk.Delta) {
|
||||
sentenceCh <- buf.String()
|
||||
buf.Reset()
|
||||
}
|
||||
}
|
||||
// 最后一段不足一句的也吐出
|
||||
if buf.Len() > 0 {
|
||||
sentenceCh <- buf.String()
|
||||
}
|
||||
// 推送 llm_done
|
||||
client.SendLLMDone(req.RequestID, fullText.String(), chunk.TokensUsed, chunk.Model)
|
||||
}()
|
||||
|
||||
// Step 3: TTS 并行消费句子流
|
||||
ttsStream, _ := o.tts.SynthesizeStream(ctx, sentenceCh, TTSOptions{
|
||||
Voice: "alloy", OutputFmt: "mp3", SampleRate: 24000,
|
||||
})
|
||||
for chunk := range ttsStream {
|
||||
client.SendTTSAudio(req.RequestID, chunk.Audio, chunk.IsLast)
|
||||
}
|
||||
}
|
||||
|
||||
// isSentenceEnd 判断 delta 中是否包含句子结束标志。
|
||||
func isSentenceEnd(delta string) bool {
|
||||
return strings.ContainsAny(delta, "。!?\n.!?\n")
|
||||
// Sender 抽象 WebSocket 消息推送能力,便于测试时 mock。
|
||||
type Sender interface {
|
||||
SendSTTResult(result models.WsSTTResult) error
|
||||
SendLLMChunk(chunk models.WsLLMChunk) error
|
||||
SendLLMDone(done models.WsLLMDone) error
|
||||
SendTTSAudio(audio models.WsTTSAudio) error
|
||||
SendError(err models.WsError) error
|
||||
}
|
||||
```
|
||||
|
||||
**Pipeline 实现**(`internal/orchestrator/pipeline.go`):
|
||||
1. Base64 解码音频/图片
|
||||
2. 调用 `stt.Recognize()` → 发送 `stt_result`
|
||||
3. 调用 `llm.ChatStream()` 获取流式输出,goroutine 消费 token → 发送 `llm_chunk` + 句子切分
|
||||
4. 另一 goroutine 从句子 channel 读取 → 调用 `tts.SynthesizeStream()` → 发送 `tts_audio`
|
||||
5. 流结束 → 发送 `llm_done`
|
||||
6. TTS 失败静默跳过,STT/LLM 失败发送对应 error 消息
|
||||
|
||||
### 并发控制
|
||||
|
||||
- 每个 `ProcessQuery` 调用在独立 goroutine 中运行
|
||||
@@ -584,9 +532,9 @@ session:{id}:history → List (对话历史)
|
||||
### 接口定义
|
||||
|
||||
```go
|
||||
// SessionManager 会话管理器。
|
||||
// WebSocket Handler 通过此接口操作会话,不直接接触 Redis。
|
||||
type SessionManager interface {
|
||||
// Manager 会话管理器接口。
|
||||
// WebSocket Handler 通过此接口操作会话,不直接接触存储层。
|
||||
type Manager interface {
|
||||
// Create 创建新会话,返回 session ID。
|
||||
Create(ctx context.Context, config models.SessionConfig) (string, error)
|
||||
|
||||
@@ -605,14 +553,20 @@ type SessionManager interface {
|
||||
// SetActiveRequest 标记当前正在处理的请求 ID(interrupt 用)。
|
||||
SetActiveRequest(ctx context.Context, sessionID string, requestID string) error
|
||||
|
||||
// GetActiveRequestID 获取当前活跃请求 ID。
|
||||
GetActiveRequestID(ctx context.Context, sessionID string) (string, error)
|
||||
|
||||
// ClearActiveRequest 清除活跃请求标记(请求完成或中断后)。
|
||||
ClearActiveRequest(ctx context.Context, sessionID string) error
|
||||
|
||||
// Touch 刷新 TTL(心跳时调用)。
|
||||
Touch(ctx context.Context, sessionID string) error
|
||||
|
||||
// Destroy 显式销毁会话(REST API DELETE 或连接断开清理)。
|
||||
// Destroy 显式销毁会话(REST API DELETE)。
|
||||
Destroy(ctx context.Context, sessionID string) error
|
||||
|
||||
// ActiveCount 返回当前活跃会话数(健康检查用)。
|
||||
ActiveCount() int
|
||||
}
|
||||
```
|
||||
|
||||
@@ -629,7 +583,8 @@ case "query":
|
||||
|
||||
history, _ := sessionMgr.GetHistory(ctx, sessionID, 20) // 获取对话上下文
|
||||
|
||||
go orchestrator.ProcessQuery(ctx, client, &msg, history) // 异步编排
|
||||
sender := &WSClient{client: client, requestID: msg.RequestID}
|
||||
go orch.ProcessQuery(ctx, sessionID, msg, history, sender) // 异步编排
|
||||
|
||||
// interrupt 分支
|
||||
case "interrupt":
|
||||
@@ -648,26 +603,30 @@ case "interrupt":
|
||||
联调阶段无 Redis 时,用同一接口的内存实现:
|
||||
|
||||
```go
|
||||
type InMemorySessionManager struct {
|
||||
mu sync.RWMutex
|
||||
sessions map[string]*sessionEntry
|
||||
type MemoryManager struct {
|
||||
mu sync.RWMutex
|
||||
sessions map[string]*sessionEntry
|
||||
ttl time.Duration
|
||||
maxHistory int
|
||||
stopCleaner chan struct{}
|
||||
}
|
||||
|
||||
type sessionEntry struct {
|
||||
session models.Session
|
||||
history []models.Message
|
||||
activeReqID string
|
||||
lastActive time.Time
|
||||
}
|
||||
```
|
||||
|
||||
注入时根据配置切换:
|
||||
|
||||
```go
|
||||
var sessionMgr SessionManager
|
||||
var sessionMgr session.Manager
|
||||
if cfg.Redis.Addr != "" {
|
||||
sessionMgr = NewRedisSessionManager(redisClient, 30*time.Minute, 20)
|
||||
sessionMgr = session.NewRedisManager(redisClient, 30*time.Minute, 20)
|
||||
} else {
|
||||
sessionMgr = NewInMemorySessionManager()
|
||||
sessionMgr = session.NewMemoryManager(30*time.Minute, 20)
|
||||
}
|
||||
```
|
||||
|
||||
@@ -931,10 +890,8 @@ type QueryRequest struct {
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
Role string `json:"role"` // "user" | "assistant"
|
||||
Content string `json:"content"`
|
||||
ImageURL string `json:"image_url,omitempty"`
|
||||
TokensUsed int `json:"tokens_used,omitempty"`
|
||||
Role string `json:"role"` // "user" | "assistant"
|
||||
Content string `json:"content"`
|
||||
}
|
||||
```
|
||||
|
||||
@@ -1085,3 +1042,25 @@ function reconnect(attempt: number) {
|
||||
}
|
||||
// attempt: 0 → 1s, 1 → 2s, 2 → 4s, 3 → 8s, ... 最大 30s
|
||||
```
|
||||
|
||||
### 跨域处理
|
||||
|
||||
采用 **Nginx 同源反代**方案,前后端统一到同一域名,浏览器层面不存在跨域问题。
|
||||
|
||||
**生产环境**:Nginx 将 `/`(前端)、`/api/*`(REST)、`/ws`(WebSocket)统一反代到同一域名,详见 `02-系统架构.md` 部署架构章节。
|
||||
|
||||
**开发环境**:Vite 内置代理,前端 :5173 的 `/api` 和 `/ws` 请求代理到后端 :8080:
|
||||
|
||||
```typescript
|
||||
// frontend/vite.config.ts
|
||||
server: {
|
||||
proxy: {
|
||||
"/api": "http://localhost:8080",
|
||||
"/ws": { target: "ws://localhost:8080", ws: true },
|
||||
},
|
||||
},
|
||||
```
|
||||
|
||||
**Go 后端 WebSocket CheckOrigin**:生产环境 Nginx 同源,`CheckOrigin` 可保持默认(拒绝跨域)。开发环境由 Vite proxy 转发,不存在跨域。因此后端无需配置 CORS 中间件,`CheckOrigin` 保持 gorilla/websocket 默认值即可。
|
||||
|
||||
> 如果未来需要支持第三方客户端直连(如移动端),再按需添加 CORS 中间件和 `CheckOrigin` 白名单。
|
||||
|
||||
221
docs/PLAN_BACKEND.md
Normal file
221
docs/PLAN_BACKEND.md
Normal file
@@ -0,0 +1,221 @@
|
||||
# CamTalk 后端完善计划
|
||||
|
||||
## Context
|
||||
|
||||
后端当前是一个骨架:`main.go` 启动 Gin 服务器,`ws/handler.go` 实现了 WebSocket 连接生命周期和消息分发,`models/models.go` 定义了所有协议消息类型,`config/config.go` 实现了 Viper 配置加载。但所有业务逻辑都是 TODO 桩——没有 Session Manager、没有 AI 服务客户端、没有编排层、没有日志/错误工具、没有测试。前端已基本完成,正在等待后端提供真实的 AI 管道。
|
||||
|
||||
**目标**:按设计文档(`docs/03-接口文档.md` 为最高依据)逐步填充所有业务模块,使端到端的 STT → LLM → TTS 流式管道可用。
|
||||
|
||||
---
|
||||
|
||||
## 分阶段实施
|
||||
|
||||
### Phase 1:基础设施(logger、errors、config 接入、graceful shutdown)
|
||||
|
||||
**目标**:为后续模块提供日志、错误码、配置等基础能力,替换 `main.go` 中的硬编码值。
|
||||
|
||||
| # | 任务 | 文件 | 说明 |
|
||||
|---|------|------|------|
|
||||
| 1.1 | 实现 Zap 日志封装 | `internal/logger/logger.go` | 提供 `Init(level, format)` 和全局 `*zap.SugaredLogger`,替换所有 `log.Printf` |
|
||||
| 1.2 | 实现错误码常量 + WS 错误发送工具 | `internal/errors/codes.go` | 10 个错误码常量 + `SendWSError(client, code, requestID, err)` |
|
||||
| 1.3 | main.go 接入 config.Load() | `cmd/server/main.go` | 用 `cfg.Server.Host:Port` 替换硬编码 `:8080`,初始化 logger |
|
||||
| 1.4 | 添加 graceful shutdown | `cmd/server/main.go` | `signal.NotifyContext` + `http.Server.Shutdown`,10s drain |
|
||||
| 1.5 | 添加 .gitignore | `backend/.gitignore` | 排除 `server` 二进制、`.env`、`tmp/` |
|
||||
|
||||
> **CORS**:不在此处实现,生产环境由 Nginx 反向代理统一处理跨域。
|
||||
|
||||
---
|
||||
|
||||
### Phase 2:Session Manager
|
||||
|
||||
**目标**:实现会话生命周期管理,让 WS handler 能追踪会话、存储对话历史。
|
||||
|
||||
| # | 任务 | 文件 | 说明 |
|
||||
|---|------|------|------|
|
||||
| 2.1 | 定义 SessionManager 接口 | `internal/session/manager.go` | 方法:`Create`, `Get`, `UpdateConfig`, `GetHistory`, `AppendMessage`, `SetActiveRequest`, `ClearActiveRequest`, `Touch`, `Destroy` |
|
||||
| 2.2 | 实现内存版 SessionManager | `internal/session/memory.go` | `sync.RWMutex` + `map[string]*sessionEntry`,TTL 30 分钟,历史上限 20 条 |
|
||||
| 2.3 | 实现 Redis 版 SessionManager | `internal/session/redis.go` | `session:{id}:meta` Hash + `session:{id}:history` List,TTL 刷新,选配 |
|
||||
| 2.4 | 编写 Session Manager 测试 | `internal/session/memory_test.go` | 覆盖 Create/Get/Expire/Destroy/AppendMessage/History 上限 |
|
||||
| 2.5 | WS handler 接入 SessionManager | `internal/ws/handler.go` | `ServeWS` 接收 `session.Manager` 参数;`connected` 消息后创建会话;`query` 时 Touch + SetActiveRequest;`config` 时 UpdateConfig;断开时不销毁(自然过期) |
|
||||
|
||||
---
|
||||
|
||||
### Phase 3:AI 服务层接口 + 实现
|
||||
|
||||
**目标**:定义并实现三个 AI 服务客户端,每个服务一个独立包。
|
||||
|
||||
| # | 任务 | 文件 | 说明 |
|
||||
|---|------|------|------|
|
||||
| **3a. STT** | | | |
|
||||
| 3.1 | STT 接口定义 | `internal/ai/stt/stt.go` | `Service` 接口:`Recognize(ctx, audio []byte, opts Options) (string, error)`。`Options`: Encoding, SampleRate, Language |
|
||||
| 3.2 | Deepgram 实现 | `internal/ai/stt/deepgram.go` | WebSocket 连接 `wss://api.deepgram.com/v1/listen`,发送 PCM 音频,接收转录结果,5s 超时 |
|
||||
| 3.3 | STT 测试(mock) | `internal/ai/stt/deepgram_test.go` | httptest/WebSocket mock,验证连接、发送、超时 |
|
||||
| **3b. LLM** | | | |
|
||||
| 3.4 | LLM 接口定义 | `internal/ai/llm/llm.go` | `Service` 接口:`ChatStream(ctx, req Request) (<-chan Chunk, error)`。`Request`: Image, Text, History, Language。`Chunk`: Delta, Done, TokensUsed, Model |
|
||||
| 3.5 | OpenAI 实现 | `internal/ai/llm/openai.go` | `POST /v1/chat/completions` + `stream: true`,SSE 解析,10s 超时,image 以 `data:image/jpeg;base64,...` 传入 |
|
||||
| 3.6 | System Prompt 定义 | `internal/ai/llm/prompt.go` | 中文视觉助手提示词,根据 Language/DetailLevel 动态构建 |
|
||||
| 3.7 | LLM 测试(mock) | `internal/ai/llm/openai_test.go` | httptest mock SSE 流,验证流式解析、超时、错误处理 |
|
||||
| **3c. TTS** | | | |
|
||||
| 3.8 | TTS 接口定义 | `internal/ai/tts/tts.go` | `Service` 接口:`SynthesizeStream(ctx, textStream <-chan string, opts Options) (<-chan Chunk, error)`。`Chunk`: Audio []byte, IsLast |
|
||||
| 3.9 | OpenAI 实现 | `internal/ai/tts/openai.go` | `POST /v1/audio/speech` 模型 `tts-1`,逐句发送,返回 MP3 流,5s/句超时 |
|
||||
| 3.10 | TTS 测试(mock) | `internal/ai/tts/openai_test.go` | httptest mock,验证逐句合成、超时 |
|
||||
|
||||
---
|
||||
|
||||
### Phase 4:AI Orchestrator(核心编排)
|
||||
|
||||
**目标**:实现 STT → LLM → TTS 流式并行管道,这是后端最关键的业务逻辑。
|
||||
|
||||
| # | 任务 | 文件 | 说明 |
|
||||
|---|------|------|------|
|
||||
| 4.1 | Orchestrator 接口 | `internal/orchestrator/orchestrator.go` | `ProcessQuery(ctx, sessionID, req, history, sender)` — 接收查询并执行管道 |
|
||||
| 4.2 | Sender 接口 | `internal/orchestrator/sender.go` | 抽象 WS 推送:`SendSTTResult`, `SendLLMChunk`, `SendLLMDone`, `SendTTSAudio`, `SendError`,便于测试 |
|
||||
| 4.3 | 管道实现 | `internal/orchestrator/pipeline.go` | ① `stt.Recognize()` → 发送 `stt_result` ② `llm.ChatStream()` 并行消费 token → 发送 `llm_chunk` + 句子切分 → channel ③ `tts.SynthesizeStream()` 从 channel 读取 → 发送 `tts_audio` ④ 流结束 → 发送 `llm_done` |
|
||||
| 4.4 | 句子切分器 | `internal/orchestrator/splitter.go` | 按 `。!?\n.!?` 切分,buffer size 4 channel |
|
||||
| 4.5 | 错误降级 | 同上文件 | STT 失败→STT_ERROR+abort;LLM 超时→LLM_TIMEOUT;TTS 失败→静默跳过 |
|
||||
| 4.6 | Interrupt 支持 | 同上文件 | context cancel 触发所有流中止 |
|
||||
| 4.7 | Orchestrator 测试 | `internal/orchestrator/pipeline_test.go` | mock 三个 AI service + mock sender,验证完整流程、中断、错误降级 |
|
||||
|
||||
---
|
||||
|
||||
### Phase 5:WS Handler 完整接入
|
||||
|
||||
**目标**:将 Session Manager + Orchestrator 串入 WebSocket handler,实现端到端消息处理。
|
||||
|
||||
| # | 任务 | 文件 | 说明 |
|
||||
|---|------|------|------|
|
||||
| 5.1 | Client 扩展 | `internal/ws/handler.go` | 添加 `session.Manager`、`orchestrator.Orchestrator`、`context.CancelFunc`(用于 interrupt) |
|
||||
| 5.2 | query 处理 | 同上 | 解码 audio Base64 → `stt.Recognize` 的输入;Touch 会话;设置 active request;启动 `orchestrator.ProcessQuery` goroutine |
|
||||
| 5.3 | config 处理 | 同上 | 调用 `session.UpdateConfig()` |
|
||||
| 5.4 | interrupt 处理 | 同上 | 查找 active request 的 cancel func,调用 `cancel()`,ClearActiveRequest |
|
||||
| 5.5 | Disconnect 处理 | 同上 | 取消当前活跃请求(如有),不销毁会话 |
|
||||
|
||||
---
|
||||
|
||||
### Phase 6:REST API 补全
|
||||
|
||||
**目标**:补全设计文档中的 REST 端点。
|
||||
|
||||
| # | 任务 | 文件 | 说明 |
|
||||
|---|------|------|------|
|
||||
| 6.1 | Session 路由 | `internal/api/session.go` | `POST /api/sessions` 创建会话,`DELETE /api/sessions/:id` 销毁会话 |
|
||||
| 6.2 | Health 更新 | `cmd/server/main.go` | 从 SessionManager 获取 `active_sessions` 真实值 |
|
||||
| 6.3 | 路由注册 | `cmd/server/main.go` | 统一注册 REST + WS 路由,注入依赖 |
|
||||
|
||||
---
|
||||
|
||||
### Phase 7:Rate Limiter + Model Router(可选/MVP 后)
|
||||
|
||||
**目标**:防止滥用 + 智能模型选择,MVP 可简化或跳过。
|
||||
|
||||
| # | 任务 | 文件 | 说明 |
|
||||
|---|------|------|------|
|
||||
| 7.1 | 令牌桶 Rate Limiter | `internal/middleware/ratelimit.go` | `golang.org/x/time/rate` 或自实现,按 session ID 限流 |
|
||||
| 7.2 | Rate Limiter 中间件 | `internal/middleware/ratelimit.go` | 在 WS query 路径上检查,超限返回 `RATE_LIMITED` |
|
||||
| 7.3 | Model Router | `internal/ai/router.go` | 规则引擎:简单识别→GPT-4o-mini,深度分析→GPT-4o,暂不实现 o1 |
|
||||
|
||||
---
|
||||
|
||||
### Phase 8:集成测试 + 文档同步
|
||||
|
||||
| # | 任务 | 文件 | 说明 |
|
||||
|---|------|------|------|
|
||||
| 8.1 | WS 集成测试 | `internal/ws/handler_test.go` | 启动 Gin test server + gorilla websocket client,验证完整 query→stt_result→llm_chunk→llm_done→tts_audio 流程 |
|
||||
| 8.2 | 文档同步 | `docs/03-接口文档.md` | 代码实现与文档有偏差时更新文档 |
|
||||
| 8.3 | go.sum 清理 | `backend/` | `go mod tidy` 清理无用依赖 |
|
||||
|
||||
---
|
||||
|
||||
## 关键文件清单
|
||||
|
||||
```
|
||||
backend/
|
||||
cmd/server/main.go ← Phase 1.3, 1.4, 1.5, 6.2, 6.3
|
||||
internal/
|
||||
config/config.go ← 已完成,Phase 1.3 接入
|
||||
logger/logger.go ← Phase 1.1(新建)
|
||||
errors/codes.go ← Phase 1.2(新建)
|
||||
models/models.go ← 已完成,可能小幅扩展
|
||||
session/
|
||||
manager.go ← Phase 2.1(新建)
|
||||
memory.go ← Phase 2.2(新建)
|
||||
redis.go ← Phase 2.3(新建)
|
||||
memory_test.go ← Phase 2.4(新建)
|
||||
ai/
|
||||
stt/
|
||||
stt.go ← Phase 3.1(新建)
|
||||
deepgram.go ← Phase 3.2(新建)
|
||||
deepgram_test.go ← Phase 3.3(新建)
|
||||
llm/
|
||||
llm.go ← Phase 3.4(新建)
|
||||
openai.go ← Phase 3.5(新建)
|
||||
prompt.go ← Phase 3.6(新建)
|
||||
openai_test.go ← Phase 3.7(新建)
|
||||
tts/
|
||||
tts.go ← Phase 3.8(新建)
|
||||
openai.go ← Phase 3.9(新建)
|
||||
openai_test.go ← Phase 3.10(新建)
|
||||
router.go ← Phase 7.3(新建)
|
||||
orchestrator/
|
||||
orchestrator.go ← Phase 4.1(新建)
|
||||
sender.go ← Phase 4.2(新建)
|
||||
pipeline.go ← Phase 4.3, 4.4, 4.5, 4.6(新建)
|
||||
pipeline_test.go ← Phase 4.7(新建)
|
||||
api/
|
||||
session.go ← Phase 6.1(新建)
|
||||
middleware/
|
||||
ratelimit.go ← Phase 7.1, 7.2(新建)
|
||||
ws/
|
||||
handler.go ← Phase 5.1-5.5(修改)
|
||||
handler_test.go ← Phase 8.1(新建)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 新增依赖
|
||||
|
||||
| 包 | 用途 | Phase |
|
||||
|----|------|-------|
|
||||
| `go.uber.org/zap` | 结构化日志 | 1 |
|
||||
| `github.com/redis/go-redis/v9` | Redis 客户端 | 2.3 |
|
||||
| `github.com/gorilla/websocket` | 已有,Deepgram WS 也复用 | 3.2 |
|
||||
|
||||
---
|
||||
|
||||
## 执行顺序与依赖关系
|
||||
|
||||
```
|
||||
Phase 1 (基础设施)
|
||||
↓
|
||||
Phase 2 (Session Manager)
|
||||
↓
|
||||
Phase 3 (AI 服务层) ← 可与 Phase 2 并行开发
|
||||
↓
|
||||
Phase 4 (Orchestrator) ← 依赖 Phase 2 + 3
|
||||
↓
|
||||
Phase 5 (WS Handler 接入) ← 依赖 Phase 4
|
||||
↓
|
||||
Phase 6 (REST API) ← 依赖 Phase 2
|
||||
↓
|
||||
Phase 7 (Rate Limiter + Router) ← 独立,可推后
|
||||
↓
|
||||
Phase 8 (集成测试 + 文档)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 验证方案
|
||||
|
||||
1. **单元测试**:每个模块独立测试,mock 外部依赖(AI API、Redis)
|
||||
2. **集成测试**:`httptest` 启动 Gin server,用 gorilla/websocket 客户端模拟完整 query 流程
|
||||
3. **端到端手动测试**:启动后端 → 打开前端 → 摄像头+麦克风对话 → 验证 stt_result / llm_chunk / tts_audio 消息流
|
||||
4. **go vet + go test ./...** 通过
|
||||
|
||||
---
|
||||
|
||||
## 设计文档参考
|
||||
|
||||
- 接口规范(最高优先级):`docs/03-接口文档.md`
|
||||
- 系统架构:`docs/02-系统架构.md`
|
||||
- 技术选型:`docs/04-技术选型.md`
|
||||
- 成本控制:`docs/08-成本控制.md`
|
||||
@@ -1,19 +1,36 @@
|
||||
/* ============================================================
|
||||
CamTalk — 主应用样式
|
||||
CamTalk — Web 端应用样式
|
||||
双栏布局:左侧视频面板 + 右侧聊天面板
|
||||
============================================================ */
|
||||
|
||||
:root {
|
||||
--color-primary: #2563eb;
|
||||
--color-primary-hover: #1d4ed8;
|
||||
--color-bg: #0f172a;
|
||||
--color-surface: #1e293b;
|
||||
--color-text: #f1f5f9;
|
||||
--color-text-muted: #94a3b8;
|
||||
--color-border: #334155;
|
||||
--color-bg: #0b0f1a;
|
||||
--color-surface: #151b2b;
|
||||
--color-surface-2: #1c2438;
|
||||
--color-text: #e2e8f0;
|
||||
--color-text-muted: #64748b;
|
||||
--color-border: #1e293b;
|
||||
--color-success: #22c55e;
|
||||
--color-warning: #f59e0b;
|
||||
--color-error: #ef4444;
|
||||
--radius: 8px;
|
||||
--radius: 10px;
|
||||
--radius-sm: 6px;
|
||||
}
|
||||
|
||||
[data-theme="light"] {
|
||||
--color-primary: #2563eb;
|
||||
--color-primary-hover: #1d4ed8;
|
||||
--color-bg: #f1f5f9;
|
||||
--color-surface: #ffffff;
|
||||
--color-surface-2: #f8fafc;
|
||||
--color-text: #0f172a;
|
||||
--color-text-muted: #64748b;
|
||||
--color-border: #e2e8f0;
|
||||
--color-success: #16a34a;
|
||||
--color-warning: #d97706;
|
||||
--color-error: #dc2626;
|
||||
}
|
||||
|
||||
* {
|
||||
@@ -22,205 +39,634 @@
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html, body, #root {
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", sans-serif;
|
||||
background: var(--color-bg);
|
||||
color: var(--color-text);
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
/* ---- App Shell ---- */
|
||||
|
||||
.app {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 100vh;
|
||||
max-width: 900px;
|
||||
margin: 0 auto;
|
||||
padding: 0 16px;
|
||||
height: 100vh;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* ---- Header ---- */
|
||||
|
||||
.app-header {
|
||||
.header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 16px 0;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.app-header h1 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.status {
|
||||
font-size: 0.85rem;
|
||||
padding: 4px 12px;
|
||||
border-radius: 12px;
|
||||
padding: 0 20px;
|
||||
height: 48px;
|
||||
background: var(--color-surface);
|
||||
}
|
||||
|
||||
.status--connected {
|
||||
color: var(--color-success);
|
||||
border: 1px solid var(--color-success);
|
||||
}
|
||||
|
||||
.status--connecting {
|
||||
color: var(--color-warning);
|
||||
border: 1px solid var(--color-warning);
|
||||
}
|
||||
|
||||
.status--disconnected {
|
||||
color: var(--color-text-muted);
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
/* ---- Main ---- */
|
||||
|
||||
.app-main {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
padding: 16px 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.video-section {
|
||||
position: relative;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.vad-indicator {
|
||||
position: absolute;
|
||||
bottom: 12px;
|
||||
left: 12px;
|
||||
background: rgba(0, 0, 0, 0.7);
|
||||
color: var(--color-success);
|
||||
padding: 6px 12px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 0.85rem;
|
||||
animation: pulse 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.vad-indicator--loading {
|
||||
color: var(--color-warning);
|
||||
}
|
||||
|
||||
.vad-indicator--error {
|
||||
color: var(--color-error);
|
||||
animation: none;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.6; }
|
||||
}
|
||||
|
||||
.chat-section {
|
||||
flex: 1;
|
||||
.header__left {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: baseline;
|
||||
gap: 12px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* ---- Video Preview ---- */
|
||||
|
||||
.video-preview {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
aspect-ratio: 4 / 3;
|
||||
background: var(--color-surface);
|
||||
border-radius: var(--radius);
|
||||
overflow: hidden;
|
||||
.header__title {
|
||||
font-size: 1.2rem;
|
||||
font-weight: 700;
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.video-preview__video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.video-preview__placeholder {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
.header__subtitle {
|
||||
font-size: 0.8rem;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
/* ---- Chat Panel ---- */
|
||||
.header__right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.chat-panel {
|
||||
.header__stats {
|
||||
font-size: 0.75rem;
|
||||
color: var(--color-text-muted);
|
||||
padding: 4px 10px;
|
||||
background: var(--color-surface-2);
|
||||
border-radius: 12px;
|
||||
}
|
||||
|
||||
.badge {
|
||||
font-size: 0.75rem;
|
||||
padding: 3px 10px;
|
||||
border-radius: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.badge--connected {
|
||||
color: var(--color-success);
|
||||
background: rgba(34, 197, 94, 0.1);
|
||||
border: 1px solid rgba(34, 197, 94, 0.3);
|
||||
}
|
||||
|
||||
.badge--connecting {
|
||||
color: var(--color-warning);
|
||||
background: rgba(245, 158, 11, 0.1);
|
||||
border: 1px solid rgba(245, 158, 11, 0.3);
|
||||
}
|
||||
|
||||
.badge--disconnected {
|
||||
color: var(--color-text-muted);
|
||||
background: var(--color-surface-2);
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.btn-icon {
|
||||
background: none;
|
||||
border: none;
|
||||
font-size: 1.1rem;
|
||||
cursor: pointer;
|
||||
padding: 6px;
|
||||
border-radius: var(--radius-sm);
|
||||
transition: background 0.15s;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
.btn-icon:hover {
|
||||
background: var(--color-surface-2);
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.lang-group {
|
||||
display: flex;
|
||||
gap: 2px;
|
||||
background: var(--color-surface-2);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 2px;
|
||||
}
|
||||
|
||||
.lang-btn {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.72rem;
|
||||
font-weight: 600;
|
||||
padding: 4px 10px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.lang-btn:hover {
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.lang-btn--active {
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
}
|
||||
|
||||
/* ---- Workspace (双栏) ---- */
|
||||
|
||||
.workspace {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* ---- 左侧视频面板 ---- */
|
||||
|
||||
.video-panel {
|
||||
width: 420px;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: 16px;
|
||||
gap: 12px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.video-container {
|
||||
position: relative;
|
||||
aspect-ratio: 4 / 3;
|
||||
background: var(--color-surface-2);
|
||||
border-radius: var(--radius);
|
||||
overflow: hidden;
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.video-container .video-preview {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border-radius: 0;
|
||||
}
|
||||
|
||||
.video-container .video-preview__video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
background: #1a1a2e;
|
||||
}
|
||||
|
||||
.video-placeholder {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.82rem;
|
||||
}
|
||||
|
||||
.video-placeholder__icon {
|
||||
font-size: 2.4rem;
|
||||
opacity: 0.2;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.video-indicator {
|
||||
position: absolute;
|
||||
bottom: 16px;
|
||||
left: 16px;
|
||||
background: rgba(0, 0, 0, 0.75);
|
||||
backdrop-filter: blur(8px);
|
||||
color: var(--color-success);
|
||||
padding: 6px 14px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.8rem;
|
||||
font-weight: 500;
|
||||
animation: pulse 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.live-badge {
|
||||
position: absolute;
|
||||
top: 12px;
|
||||
left: 12px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
background: rgba(0, 0, 0, 0.75);
|
||||
backdrop-filter: blur(8px);
|
||||
color: white;
|
||||
padding: 4px 12px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.live-badge__dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: var(--color-success);
|
||||
animation: pulse 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.video-indicator--loading { color: var(--color-warning); }
|
||||
.video-indicator--audio { left: auto; right: 16px; color: var(--color-primary); }
|
||||
.video-indicator--error { color: var(--color-error); animation: none; }
|
||||
|
||||
.detail-badge {
|
||||
position: absolute;
|
||||
top: 12px;
|
||||
right: 12px;
|
||||
background: rgba(37, 99, 235, 0.85);
|
||||
backdrop-filter: blur(8px);
|
||||
color: white;
|
||||
padding: 3px 10px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.7rem;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.observation-badge {
|
||||
position: absolute;
|
||||
top: 12px;
|
||||
left: 12px;
|
||||
background: rgba(34, 197, 94, 0.85);
|
||||
backdrop-filter: blur(8px);
|
||||
color: white;
|
||||
padding: 3px 10px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.7rem;
|
||||
font-weight: 700;
|
||||
animation: pulse 2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.video-overlay {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.35);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
backdrop-filter: blur(2px);
|
||||
}
|
||||
|
||||
.video-overlay__spinner {
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
border: 3px solid rgba(255, 255, 255, 0.2);
|
||||
border-top-color: white;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
/* ---- 视频控制栏 ---- */
|
||||
|
||||
.video-controls {
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: 4px 0;
|
||||
}
|
||||
|
||||
.video-controls__row {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.btn--ctrl {
|
||||
padding: 6px 12px;
|
||||
font-size: 0.78rem;
|
||||
border-radius: var(--radius-sm);
|
||||
border: 1px solid var(--color-border);
|
||||
background: var(--color-surface-2);
|
||||
color: var(--color-text-muted);
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.btn--ctrl:hover {
|
||||
background: var(--color-border);
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.btn--ctrl-on {
|
||||
background: rgba(34, 197, 94, 0.15);
|
||||
color: var(--color-success);
|
||||
border-color: rgba(34, 197, 94, 0.3);
|
||||
}
|
||||
|
||||
.btn--ctrl-off {
|
||||
background: rgba(239, 68, 68, 0.1);
|
||||
color: var(--color-error);
|
||||
border-color: rgba(239, 68, 68, 0.2);
|
||||
}
|
||||
|
||||
.btn--speaking {
|
||||
animation: micPulse 0.8s ease-in-out infinite;
|
||||
box-shadow: 0 0 12px rgba(34, 197, 94, 0.5);
|
||||
}
|
||||
|
||||
@keyframes micPulse {
|
||||
0%, 100% { box-shadow: 0 0 8px rgba(34, 197, 94, 0.4); transform: scale(1); }
|
||||
50% { box-shadow: 0 0 16px rgba(34, 197, 94, 0.8); transform: scale(1.05); }
|
||||
}
|
||||
|
||||
/* ---- 右侧聊天面板 ---- */
|
||||
|
||||
.chat-panel-wrapper {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background: var(--color-surface);
|
||||
border-left: 1px solid var(--color-border);
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.chat-panel-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 12px 16px;
|
||||
font-weight: 600;
|
||||
font-size: 0.85rem;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
flex-shrink: 0;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
.chat-panel-header__mode {
|
||||
font-size: 0.7rem;
|
||||
font-weight: 500;
|
||||
color: var(--color-success);
|
||||
background: rgba(34, 197, 94, 0.1);
|
||||
padding: 2px 8px;
|
||||
border-radius: 10px;
|
||||
}
|
||||
|
||||
.chat-panel-body {
|
||||
flex: 1;
|
||||
overflow: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
/* ---- Chat Panel (覆盖子组件样式) ---- */
|
||||
|
||||
.chat-panel {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 12px 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.chat-panel--empty {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 48px 0;
|
||||
flex: 1;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.chat-message {
|
||||
padding: 12px 16px;
|
||||
border-radius: var(--radius);
|
||||
background: var(--color-surface);
|
||||
padding: 10px 14px;
|
||||
border-radius: var(--radius-sm);
|
||||
background: var(--color-surface-2);
|
||||
max-width: 92%;
|
||||
}
|
||||
|
||||
.chat-message--user {
|
||||
border-left: 3px solid var(--color-primary);
|
||||
border-left: 2px solid var(--color-primary);
|
||||
align-self: flex-end;
|
||||
background: rgba(37, 99, 235, 0.08);
|
||||
}
|
||||
|
||||
.chat-message--assistant {
|
||||
border-left: 3px solid var(--color-success);
|
||||
border-left: 2px solid var(--color-success);
|
||||
}
|
||||
|
||||
.chat-message--streaming {
|
||||
opacity: 0.8;
|
||||
opacity: 0.85;
|
||||
}
|
||||
|
||||
.chat-message__role {
|
||||
font-size: 0.75rem;
|
||||
font-size: 0.65rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
color: var(--color-text-muted);
|
||||
margin-bottom: 4px;
|
||||
margin-bottom: 3px;
|
||||
}
|
||||
|
||||
.chat-message__content {
|
||||
font-size: 0.95rem;
|
||||
line-height: 1.5;
|
||||
font-size: 0.82rem;
|
||||
line-height: 1.55;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
/* ---- Footer ---- */
|
||||
|
||||
.app-footer {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
justify-content: center;
|
||||
padding: 16px 0;
|
||||
border-top: 1px solid var(--color-border);
|
||||
.chat-message__meta {
|
||||
margin-top: 6px;
|
||||
font-size: 0.65rem;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
/* ---- Streaming Cursor ---- */
|
||||
|
||||
.cursor {
|
||||
display: inline;
|
||||
animation: blink 0.8s step-end infinite;
|
||||
color: var(--color-success);
|
||||
}
|
||||
|
||||
/* ---- System Message ---- */
|
||||
|
||||
.system-message {
|
||||
text-align: center;
|
||||
padding: 8px 16px;
|
||||
margin: 0 20px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
.system-message--warning {
|
||||
background: rgba(245, 158, 11, 0.1);
|
||||
color: var(--color-warning);
|
||||
border: 1px solid rgba(245, 158, 11, 0.2);
|
||||
}
|
||||
|
||||
/* ---- Drawer (Config Panel) ---- */
|
||||
|
||||
.drawer-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
z-index: 100;
|
||||
background: rgba(0, 0, 0, 0.4);
|
||||
backdrop-filter: blur(2px);
|
||||
animation: fadeIn 0.2s ease;
|
||||
}
|
||||
|
||||
.drawer {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
width: 320px;
|
||||
background: var(--color-surface);
|
||||
border-left: 1px solid var(--color-border);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
animation: slideInRight 0.25s ease;
|
||||
box-shadow: -8px 0 24px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
.drawer__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 14px 16px;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.drawer__title {
|
||||
font-weight: 600;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.drawer__close {
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--color-text-muted);
|
||||
cursor: pointer;
|
||||
font-size: 1rem;
|
||||
padding: 4px 8px;
|
||||
border-radius: var(--radius-sm);
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.drawer__close:hover {
|
||||
background: var(--color-surface-2);
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.drawer__body {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
/* ---- Config Group ---- */
|
||||
|
||||
.config-group {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.config-group__title {
|
||||
font-size: 0.7rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.8px;
|
||||
color: var(--color-text-muted);
|
||||
margin-bottom: 12px;
|
||||
padding-bottom: 8px;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.config-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 10px 0;
|
||||
font-size: 0.82rem;
|
||||
color: var(--color-text);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.config-row__info {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.config-row__label {
|
||||
font-weight: 500;
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.config-row__desc {
|
||||
font-size: 0.72rem;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
.config-row select,
|
||||
.config-row input[type="checkbox"] {
|
||||
background: var(--color-surface-2);
|
||||
color: var(--color-text);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 5px 10px;
|
||||
font-size: 0.8rem;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.config-row input[type="checkbox"] {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* ---- Toast ---- */
|
||||
|
||||
.toast-container {
|
||||
position: fixed;
|
||||
top: 68px;
|
||||
right: 20px;
|
||||
z-index: 1000;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.toast {
|
||||
padding: 10px 16px;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.82rem;
|
||||
cursor: pointer;
|
||||
animation: slideIn 0.3s ease-out;
|
||||
max-width: 320px;
|
||||
backdrop-filter: blur(8px);
|
||||
}
|
||||
|
||||
.toast--error { background: rgba(239, 68, 68, 0.9); color: white; }
|
||||
.toast--warning { background: rgba(245, 158, 11, 0.9); color: #000; }
|
||||
.toast--info { background: rgba(37, 99, 235, 0.9); color: white; }
|
||||
|
||||
/* ---- Buttons ---- */
|
||||
|
||||
.btn {
|
||||
padding: 10px 24px;
|
||||
padding: 8px 20px;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
font-size: 0.95rem;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.85rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: background 0.2s;
|
||||
transition: all 0.15s;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.btn--lg {
|
||||
padding: 12px 32px;
|
||||
font-size: 0.95rem;
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
|
||||
.btn--primary {
|
||||
@@ -233,7 +679,7 @@ body {
|
||||
}
|
||||
|
||||
.btn--secondary {
|
||||
background: var(--color-surface);
|
||||
background: var(--color-surface-2);
|
||||
color: var(--color-text);
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
@@ -247,16 +693,35 @@ body {
|
||||
color: #000;
|
||||
}
|
||||
|
||||
.btn--warning:hover {
|
||||
opacity: 0.9;
|
||||
.btn--warning:hover { opacity: 0.9; }
|
||||
|
||||
.btn--danger {
|
||||
background: rgba(239, 68, 68, 0.15);
|
||||
color: var(--color-error);
|
||||
border: 1px solid rgba(239, 68, 68, 0.3);
|
||||
}
|
||||
|
||||
/* ---- Streaming Cursor ---- */
|
||||
.btn--danger:hover {
|
||||
background: rgba(239, 68, 68, 0.25);
|
||||
}
|
||||
|
||||
.cursor {
|
||||
display: inline;
|
||||
animation: blink 0.8s step-end infinite;
|
||||
color: var(--color-success);
|
||||
.btn--active {
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
box-shadow: 0 0 12px rgba(37, 99, 235, 0.4);
|
||||
animation: pulseBtn 2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
/* ---- Animations ---- */
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.6; }
|
||||
}
|
||||
|
||||
@keyframes pulseBtn {
|
||||
0%, 100% { box-shadow: 0 0 12px rgba(37, 99, 235, 0.4); }
|
||||
50% { box-shadow: 0 0 20px rgba(37, 99, 235, 0.7); }
|
||||
}
|
||||
|
||||
@keyframes blink {
|
||||
@@ -264,97 +729,40 @@ body {
|
||||
50% { opacity: 0; }
|
||||
}
|
||||
|
||||
/* ---- Message Meta ---- */
|
||||
|
||||
.chat-message__meta {
|
||||
margin-top: 6px;
|
||||
font-size: 0.75rem;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
/* ---- System Message ---- */
|
||||
|
||||
.system-message {
|
||||
text-align: center;
|
||||
padding: 8px 16px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.system-message--warning {
|
||||
background: rgba(245, 158, 11, 0.15);
|
||||
color: var(--color-warning);
|
||||
border: 1px solid rgba(245, 158, 11, 0.3);
|
||||
}
|
||||
|
||||
/* ---- Toast ---- */
|
||||
|
||||
.toast-container {
|
||||
position: fixed;
|
||||
top: 16px;
|
||||
right: 16px;
|
||||
z-index: 1000;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.toast {
|
||||
padding: 10px 16px;
|
||||
border-radius: var(--radius);
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
animation: slideIn 0.3s ease-out;
|
||||
max-width: 320px;
|
||||
}
|
||||
|
||||
.toast--error {
|
||||
background: rgba(239, 68, 68, 0.9);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.toast--warning {
|
||||
background: rgba(245, 158, 11, 0.9);
|
||||
color: #000;
|
||||
}
|
||||
|
||||
.toast--info {
|
||||
background: rgba(37, 99, 235, 0.9);
|
||||
color: white;
|
||||
}
|
||||
|
||||
@keyframes slideIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateX(20px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateX(0);
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- Video Overlay ---- */
|
||||
|
||||
.video-overlay {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.3);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
|
||||
.video-overlay__spinner {
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border: 3px solid rgba(255, 255, 255, 0.3);
|
||||
border-top-color: white;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
@keyframes slideIn {
|
||||
from { opacity: 0; transform: translateX(20px); }
|
||||
to { opacity: 1; transform: translateX(0); }
|
||||
}
|
||||
|
||||
@keyframes fadeIn {
|
||||
from { opacity: 0; }
|
||||
to { opacity: 1; }
|
||||
}
|
||||
|
||||
@keyframes slideInRight {
|
||||
from { transform: translateX(100%); }
|
||||
to { transform: translateX(0); }
|
||||
}
|
||||
|
||||
/* ---- Scrollbar ---- */
|
||||
|
||||
::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: var(--color-border);
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb:hover {
|
||||
background: var(--color-text-muted);
|
||||
}
|
||||
|
||||
@@ -1,97 +1,235 @@
|
||||
// ============================================================
|
||||
// CamTalk — 主应用组件
|
||||
// CamTalk — 主应用组件(Web 端双栏布局)
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useVisionSession } from "./hooks/useVisionSession";
|
||||
import { VideoPreview } from "./components/VideoPreview";
|
||||
import { ChatPanel } from "./components/ChatPanel";
|
||||
import { ConfigPanel } from "./components/ConfigPanel";
|
||||
import { ToastContainer } from "./components/Toast";
|
||||
import { loadTheme, saveTheme } from "./lib/storage";
|
||||
import type { Theme } from "./types";
|
||||
import "./App.css";
|
||||
|
||||
function App() {
|
||||
const [showConfig, setShowConfig] = useState(false);
|
||||
const [theme, setTheme] = useState<Theme>(loadTheme);
|
||||
const [elapsed, setElapsed] = useState(0);
|
||||
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
// 切换主题时更新 <html> 的 data-theme 属性
|
||||
useEffect(() => {
|
||||
document.documentElement.setAttribute("data-theme", theme);
|
||||
}, [theme]);
|
||||
|
||||
const handleThemeChange = useCallback((t: Theme) => {
|
||||
setTheme(t);
|
||||
saveTheme(t);
|
||||
}, []);
|
||||
|
||||
const formatTime = (s: number) => {
|
||||
const m = Math.floor(s / 60);
|
||||
const sec = s % 60;
|
||||
return `${m.toString().padStart(2, "0")}:${sec.toString().padStart(2, "0")}`;
|
||||
};
|
||||
|
||||
const {
|
||||
messages,
|
||||
currentReply,
|
||||
isProcessing,
|
||||
isAudioPlaying,
|
||||
isSpeaking,
|
||||
isVADReady,
|
||||
vadError,
|
||||
connectionStatus,
|
||||
videoRef,
|
||||
stream,
|
||||
config,
|
||||
updateConfig,
|
||||
stats,
|
||||
mode,
|
||||
isObserving,
|
||||
toggleMode,
|
||||
startSession,
|
||||
stopSession,
|
||||
interrupt,
|
||||
isCameraOn,
|
||||
isMicOn,
|
||||
toggleCamera,
|
||||
toggleMic,
|
||||
} = useVisionSession();
|
||||
|
||||
const isConnected = connectionStatus === "connected";
|
||||
|
||||
// 连接计时器
|
||||
const elapsedRef = useRef(0);
|
||||
useEffect(() => {
|
||||
if (isConnected) {
|
||||
elapsedRef.current = 0;
|
||||
setElapsed(0); // eslint-disable-line react-hooks/set-state-in-effect -- 连接时重置计时器
|
||||
}
|
||||
}, [isConnected]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isConnected) {
|
||||
if (timerRef.current) {
|
||||
clearInterval(timerRef.current);
|
||||
timerRef.current = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const id = setInterval(() => {
|
||||
elapsedRef.current += 1;
|
||||
setElapsed(elapsedRef.current);
|
||||
}, 1000);
|
||||
timerRef.current = id;
|
||||
return () => clearInterval(id);
|
||||
}, [isConnected]);
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<header className="app-header">
|
||||
<h1>CamTalk</h1>
|
||||
<span className={`status status--${connectionStatus}`}>
|
||||
{isConnected
|
||||
? "已连接"
|
||||
: connectionStatus === "connecting"
|
||||
? "连接中..."
|
||||
: "未连接"}
|
||||
</span>
|
||||
{/* ---- 顶部导航栏 ---- */}
|
||||
<header className="header">
|
||||
<div className="header__left">
|
||||
<h1 className="header__title">CamTalk</h1>
|
||||
<span className="header__subtitle">AI 视觉对话助手</span>
|
||||
</div>
|
||||
<div className="header__right">
|
||||
{isConnected && (stats.queryCount > 0 || stats.totalTokens > 0) && (
|
||||
<span className="header__stats">
|
||||
{stats.queryCount} 次请求 · {stats.totalTokens} tokens
|
||||
</span>
|
||||
)}
|
||||
<span className={`badge badge--${connectionStatus}`}>
|
||||
{isConnected ? "已连接" : connectionStatus === "connecting" ? "连接中..." : "未连接"}
|
||||
</span>
|
||||
<button
|
||||
className="btn-icon"
|
||||
onClick={() => setShowConfig((v) => !v)}
|
||||
title="设置"
|
||||
>
|
||||
⚙️
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main className="app-main">
|
||||
<div className="video-section">
|
||||
<VideoPreview ref={videoRef} isStreaming={!!stream} />
|
||||
{isSpeaking && <div className="vad-indicator">🎤 正在聆听...</div>}
|
||||
{isConnected && !isVADReady && !vadError && (
|
||||
<div className="vad-indicator vad-indicator--loading">
|
||||
正在初始化语音检测...
|
||||
</div>
|
||||
)}
|
||||
{vadError && (
|
||||
<div className="vad-indicator vad-indicator--error">
|
||||
⚠️ {vadError}
|
||||
</div>
|
||||
)}
|
||||
{isProcessing && (
|
||||
<div className="video-overlay">
|
||||
<div className="video-overlay__spinner" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{showConfig && (
|
||||
<ConfigPanel
|
||||
config={config}
|
||||
theme={theme}
|
||||
onUpdate={updateConfig}
|
||||
onThemeChange={handleThemeChange}
|
||||
onClose={() => setShowConfig(false)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="chat-section">
|
||||
{connectionStatus === "disconnected" && messages.length > 0 && (
|
||||
<div className="system-message system-message--warning">
|
||||
连接已断开,正在重连...
|
||||
</div>
|
||||
)}
|
||||
<ChatPanel
|
||||
messages={messages}
|
||||
currentReply={currentReply}
|
||||
connectionStatus={connectionStatus}
|
||||
/>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<footer className="app-footer">
|
||||
{!isConnected ? (
|
||||
<button className="btn btn--primary" onClick={startSession}>
|
||||
{connectionStatus === "connecting" ? "连接中..." : "开始对话"}
|
||||
</button>
|
||||
) : (
|
||||
<>
|
||||
<button className="btn btn--secondary" onClick={stopSession}>
|
||||
结束对话
|
||||
</button>
|
||||
{isProcessing && (
|
||||
<button className="btn btn--warning" onClick={interrupt}>
|
||||
打断
|
||||
</button>
|
||||
{/* ---- 主体:左侧视频 + 右侧聊天 ---- */}
|
||||
<div className="workspace">
|
||||
{/* 左侧:视频预览 + 控制栏 */}
|
||||
<div className="video-panel">
|
||||
<div className="video-container">
|
||||
<VideoPreview ref={videoRef} isStreaming={!!stream} />
|
||||
{isConnected && (
|
||||
<div className="live-badge">
|
||||
<span className="live-badge__dot" />
|
||||
LIVE {formatTime(elapsed)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</footer>
|
||||
{config.detailLevel === "high" && (
|
||||
<div className="detail-badge">HD</div>
|
||||
)}
|
||||
{isObserving && (
|
||||
<div className="observation-badge">👁️ 观察中</div>
|
||||
)}
|
||||
{isSpeaking && (
|
||||
<div className="video-indicator">🎤 正在聆听...</div>
|
||||
)}
|
||||
{isAudioPlaying && config.ttsEnabled && (
|
||||
<div className="video-indicator video-indicator--audio">🔊 正在播放...</div>
|
||||
)}
|
||||
{isConnected && !isVADReady && !vadError && (
|
||||
<div className="video-indicator video-indicator--loading">正在初始化语音检测...</div>
|
||||
)}
|
||||
{vadError && (
|
||||
<div className="video-indicator video-indicator--error">⚠️ {vadError}</div>
|
||||
)}
|
||||
{isProcessing && (
|
||||
<div className="video-overlay">
|
||||
<div className="video-overlay__spinner" />
|
||||
</div>
|
||||
)}
|
||||
{!isConnected && !stream && (
|
||||
<div className="video-placeholder">
|
||||
<span className="video-placeholder__icon">📷</span>
|
||||
<span>点击右侧按钮开始对话</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 视频下方控制栏 */}
|
||||
<div className="video-controls">
|
||||
{!isConnected ? (
|
||||
<button className="btn btn--primary btn--lg" onClick={startSession}>
|
||||
{connectionStatus === "connecting" ? "连接中..." : "🎙️ 开始对话"}
|
||||
</button>
|
||||
) : (
|
||||
<div className="video-controls__row">
|
||||
<button
|
||||
className={`btn btn--ctrl ${isCameraOn ? "btn--ctrl-on" : "btn--ctrl-off"}`}
|
||||
onClick={toggleCamera}
|
||||
title={isCameraOn ? "关闭摄像头" : "开启摄像头"}
|
||||
>
|
||||
📷
|
||||
</button>
|
||||
<button
|
||||
className={`btn btn--ctrl ${isMicOn ? "btn--ctrl-on" : "btn--ctrl-off"} ${isSpeaking ? "btn--speaking" : ""}`}
|
||||
onClick={toggleMic}
|
||||
title={isMicOn ? "关闭麦克风" : "开启麦克风"}
|
||||
>
|
||||
🎤
|
||||
</button>
|
||||
<button
|
||||
className={`btn ${mode === "observation" ? "btn--active" : "btn--secondary"}`}
|
||||
onClick={toggleMode}
|
||||
>
|
||||
{mode === "observation" ? "👁️ 观察中" : "👁️ 观察模式"}
|
||||
</button>
|
||||
{isProcessing && (
|
||||
<button className="btn btn--warning" onClick={interrupt}>
|
||||
⏹ 打断
|
||||
</button>
|
||||
)}
|
||||
<button className="btn btn--danger" onClick={stopSession}>
|
||||
结束对话
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* 右侧:聊天面板 */}
|
||||
<div className="chat-panel-wrapper">
|
||||
<div className="chat-panel-header">
|
||||
<span>对话</span>
|
||||
{isConnected && mode === "observation" && (
|
||||
<span className="chat-panel-header__mode">观察模式</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="chat-panel-body">
|
||||
{connectionStatus === "disconnected" && messages.length > 0 && (
|
||||
<div className="system-message system-message--warning">
|
||||
连接已断开,正在重连...
|
||||
</div>
|
||||
)}
|
||||
<ChatPanel
|
||||
messages={messages}
|
||||
currentReply={currentReply}
|
||||
connectionStatus={connectionStatus}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ToastContainer />
|
||||
</div>
|
||||
|
||||
92
frontend/src/components/ConfigPanel/index.tsx
Normal file
92
frontend/src/components/ConfigPanel/index.tsx
Normal file
@@ -0,0 +1,92 @@
|
||||
// ============================================================
|
||||
// ConfigPanel — 右侧抽屉式配置面板
|
||||
// 职责:主题切换、TTS 开关、detail level 切换、语言选择
|
||||
// ============================================================
|
||||
|
||||
import type { SessionConfig, Theme } from "../../types";
|
||||
|
||||
interface ConfigPanelProps {
|
||||
config: SessionConfig;
|
||||
theme: Theme;
|
||||
onUpdate: (partial: Partial<SessionConfig>) => void;
|
||||
onThemeChange: (theme: Theme) => void;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function ConfigPanel({ config, theme, onUpdate, onThemeChange, onClose }: ConfigPanelProps) {
|
||||
return (
|
||||
<div className="drawer-overlay" onClick={onClose}>
|
||||
<div className="drawer" onClick={(e) => e.stopPropagation()}>
|
||||
<div className="drawer__header">
|
||||
<span className="drawer__title">设置</span>
|
||||
<button className="drawer__close" onClick={onClose}>✕</button>
|
||||
</div>
|
||||
|
||||
<div className="drawer__body">
|
||||
<div className="config-group">
|
||||
<div className="config-group__title">外观</div>
|
||||
|
||||
<label className="config-row">
|
||||
<div className="config-row__info">
|
||||
<span className="config-row__label">主题</span>
|
||||
<span className="config-row__desc">切换明暗主题</span>
|
||||
</div>
|
||||
<select
|
||||
value={theme}
|
||||
onChange={(e) => onThemeChange(e.target.value as Theme)}
|
||||
>
|
||||
<option value="dark">深色</option>
|
||||
<option value="light">浅色</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className="config-group">
|
||||
<div className="config-group__title">会话</div>
|
||||
|
||||
<label className="config-row">
|
||||
<div className="config-row__info">
|
||||
<span className="config-row__label">语音回答</span>
|
||||
<span className="config-row__desc">AI 回答时同步播放语音</span>
|
||||
</div>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={config.ttsEnabled}
|
||||
onChange={(e) => onUpdate({ ttsEnabled: e.target.checked })}
|
||||
/>
|
||||
</label>
|
||||
|
||||
<label className="config-row">
|
||||
<div className="config-row__info">
|
||||
<span className="config-row__label">图像精度</span>
|
||||
<span className="config-row__desc">高精度适合识别文字,低精度省流量</span>
|
||||
</div>
|
||||
<select
|
||||
value={config.detailLevel}
|
||||
onChange={(e) => onUpdate({ detailLevel: e.target.value as "low" | "high" })}
|
||||
>
|
||||
<option value="low">低</option>
|
||||
<option value="high">高</option>
|
||||
</select>
|
||||
</label>
|
||||
|
||||
<label className="config-row">
|
||||
<div className="config-row__info">
|
||||
<span className="config-row__label">语言</span>
|
||||
<span className="config-row__desc">交互语言偏好</span>
|
||||
</div>
|
||||
<select
|
||||
value={config.language}
|
||||
onChange={(e) => onUpdate({ language: e.target.value })}
|
||||
>
|
||||
<option value="zh-CN">中文</option>
|
||||
<option value="en-US">English</option>
|
||||
<option value="ja-JP">日本語</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -103,19 +103,61 @@ export function useVAD(options?: VADOptions) {
|
||||
return { isSpeaking, isReady, error, start, stop };
|
||||
}
|
||||
|
||||
// ---- 关键帧检测(ONNX Runtime Web)----
|
||||
// ---- 关键帧检测 ----
|
||||
|
||||
export function useKeyframeDetection() {
|
||||
// TODO: 加载 ONNX 模型后设为 true
|
||||
const isReady = false;
|
||||
const DETECT_WIDTH = 160;
|
||||
const DETECT_HEIGHT = 120;
|
||||
const DIFF_THRESHOLD = 30;
|
||||
|
||||
const isKeyframe = useCallback(
|
||||
(_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
|
||||
// TODO: 实现像素差异对比
|
||||
return true; // 暂时所有帧都视为关键帧
|
||||
},
|
||||
[],
|
||||
);
|
||||
/** 离屏 canvas,用于降采样 */
|
||||
let offscreen: HTMLCanvasElement | null = null;
|
||||
|
||||
return { isReady, isKeyframe };
|
||||
function getOffscreen(): HTMLCanvasElement {
|
||||
if (!offscreen) {
|
||||
offscreen = document.createElement("canvas");
|
||||
offscreen.width = DETECT_WIDTH;
|
||||
offscreen.height = DETECT_HEIGHT;
|
||||
}
|
||||
return offscreen;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 video 元素缩放采样为 Uint8ClampedArray(RGBA)
|
||||
* 返回 null 如果 video 未就绪
|
||||
*/
|
||||
export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null {
|
||||
if (video.readyState < 2) return null;
|
||||
const canvas = getOffscreen();
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return null;
|
||||
ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT);
|
||||
return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data;
|
||||
}
|
||||
|
||||
/**
|
||||
* 对比两帧像素差异
|
||||
* @returns { isKeyframe: boolean, similarity: number }
|
||||
*/
|
||||
export function compareFrames(
|
||||
prev: Uint8ClampedArray,
|
||||
curr: Uint8ClampedArray,
|
||||
): { isKeyframe: boolean; similarity: number } {
|
||||
let diffSum = 0;
|
||||
const len = Math.min(prev.length, curr.length);
|
||||
const pixelCount = len / 4;
|
||||
|
||||
for (let i = 0; i < len; i += 4) {
|
||||
// 只比较 RGB,跳过 Alpha
|
||||
diffSum += Math.abs(prev[i] - curr[i]);
|
||||
diffSum += Math.abs(prev[i + 1] - curr[i + 1]);
|
||||
diffSum += Math.abs(prev[i + 2] - curr[i + 2]);
|
||||
}
|
||||
|
||||
const avgDiff = diffSum / (pixelCount * 3);
|
||||
const similarity = 1 - avgDiff / 255;
|
||||
|
||||
return {
|
||||
isKeyframe: avgDiff > DIFF_THRESHOLD,
|
||||
similarity,
|
||||
};
|
||||
}
|
||||
|
||||
88
frontend/src/hooks/useObservationMode.ts
Normal file
88
frontend/src/hooks/useObservationMode.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
// ============================================================
|
||||
// useObservationMode — 观察模式 Hook
|
||||
// 职责:定时采帧 → 关键帧检测 → 画面变化时触发回调
|
||||
// 来源:docs/05-用户故事.md US-05(持续场景监控)
|
||||
// ============================================================
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { sampleFrame, compareFrames } from "../components/EdgeProcessor";
|
||||
|
||||
/** 画面变化显著阈值 */
|
||||
const CHANGE_THRESHOLD = 0.85;
|
||||
/** 采样间隔(ms) */
|
||||
const SAMPLE_INTERVAL = 5000;
|
||||
|
||||
export interface ObservationOptions {
|
||||
/** 画面变化回调,携带当前帧的 DataURL */
|
||||
onChange?: (frameDataUrl: string) => void;
|
||||
}
|
||||
|
||||
export function useObservationMode(options?: ObservationOptions) {
|
||||
const [isObserving, setIsObserving] = useState(false);
|
||||
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
|
||||
const optionsRef = useRef(options);
|
||||
|
||||
useEffect(() => {
|
||||
optionsRef.current = options;
|
||||
}, [options]);
|
||||
|
||||
/**
|
||||
* 启动观察模式
|
||||
* @param video 摄像头 video 元素
|
||||
* @param captureFrame 从 video 捕获 DataURL 的函数
|
||||
*/
|
||||
const startObserving = useCallback(
|
||||
(video: HTMLVideoElement | null, captureFrame: () => string | null) => {
|
||||
if (!video) return;
|
||||
|
||||
// 立即采一帧作为基准
|
||||
prevFrameRef.current = sampleFrame(video);
|
||||
|
||||
intervalRef.current = setInterval(() => {
|
||||
const current = sampleFrame(video);
|
||||
if (!current) return;
|
||||
|
||||
if (prevFrameRef.current) {
|
||||
const { similarity } = compareFrames(prevFrameRef.current, current);
|
||||
|
||||
if (similarity < CHANGE_THRESHOLD) {
|
||||
console.log(
|
||||
`[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`,
|
||||
);
|
||||
const frameDataUrl = captureFrame();
|
||||
if (frameDataUrl) {
|
||||
optionsRef.current?.onChange?.(frameDataUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prevFrameRef.current = current;
|
||||
}, SAMPLE_INTERVAL);
|
||||
|
||||
setIsObserving(true);
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
/** 停止观察模式 */
|
||||
const stopObserving = useCallback(() => {
|
||||
if (intervalRef.current) {
|
||||
clearInterval(intervalRef.current);
|
||||
intervalRef.current = null;
|
||||
}
|
||||
prevFrameRef.current = null;
|
||||
setIsObserving(false);
|
||||
}, []);
|
||||
|
||||
// 组件卸载时清理
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (intervalRef.current) {
|
||||
clearInterval(intervalRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
return { isObserving, startObserving, stopObserving };
|
||||
}
|
||||
@@ -9,17 +9,52 @@ import { v4 as uuidv4 } from "uuid";
|
||||
import { wsClient } from "../lib/websocket";
|
||||
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
|
||||
import { getErrorMessage } from "../lib/errors";
|
||||
import { TTSPlayer } from "../lib/ttsPlayer";
|
||||
import { showToast } from "../lib/toast";
|
||||
import { loadConfig, saveConfig } from "../lib/storage";
|
||||
import { useCamera } from "../components/CameraManager";
|
||||
import { useMicrophone } from "../components/MicManager";
|
||||
import { useVAD } from "../components/EdgeProcessor";
|
||||
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
|
||||
import { useWebSocketManager } from "../components/WebSocketManager";
|
||||
import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";
|
||||
import { useObservationMode } from "./useObservationMode";
|
||||
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
|
||||
|
||||
export type SessionMode = "dialogue" | "observation";
|
||||
|
||||
const MAX_HISTORY_ROUNDS = 10;
|
||||
|
||||
export interface SessionStats {
|
||||
queryCount: number;
|
||||
totalTokens: number;
|
||||
}
|
||||
|
||||
export function useVisionSession() {
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const [currentReply, setCurrentReply] = useState<string>("");
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
|
||||
const [config, setConfig] = useState<SessionConfig>(loadConfig);
|
||||
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
|
||||
const [mode, setMode] = useState<SessionMode>("dialogue");
|
||||
const [isCameraOn, setIsCameraOn] = useState(false);
|
||||
const [isMicOn, setIsMicOn] = useState(false);
|
||||
|
||||
// 上一帧采样数据(用于关键帧检测)
|
||||
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
|
||||
|
||||
// 对话历史(role + content),用于多轮上下文
|
||||
const historyRef = useRef<Array<{ role: string; content: string }>>([]);
|
||||
|
||||
// TTS 播放器
|
||||
const ttsPlayerRef = useRef<TTSPlayer | null>(null);
|
||||
const getTTSPlayer = useCallback(() => {
|
||||
if (!ttsPlayerRef.current) {
|
||||
const player = new TTSPlayer();
|
||||
player.onEnd(() => setIsAudioPlaying(false));
|
||||
ttsPlayerRef.current = player;
|
||||
}
|
||||
return ttsPlayerRef.current;
|
||||
}, []);
|
||||
|
||||
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
|
||||
const { startMic, stopMic } = useMicrophone();
|
||||
@@ -31,6 +66,82 @@ export function useVisionSession() {
|
||||
isProcessingRef.current = isProcessing;
|
||||
}, [isProcessing]);
|
||||
|
||||
// 观察模式:画面变化时自动发送 query
|
||||
const { isObserving, startObserving, stopObserving } = useObservationMode({
|
||||
onChange: useCallback(
|
||||
(frameDataUrl: string) => {
|
||||
if (isProcessingRef.current) return;
|
||||
|
||||
const requestId = uuidv4();
|
||||
send({
|
||||
type: "query",
|
||||
request_id: requestId,
|
||||
image: dataUrlToBase64(frameDataUrl),
|
||||
audio: "", // 观察模式无音频
|
||||
});
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
role: "user",
|
||||
content: "👁️ 画面变化检测",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
]);
|
||||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||||
setIsProcessing(true);
|
||||
},
|
||||
[send],
|
||||
),
|
||||
});
|
||||
|
||||
/** 切换对话/观察模式 */
|
||||
const toggleMode = useCallback(() => {
|
||||
setMode((prev) => {
|
||||
const next = prev === "dialogue" ? "observation" : "dialogue";
|
||||
if (next === "observation") {
|
||||
startObserving(videoRef.current, captureFrame);
|
||||
} else {
|
||||
stopObserving();
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [videoRef, captureFrame, startObserving, stopObserving]);
|
||||
|
||||
// WebSocket 连接成功后发送 config
|
||||
useEffect(() => {
|
||||
if (status === "connected") {
|
||||
send({
|
||||
type: "config",
|
||||
payload: {
|
||||
tts_enabled: config.ttsEnabled,
|
||||
detail_level: config.detailLevel,
|
||||
language: config.language,
|
||||
},
|
||||
});
|
||||
}
|
||||
}, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送
|
||||
|
||||
/** 更新会话配置 */
|
||||
const updateConfig = useCallback((partial: Partial<SessionConfig>) => {
|
||||
setConfig((prev) => {
|
||||
const next = { ...prev, ...partial };
|
||||
saveConfig(next);
|
||||
// 如果已连接,立即发送更新
|
||||
if (status === "connected") {
|
||||
send({
|
||||
type: "config",
|
||||
payload: {
|
||||
tts_enabled: next.ttsEnabled,
|
||||
detail_level: next.detailLevel,
|
||||
language: next.language,
|
||||
},
|
||||
});
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [status, send]);
|
||||
|
||||
// VAD:语音结束时自动发送 query
|
||||
const {
|
||||
isSpeaking,
|
||||
@@ -53,6 +164,23 @@ export function useVisionSession() {
|
||||
return;
|
||||
}
|
||||
|
||||
// 关键帧检测:与上一帧对比,相似度过高则跳过
|
||||
const video = videoRef.current;
|
||||
if (video) {
|
||||
const currentSample = sampleFrame(video);
|
||||
if (currentSample && prevFrameRef.current) {
|
||||
const { similarity } = compareFrames(prevFrameRef.current, currentSample);
|
||||
if (similarity > 0.9) {
|
||||
console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)}),跳过`);
|
||||
prevFrameRef.current = currentSample;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (currentSample) {
|
||||
prevFrameRef.current = currentSample;
|
||||
}
|
||||
}
|
||||
|
||||
const requestId = uuidv4();
|
||||
send({
|
||||
type: "query",
|
||||
@@ -61,6 +189,9 @@ export function useVisionSession() {
|
||||
audio: encodeAudioToBase64(audio),
|
||||
});
|
||||
|
||||
// 更新请求统计
|
||||
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
|
||||
|
||||
// 添加用户消息(STT 流式结果会逐步更新文本)
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
@@ -68,7 +199,7 @@ export function useVisionSession() {
|
||||
]);
|
||||
setIsProcessing(true);
|
||||
},
|
||||
[captureFrame, send]
|
||||
[captureFrame, send, videoRef]
|
||||
),
|
||||
});
|
||||
|
||||
@@ -98,6 +229,21 @@ export function useVisionSession() {
|
||||
|
||||
case "llm_done": {
|
||||
const done = msg as LLMDoneMessage;
|
||||
// 记录到对话历史
|
||||
historyRef.current.push({ role: "assistant", content: done.full_text });
|
||||
// 裁剪历史到最近 N 轮
|
||||
if (historyRef.current.length > MAX_HISTORY_ROUNDS * 2) {
|
||||
historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
|
||||
}
|
||||
|
||||
// 累计 token 统计
|
||||
if (done.tokens_used?.total) {
|
||||
setStats((prev) => ({
|
||||
...prev,
|
||||
totalTokens: prev.totalTokens + done.tokens_used.total,
|
||||
}));
|
||||
}
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
@@ -115,7 +261,10 @@ export function useVisionSession() {
|
||||
}
|
||||
|
||||
case "tts_audio":
|
||||
// TODO: 阶段 5 音频流播放
|
||||
getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
|
||||
if (!msg.is_last) {
|
||||
setIsAudioPlaying(true);
|
||||
}
|
||||
break;
|
||||
|
||||
case "error":
|
||||
@@ -127,25 +276,19 @@ export function useVisionSession() {
|
||||
});
|
||||
|
||||
return unsub;
|
||||
}, []);
|
||||
|
||||
// 连接断开时显示提示
|
||||
useEffect(() => {
|
||||
if (status === "disconnected") {
|
||||
// 只在非主动断开时提示(通过检查是否有活跃会话判断)
|
||||
// 这里简单处理,由 App 层根据状态显示
|
||||
}
|
||||
}, [status]);
|
||||
}, [getTTSPlayer]);
|
||||
|
||||
/** 启动会话 */
|
||||
const startSession = useCallback(async () => {
|
||||
// 1. 获取摄像头和麦克风
|
||||
await startCamera();
|
||||
setIsCameraOn(true);
|
||||
const micStream = await startMic();
|
||||
if (!micStream) {
|
||||
showToast("无法获取麦克风权限", "error");
|
||||
return;
|
||||
}
|
||||
setIsMicOn(true);
|
||||
|
||||
// 2. 连接 WebSocket
|
||||
connect();
|
||||
@@ -156,24 +299,60 @@ export function useVisionSession() {
|
||||
|
||||
/** 结束会话 */
|
||||
const stopSession = useCallback(async () => {
|
||||
stopObserving();
|
||||
setMode("dialogue");
|
||||
await stopVAD();
|
||||
stopMic();
|
||||
stopCamera();
|
||||
disconnect();
|
||||
// 清理所有对话状态
|
||||
// 停止 TTS 并清理状态
|
||||
ttsPlayerRef.current?.stop();
|
||||
setIsAudioPlaying(false);
|
||||
setMessages([]);
|
||||
setCurrentReply("");
|
||||
setIsProcessing(false);
|
||||
}, [stopVAD, stopMic, stopCamera, disconnect]);
|
||||
setStats({ queryCount: 0, totalTokens: 0 });
|
||||
historyRef.current = [];
|
||||
prevFrameRef.current = null;
|
||||
setIsCameraOn(false);
|
||||
setIsMicOn(false);
|
||||
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
|
||||
|
||||
/** 摄像头开关 */
|
||||
const toggleCamera = useCallback(async () => {
|
||||
if (isCameraOn) {
|
||||
stopCamera();
|
||||
setIsCameraOn(false);
|
||||
} else {
|
||||
await startCamera();
|
||||
setIsCameraOn(true);
|
||||
}
|
||||
}, [isCameraOn, startCamera, stopCamera]);
|
||||
|
||||
/** 麦克风开关 */
|
||||
const toggleMic = useCallback(async () => {
|
||||
if (isMicOn) {
|
||||
stopMic();
|
||||
setIsMicOn(false);
|
||||
} else {
|
||||
const micStream = await startMic();
|
||||
setIsMicOn(!!micStream);
|
||||
}
|
||||
}, [isMicOn, startMic, stopMic]);
|
||||
|
||||
/** 打断当前回复 */
|
||||
const interrupt = useCallback(() => {
|
||||
send({ type: "interrupt" });
|
||||
// 停止 TTS 播放
|
||||
ttsPlayerRef.current?.stop();
|
||||
setIsAudioPlaying(false);
|
||||
// 将未完成的流式内容保存为最终消息
|
||||
if (currentReply) {
|
||||
const interrupted = currentReply + "(已打断)";
|
||||
historyRef.current.push({ role: "assistant", content: interrupted });
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{ role: "assistant", content: currentReply + "(已打断)", timestamp: Date.now() },
|
||||
{ role: "assistant", content: interrupted, timestamp: Date.now() },
|
||||
]);
|
||||
}
|
||||
setCurrentReply("");
|
||||
@@ -184,14 +363,25 @@ export function useVisionSession() {
|
||||
messages,
|
||||
currentReply,
|
||||
isProcessing,
|
||||
isAudioPlaying,
|
||||
isSpeaking,
|
||||
isVADReady,
|
||||
vadError,
|
||||
connectionStatus: status,
|
||||
videoRef,
|
||||
stream,
|
||||
config,
|
||||
updateConfig,
|
||||
stats,
|
||||
mode,
|
||||
isObserving,
|
||||
toggleMode,
|
||||
startSession,
|
||||
stopSession,
|
||||
interrupt,
|
||||
isCameraOn,
|
||||
isMicOn,
|
||||
toggleCamera,
|
||||
toggleMic,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,7 +1 @@
|
||||
/* 全局重置 — 详细样式在 App.css 中定义 */
|
||||
|
||||
#root {
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
43
frontend/src/lib/sampling.ts
Normal file
43
frontend/src/lib/sampling.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
// ============================================================
|
||||
// Sampling — 混合采样策略
|
||||
// 来源:docs/08-成本控制.md — "定时低频 + 事件高频"
|
||||
// ============================================================
|
||||
|
||||
/** 静默时采样间隔(ms) */
|
||||
const IDLE_INTERVAL = 5000;
|
||||
/** 用户说话时采样间隔(ms) */
|
||||
const ACTIVE_INTERVAL = 1000;
|
||||
|
||||
export class SamplingController {
|
||||
private lastSampleTime = 0;
|
||||
private _isUserSpeaking = false;
|
||||
|
||||
/** 设置用户是否正在说话 */
|
||||
set speaking(value: boolean) {
|
||||
this._isUserSpeaking = value;
|
||||
}
|
||||
|
||||
get speaking(): boolean {
|
||||
return this._isUserSpeaking;
|
||||
}
|
||||
|
||||
/** 当前采样间隔 */
|
||||
get interval(): number {
|
||||
return this._isUserSpeaking ? ACTIVE_INTERVAL : IDLE_INTERVAL;
|
||||
}
|
||||
|
||||
/** 是否应该采样(基于时间间隔) */
|
||||
shouldSample(): boolean {
|
||||
const now = Date.now();
|
||||
if (now - this.lastSampleTime < this.interval) {
|
||||
return false;
|
||||
}
|
||||
this.lastSampleTime = now;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** 强制允许下次采样(如用户刚说完话时) */
|
||||
resetTimer(): void {
|
||||
this.lastSampleTime = 0;
|
||||
}
|
||||
}
|
||||
52
frontend/src/lib/storage.ts
Normal file
52
frontend/src/lib/storage.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
// ============================================================
|
||||
// Storage — localStorage 封装
|
||||
// 职责:会话配置持久化
|
||||
// ============================================================
|
||||
|
||||
import type { SessionConfig, Theme } from "../types";
|
||||
|
||||
const CONFIG_KEY = "camtalk:config";
|
||||
const THEME_KEY = "camtalk:theme";
|
||||
|
||||
const DEFAULT_CONFIG: SessionConfig = {
|
||||
ttsEnabled: true,
|
||||
detailLevel: "low",
|
||||
language: "zh-CN",
|
||||
};
|
||||
|
||||
/** 加载配置,无存储时返回默认值 */
|
||||
export function loadConfig(): SessionConfig {
|
||||
try {
|
||||
const raw = localStorage.getItem(CONFIG_KEY);
|
||||
if (!raw) return DEFAULT_CONFIG;
|
||||
const parsed = JSON.parse(raw) as Partial<SessionConfig>;
|
||||
return { ...DEFAULT_CONFIG, ...parsed };
|
||||
} catch {
|
||||
return DEFAULT_CONFIG;
|
||||
}
|
||||
}
|
||||
|
||||
/** 保存配置到 localStorage */
|
||||
export function saveConfig(config: SessionConfig): void {
|
||||
try {
|
||||
localStorage.setItem(CONFIG_KEY, JSON.stringify(config));
|
||||
} catch {
|
||||
// localStorage 不可用时静默失败
|
||||
}
|
||||
}
|
||||
|
||||
/** 加载主题偏好 */
|
||||
export function loadTheme(): Theme {
|
||||
try {
|
||||
const raw = localStorage.getItem(THEME_KEY);
|
||||
if (raw === "light" || raw === "dark") return raw;
|
||||
} catch { /* ignore */ }
|
||||
return "dark";
|
||||
}
|
||||
|
||||
/** 保存主题偏好 */
|
||||
export function saveTheme(theme: Theme): void {
|
||||
try {
|
||||
localStorage.setItem(THEME_KEY, theme);
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
104
frontend/src/lib/ttsPlayer.ts
Normal file
104
frontend/src/lib/ttsPlayer.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
// ============================================================
|
||||
// TTS Player — 语音播放器
|
||||
// 职责:收集后端流式 tts_audio 片段,拼接后播放
|
||||
// 格式:MVP 仅支持 audio/mp3,pcm 为 TODO
|
||||
// ============================================================
|
||||
|
||||
type OnEndCallback = () => void;
|
||||
|
||||
export class TTSPlayer {
|
||||
private chunks: string[] = [];
|
||||
private audio: HTMLAudioElement | null = null;
|
||||
private _isPlaying = false;
|
||||
private onEndCallback: OnEndCallback | null = null;
|
||||
|
||||
/** 注册播放完成回调 */
|
||||
onEnd(cb: OnEndCallback): void {
|
||||
this.onEndCallback = cb;
|
||||
}
|
||||
|
||||
/** 当前是否正在播放 */
|
||||
get isPlaying(): boolean {
|
||||
return this._isPlaying;
|
||||
}
|
||||
|
||||
/**
|
||||
* 入队一个 TTS 音频片段
|
||||
* @param base64 Base64 编码的音频数据
|
||||
* @param mimeType 音频格式("audio/mp3" 或 "audio/pcm")
|
||||
* @param isLast 是否为最后一个片段
|
||||
*/
|
||||
enqueue(base64: string, mimeType: string, isLast: boolean): void {
|
||||
this.chunks.push(base64);
|
||||
|
||||
if (isLast) {
|
||||
this.play(mimeType);
|
||||
}
|
||||
}
|
||||
|
||||
/** 停止播放并清空缓冲区 */
|
||||
stop(): void {
|
||||
if (this.audio) {
|
||||
this.audio.pause();
|
||||
this.audio.removeAttribute("src");
|
||||
this.audio = null;
|
||||
}
|
||||
this.chunks = [];
|
||||
this._isPlaying = false;
|
||||
}
|
||||
|
||||
/** 暂停播放 */
|
||||
pause(): void {
|
||||
this.audio?.pause();
|
||||
}
|
||||
|
||||
/** 恢复播放 */
|
||||
resume(): void {
|
||||
this.audio?.play();
|
||||
}
|
||||
|
||||
/** 拼接所有片段并播放 */
|
||||
private play(mimeType: string): void {
|
||||
if (this.chunks.length === 0) return;
|
||||
|
||||
// 拼接所有 Base64 片段
|
||||
const combined = this.chunks.join("");
|
||||
this.chunks = [];
|
||||
|
||||
// Base64 → Uint8Array → Blob
|
||||
const binary = atob(combined);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
const blob = new Blob([bytes], { type: mimeType });
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
// 播放
|
||||
const audio = new Audio(url);
|
||||
this.audio = audio;
|
||||
this._isPlaying = true;
|
||||
|
||||
audio.onended = () => {
|
||||
URL.revokeObjectURL(url);
|
||||
this._isPlaying = false;
|
||||
this.audio = null;
|
||||
this.onEndCallback?.();
|
||||
};
|
||||
|
||||
audio.onerror = () => {
|
||||
console.error("[TTS] 播放失败");
|
||||
URL.revokeObjectURL(url);
|
||||
this._isPlaying = false;
|
||||
this.audio = null;
|
||||
this.onEndCallback?.();
|
||||
};
|
||||
|
||||
audio.play().catch((err) => {
|
||||
console.error("[TTS] play() 被拒绝:", err);
|
||||
this._isPlaying = false;
|
||||
this.audio = null;
|
||||
this.onEndCallback?.();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,8 @@ export interface SessionConfig {
|
||||
language: string;
|
||||
}
|
||||
|
||||
export type Theme = "dark" | "light";
|
||||
|
||||
export interface Session {
|
||||
sessionId: string;
|
||||
createdAt: string;
|
||||
|
||||
Reference in New Issue
Block a user