diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3be820f --- /dev/null +++ b/.env.example @@ -0,0 +1,21 @@ +# CamTalk 环境变量模板 +# 复制为 .env 并填入实际值：cp .env.example .env +# .env 已在 .gitignore 中，不会提交到版本控制 + +# ---- AI 服务 API Key ---- +CAMTALK_AI_LLM_API_KEY=sk-xxx +CAMTALK_AI_STT_API_KEY= +CAMTALK_AI_TTS_API_KEY= + +# ---- 可选覆盖（默认值见 config.yaml）---- +# CAMTALK_AI_LLM_MODEL=gpt-4o +# CAMTALK_AI_LLM_ENDPOINT=https://api.openai.com/v1 +# CAMTALK_AI_LLM_TIMEOUT=10 +# CAMTALK_AI_STT_ENDPOINT=wss://api.deepgram.com/v1/listen +# CAMTALK_AI_TTS_ENDPOINT=https://api.openai.com/v1 +# CAMTALK_AI_TTS_VOICE=alloy +# CAMTALK_AI_TTS_SPEED=1.0 +# CAMTALK_AI_TTS_TIMEOUT=5 + +# ---- 应用 ---- +# APP_ENV=dev diff --git a/.gitignore b/.gitignore index d37bf7a..4c77b4c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ frontend/dist/ # ---- 后端 ---- backend/bin/ +backend/server # ---- 环境变量 ---- .env @@ -18,5 +19,11 @@ Thumbs.db .idea/ .vscode/ +# ---- Playwright MCP ---- +.playwright-mcp/ + +# ---- 截图 ---- +*.png + # ---- Obsidian ---- .obsidian/ diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..8304260 --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,17 @@ +# 编译产物 +/server +bin/ + +# 环境配置 +.env +config.dev.yaml +config.prod.yaml + +# 临时文件 +tmp/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 18aafe2..134b791 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -1,40 +1,118 @@ package main import ( - "log" + "context" + "errors" + "net/http" + "os/signal" + "syscall" "time" "github.com/gin-gonic/gin" + "github.com/hhs/camtalk/internal/api" + "github.com/hhs/camtalk/internal/ai/llm" + "github.com/hhs/camtalk/internal/ai/stt" + "github.com/hhs/camtalk/internal/ai/tts" + "github.com/hhs/camtalk/internal/config" + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/orchestrator" + "github.com/hhs/camtalk/internal/session" "github.com/hhs/camtalk/internal/ws" ) var startTime = time.Now() func main() { - r := gin.Default() + // 加载配置 + cfg, err := config.Load() + if err != nil { + panic("failed to load config: " + err.Error()) + } + + // 初始化日志 + logger.Init(cfg.Log.Level, cfg.Log.Format) + defer logger.Sync() + + logger.Log.Infow("config loaded", + "env", cfg.App.Env, + "addr", cfg.Server.Addr(), + ) + + // 初始化 Session Manager（MVP 默认内存实现） + var sessionMgr session.Manager + // TODO: 当 Redis 配置非空时切换为 RedisManager + sessionMgr = session.NewMemoryManager(30*time.Minute, 20) + defer sessionMgr.(*session.MemoryManager).Stop() + + // 初始化 AI 服务 + sttService := stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, logger.Log) + llmService := llm.NewOpenAIService(cfg.AI.LLM.APIKey, cfg.AI.LLM.Model, cfg.AI.LLM.Endpoint, cfg.AI.LLM.Timeout, logger.Log) + ttsService := tts.NewOpenAIService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Speed, cfg.AI.TTS.Timeout, logger.Log) + + // 初始化 Orchestrator + orch := orchestrator.New(sttService, llmService, ttsService, sessionMgr, cfg.AI.LLM.Model) + + // Gin 模式 + if cfg.App.Env == "prod" { + gin.SetMode(gin.ReleaseMode) + } + + r := gin.New() + r.Use(gin.Recovery()) // REST API - api := r.Group("/api") + apiGroup := r.Group("/api") { - api.GET("/health", healthHandler) + apiGroup.GET("/health", healthHandler(sessionMgr)) } + // Session REST 端点 + sessionHandler := api.NewSessionHandler(sessionMgr) + sessionHandler.RegisterRoutes(apiGroup) + // WebSocket - r.GET("/ws", ws.ServeWS) + r.GET("/ws", ws.ServeWS(sessionMgr, orch)) - log.Println("CamTalk gateway starting on :8080") - if err := r.Run(":8080"); err != nil { - log.Fatalf("failed to start server: %v", err) + // HTTP Server + srv := &http.Server{ + Addr: cfg.Server.Addr(), + Handler: r, + ReadTimeout: time.Duration(cfg.Server.ReadTimeout) * time.Second, + WriteTimeout: time.Duration(cfg.Server.WriteTimeout) * time.Second, } + + // Graceful shutdown + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + go func() { + logger.Log.Infow("server starting", "addr", srv.Addr) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Log.Fatalw("listen failed", "error", err) + } + }() + + <-ctx.Done() + logger.Log.Info("shutting down...") + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := srv.Shutdown(shutdownCtx); err != nil { + logger.Log.Errorw("shutdown error", "error", err) + } + logger.Log.Info("server stopped") } // healthHandler 健康检查。 -func healthHandler(c *gin.Context) { - c.JSON(200, gin.H{ - "status": "ok", - "version": "0.1.0", - "uptime": time.Since(startTime).String(), - "active_sessions": 0, // TODO: 接入 Session Manager - }) +func healthHandler(sessionMgr session.Manager) gin.HandlerFunc { + return func(c *gin.Context) { + c.JSON(200, gin.H{ + "status": "ok", + "version": "0.1.0", + "uptime_seconds": int(time.Since(startTime).Seconds()), + "active_sessions": sessionMgr.ActiveCount(), + }) + } } diff --git a/backend/config.yaml b/backend/config.yaml new file mode 100644 index 0000000..988f336 --- /dev/null +++ b/backend/config.yaml @@ -0,0 +1,39 @@ +# config.yaml — 默认配置 +app: + env: dev + +server: + host: "0.0.0.0" + port: 8080 + read_timeout: 30 + write_timeout: 30 + +redis: + addr: "localhost:6379" + password: "" + db: 0 + +ai: + stt: + provider: Xiaomi MiMo + model: mimo-v2.5 + endpoint: "https://token-plan-cn.xiaomimimo.com/v1" + llm: + provider: dashscope + model: qwen3-vl-plus + endpoint: "https://dashscope.aliyuncs.com/compatible-mode/v1" + timeout: 30 + tts: + provider: Xiaomi MiMo + model: mimo-v2.5 + voice: alloy + speed: 1.0 + endpoint: "https://token-plan-cn.xiaomimimo.com/v1" + timeout: 5 + +storage: + driver: memory + +log: + level: info + format: console diff --git a/backend/go.mod b/backend/go.mod index 6204b55..31577ab 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,38 +1,58 @@ module github.com/hhs/camtalk -go 1.23 +go 1.24 require ( github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/redis/go-redis/v9 v9.20.1 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 + go.uber.org/zap v1.28.0 ) require ( github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.28.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index c5a8acc..17b84e6 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,7 +1,13 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -9,6 +15,10 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -23,21 +33,29 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -47,26 +65,56 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.20.1 h1:sfCU6A8P3dXbKyWes02uxA2baehGux9dZHfEKtsTB1w= +github.com/redis/go-redis/v9 v9.20.1/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= @@ -74,18 +122,16 @@ golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/backend/internal/ai/llm/llm.go b/backend/internal/ai/llm/llm.go new file mode 100644 index 0000000..13f3990 --- /dev/null +++ b/backend/internal/ai/llm/llm.go @@ -0,0 +1,37 @@ +package llm + +import ( + "context" + + "github.com/hhs/camtalk/internal/models" +) + +// Service 多模态大模型服务契约。 +type Service interface { + // ChatStream 流式推理，返回增量文本的 channel。 + // 调用方必须消费 channel 直到 Done=true，否则需 cancel ctx 以释放连接。 + ChatStream(ctx context.Context, req Request) (<-chan Chunk, error) +} + +// Request 推理请求。 +type Request struct { + Image []byte // JPEG 图片（已从 Base64 解码） + Text string // 用户语音识别后的文本 + History []models.Message // 最近 N 轮对话历史 + Language string // 语言，如 "zh-CN" +} + +// Chunk 流式推理的一个增量片段。 +type Chunk struct { + Delta string // 增量文本 + Done bool // 是否结束 + TokensUsed *TokenUsage // 仅 Done=true 时有值 + Model string // 实际使用的模型名 +} + +// TokenUsage 用量统计。 +type TokenUsage struct { + Prompt int + Completion int + Total int +} diff --git a/backend/internal/ai/llm/openai.go b/backend/internal/ai/llm/openai.go new file mode 100644 index 0000000..bbc9444 --- /dev/null +++ b/backend/internal/ai/llm/openai.go @@ -0,0 +1,237 @@ +package llm + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "go.uber.org/zap" +) + +// OpenAIService 基于 OpenAI Chat Completions API 的 LLM 实现。 +type OpenAIService struct { + apiKey string + model string + endpoint string + timeout time.Duration + logger *zap.SugaredLogger + client *http.Client +} + +// NewOpenAIService 创建 OpenAI LLM 服务。 +func NewOpenAIService(apiKey, model, endpoint string, timeoutSec int, logger *zap.SugaredLogger) *OpenAIService { + if model == "" { + model = "gpt-4o" + } + if endpoint == "" { + endpoint = "https://api.openai.com/v1" + } + timeout := time.Duration(timeoutSec) * time.Second + if timeout <= 0 { + timeout = 10 * time.Second + } + return &OpenAIService{ + apiKey: apiKey, + model: model, + endpoint: endpoint, + timeout: timeout, + logger: logger, + client: &http.Client{Timeout: 60 * time.Second}, // HTTP client timeout > LLM timeout + } +} + +// --- OpenAI API 请求/响应结构 --- + +type chatRequest struct { + Model string `json:"model"` + Messages []chatMessage `json:"messages"` + Stream bool `json:"stream"` +} + +type chatMessage struct { + Role string `json:"role"` + Content []contentPart `json:"content"` +} + +type contentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *imageURL `json:"image_url,omitempty"` +} + +type imageURL struct { + URL string `json:"url"` +} + +// streamDelta SSE 流式响应的单个 delta。 +type streamDelta struct { + Choices []struct { + Delta struct { + Content string `json:"content"` + } `json:"delta"` + FinishReason *string `json:"finish_reason"` + } `json:"choices"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + Model string `json:"model"` +} + +// ChatStream 实现 llm.Service。调用 OpenAI Chat Completions API 流式推理。 +func (o *OpenAIService) ChatStream(ctx context.Context, req Request) (<-chan Chunk, error) { + // 构建请求 + messages := o.buildMessages(req) + + body := chatRequest{ + Model: o.model, + Messages: messages, + Stream: true, + } + + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("llm: marshal request: %w", err) + } + + // 创建带超时的 context + ctx, cancel := context.WithTimeout(ctx, o.timeout) + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, o.endpoint+"/chat/completions", bytes.NewReader(payload)) + if err != nil { + cancel() + return nil, fmt.Errorf("llm: create request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+o.apiKey) + + resp, err := o.client.Do(httpReq) + if err != nil { + cancel() + return nil, fmt.Errorf("llm: send request: %w", err) + } + + if resp.StatusCode != http.StatusOK { + cancel() + bodyBytes, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("llm: api error (status %d): %s", resp.StatusCode, string(bodyBytes)) + } + + // 启动 goroutine 解析 SSE 流 + ch := make(chan Chunk, 64) + go func() { + defer close(ch) + defer cancel() + defer resp.Body.Close() + + o.parseSSEStream(resp.Body, ch) + }() + + return ch, nil +} + +// parseSSEStream 解析 SSE 流，将 delta 发送到 channel。 +func (o *OpenAIService) parseSSEStream(body io.Reader, ch chan<- Chunk) { + scanner := bufio.NewScanner(body) + scanner.Buffer(make([]byte, 0, 64*1024), 256*1024) + + var fullText strings.Builder + var lastModel string + + for scanner.Scan() { + line := scanner.Text() + + // SSE 格式：data: {...} + if !strings.HasPrefix(line, "data: ") { + continue + } + data := strings.TrimPrefix(line, "data: ") + if data == "[DONE]" { + // 流结束，发送最终 chunk + ch <- Chunk{Delta: "", Done: true, Model: lastModel} + return + } + + var delta streamDelta + if err := json.Unmarshal([]byte(data), &delta); err != nil { + o.logger.Warnw("llm: unmarshal delta failed", "error", err, "data", data) + continue + } + + if delta.Model != "" { + lastModel = delta.Model + } + + // 提取增量文本 + if len(delta.Choices) > 0 { + content := delta.Choices[0].Delta.Content + if content != "" { + fullText.WriteString(content) + ch <- Chunk{Delta: content, Done: false, Model: lastModel} + } + + // 某些模型在最后一个 choice 中携带 usage + if delta.Choices[0].FinishReason != nil && delta.Usage != nil { + ch <- Chunk{ + Delta: "", + Done: true, + Model: lastModel, + TokensUsed: &TokenUsage{ + Prompt: delta.Usage.PromptTokens, + Completion: delta.Usage.CompletionTokens, + Total: delta.Usage.TotalTokens, + }, + } + return + } + } + } + + // scanner 结束但没收到 [DONE] + if err := scanner.Err(); err != nil { + o.logger.Warnw("llm: scan error", "error", err) + } + ch <- Chunk{Delta: "", Done: true, Model: lastModel} +} + +// buildMessages 构建 OpenAI Chat API 的 messages 数组。 +func (o *OpenAIService) buildMessages(req Request) []chatMessage { + var messages []chatMessage + + // System prompt + messages = append(messages, chatMessage{ + Role: "system", + Content: []contentPart{{Type: "text", Text: BuildSystemPrompt(req.Language, "")}}, + }) + + // 历史消息 + for _, msg := range req.History { + messages = append(messages, chatMessage{ + Role: msg.Role, + Content: []contentPart{{Type: "text", Text: msg.Content}}, + }) + } + + // 当前用户消息（图像 + 文本） + var parts []contentPart + if len(req.Image) > 0 { + b64 := base64.StdEncoding.EncodeToString(req.Image) + parts = append(parts, contentPart{ + Type: "image_url", + ImageURL: &imageURL{URL: "data:image/jpeg;base64," + b64}, + }) + } + parts = append(parts, contentPart{Type: "text", Text: req.Text}) + messages = append(messages, chatMessage{Role: "user", Content: parts}) + + return messages +} diff --git a/backend/internal/ai/llm/openai_test.go b/backend/internal/ai/llm/openai_test.go new file mode 100644 index 0000000..9f2eee2 --- /dev/null +++ b/backend/internal/ai/llm/openai_test.go @@ -0,0 +1,251 @@ +package llm + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "go.uber.org/zap" + + "github.com/hhs/camtalk/internal/models" +) + +// mockLLMServer 创建模拟 OpenAI SSE 流式响应的 HTTP 服务器。 +func mockLLMServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + return httptest.NewServer(handler) +} + +func TestOpenAIService_ChatStream_Success(t *testing.T) { + srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { + // 验证请求 + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + if !strings.Contains(r.URL.Path, "/chat/completions") { + t.Errorf("path = %s, should contain /chat/completions", r.URL.Path) + } + auth := r.Header.Get("Authorization") + if auth != "Bearer test-key" { + t.Errorf("Authorization = %q, want %q", auth, "Bearer test-key") + } + + w.Header().Set("Content-Type", "text/event-stream") + flusher, ok := w.(http.Flusher) + if !ok { + t.Fatal("ResponseWriter does not support Flusher") + } + + // 发送几个 delta + deltas := []string{"你好", "世界", "！"} + for _, d := range deltas { + fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"%s\"}}],\"model\":\"gpt-4o\"}\n\n", d) + flusher.Flush() + } + + // 发送 [DONE] + fmt.Fprintf(w, "data: [DONE]\n\n") + flusher.Flush() + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + + ch, err := svc.ChatStream(context.Background(), Request{ + Text: "这是什么？", + Language: "zh-CN", + }) + if err != nil { + t.Fatalf("ChatStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 应该有 3 个文本 chunk + 1 个 Done chunk + if len(chunks) != 4 { + t.Fatalf("got %d chunks, want 4", len(chunks)) + } + + // 验证文本内容 + if chunks[0].Delta != "你好" { + t.Errorf("chunk[0].Delta = %q, want %q", chunks[0].Delta, "你好") + } + if chunks[1].Delta != "世界" { + t.Errorf("chunk[1].Delta = %q, want %q", chunks[1].Delta, "世界") + } + + // 验证最后一个 chunk 是 Done + last := chunks[len(chunks)-1] + if !last.Done { + t.Error("last chunk should be Done") + } + if last.Model != "gpt-4o" { + t.Errorf("last chunk Model = %q, want %q", last.Model, "gpt-4o") + } +} + +func TestOpenAIService_ChatStream_WithImage(t *testing.T) { + srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}],\"model\":\"gpt-4o\"}\n\n") + fmt.Fprintf(w, "data: [DONE]\n\n") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + + ch, err := svc.ChatStream(context.Background(), Request{ + Image: []byte("fake-jpeg-data"), + Text: "描述图片", + Language: "zh-CN", + }) + if err != nil { + t.Fatalf("ChatStream() error: %v", err) + } + + // 消费 channel + for range ch { + } +} + +func TestOpenAIService_ChatStream_WithHistory(t *testing.T) { + srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}],\"model\":\"gpt-4o\"}\n\n") + fmt.Fprintf(w, "data: [DONE]\n\n") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + + ch, err := svc.ChatStream(context.Background(), Request{ + Text: "继续", + Language: "zh-CN", + History: []models.Message{ + {Role: "user", Content: "你好"}, + {Role: "assistant", Content: "你好！有什么可以帮助你的吗？"}, + }, + }) + if err != nil { + t.Fatalf("ChatStream() error: %v", err) + } + + for range ch { + } +} + +func TestOpenAIService_ChatStream_APIError(t *testing.T) { + srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprintf(w, `{"error":{"message":"Invalid API key"}}`) + }) + defer srv.Close() + + svc := NewOpenAIService("bad-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + + _, err := svc.ChatStream(context.Background(), Request{ + Text: "test", + }) + if err == nil { + t.Fatal("ChatStream() should return error for 401") + } + if !strings.Contains(err.Error(), "401") { + t.Errorf("error should mention 401, got: %v", err) + } +} + +func TestOpenAIService_ChatStream_Timeout(t *testing.T) { + srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { + // 模拟慢响应 + time.Sleep(5 * time.Second) + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"late\"}}]}\n\n") + fmt.Fprintf(w, "data: [DONE]\n\n") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 1, zap.NewNop().Sugar()) // 1s timeout + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + ch, err := svc.ChatStream(ctx, Request{Text: "test"}) + if err != nil { + // 超时可能在建立连接时或读取时发生 + return + } + + // 如果连接成功，消费 channel 应该超时 + var gotContent bool + for c := range ch { + if c.Delta != "" { + gotContent = true + } + } + if gotContent { + t.Error("should not receive content before timeout") + } +} + +func TestOpenAIService_ChatStream_UsageInResponse(t *testing.T) { + srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + // 带 usage 的最后一个 chunk + fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"},\"finish_reason\":\"stop\"}],\"model\":\"gpt-4o\",\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":5,\"total_tokens\":15}}\n\n") + fmt.Fprintf(w, "data: [DONE]\n\n") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar()) + + ch, err := svc.ChatStream(context.Background(), Request{Text: "test"}) + if err != nil { + t.Fatalf("ChatStream() error: %v", err) + } + + var last Chunk + for c := range ch { + last = c + } + + if !last.Done { + t.Error("last chunk should be Done") + } + if last.TokensUsed == nil { + t.Fatal("last chunk should have TokensUsed") + } + if last.TokensUsed.Total != 15 { + t.Errorf("TokensUsed.Total = %d, want 15", last.TokensUsed.Total) + } +} + +func TestBuildSystemPrompt(t *testing.T) { + tests := []struct { + name string + language string + detailLevel string + wantContain string + }{ + {"chinese default", "zh-CN", "", "视觉助手"}, + {"chinese high", "zh-CN", "high", "更详细"}, + {"english default", "en", "", "visual assistant"}, + {"english high", "en", "high", "detailed"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BuildSystemPrompt(tt.language, tt.detailLevel) + if !strings.Contains(got, tt.wantContain) { + t.Errorf("BuildSystemPrompt(%q, %q) should contain %q", tt.language, tt.detailLevel, tt.wantContain) + } + }) + } +} diff --git a/backend/internal/ai/llm/prompt.go b/backend/internal/ai/llm/prompt.go new file mode 100644 index 0000000..417288a --- /dev/null +++ b/backend/internal/ai/llm/prompt.go @@ -0,0 +1,25 @@ +package llm + +import "strings" + +// BuildSystemPrompt 根据语言和细节级别构建系统提示词。 +func BuildSystemPrompt(language, detailLevel string) string { + isChinese := strings.HasPrefix(language, "zh") + + var prompt strings.Builder + if isChinese { + prompt.WriteString("你是一个视觉助手。用户通过摄像头看到一个场景，并用语音向你提问。请用简洁自然的中文回答。如果涉及视觉描述，先说\"我看到……\"。回答控制在3-5句话以内，除非用户要求详细说明。") + } else { + prompt.WriteString("You are a visual assistant. The user sees a scene through their camera and asks questions by voice. Answer concisely and naturally. If describing visual content, start with 'I see...'. Keep answers to 3-5 sentences unless the user asks for detail.") + } + + if detailLevel == "high" { + if isChinese { + prompt.WriteString("请提供更详细的视觉描述，包括颜色、位置、数量等细节。") + } else { + prompt.WriteString(" Provide detailed visual descriptions including colors, positions, quantities, and other details.") + } + } + + return prompt.String() +} diff --git a/backend/internal/ai/stt/deepgram.go b/backend/internal/ai/stt/deepgram.go new file mode 100644 index 0000000..54da674 --- /dev/null +++ b/backend/internal/ai/stt/deepgram.go @@ -0,0 +1,141 @@ +package stt + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/gorilla/websocket" + "go.uber.org/zap" +) + +// DeepgramService 基于 Deepgram WebSocket API 的语音识别实现。 +type DeepgramService struct { + apiKey string + model string + endpoint string + logger *zap.SugaredLogger +} + +// NewDeepgramService 创建 Deepgram STT 服务。 +func NewDeepgramService(apiKey, model, endpoint string, logger *zap.SugaredLogger) *DeepgramService { + if model == "" { + model = "nova-2" + } + if endpoint == "" { + endpoint = "wss://api.deepgram.com/v1/listen" + } + return &DeepgramService{ + apiKey: apiKey, + model: model, + endpoint: endpoint, + logger: logger, + } +} + +// deepgramResponse Deepgram WebSocket 响应。 +type deepgramResponse struct { + Channel struct { + Alternatives []struct { + Transcript string `json:"transcript"` + Confidence float64 `json:"confidence"` + } `json:"alternatives"` + } `json:"channel"` + IsFinal bool `json:"is_final"` +} + +// Recognize 实现 stt.Service。通过 WebSocket 发送音频到 Deepgram，返回最终识别文本。 +func (d *DeepgramService) Recognize(ctx context.Context, audio []byte, opts Options) (string, error) { + if len(audio) == 0 { + return "", fmt.Errorf("stt: empty audio") + } + + // 构建 WebSocket URL，附带查询参数 + wsURL := d.buildURL(opts) + + // 5 秒总超时 + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + // 建立 WebSocket 连接 + conn, _, err := websocket.DefaultDialer.DialContext(ctx, wsURL, http.Header{ + "Authorization": []string{"Token " + d.apiKey}, + }) + if err != nil { + return "", fmt.Errorf("stt: connect deepgram: %w", err) + } + defer conn.Close() + + // 发送音频数据（一次性） + if err := conn.WriteMessage(websocket.BinaryMessage, audio); err != nil { + return "", fmt.Errorf("stt: send audio: %w", err) + } + + // 发送 Close 消息通知服务端音频已发送完毕 + closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") + _ = conn.WriteMessage(websocket.CloseMessage, closeMsg) + + // 读取识别结果 + var transcript strings.Builder + for { + _, message, err := conn.ReadMessage() + if err != nil { + // Close 帧是正常的结束信号 + if websocket.IsCloseError(err, websocket.CloseNormalClosure) { + break + } + if websocket.IsUnexpectedCloseError(err, websocket.CloseNormalClosure) { + break + } + return "", fmt.Errorf("stt: read response: %w", err) + } + + var resp deepgramResponse + if err := json.Unmarshal(message, &resp); err != nil { + d.logger.Warnw("stt: unmarshal response failed", "error", err) + continue + } + + // 只累积 final 结果，跳过中间结果 + if resp.IsFinal && len(resp.Channel.Alternatives) > 0 { + text := strings.TrimSpace(resp.Channel.Alternatives[0].Transcript) + if text != "" { + transcript.WriteString(text) + } + } + } + + return strings.TrimSpace(transcript.String()), nil +} + +// buildURL 构建 Deepgram WebSocket URL，包含音频格式参数。 +func (d *DeepgramService) buildURL(opts Options) string { + u, _ := url.Parse(d.endpoint) + + encoding := opts.Encoding + if encoding == "" { + encoding = "pcm_s16le" + } + sampleRate := opts.SampleRate + if sampleRate == 0 { + sampleRate = 16000 + } + language := opts.Language + if language == "" { + language = "zh-CN" + } + + q := u.Query() + q.Set("encoding", encoding) + q.Set("sample_rate", fmt.Sprintf("%d", sampleRate)) + q.Set("language", language) + q.Set("model", d.model) + q.Set("punctuate", "true") + u.RawQuery = q.Encode() + + return u.String() +} diff --git a/backend/internal/ai/stt/deepgram_test.go b/backend/internal/ai/stt/deepgram_test.go new file mode 100644 index 0000000..2d00e53 --- /dev/null +++ b/backend/internal/ai/stt/deepgram_test.go @@ -0,0 +1,191 @@ +package stt + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gorilla/websocket" + "go.uber.org/zap" +) + +var upgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, +} + +// newMockDeepgram 创建模拟 Deepgram WebSocket 服务。 +// 返回 httptest.Server 和对应的 ws:// URL。 +func newMockDeepgram(t *testing.T, handler func(conn *websocket.Conn)) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Logf("upgrade error: %v", err) + return + } + defer conn.Close() + handler(conn) + })) + return srv +} + +// wsToWss 将 http:// 转换为 ws://。 +func wsToWss(httpURL string) string { + return "ws" + strings.TrimPrefix(httpURL, "http") +} + +func TestDeepgramService_Recognize_Success(t *testing.T) { + srv := newMockDeepgram(t, func(conn *websocket.Conn) { + // 读取音频数据 + _, _, err := conn.ReadMessage() + if err != nil { + t.Errorf("read audio: %v", err) + return + } + + // 发送中间结果（非 final） + intermediate := deepgramResponse{ + IsFinal: false, + } + intermediate.Channel.Alternatives = []struct { + Transcript string `json:"transcript"` + Confidence float64 `json:"confidence"` + }{{Transcript: "你好", Confidence: 0.9}} + data, _ := json.Marshal(intermediate) + _ = conn.WriteMessage(websocket.TextMessage, data) + + // 发送最终结果 + final := deepgramResponse{ + IsFinal: true, + } + final.Channel.Alternatives = []struct { + Transcript string `json:"transcript"` + Confidence float64 `json:"confidence"` + }{{Transcript: "你好世界", Confidence: 0.95}} + data, _ = json.Marshal(final) + _ = conn.WriteMessage(websocket.TextMessage, data) + + // 等待客户端关闭 + _, _, _ = conn.ReadMessage() + }) + defer srv.Close() + + svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar()) + + text, err := svc.Recognize(context.Background(), []byte("fake-pcm-audio"), Options{ + Encoding: "pcm_s16le", + SampleRate: 16000, + Language: "zh-CN", + }) + if err != nil { + t.Fatalf("Recognize() error: %v", err) + } + if text != "你好世界" { + t.Errorf("Recognize() = %q, want %q", text, "你好世界") + } +} + +func TestDeepgramService_Recognize_EmptyAudio(t *testing.T) { + svc := NewDeepgramService("test-key", "", "ws://localhost", zap.NewNop().Sugar()) + _, err := svc.Recognize(context.Background(), nil, Options{}) + if err == nil { + t.Fatal("Recognize() with empty audio should return error") + } +} + +func TestDeepgramService_Recognize_ConnectError(t *testing.T) { + svc := NewDeepgramService("test-key", "", "ws://localhost:1", zap.NewNop().Sugar()) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + _, err := svc.Recognize(ctx, []byte("audio"), Options{}) + if err == nil { + t.Fatal("Recognize() with bad endpoint should return error") + } +} + +func TestDeepgramService_Recognize_Timeout(t *testing.T) { + // 模拟一个永不响应的服务端 + srv := newMockDeepgram(t, func(conn *websocket.Conn) { + // 读取音频但不发送任何结果，让客户端超时 + _, _, _ = conn.ReadMessage() + time.Sleep(10 * time.Second) + }) + defer srv.Close() + + svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar()) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + _, err := svc.Recognize(ctx, []byte("audio"), Options{}) + if err == nil { + t.Fatal("Recognize() should timeout") + } +} + +func TestDeepgramService_Recognize_MultipleFinals(t *testing.T) { + srv := newMockDeepgram(t, func(conn *websocket.Conn) { + _, _, _ = conn.ReadMessage() + + // 发送多个 final 结果（多句话场景） + for _, text := range []string{"你好", "世界"} { + resp := deepgramResponse{IsFinal: true} + resp.Channel.Alternatives = []struct { + Transcript string `json:"transcript"` + Confidence float64 `json:"confidence"` + }{{Transcript: text, Confidence: 0.9}} + data, _ := json.Marshal(resp) + _ = conn.WriteMessage(websocket.TextMessage, data) + } + + _, _, _ = conn.ReadMessage() + }) + defer srv.Close() + + svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar()) + + text, err := svc.Recognize(context.Background(), []byte("audio"), Options{}) + if err != nil { + t.Fatalf("Recognize() error: %v", err) + } + if text != "你好世界" { + t.Errorf("Recognize() = %q, want %q", text, "你好世界") + } +} + +func TestDeepgramService_buildURL(t *testing.T) { + svc := NewDeepgramService("key", "", "wss://api.deepgram.com/v1/listen", zap.NewNop().Sugar()) + + tests := []struct { + name string + opts Options + want []string // URL 中应包含的参数 + }{ + { + name: "defaults", + opts: Options{}, + want: []string{"encoding=pcm_s16le", "sample_rate=16000", "language=zh-CN"}, + }, + { + name: "custom", + opts: Options{Encoding: "wav", SampleRate: 44100, Language: "en"}, + want: []string{"encoding=wav", "sample_rate=44100", "language=en"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := svc.buildURL(tt.opts) + for _, param := range tt.want { + if !strings.Contains(u, param) { + t.Errorf("buildURL() = %q, should contain %q", u, param) + } + } + }) + } +} diff --git a/backend/internal/ai/stt/stt.go b/backend/internal/ai/stt/stt.go new file mode 100644 index 0000000..33fe57b --- /dev/null +++ b/backend/internal/ai/stt/stt.go @@ -0,0 +1,16 @@ +package stt + +import "context" + +// Service 语音识别服务契约。 +type Service interface { + // Recognize 识别一段完整音频，返回最终文本。 + Recognize(ctx context.Context, audio []byte, opts Options) (string, error) +} + +// Options 语音识别参数。 +type Options struct { + Encoding string // 音频编码，如 "pcm_s16le" + SampleRate int // 采样率，如 16000 + Language string // 语言，如 "zh-CN" +} diff --git a/backend/internal/ai/tts/openai.go b/backend/internal/ai/tts/openai.go new file mode 100644 index 0000000..6e0123e --- /dev/null +++ b/backend/internal/ai/tts/openai.go @@ -0,0 +1,154 @@ +package tts + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "go.uber.org/zap" +) + +// OpenAIService 基于 OpenAI TTS API 的语音合成实现。 +type OpenAIService struct { + apiKey string + model string + voice string + speed float64 + endpoint string + timeout time.Duration + logger *zap.SugaredLogger + client *http.Client +} + +// NewOpenAIService 创建 OpenAI TTS 服务。 +func NewOpenAIService(apiKey, model, voice, endpoint string, speed float64, timeoutSec int, logger *zap.SugaredLogger) *OpenAIService { + if model == "" { + model = "tts-1" + } + if voice == "" { + voice = "alloy" + } + if speed <= 0 { + speed = 1.0 + } + if endpoint == "" { + endpoint = "https://api.openai.com/v1" + } + timeout := time.Duration(timeoutSec) * time.Second + if timeout <= 0 { + timeout = 5 * time.Second + } + return &OpenAIService{ + apiKey: apiKey, + model: model, + voice: voice, + speed: speed, + endpoint: endpoint, + timeout: timeout, + logger: logger, + client: &http.Client{Timeout: 30 * time.Second}, + } +} + +// ttsRequest OpenAI TTS API 请求。 +type ttsRequest struct { + Model string `json:"model"` + Input string `json:"input"` + Voice string `json:"voice"` + ResponseFormat string `json:"response_format"` + Speed float64 `json:"speed"` +} + +// SynthesizeStream 实现 tts.Service。从 textStream 读取句子，逐句调用 OpenAI TTS API。 +func (o *OpenAIService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error) { + voice := opts.Voice + if voice == "" { + voice = o.voice + } + speed := opts.Speed + if speed <= 0 { + speed = o.speed + } + + ch := make(chan Chunk, 4) + go func() { + defer close(ch) + + for text := range textStream { + if text == "" { + continue + } + + audio, err := o.synthesize(ctx, text, voice, speed) + if err != nil { + o.logger.Warnw("tts: synthesize failed", "error", err, "text", text) + // 静默跳过，不中断整个流 + continue + } + + select { + case ch <- Chunk{Audio: audio, IsLast: false}: + case <-ctx.Done(): + return + } + } + + // textStream 关闭，发送 IsLast 标记 + select { + case ch <- Chunk{Audio: nil, IsLast: true}: + case <-ctx.Done(): + } + }() + + return ch, nil +} + +// synthesize 调用 OpenAI TTS API 合成单个句子。 +func (o *OpenAIService) synthesize(ctx context.Context, text, voice string, speed float64) ([]byte, error) { + // 单句超时 + ctx, cancel := context.WithTimeout(ctx, o.timeout) + defer cancel() + + body := ttsRequest{ + Model: o.model, + Input: text, + Voice: voice, + ResponseFormat: "mp3", + Speed: speed, + } + + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("tts: marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, o.endpoint+"/audio/speech", bytes.NewReader(payload)) + if err != nil { + return nil, fmt.Errorf("tts: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+o.apiKey) + + resp, err := o.client.Do(req) + if err != nil { + return nil, fmt.Errorf("tts: send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("tts: api error (status %d): %s", resp.StatusCode, string(errBody)) + } + + // 读取整个 MP3 响应 + audio, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("tts: read response: %w", err) + } + + return audio, nil +} diff --git a/backend/internal/ai/tts/openai_test.go b/backend/internal/ai/tts/openai_test.go new file mode 100644 index 0000000..36afd31 --- /dev/null +++ b/backend/internal/ai/tts/openai_test.go @@ -0,0 +1,300 @@ +package tts + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "go.uber.org/zap" +) + +// mockTTSServer 创建模拟 OpenAI TTS API 的 HTTP 服务器。 +func mockTTSServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + return httptest.NewServer(handler) +} + +// sendSentences 向 channel 发送句子并关闭。 +func sendSentences(sentences ...string) <-chan string { + ch := make(chan string, len(sentences)) + for _, s := range sentences { + ch <- s + } + close(ch) + return ch +} + +func TestOpenAIService_SynthesizeStream_Success(t *testing.T) { + var callCount int32 + srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&callCount, 1) + + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + if !strings.Contains(r.URL.Path, "/audio/speech") { + t.Errorf("path = %s, should contain /audio/speech", r.URL.Path) + } + auth := r.Header.Get("Authorization") + if auth != "Bearer test-key" { + t.Errorf("Authorization = %q, want %q", auth, "Bearer test-key") + } + + // 验证请求体 + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), "tts-1") { + t.Errorf("request body should contain model tts-1") + } + + // 返回假 MP3 数据 + w.Header().Set("Content-Type", "audio/mpeg") + fmt.Fprintf(w, "fake-mp3-data") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("你好", "世界", "！") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{ + Voice: "alloy", Speed: 1.0, OutputFmt: "mp3", SampleRate: 24000, + }) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 应该有 3 个音频 chunk + 1 个 IsLast 标记 + if len(chunks) != 4 { + t.Fatalf("got %d chunks, want 4", len(chunks)) + } + + // 验证前 3 个有音频数据 + for i := 0; i < 3; i++ { + if string(chunks[i].Audio) != "fake-mp3-data" { + t.Errorf("chunk[%d].Audio = %q, want %q", i, string(chunks[i].Audio), "fake-mp3-data") + } + if chunks[i].IsLast { + t.Errorf("chunk[%d].IsLast should be false", i) + } + } + + // 验证最后一个是 IsLast + if !chunks[3].IsLast { + t.Error("last chunk should be IsLast") + } + if chunks[3].Audio != nil { + t.Error("last chunk Audio should be nil") + } + + // 验证调用了 3 次 API（3 个句子） + if atomic.LoadInt32(&callCount) != 3 { + t.Errorf("API called %d times, want 3", callCount) + } +} + +func TestOpenAIService_SynthesizeStream_APIError(t *testing.T) { + srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "internal error") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + // 应该只有一个 IsLast chunk（音频被跳过） + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1 (IsLast only)", len(chunks)) + } + if !chunks[0].IsLast { + t.Error("chunk should be IsLast") + } +} + +func TestOpenAIService_SynthesizeStream_Timeout(t *testing.T) { + srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(3 * time.Second) + w.Header().Set("Content-Type", "audio/mpeg") + fmt.Fprintf(w, "late-mp3") + }) + defer srv.Close() + + // 1 秒超时 + svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 1, zap.NewNop().Sugar()) + + textStream := sendSentences("很长的句子") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ch, err := svc.SynthesizeStream(ctx, textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 超时后音频被跳过，只有 IsLast + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1", len(chunks)) + } + if !chunks[0].IsLast { + t.Error("chunk should be IsLast") + } +} + +func TestOpenAIService_SynthesizeStream_EmptyText(t *testing.T) { + var callCount int32 + srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&callCount, 1) + w.Header().Set("Content-Type", "audio/mpeg") + fmt.Fprintf(w, "mp3") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + + // 空句子应该被跳过 + textStream := sendSentences("", "你好", "") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 只有 "你好" 应该被合成 + if atomic.LoadInt32(&callCount) != 1 { + t.Errorf("API called %d times, want 1", callCount) + } + + // 1 个音频 + 1 个 IsLast + if len(chunks) != 2 { + t.Fatalf("got %d chunks, want 2", len(chunks)) + } +} + +func TestOpenAIService_SynthesizeStream_ContextCancelled(t *testing.T) { + srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "audio/mpeg") + fmt.Fprintf(w, "mp3") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + + // 发送多个句子，但在第一个后取消 + textStream := make(chan string, 3) + textStream <- "第一句" + textStream <- "第二句" + textStream <- "第三句" + close(textStream) + + ctx, cancel := context.WithCancel(context.Background()) + // 立即取消 + cancel() + + ch, err := svc.SynthesizeStream(ctx, textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + // 消费 channel，应该很快结束 + var count int + for range ch { + count++ + } + // 可能收到 0 个或 1 个 chunk，取决于时序 + t.Logf("received %d chunks after context cancel", count) +} + +func TestOpenAIService_SynthesizeStream_PartialFailure(t *testing.T) { + var callCount int32 + srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&callCount, 1) + if n == 2 { + // 第二个句子失败 + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "error") + return + } + w.Header().Set("Content-Type", "audio/mpeg") + fmt.Fprintf(w, "mp3-%d", n) + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("第一句", "第二句", "第三句") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + var chunks []Chunk + for c := range ch { + chunks = append(chunks, c) + } + + // 2 个成功音频 + 1 个 IsLast（第二句被跳过） + if len(chunks) != 3 { + t.Fatalf("got %d chunks, want 3", len(chunks)) + } + if !chunks[len(chunks)-1].IsLast { + t.Error("last chunk should be IsLast") + } +} + +func TestOpenAIService_SynthesizeStream_CustomVoice(t *testing.T) { + srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), "nova") { + t.Errorf("request body should contain voice 'nova', got: %s", string(body)) + } + w.Header().Set("Content-Type", "audio/mpeg") + fmt.Fprintf(w, "mp3") + }) + defer srv.Close() + + svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar()) + + textStream := sendSentences("你好") + + ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{Voice: "nova"}) + if err != nil { + t.Fatalf("SynthesizeStream() error: %v", err) + } + + for range ch { + } +} diff --git a/backend/internal/ai/tts/tts.go b/backend/internal/ai/tts/tts.go new file mode 100644 index 0000000..9b08d0e --- /dev/null +++ b/backend/internal/ai/tts/tts.go @@ -0,0 +1,25 @@ +package tts + +import "context" + +// Service 语音合成服务契约。 +type Service interface { + // SynthesizeStream 流式合成。 + // textStream 接收句子级文本（由 Orchestrator 的句子切分器产出）， + // 返回的 channel 持续输出 MP3 音频 chunk。 + SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error) +} + +// Options 合成参数。 +type Options struct { + Voice string // "alloy" | "nova" | "shimmer" 等 + Speed float64 // 1.0 为正常语速 + OutputFmt string // "mp3" — 固定使用 MP3 + SampleRate int // 24000 +} + +// Chunk 一个音频片段。 +type Chunk struct { + Audio []byte // MP3 音频数据（未 Base64 编码） + IsLast bool // 是否为最后一片 +} diff --git a/backend/internal/api/session.go b/backend/internal/api/session.go new file mode 100644 index 0000000..135758b --- /dev/null +++ b/backend/internal/api/session.go @@ -0,0 +1,91 @@ +// Package api 提供 REST API 处理函数。 +package api + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/session" +) + +// SessionHandler 提供会话相关的 REST 端点。 +type SessionHandler struct { + sessionMgr session.Manager +} + +// NewSessionHandler 创建 SessionHandler。 +func NewSessionHandler(sessionMgr session.Manager) *SessionHandler { + return &SessionHandler{sessionMgr: sessionMgr} +} + +// CreateSessionRequest POST /api/sessions 请求体（所有字段可选）。 +type CreateSessionRequest struct { + Config *models.SessionConfig `json:"config,omitempty"` +} + +// CreateSession POST /api/sessions — 创建新会话。 +func (h *SessionHandler) CreateSession(c *gin.Context) { + var req CreateSessionRequest + // 请求体可选，解析失败不报错（使用默认配置） + _ = c.ShouldBindJSON(&req) + + cfg := models.DefaultConfig() + if req.Config != nil { + cfg = *req.Config + } + + sessionID, err := h.sessionMgr.Create(c.Request.Context(), cfg) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "code": "INTERNAL_ERROR", + "message": "failed to create session", + }) + return + } + + // 获取创建后的会话以返回 created_at + sess, err := h.sessionMgr.Get(c.Request.Context(), sessionID) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "code": "INTERNAL_ERROR", + "message": "failed to retrieve created session", + }) + return + } + + c.JSON(http.StatusCreated, gin.H{ + "session_id": sess.ID, + "created_at": sess.CreatedAt, + }) +} + +// DestroySession DELETE /api/sessions/:id — 销毁会话。 +func (h *SessionHandler) DestroySession(c *gin.Context) { + sessionID := c.Param("id") + + err := h.sessionMgr.Destroy(c.Request.Context(), sessionID) + if err != nil { + if err == session.ErrSessionNotFound { + c.JSON(http.StatusNotFound, gin.H{ + "code": "SESSION_NOT_FOUND", + "message": "session not found or already expired", + }) + return + } + c.JSON(http.StatusInternalServerError, gin.H{ + "code": "INTERNAL_ERROR", + "message": "failed to destroy session", + }) + return + } + + c.Status(http.StatusNoContent) +} + +// RegisterRoutes 注册会话相关路由到给定的路由组。 +func (h *SessionHandler) RegisterRoutes(rg *gin.RouterGroup) { + rg.POST("/sessions", h.CreateSession) + rg.DELETE("/sessions/:id", h.DestroySession) +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go new file mode 100644 index 0000000..1a5cac8 --- /dev/null +++ b/backend/internal/config/config.go @@ -0,0 +1,160 @@ +package config + +import ( + "fmt" + "os" + "strings" + + "github.com/joho/godotenv" + "github.com/spf13/viper" +) + +// Config 应用配置。 +type Config struct { + App AppConfig `mapstructure:"app"` + Server ServerConfig `mapstructure:"server"` + Redis RedisConfig `mapstructure:"redis"` + AI AIConfig `mapstructure:"ai"` + Storage StorageConfig `mapstructure:"storage"` + Log LogConfig `mapstructure:"log"` +} + +type AppConfig struct { + Env string `mapstructure:"env"` + Version string `mapstructure:"version"` +} + +type ServerConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + ReadTimeout int `mapstructure:"read_timeout"` + WriteTimeout int `mapstructure:"write_timeout"` +} + +// Addr 返回 host:port 地址。 +func (s ServerConfig) Addr() string { + return fmt.Sprintf("%s:%d", s.Host, s.Port) +} + +type RedisConfig struct { + Addr string `mapstructure:"addr"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` +} + +type AIConfig struct { + STT STTConfig `mapstructure:"stt"` + LLM LLMConfig `mapstructure:"llm"` + TTS TTSConfig `mapstructure:"tts"` +} + +type STTConfig struct { + Provider string `mapstructure:"provider"` + APIKey string `mapstructure:"api_key"` + Model string `mapstructure:"model"` + Endpoint string `mapstructure:"endpoint"` +} + +type LLMConfig struct { + Provider string `mapstructure:"provider"` + APIKey string `mapstructure:"api_key"` + Model string `mapstructure:"model"` + Endpoint string `mapstructure:"endpoint"` + Timeout int `mapstructure:"timeout"` +} + +type TTSConfig struct { + Provider string `mapstructure:"provider"` + APIKey string `mapstructure:"api_key"` + Model string `mapstructure:"model"` + Voice string `mapstructure:"voice"` + Speed float64 `mapstructure:"speed"` + Endpoint string `mapstructure:"endpoint"` + Timeout int `mapstructure:"timeout"` +} + +type StorageConfig struct { + Driver string `mapstructure:"driver"` + DSN string `mapstructure:"dsn"` +} + +type LogConfig struct { + Level string `mapstructure:"level"` + Format string `mapstructure:"format"` +} + +// Load 加载配置。优先级：环境变量 > config.{env}.yaml > config.yaml。 +func Load() (*Config, error) { + v := viper.New() + v.SetConfigName("config") + v.SetConfigType("yaml") + v.AddConfigPath(".") + v.AddConfigPath("./config") + v.AddConfigPath("./backend") + + // 默认值 + v.SetDefault("app.env", "dev") + v.SetDefault("server.host", "0.0.0.0") + v.SetDefault("server.port", 8080) + v.SetDefault("server.read_timeout", 30) + v.SetDefault("server.write_timeout", 30) + v.SetDefault("redis.addr", "localhost:6379") + v.SetDefault("redis.db", 0) + v.SetDefault("ai.stt.provider", "deepgram") + v.SetDefault("ai.stt.model", "nova-2") + v.SetDefault("ai.stt.endpoint", "wss://api.deepgram.com/v1/listen") + v.SetDefault("ai.llm.provider", "openai") + v.SetDefault("ai.llm.model", "gpt-4o") + v.SetDefault("ai.llm.endpoint", "https://api.openai.com/v1") + v.SetDefault("ai.llm.timeout", 10) + v.SetDefault("ai.tts.provider", "openai") + v.SetDefault("ai.tts.model", "tts-1") + v.SetDefault("ai.tts.voice", "alloy") + v.SetDefault("ai.tts.speed", 1.0) + v.SetDefault("ai.tts.endpoint", "https://api.openai.com/v1") + v.SetDefault("ai.tts.timeout", 5) + v.SetDefault("storage.driver", "memory") + v.SetDefault("log.level", "info") + v.SetDefault("log.format", "console") + + // 读取基础配置文件 + _ = v.ReadInConfig() // 文件不存在不报错 + + // 根据 APP_ENV 覆盖 + env := os.Getenv("APP_ENV") + if env == "" { + env = v.GetString("app.env") + } + if env != "" { + v.SetConfigName("config." + env) + _ = v.MergeInConfig() + } + + // 加载 .env 文件（不覆盖已有环境变量） + // 按优先级尝试：当前目录、上级目录（兼容从 backend/ 或项目根目录启动） + _ = godotenv.Load() + _ = godotenv.Load("../.env") + + // 环境变量覆盖 + v.SetEnvPrefix("CAMTALK") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.AutomaticEnv() + + var cfg Config + if err := v.Unmarshal(&cfg); err != nil { + return nil, fmt.Errorf("config unmarshal: %w", err) + } + + // 填充默认值 + if cfg.Server.Host == "" { + cfg.Server.Host = "0.0.0.0" + } + if cfg.Server.Port == 0 { + cfg.Server.Port = 8080 + } + if cfg.App.Env == "" { + cfg.App.Env = "dev" + } + + return &cfg, nil +} diff --git a/backend/internal/errors/codes.go b/backend/internal/errors/codes.go new file mode 100644 index 0000000..739cfdb --- /dev/null +++ b/backend/internal/errors/codes.go @@ -0,0 +1,33 @@ +package errors + +import "github.com/hhs/camtalk/internal/models" + +// 错误码常量，与 docs/03-接口文档.md 保持一致。 +const ( + CodeInvalidMessage = "INVALID_MESSAGE" + CodeSessionNotFound = "SESSION_NOT_FOUND" + CodeRateLimited = "RATE_LIMITED" + CodeImageTooLarge = "IMAGE_TOO_LARGE" + CodeAudioTooShort = "AUDIO_TOO_SHORT" + CodeLLMTimeout = "LLM_TIMEOUT" + CodeLLMError = "LLM_ERROR" + CodeSTTError = "STT_ERROR" + CodeTTSError = "TTS_ERROR" + CodeInternalError = "INTERNAL_ERROR" +) + +// Sender 定义发送 WS 错误消息的接口，便于测试 mock。 +type Sender interface { + SendError(code, requestID, message string) +} + +// SendWSError 向客户端发送 error 消息。 +// sender 是一个具有 sendJSON 方法的对象，这里用接口抽象。 +func SendWSError(sender interface{ SendJSON(v any) error }, code, requestID string, err error) { + _ = sender.SendJSON(models.WsError{ + Type: "error", + Code: code, + RequestID: requestID, + Message: err.Error(), + }) +} diff --git a/backend/internal/logger/logger.go b/backend/internal/logger/logger.go new file mode 100644 index 0000000..4c113f3 --- /dev/null +++ b/backend/internal/logger/logger.go @@ -0,0 +1,57 @@ +package logger + +import ( + "os" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// Log 是全局 SugaredLogger，由 Init 初始化。 +var Log *zap.SugaredLogger + +// Init 初始化全局日志器。 +// level: "debug", "info", "warn", "error" +// format: "json" 或 "console" +func Init(level, format string) { + var lvl zapcore.Level + switch level { + case "debug": + lvl = zapcore.DebugLevel + case "warn": + lvl = zapcore.WarnLevel + case "error": + lvl = zapcore.ErrorLevel + default: + lvl = zapcore.InfoLevel + } + + encoderCfg := zap.NewProductionEncoderConfig() + encoderCfg.TimeKey = "ts" + encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder + + var core zapcore.Core + if format == "console" { + core = zapcore.NewCore( + zapcore.NewConsoleEncoder(encoderCfg), + zapcore.AddSync(os.Stdout), + lvl, + ) + } else { + core = zapcore.NewCore( + zapcore.NewJSONEncoder(encoderCfg), + zapcore.AddSync(os.Stdout), + lvl, + ) + } + + logger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel)) + Log = logger.Sugar() +} + +// Sync 刷新缓冲区，退出前调用。 +func Sync() { + if Log != nil { + _ = Log.Sync() + } +} diff --git a/backend/internal/models/models.go b/backend/internal/models/models.go index 2508195..73d017a 100644 --- a/backend/internal/models/models.go +++ b/backend/internal/models/models.go @@ -21,6 +21,26 @@ func DefaultConfig() SessionConfig { return SessionConfig{TTSEnabled: true, DetailLevel: "low", Language: "zh-CN"} } +// SessionConfigPatch 会话配置增量更新（指针字段表示"未传则不更新"）。 +type SessionConfigPatch struct { + TTSEnabled *bool `json:"tts_enabled,omitempty"` + DetailLevel *string `json:"detail_level,omitempty"` + Language *string `json:"language,omitempty"` +} + +// Apply 将 patch 中的非 nil 字段覆盖到 cfg。 +func (p SessionConfigPatch) Apply(cfg *SessionConfig) { + if p.TTSEnabled != nil { + cfg.TTSEnabled = *p.TTSEnabled + } + if p.DetailLevel != nil { + cfg.DetailLevel = *p.DetailLevel + } + if p.Language != nil { + cfg.Language = *p.Language + } +} + // Message 对话消息。 type Message struct { Role string `json:"role"` // "user" | "assistant" diff --git a/backend/internal/orchestrator/orchestrator.go b/backend/internal/orchestrator/orchestrator.go new file mode 100644 index 0000000..d038603 --- /dev/null +++ b/backend/internal/orchestrator/orchestrator.go @@ -0,0 +1,33 @@ +// Package orchestrator 实现 STT → LLM → TTS 流式并行管道。 +package orchestrator + +import ( + "context" + + "github.com/hhs/camtalk/internal/models" +) + +// Orchestrator AI 编排器接口。 +// 接收查询并执行完整的 STT → LLM → TTS 管道。 +type Orchestrator interface { + // ProcessQuery 处理一次用户查询。 + // ctx 用于整体超时和中断控制。 + // sessionID 用于会话管理和历史获取。 + // req 包含图像和音频数据。 + // history 是最近的对话历史。 + // sender 用于向客户端推送消息。 + ProcessQuery( + ctx context.Context, + sessionID string, + req models.WsQuery, + history []models.Message, + sender Sender, + ) error +} + +// QueryRequest 查询请求（内部使用）。 +type QueryRequest struct { + Image []byte // JPEG 图片（已从 Base64 解码） + Audio []byte // 音频数据（已从 Base64 解码） + Language string // 语言，如 "zh-CN" +} diff --git a/backend/internal/orchestrator/pipeline.go b/backend/internal/orchestrator/pipeline.go new file mode 100644 index 0000000..0186108 --- /dev/null +++ b/backend/internal/orchestrator/pipeline.go @@ -0,0 +1,328 @@ +package orchestrator + +import ( + "context" + "encoding/base64" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/hhs/camtalk/internal/ai/llm" + "github.com/hhs/camtalk/internal/ai/stt" + "github.com/hhs/camtalk/internal/ai/tts" + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/session" +) + +// Pipeline 实现 Orchestrator 接口，管理 STT → LLM → TTS 流式管道。 +type Pipeline struct { + sttService stt.Service + llmService llm.Service + ttsService tts.Service + sessionMgr session.Manager + model string // LLM 模型名，用于 llm_done 上报 +} + +// New 创建 Pipeline 实例。 +func New( + sttService stt.Service, + llmService llm.Service, + ttsService tts.Service, + sessionMgr session.Manager, + model string, +) *Pipeline { + return &Pipeline{ + sttService: sttService, + llmService: llmService, + ttsService: ttsService, + sessionMgr: sessionMgr, + model: model, + } +} + +// ProcessQuery 实现 Orchestrator 接口。 +func (p *Pipeline) ProcessQuery( + ctx context.Context, + sessionID string, + req models.WsQuery, + history []models.Message, + sender Sender, +) error { + log := logger.Log + startTime := time.Now() + + // 解码音频数据 + audio, err := base64.StdEncoding.DecodeString(req.Audio) + if err != nil { + log.Errorw("音频解码失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "INVALID_MESSAGE", + Message: "音频数据解码失败", + }) + return err + } + + // 解码图片数据（可选） + var image []byte + if req.Image != "" { + image, err = base64.StdEncoding.DecodeString(req.Image) + if err != nil { + log.Errorw("图片解码失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "INVALID_MESSAGE", + Message: "图片数据解码失败", + }) + return err + } + } + + // 设置活跃请求 + if err := p.sessionMgr.SetActiveRequest(ctx, sessionID, req.RequestID); err != nil { + log.Errorw("设置活跃请求失败", "error", err) + } + defer p.sessionMgr.ClearActiveRequest(ctx, sessionID) + + // 获取会话配置 + sess, err := p.sessionMgr.Get(ctx, sessionID) + if err != nil { + log.Errorw("获取会话失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "SESSION_NOT_FOUND", + Message: "会话不存在", + }) + return err + } + + // Step 1: STT 语音识别 + log.Infow("开始语音识别", "request_id", req.RequestID) + sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{ + Encoding: "pcm_s16le", + SampleRate: 16000, + Language: sess.Config.Language, + }) + if err != nil { + log.Errorw("语音识别失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "STT_ERROR", + Message: "语音识别失败", + }) + return err + } + + // 发送 STT 结果 + if err := sender.SendSTTResult(models.WsSTTResult{ + Type: "stt_result", + RequestID: req.RequestID, + Text: sttResult, + IsFinal: true, + }); err != nil { + log.Errorw("发送 STT 结果失败", "error", err) + } + + // 追加用户消息到历史 + p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{ + Role: "user", + Content: sttResult, + }) + + // Step 2+3: LLM 流式推理 + TTS 并行合成 + log.Infow("开始 LLM 推理", "request_id", req.RequestID) + llmReq := llm.Request{ + Image: image, + Text: sttResult, + History: history, + Language: sess.Config.Language, + } + + llmStream, err := p.llmService.ChatStream(ctx, llmReq) + if err != nil { + log.Errorw("LLM 流式推理启动失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "LLM_ERROR", + Message: "LLM 推理失败", + }) + return err + } + + // 创建句子切分器 + sentenceCh := make(chan string, 4) + splitter := NewSplitter(sentenceCh) + + // 并行：LLM 消费 + TTS 合成 + var wg sync.WaitGroup + var fullText string + var ttsErr error + + // goroutine 1: 消费 LLM token + 句子切分 + wg.Add(1) + go func() { + defer wg.Done() + defer close(sentenceCh) + fullText = p.consumeLLMStream(ctx, llmStream, req.RequestID, sender, splitter) + }() + + // goroutine 2: TTS 合成（如果启用） + if sess.Config.TTSEnabled { + wg.Add(1) + go func() { + defer wg.Done() + log.Infow("开始 TTS 合成", "request_id", req.RequestID) + ttsErr = p.synthesizeTTS(ctx, sentenceCh, req.RequestID, sender) + }() + } else { + // 如果 TTS 未启用，需要消费 sentenceCh 防止阻塞 + go func() { + for range sentenceCh { + } + }() + } + + // 等待所有 goroutine 完成 + wg.Wait() + + // TTS 失败静默跳过 + if ttsErr != nil { + log.Warnw("TTS 合成失败（已跳过）", "error", ttsErr) + } + + // 追加助手消息到历史 + p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{ + Role: "assistant", + Content: fullText, + }) + + // 发送 llm_done + latency := time.Since(startTime).Milliseconds() + if err := sender.SendLLMDone(models.WsLLMDone{ + Type: "llm_done", + RequestID: req.RequestID, + FullText: fullText, + Model: p.model, + LatencyMs: latency, + }); err != nil { + log.Errorw("发送 llm_done 失败", "error", err) + } + + log.Infow("查询处理完成", + "request_id", req.RequestID, + "latency_ms", latency, + "text_length", utf8.RuneCountInString(fullText), + ) + + return nil +} + +// consumeLLMStream 消费 LLM 流式输出，发送 llm_chunk 并进行句子切分。 +func (p *Pipeline) consumeLLMStream( + ctx context.Context, + stream <-chan llm.Chunk, + requestID string, + sender Sender, + splitter *Splitter, +) string { + log := logger.Log + var fullText strings.Builder + + for chunk := range stream { + // 检查上下文是否已取消 + select { + case <-ctx.Done(): + log.Infow("LLM 流被中断", "request_id", requestID) + return fullText.String() + default: + } + + if chunk.Done { + // 流结束 + if chunk.TokensUsed != nil { + log.Infow("LLM 用量统计", + "request_id", requestID, + "prompt_tokens", chunk.TokensUsed.Prompt, + "completion_tokens", chunk.TokensUsed.Completion, + "total_tokens", chunk.TokensUsed.Total, + ) + } + break + } + + // 累积全文 + fullText.WriteString(chunk.Delta) + + // 发送 llm_chunk + if err := sender.SendLLMChunk(models.WsLLMChunk{ + Type: "llm_chunk", + RequestID: requestID, + Delta: chunk.Delta, + Role: "assistant", + }); err != nil { + log.Errorw("发送 llm_chunk 失败", "error", err) + } + + // 句子切分 + splitter.Feed(chunk.Delta) + } + + // 刷新切分器中的剩余文本 + splitter.Flush() + + return fullText.String() +} + +// synthesizeTTS 从句子 channel 读取文本，进行 TTS 合成并发送音频。 +func (p *Pipeline) synthesizeTTS( + ctx context.Context, + sentenceCh <-chan string, + requestID string, + sender Sender, +) error { + log := logger.Log + + ttsStream, err := p.ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{ + Voice: "alloy", + Speed: 1.0, + OutputFmt: "mp3", + SampleRate: 24000, + }) + if err != nil { + log.Errorw("TTS 合成启动失败", "error", err) + return err + } + + // 消费 TTS 音频流 + for chunk := range ttsStream { + // 检查上下文是否已取消 + select { + case <-ctx.Done(): + log.Infow("TTS 流被中断", "request_id", requestID) + return ctx.Err() + default: + } + + // Base64 编码音频数据 + audioBase64 := base64.StdEncoding.EncodeToString(chunk.Audio) + + if err := sender.SendTTSAudio(models.WsTTSAudio{ + Type: "tts_audio", + RequestID: requestID, + Audio: audioBase64, + MimeType: "audio/mp3", + IsLast: chunk.IsLast, + }); err != nil { + log.Errorw("发送 tts_audio 失败", "error", err) + } + } + + return nil +} diff --git a/backend/internal/orchestrator/pipeline_test.go b/backend/internal/orchestrator/pipeline_test.go new file mode 100644 index 0000000..4a3f2a1 --- /dev/null +++ b/backend/internal/orchestrator/pipeline_test.go @@ -0,0 +1,661 @@ +package orchestrator + +import ( + "context" + "encoding/base64" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "github.com/hhs/camtalk/internal/ai/llm" + "github.com/hhs/camtalk/internal/ai/stt" + "github.com/hhs/camtalk/internal/ai/tts" + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" +) + +func init() { + logger.Init("debug", "console") +} + +// MockSTTService mock STT 服务 +type MockSTTService struct { + mock.Mock +} + +func (m *MockSTTService) Recognize(ctx context.Context, audio []byte, opts stt.Options) (string, error) { + args := m.Called(ctx, audio, opts) + return args.String(0), args.Error(1) +} + +// MockLLMService mock LLM 服务 +type MockLLMService struct { + mock.Mock +} + +func (m *MockLLMService) ChatStream(ctx context.Context, req llm.Request) (<-chan llm.Chunk, error) { + args := m.Called(ctx, req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(<-chan llm.Chunk), args.Error(1) +} + +// MockTTSService mock TTS 服务 +type MockTTSService struct { + mock.Mock +} + +func (m *MockTTSService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts tts.Options) (<-chan tts.Chunk, error) { + args := m.Called(ctx, textStream, opts) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(<-chan tts.Chunk), args.Error(1) +} + +// MockSessionManager mock 会话管理器 +type MockSessionManager struct { + mock.Mock +} + +func (m *MockSessionManager) Create(ctx context.Context, config models.SessionConfig) (string, error) { + args := m.Called(ctx, config) + return args.String(0), args.Error(1) +} + +func (m *MockSessionManager) Get(ctx context.Context, sessionID string) (*models.Session, error) { + args := m.Called(ctx, sessionID) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*models.Session), args.Error(1) +} + +func (m *MockSessionManager) UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error { + args := m.Called(ctx, sessionID, patch) + return args.Error(0) +} + +func (m *MockSessionManager) GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error) { + args := m.Called(ctx, sessionID, limit) + return args.Get(0).([]models.Message), args.Error(1) +} + +func (m *MockSessionManager) AppendMessage(ctx context.Context, sessionID string, msg models.Message) error { + args := m.Called(ctx, sessionID, msg) + return args.Error(0) +} + +func (m *MockSessionManager) SetActiveRequest(ctx context.Context, sessionID string, requestID string) error { + args := m.Called(ctx, sessionID, requestID) + return args.Error(0) +} + +func (m *MockSessionManager) GetActiveRequestID(ctx context.Context, sessionID string) (string, error) { + args := m.Called(ctx, sessionID) + return args.String(0), args.Error(1) +} + +func (m *MockSessionManager) ClearActiveRequest(ctx context.Context, sessionID string) error { + args := m.Called(ctx, sessionID) + return args.Error(0) +} + +func (m *MockSessionManager) Touch(ctx context.Context, sessionID string) error { + args := m.Called(ctx, sessionID) + return args.Error(0) +} + +func (m *MockSessionManager) Destroy(ctx context.Context, sessionID string) error { + args := m.Called(ctx, sessionID) + return args.Error(0) +} + +func (m *MockSessionManager) ActiveCount() int { + args := m.Called() + return args.Int(0) +} + +// MockSender mock WebSocket 发送器 +type MockSender struct { + mock.Mock + STTResults []models.WsSTTResult + LLMChunks []models.WsLLMChunk + LLMDones []models.WsLLMDone + TTSAudios []models.WsTTSAudio + Errors []models.WsError +} + +func NewMockSender() *MockSender { + return &MockSender{ + STTResults: make([]models.WsSTTResult, 0), + LLMChunks: make([]models.WsLLMChunk, 0), + LLMDones: make([]models.WsLLMDone, 0), + TTSAudios: make([]models.WsTTSAudio, 0), + Errors: make([]models.WsError, 0), + } +} + +func (m *MockSender) SendSTTResult(result models.WsSTTResult) error { + m.STTResults = append(m.STTResults, result) + args := m.Called(result) + return args.Error(0) +} + +func (m *MockSender) SendLLMChunk(chunk models.WsLLMChunk) error { + m.LLMChunks = append(m.LLMChunks, chunk) + args := m.Called(chunk) + return args.Error(0) +} + +func (m *MockSender) SendLLMDone(done models.WsLLMDone) error { + m.LLMDones = append(m.LLMDones, done) + args := m.Called(done) + return args.Error(0) +} + +func (m *MockSender) SendTTSAudio(audio models.WsTTSAudio) error { + m.TTSAudios = append(m.TTSAudios, audio) + args := m.Called(audio) + return args.Error(0) +} + +func (m *MockSender) SendError(err models.WsError) error { + m.Errors = append(m.Errors, err) + args := m.Called(err) + return args.Error(0) +} + +// 辅助函数：创建 LLM 流式响应 +func createLLMStream(chunks []llm.Chunk) <-chan llm.Chunk { + ch := make(chan llm.Chunk, len(chunks)) + for _, chunk := range chunks { + ch <- chunk + } + close(ch) + return ch +} + +// 辅助函数：创建 TTS 流式响应 +func createTTSStream(chunks []tts.Chunk) <-chan tts.Chunk { + ch := make(chan tts.Chunk, len(chunks)) + for _, chunk := range chunks { + ch <- chunk + } + close(ch) + return ch +} + +// TestProcessQuery_Success 测试完整流程 +func TestProcessQuery_Success(t *testing.T) { + // 准备测试数据 + audioData := []byte("test audio") + imageData := []byte("test image") + audioBase64 := base64.StdEncoding.EncodeToString(audioData) + imageBase64 := base64.StdEncoding.EncodeToString(imageData) + + req := models.WsQuery{ + Type: "query", + RequestID: "req-123", + Image: imageBase64, + Audio: audioBase64, + } + + session := &models.Session{ + ID: "session-123", + Config: models.SessionConfig{ + TTSEnabled: true, + DetailLevel: "low", + Language: "zh-CN", + }, + } + + // 创建 mock + mockSTT := new(MockSTTService) + mockLLM := new(MockLLMService) + mockTTS := new(MockTTSService) + mockSession := new(MockSessionManager) + mockSender := NewMockSender() + + // 设置 mock 期望 + mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) + mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) + mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) + mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) + + mockSTT.On("Recognize", mock.Anything, audioData, stt.Options{ + Encoding: "pcm_s16le", + SampleRate: 16000, + Language: "zh-CN", + }).Return("你好，世界", nil) + + mockSender.On("SendSTTResult", mock.Anything).Return(nil) + + llmChunks := []llm.Chunk{ + {Delta: "你好"}, + {Delta: "，世界！"}, + {Done: true, TokensUsed: &llm.TokenUsage{Prompt: 10, Completion: 5, Total: 15}}, + } + mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil) + + mockSender.On("SendLLMChunk", mock.Anything).Return(nil) + mockSender.On("SendLLMDone", mock.Anything).Return(nil) + + ttsChunks := []tts.Chunk{ + {Audio: []byte("audio1"), IsLast: false}, + {Audio: []byte("audio2"), IsLast: true}, + } + mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return(createTTSStream(ttsChunks), nil) + + mockSender.On("SendTTSAudio", mock.Anything).Return(nil) + + // 创建 Pipeline + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + + // 执行 + ctx := context.Background() + err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) + + // 验证 + assert.NoError(t, err) + assert.Len(t, mockSender.STTResults, 1) + assert.Equal(t, "你好，世界", mockSender.STTResults[0].Text) + assert.Len(t, mockSender.LLMChunks, 2) + assert.Len(t, mockSender.LLMDones, 1) + assert.Len(t, mockSender.TTSAudios, 2) + + mockSTT.AssertExpectations(t) + mockLLM.AssertExpectations(t) + mockTTS.AssertExpectations(t) + mockSession.AssertExpectations(t) +} + +// TestProcessQuery_STTError 测试 STT 失败降级 +func TestProcessQuery_STTError(t *testing.T) { + audioData := []byte("test audio") + audioBase64 := base64.StdEncoding.EncodeToString(audioData) + + req := models.WsQuery{ + Type: "query", + RequestID: "req-123", + Audio: audioBase64, + } + + mockSTT := new(MockSTTService) + mockLLM := new(MockLLMService) + mockTTS := new(MockTTSService) + mockSession := new(MockSessionManager) + mockSender := NewMockSender() + + mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) + mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) + mockSession.On("Get", mock.Anything, "session-123").Return(&models.Session{ + ID: "session-123", + Config: models.SessionConfig{ + Language: "zh-CN", + }, + }, nil) + + mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything). + Return("", errors.New("STT service unavailable")) + + mockSender.On("SendError", mock.Anything).Return(nil) + + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + + ctx := context.Background() + err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) + + assert.Error(t, err) + assert.Len(t, mockSender.Errors, 1) + assert.Equal(t, "STT_ERROR", mockSender.Errors[0].Code) + + mockSTT.AssertExpectations(t) + mockLLM.AssertNotCalled(t, "ChatStream") + mockTTS.AssertNotCalled(t, "SynthesizeStream") +} + +// TestProcessQuery_LLMError 测试 LLM 失败降级 +func TestProcessQuery_LLMError(t *testing.T) { + audioData := []byte("test audio") + audioBase64 := base64.StdEncoding.EncodeToString(audioData) + + req := models.WsQuery{ + Type: "query", + RequestID: "req-123", + Audio: audioBase64, + } + + session := &models.Session{ + ID: "session-123", + Config: models.SessionConfig{ + TTSEnabled: true, + Language: "zh-CN", + }, + } + + mockSTT := new(MockSTTService) + mockLLM := new(MockLLMService) + mockTTS := new(MockTTSService) + mockSession := new(MockSessionManager) + mockSender := NewMockSender() + + mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) + mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) + mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) + mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) + + mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil) + mockSender.On("SendSTTResult", mock.Anything).Return(nil) + + mockLLM.On("ChatStream", mock.Anything, mock.Anything). + Return(nil, errors.New("LLM service unavailable")) + + mockSender.On("SendError", mock.Anything).Return(nil) + + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + + ctx := context.Background() + err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) + + assert.Error(t, err) + assert.Len(t, mockSender.Errors, 1) + assert.Equal(t, "LLM_ERROR", mockSender.Errors[0].Code) + + mockSTT.AssertExpectations(t) + mockLLM.AssertExpectations(t) + mockTTS.AssertNotCalled(t, "SynthesizeStream") +} + +// TestProcessQuery_TTSError 测试 TTS 失败静默跳过 +func TestProcessQuery_TTSError(t *testing.T) { + audioData := []byte("test audio") + audioBase64 := base64.StdEncoding.EncodeToString(audioData) + + req := models.WsQuery{ + Type: "query", + RequestID: "req-123", + Audio: audioBase64, + } + + session := &models.Session{ + ID: "session-123", + Config: models.SessionConfig{ + TTSEnabled: true, + Language: "zh-CN", + }, + } + + mockSTT := new(MockSTTService) + mockLLM := new(MockLLMService) + mockTTS := new(MockTTSService) + mockSession := new(MockSessionManager) + mockSender := NewMockSender() + + mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) + mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) + mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) + mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) + + mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil) + mockSender.On("SendSTTResult", mock.Anything).Return(nil) + + llmChunks := []llm.Chunk{ + {Delta: "你好"}, + {Done: true}, + } + mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil) + mockSender.On("SendLLMChunk", mock.Anything).Return(nil) + mockSender.On("SendLLMDone", mock.Anything).Return(nil) + + mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything). + Return(nil, errors.New("TTS service unavailable")) + + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + + ctx := context.Background() + err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) + + // TTS 失败应该静默跳过，不返回错误 + assert.NoError(t, err) + assert.Len(t, mockSender.LLMDones, 1) + assert.Len(t, mockSender.TTSAudios, 0) + + mockSTT.AssertExpectations(t) + mockLLM.AssertExpectations(t) + mockTTS.AssertExpectations(t) +} + +// TestProcessQuery_ContextCancelled 测试上下文取消（Interrupt） +func TestProcessQuery_ContextCancelled(t *testing.T) { + audioData := []byte("test audio") + audioBase64 := base64.StdEncoding.EncodeToString(audioData) + + req := models.WsQuery{ + Type: "query", + RequestID: "req-123", + Audio: audioBase64, + } + + session := &models.Session{ + ID: "session-123", + Config: models.SessionConfig{ + TTSEnabled: true, + Language: "zh-CN", + }, + } + + mockSTT := new(MockSTTService) + mockLLM := new(MockLLMService) + mockTTS := new(MockTTSService) + mockSession := new(MockSessionManager) + mockSender := NewMockSender() + + mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) + mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) + mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) + mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) + + mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil) + mockSender.On("SendSTTResult", mock.Anything).Return(nil) + + // 创建一个会延迟的 LLM 流，以便我们可以取消上下文 + llmCh := make(chan llm.Chunk) + go func() { + time.Sleep(100 * time.Millisecond) + llmCh <- llm.Chunk{Delta: "你"} + time.Sleep(100 * time.Millisecond) + llmCh <- llm.Chunk{Delta: "好"} + close(llmCh) + }() + + mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return((<-chan llm.Chunk)(llmCh), nil) + mockSender.On("SendLLMChunk", mock.Anything).Return(nil) + mockSender.On("SendLLMDone", mock.Anything).Return(nil) + + // 创建一个会延迟的 TTS 流 + ttsCh := make(chan tts.Chunk) + go func() { + time.Sleep(200 * time.Millisecond) + close(ttsCh) + }() + mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return((<-chan tts.Chunk)(ttsCh), nil) + + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + + // 创建可取消的上下文 + ctx, cancel := context.WithCancel(context.Background()) + + // 在 50ms 后取消 + go func() { + time.Sleep(50 * time.Millisecond) + cancel() + }() + + err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) + + // 上下文取消后，流程应该正常完成（中断流但不返回错误） + assert.NoError(t, err) + + mockSTT.AssertExpectations(t) +} + +// TestProcessQuery_DisabledTTS 测试 TTS 未启用的情况 +func TestProcessQuery_DisabledTTS(t *testing.T) { + audioData := []byte("test audio") + audioBase64 := base64.StdEncoding.EncodeToString(audioData) + + req := models.WsQuery{ + Type: "query", + RequestID: "req-123", + Audio: audioBase64, + } + + session := &models.Session{ + ID: "session-123", + Config: models.SessionConfig{ + TTSEnabled: false, // TTS 未启用 + Language: "zh-CN", + }, + } + + mockSTT := new(MockSTTService) + mockLLM := new(MockLLMService) + mockTTS := new(MockTTSService) + mockSession := new(MockSessionManager) + mockSender := NewMockSender() + + mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) + mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) + mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) + mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) + + mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil) + mockSender.On("SendSTTResult", mock.Anything).Return(nil) + + llmChunks := []llm.Chunk{ + {Delta: "你好"}, + {Done: true}, + } + mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil) + mockSender.On("SendLLMChunk", mock.Anything).Return(nil) + mockSender.On("SendLLMDone", mock.Anything).Return(nil) + + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + + ctx := context.Background() + err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) + + assert.NoError(t, err) + assert.Len(t, mockSender.LLMDones, 1) + assert.Len(t, mockSender.TTSAudios, 0) + + // TTS 不应该被调用 + mockTTS.AssertNotCalled(t, "SynthesizeStream") +} + +// TestSplitter 测试句子切分器 +func TestSplitter(t *testing.T) { + ch := make(chan string, 10) + splitter := NewSplitter(ch) + + // 输入包含多个句子的文本 + splitter.Feed("你好。") + splitter.Feed("世界！") + splitter.Feed("这是") + splitter.Feed("一个测试。") + splitter.Flush() + + // 应该有 3 个句子 + assert.Equal(t, 3, len(ch)) + assert.Equal(t, "你好。", <-ch) + assert.Equal(t, "世界！", <-ch) + assert.Equal(t, "这是一个测试。", <-ch) +} + +// TestSplitter_NoDelimiter 测试没有分隔符的情况 +func TestSplitter_NoDelimiter(t *testing.T) { + ch := make(chan string, 10) + splitter := NewSplitter(ch) + + splitter.Feed("没有分隔符的文本") + splitter.Flush() + + // 应该有 1 个句子（Flush 会发送剩余内容） + assert.Equal(t, 1, len(ch)) + assert.Equal(t, "没有分隔符的文本", <-ch) +} + +// TestSplitter_Empty 测试空输入 +func TestSplitter_Empty(t *testing.T) { + ch := make(chan string, 10) + splitter := NewSplitter(ch) + + splitter.Flush() + + // 应该没有句子 + assert.Equal(t, 0, len(ch)) +} + +// TestProcessQuery_InvalidAudio 测试无效音频数据 +func TestProcessQuery_InvalidAudio(t *testing.T) { + req := models.WsQuery{ + Type: "query", + RequestID: "req-123", + Audio: "invalid-base64!!!", + } + + mockSTT := new(MockSTTService) + mockLLM := new(MockLLMService) + mockTTS := new(MockTTSService) + mockSession := new(MockSessionManager) + mockSender := NewMockSender() + + mockSender.On("SendError", mock.Anything).Return(nil) + + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + + ctx := context.Background() + err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) + + assert.Error(t, err) + assert.Len(t, mockSender.Errors, 1) + assert.Equal(t, "INVALID_MESSAGE", mockSender.Errors[0].Code) +} + +// TestProcessQuery_SessionNotFound 测试会话不存在 +func TestProcessQuery_SessionNotFound(t *testing.T) { + audioData := []byte("test audio") + audioBase64 := base64.StdEncoding.EncodeToString(audioData) + + req := models.WsQuery{ + Type: "query", + RequestID: "req-123", + Audio: audioBase64, + } + + mockSTT := new(MockSTTService) + mockLLM := new(MockLLMService) + mockTTS := new(MockTTSService) + mockSession := new(MockSessionManager) + mockSender := NewMockSender() + + mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) + mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) + mockSession.On("Get", mock.Anything, "session-123").Return(nil, errors.New("session not found")) + + mockSender.On("SendError", mock.Anything).Return(nil) + + pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o") + + ctx := context.Background() + err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) + + assert.Error(t, err) + assert.Len(t, mockSender.Errors, 1) + assert.Equal(t, "SESSION_NOT_FOUND", mockSender.Errors[0].Code) +} diff --git a/backend/internal/orchestrator/sender.go b/backend/internal/orchestrator/sender.go new file mode 100644 index 0000000..4a9bbcd --- /dev/null +++ b/backend/internal/orchestrator/sender.go @@ -0,0 +1,22 @@ +package orchestrator + +import "github.com/hhs/camtalk/internal/models" + +// Sender 抽象 WebSocket 消息推送能力。 +// 便于测试时 mock，避免依赖真实 WebSocket 连接。 +type Sender interface { + // SendSTTResult 发送语音识别结果。 + SendSTTResult(result models.WsSTTResult) error + + // SendLLMChunk 发送 LLM 流式文本增量。 + SendLLMChunk(chunk models.WsLLMChunk) error + + // SendLLMDone 发送 LLM 流结束信号。 + SendLLMDone(done models.WsLLMDone) error + + // SendTTSAudio 发送 TTS 音频数据。 + SendTTSAudio(audio models.WsTTSAudio) error + + // SendError 发送错误消息。 + SendError(err models.WsError) error +} diff --git a/backend/internal/orchestrator/splitter.go b/backend/internal/orchestrator/splitter.go new file mode 100644 index 0000000..57b8b26 --- /dev/null +++ b/backend/internal/orchestrator/splitter.go @@ -0,0 +1,55 @@ +package orchestrator + +import "strings" + +// sentenceDelimiters 句子分隔符集合。 +var sentenceDelimiters = map[rune]bool{ + '。': true, + '！': true, + '？': true, + '\n': true, + '.': true, + '!': true, + '?': true, +} + +// Splitter 句子切分器。 +// 将流式文本按句子边界切分，发送到 channel 供 TTS 合成。 +type Splitter struct { + ch chan<- string + buffer strings.Builder +} + +// NewSplitter 创建句子切分器。 +// ch 用于接收切分后的句子文本。 +func NewSplitter(ch chan<- string) *Splitter { + return &Splitter{ + ch: ch, + } +} + +// Feed 输入增量文本，遇到句子分隔符时发送完整句子。 +func (s *Splitter) Feed(delta string) { + for _, r := range delta { + s.buffer.WriteRune(r) + if sentenceDelimiters[r] { + s.flushBuffer() + } + } +} + +// Flush 刷新缓冲区中的剩余文本（即使没有句子分隔符）。 +func (s *Splitter) Flush() { + if s.buffer.Len() > 0 { + s.flushBuffer() + } +} + +// flushBuffer 将缓冲区内容发送到 channel 并清空。 +func (s *Splitter) flushBuffer() { + text := strings.TrimSpace(s.buffer.String()) + if text != "" { + s.ch <- text + } + s.buffer.Reset() +} diff --git a/backend/internal/session/manager.go b/backend/internal/session/manager.go new file mode 100644 index 0000000..6e7f0eb --- /dev/null +++ b/backend/internal/session/manager.go @@ -0,0 +1,49 @@ +// Package session 提供会话生命周期管理能力。 +package session + +import ( + "context" + "errors" + + "github.com/hhs/camtalk/internal/models" +) + +// ErrSessionNotFound 会话不存在或已过期。 +var ErrSessionNotFound = errors.New("session not found") + +// Manager 会话管理器接口。 +// WebSocket Handler 通过此接口操作会话，不直接接触存储层。 +type Manager interface { + // Create 创建新会话，返回 session ID。 + Create(ctx context.Context, config models.SessionConfig) (string, error) + + // Get 获取会话（含 config）。不存在返回 ErrSessionNotFound。 + Get(ctx context.Context, sessionID string) (*models.Session, error) + + // UpdateConfig 更新会话配置（config 消息触发）。 + UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error + + // GetHistory 获取最近 N 轮对话历史（供 Orchestrator 构建 LLM 上下文）。 + GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error) + + // AppendMessage 追加一条对话消息，同时刷新 TTL。 + AppendMessage(ctx context.Context, sessionID string, msg models.Message) error + + // SetActiveRequest 标记当前正在处理的请求 ID（interrupt 用）。 + SetActiveRequest(ctx context.Context, sessionID string, requestID string) error + + // GetActiveRequestID 获取当前活跃请求 ID。 + GetActiveRequestID(ctx context.Context, sessionID string) (string, error) + + // ClearActiveRequest 清除活跃请求标记（请求完成或中断后）。 + ClearActiveRequest(ctx context.Context, sessionID string) error + + // Touch 刷新 TTL（心跳时调用）。 + Touch(ctx context.Context, sessionID string) error + + // Destroy 显式销毁会话（REST API DELETE 或连接断开清理）。 + Destroy(ctx context.Context, sessionID string) error + + // ActiveCount 返回当前活跃会话数（健康检查用）。 + ActiveCount() int +} diff --git a/backend/internal/session/memory.go b/backend/internal/session/memory.go new file mode 100644 index 0000000..edea8cb --- /dev/null +++ b/backend/internal/session/memory.go @@ -0,0 +1,275 @@ +package session + +import ( + "context" + "sync" + "time" + + "github.com/google/uuid" + + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" +) + +const ( + defaultTTL = 30 * time.Minute + defaultHistorySize = 20 +) + +// sessionEntry 内部会话条目。 +type sessionEntry struct { + session models.Session + history []models.Message + activeReqID string + lastActive time.Time +} + +// MemoryManager 基于内存的 SessionManager 实现。 +// 适用于 MVP 和无 Redis 的开发环境。 +type MemoryManager struct { + mu sync.RWMutex + sessions map[string]*sessionEntry + ttl time.Duration + maxHistory int + stopCleaner chan struct{} +} + +// NewMemoryManager 创建内存版 SessionManager。 +// ttl 为会话过期时间，maxHistory 为对话历史上限（0 表示使用默认值 20）。 +func NewMemoryManager(ttl time.Duration, maxHistory int) *MemoryManager { + if ttl <= 0 { + ttl = defaultTTL + } + if maxHistory <= 0 { + maxHistory = defaultHistorySize + } + + m := &MemoryManager{ + sessions: make(map[string]*sessionEntry), + ttl: ttl, + maxHistory: maxHistory, + stopCleaner: make(chan struct{}), + } + + // 启动后台清理 goroutine，每分钟清除过期会话。 + go m.cleanLoop() + + return m +} + +// cleanLoop 后台定期清理过期会话。 +func (m *MemoryManager) cleanLoop() { + ticker := time.NewTicker(1 * time.Minute) + defer ticker.Stop() + for { + select { + case <-ticker.C: + m.cleanExpired() + case <-m.stopCleaner: + return + } + } +} + +// cleanExpired 清除所有过期会话。 +func (m *MemoryManager) cleanExpired() { + m.mu.Lock() + defer m.mu.Unlock() + + now := time.Now() + for id, entry := range m.sessions { + if now.Sub(entry.lastActive) > m.ttl { + delete(m.sessions, id) + logger.Log.Debugw("session expired (cleaner)", "session", id) + } + } +} + +// Stop 停止后台清理 goroutine。应用退出前调用。 +func (m *MemoryManager) Stop() { + close(m.stopCleaner) +} + +// isExpired 检查会话是否过期（调用方需持锁或在已知 entry 存在时调用）。 +func (m *MemoryManager) isExpired(entry *sessionEntry) bool { + return time.Since(entry.lastActive) > m.ttl +} + +// Create 创建新会话。 +func (m *MemoryManager) Create(_ context.Context, config models.SessionConfig) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + + id := uuid.New().String() + now := time.Now() + m.sessions[id] = &sessionEntry{ + session: models.Session{ + ID: id, + CreatedAt: now, + Config: config, + }, + history: make([]models.Message, 0), + lastActive: now, + } + + logger.Log.Debugw("session created", "session", id) + return id, nil +} + +// Get 获取会话。 +func (m *MemoryManager) Get(_ context.Context, sessionID string) (*models.Session, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + entry, ok := m.sessions[sessionID] + if !ok || m.isExpired(entry) { + return nil, ErrSessionNotFound + } + + sess := entry.session // 复制一份返回 + return &sess, nil +} + +// UpdateConfig 更新会话配置。 +func (m *MemoryManager) UpdateConfig(_ context.Context, sessionID string, patch models.SessionConfigPatch) error { + m.mu.Lock() + defer m.mu.Unlock() + + entry, ok := m.sessions[sessionID] + if !ok || m.isExpired(entry) { + return ErrSessionNotFound + } + + patch.Apply(&entry.session.Config) + entry.lastActive = time.Now() + + logger.Log.Debugw("session config updated", "session", sessionID) + return nil +} + +// GetHistory 获取最近 N 轮对话历史。 +func (m *MemoryManager) GetHistory(_ context.Context, sessionID string, limit int) ([]models.Message, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + entry, ok := m.sessions[sessionID] + if !ok || m.isExpired(entry) { + return nil, ErrSessionNotFound + } + + if limit <= 0 || limit > len(entry.history) { + limit = len(entry.history) + } + + // 返回最近 limit 条的副本 + result := make([]models.Message, limit) + copy(result, entry.history[len(entry.history)-limit:]) + return result, nil +} + +// AppendMessage 追加一条对话消息，同时刷新 TTL。 +func (m *MemoryManager) AppendMessage(_ context.Context, sessionID string, msg models.Message) error { + m.mu.Lock() + defer m.mu.Unlock() + + entry, ok := m.sessions[sessionID] + if !ok || m.isExpired(entry) { + return ErrSessionNotFound + } + + entry.history = append(entry.history, msg) + + // 超过上限时裁剪，保留最新的 maxHistory 条 + if len(entry.history) > m.maxHistory { + entry.history = entry.history[len(entry.history)-m.maxHistory:] + } + + entry.lastActive = time.Now() + return nil +} + +// SetActiveRequest 标记当前正在处理的请求 ID。 +func (m *MemoryManager) SetActiveRequest(_ context.Context, sessionID string, requestID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + entry, ok := m.sessions[sessionID] + if !ok || m.isExpired(entry) { + return ErrSessionNotFound + } + + entry.activeReqID = requestID + entry.lastActive = time.Now() + return nil +} + +// GetActiveRequestID 获取当前活跃请求 ID。 +func (m *MemoryManager) GetActiveRequestID(_ context.Context, sessionID string) (string, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + entry, ok := m.sessions[sessionID] + if !ok || m.isExpired(entry) { + return "", ErrSessionNotFound + } + + return entry.activeReqID, nil +} + +// ClearActiveRequest 清除活跃请求标记。 +func (m *MemoryManager) ClearActiveRequest(_ context.Context, sessionID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + entry, ok := m.sessions[sessionID] + if !ok || m.isExpired(entry) { + return ErrSessionNotFound + } + + entry.activeReqID = "" + entry.lastActive = time.Now() + return nil +} + +// Touch 刷新 TTL。 +func (m *MemoryManager) Touch(_ context.Context, sessionID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + entry, ok := m.sessions[sessionID] + if !ok || m.isExpired(entry) { + return ErrSessionNotFound + } + + entry.lastActive = time.Now() + return nil +} + +// Destroy 显式销毁会话。 +func (m *MemoryManager) Destroy(_ context.Context, sessionID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.sessions[sessionID]; !ok { + return ErrSessionNotFound + } + + delete(m.sessions, sessionID) + logger.Log.Debugw("session destroyed", "session", sessionID) + return nil +} + +// ActiveCount 返回当前活跃会话数。 +func (m *MemoryManager) ActiveCount() int { + m.mu.RLock() + defer m.mu.RUnlock() + + now := time.Now() + count := 0 + for _, entry := range m.sessions { + if now.Sub(entry.lastActive) <= m.ttl { + count++ + } + } + return count +} diff --git a/backend/internal/session/memory_test.go b/backend/internal/session/memory_test.go new file mode 100644 index 0000000..acfa433 --- /dev/null +++ b/backend/internal/session/memory_test.go @@ -0,0 +1,286 @@ +package session + +import ( + "context" + "testing" + "time" + + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" +) + +func init() { + logger.Init("debug", "console") +} + +func TestCreateAndGet(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + config := models.DefaultConfig() + id, err := m.Create(ctx, config) + if err != nil { + t.Fatalf("Create: %v", err) + } + if id == "" { + t.Fatal("Create returned empty ID") + } + + sess, err := m.Get(ctx, id) + if err != nil { + t.Fatalf("Get: %v", err) + } + if sess.ID != id { + t.Errorf("ID = %q, want %q", sess.ID, id) + } + if sess.Config.Language != "zh-CN" { + t.Errorf("Language = %q, want %q", sess.Config.Language, "zh-CN") + } +} + +func TestGetNotFound(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + _, err := m.Get(ctx, "nonexistent") + if err != ErrSessionNotFound { + t.Errorf("Get nonexistent: err = %v, want ErrSessionNotFound", err) + } +} + +func TestExpire(t *testing.T) { + // 使用极短 TTL 测试过期 + m := NewMemoryManager(50*time.Millisecond, 20) + defer m.Stop() + ctx := context.Background() + + id, _ := m.Create(ctx, models.DefaultConfig()) + + // 未过期时应能获取 + _, err := m.Get(ctx, id) + if err != nil { + t.Fatalf("Get before expire: %v", err) + } + + // 等待过期 + time.Sleep(80 * time.Millisecond) + + _, err = m.Get(ctx, id) + if err != ErrSessionNotFound { + t.Errorf("Get after expire: err = %v, want ErrSessionNotFound", err) + } +} + +func TestDestroy(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + id, _ := m.Create(ctx, models.DefaultConfig()) + + if err := m.Destroy(ctx, id); err != nil { + t.Fatalf("Destroy: %v", err) + } + + _, err := m.Get(ctx, id) + if err != ErrSessionNotFound { + t.Errorf("Get after Destroy: err = %v, want ErrSessionNotFound", err) + } +} + +func TestDestroyNotFound(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + err := m.Destroy(ctx, "nonexistent") + if err != ErrSessionNotFound { + t.Errorf("Destroy nonexistent: err = %v, want ErrSessionNotFound", err) + } +} + +func TestAppendMessageAndGetHistory(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + id, _ := m.Create(ctx, models.DefaultConfig()) + + msgs := []models.Message{ + {Role: "user", Content: "你好"}, + {Role: "assistant", Content: "你好！有什么可以帮你的吗？"}, + {Role: "user", Content: "这是什么？"}, + {Role: "assistant", Content: "这是一朵花。"}, + } + + for _, msg := range msgs { + if err := m.AppendMessage(ctx, id, msg); err != nil { + t.Fatalf("AppendMessage: %v", err) + } + } + + history, err := m.GetHistory(ctx, id, 0) + if err != nil { + t.Fatalf("GetHistory: %v", err) + } + if len(history) != 4 { + t.Fatalf("GetHistory len = %d, want 4", len(history)) + } + if history[0].Content != "你好" { + t.Errorf("history[0] = %q, want %q", history[0].Content, "你好") + } +} + +func TestGetHistoryLimit(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + id, _ := m.Create(ctx, models.DefaultConfig()) + + for i := 0; i < 10; i++ { + m.AppendMessage(ctx, id, models.Message{Role: "user", Content: "msg"}) + } + + history, err := m.GetHistory(ctx, id, 3) + if err != nil { + t.Fatalf("GetHistory: %v", err) + } + if len(history) != 3 { + t.Fatalf("GetHistory limit=3: len = %d, want 3", len(history)) + } +} + +func TestHistoryLimit(t *testing.T) { + const maxHistory = 5 + m := NewMemoryManager(30*time.Minute, maxHistory) + defer m.Stop() + ctx := context.Background() + + id, _ := m.Create(ctx, models.DefaultConfig()) + + // 插入超过上限的消息 + for i := 0; i < 10; i++ { + m.AppendMessage(ctx, id, models.Message{Role: "user", Content: "msg"}) + } + + history, err := m.GetHistory(ctx, id, 0) + if err != nil { + t.Fatalf("GetHistory: %v", err) + } + if len(history) != maxHistory { + t.Fatalf("GetHistory after overflow: len = %d, want %d", len(history), maxHistory) + } +} + +func TestUpdateConfig(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + id, _ := m.Create(ctx, models.DefaultConfig()) + + ttsEnabled := false + detailLevel := "high" + patch := models.SessionConfigPatch{ + TTSEnabled: &ttsEnabled, + DetailLevel: &detailLevel, + } + + if err := m.UpdateConfig(ctx, id, patch); err != nil { + t.Fatalf("UpdateConfig: %v", err) + } + + sess, _ := m.Get(ctx, id) + if sess.Config.TTSEnabled != false { + t.Errorf("TTSEnabled = %v, want false", sess.Config.TTSEnabled) + } + if sess.Config.DetailLevel != "high" { + t.Errorf("DetailLevel = %q, want %q", sess.Config.DetailLevel, "high") + } + // Language 未传，应保持原值 + if sess.Config.Language != "zh-CN" { + t.Errorf("Language = %q, want %q", sess.Config.Language, "zh-CN") + } +} + +func TestActiveRequest(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + id, _ := m.Create(ctx, models.DefaultConfig()) + + // 初始应为空 + reqID, err := m.GetActiveRequestID(ctx, id) + if err != nil { + t.Fatalf("GetActiveRequestID: %v", err) + } + if reqID != "" { + t.Errorf("initial active request = %q, want empty", reqID) + } + + // 设置 + if err := m.SetActiveRequest(ctx, id, "req-123"); err != nil { + t.Fatalf("SetActiveRequest: %v", err) + } + reqID, _ = m.GetActiveRequestID(ctx, id) + if reqID != "req-123" { + t.Errorf("active request = %q, want %q", reqID, "req-123") + } + + // 清除 + if err := m.ClearActiveRequest(ctx, id); err != nil { + t.Fatalf("ClearActiveRequest: %v", err) + } + reqID, _ = m.GetActiveRequestID(ctx, id) + if reqID != "" { + t.Errorf("active request after clear = %q, want empty", reqID) + } +} + +func TestTouchRefreshesTTL(t *testing.T) { + m := NewMemoryManager(100*time.Millisecond, 20) + defer m.Stop() + ctx := context.Background() + + id, _ := m.Create(ctx, models.DefaultConfig()) + + // 50ms 后 Touch，应重置 TTL + time.Sleep(50 * time.Millisecond) + if err := m.Touch(ctx, id); err != nil { + t.Fatalf("Touch: %v", err) + } + + // 再等 70ms（距创建 120ms，但距 Touch 只有 70ms），不应过期 + time.Sleep(70 * time.Millisecond) + _, err := m.Get(ctx, id) + if err != nil { + t.Errorf("Get after Touch: %v, want nil (should not expire yet)", err) + } + + // 再等 50ms（距 Touch 120ms），应过期 + time.Sleep(50 * time.Millisecond) + _, err = m.Get(ctx, id) + if err != ErrSessionNotFound { + t.Errorf("Get after TTL: err = %v, want ErrSessionNotFound", err) + } +} + +func TestActiveCount(t *testing.T) { + m := NewMemoryManager(30*time.Minute, 20) + defer m.Stop() + ctx := context.Background() + + if m.ActiveCount() != 0 { + t.Errorf("initial ActiveCount = %d, want 0", m.ActiveCount()) + } + + m.Create(ctx, models.DefaultConfig()) + m.Create(ctx, models.DefaultConfig()) + if m.ActiveCount() != 2 { + t.Errorf("ActiveCount = %d, want 2", m.ActiveCount()) + } +} diff --git a/backend/internal/session/redis.go b/backend/internal/session/redis.go new file mode 100644 index 0000000..ae24f11 --- /dev/null +++ b/backend/internal/session/redis.go @@ -0,0 +1,324 @@ +package session + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" +) + +// RedisManager 基于 Redis 的 SessionManager 实现。 +// 数据结构： +// - session:{id}:meta → Hash（会话元数据） +// - session:{id}:history → List（对话历史） +type RedisManager struct { + rdb *redis.Client + ttl time.Duration + maxHistory int +} + +// NewRedisManager 创建 Redis 版 SessionManager。 +func NewRedisManager(rdb *redis.Client, ttl time.Duration, maxHistory int) *RedisManager { + if ttl <= 0 { + ttl = defaultTTL + } + if maxHistory <= 0 { + maxHistory = defaultHistorySize + } + return &RedisManager{rdb: rdb, ttl: ttl, maxHistory: maxHistory} +} + +func metaKey(id string) string { return fmt.Sprintf("session:%s:meta", id) } +func histKey(id string) string { return fmt.Sprintf("session:%s:history", id) } + +// Create 创建新会话。 +func (m *RedisManager) Create(ctx context.Context, config models.SessionConfig) (string, error) { + id := uuidNew() + now := time.Now().UTC() + + pipe := m.rdb.Pipeline() + + // 写入 meta Hash + pipe.HSet(ctx, metaKey(id), map[string]interface{}{ + "session_id": id, + "config.tts_enabled": strconv.FormatBool(config.TTSEnabled), + "config.detail_level": config.DetailLevel, + "config.language": config.Language, + "created_at": now.Format(time.RFC3339), + "last_active": now.Format(time.RFC3339), + "active_request_id": "", + }) + pipe.Expire(ctx, metaKey(id), m.ttl) + + // 初始化空 history List + pipe.RPush(ctx, histKey(id), placeholderHistoryMark) + pipe.Expire(ctx, histKey(id), m.ttl) + + if _, err := pipe.Exec(ctx); err != nil { + return "", fmt.Errorf("redis create session: %w", err) + } + + logger.Log.Debugw("redis session created", "session", id) + return id, nil +} + +// placeholderHistoryMark 占位符，避免 Redis 对空 key 的特殊行为。 +const placeholderHistoryMark = "__placeholder__" + +// Get 获取会话。 +func (m *RedisManager) Get(ctx context.Context, sessionID string) (*models.Session, error) { + vals, err := m.rdb.HGetAll(ctx, metaKey(sessionID)).Result() + if err != nil { + return nil, fmt.Errorf("redis get session: %w", err) + } + if len(vals) == 0 { + return nil, ErrSessionNotFound + } + + sess := &models.Session{ + ID: vals["session_id"], + } + sess.CreatedAt, _ = time.Parse(time.RFC3339, vals["created_at"]) + sess.Config.TTSEnabled, _ = strconv.ParseBool(vals["config.tts_enabled"]) + sess.Config.DetailLevel = vals["config.detail_level"] + sess.Config.Language = vals["config.language"] + + return sess, nil +} + +// UpdateConfig 更新会话配置。 +func (m *RedisManager) UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error { + // 先检查会话是否存在 + exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result() + if err != nil { + return fmt.Errorf("redis check session: %w", err) + } + if exists == 0 { + return ErrSessionNotFound + } + + fields := map[string]interface{}{ + "last_active": time.Now().UTC().Format(time.RFC3339), + } + if patch.TTSEnabled != nil { + fields["config.tts_enabled"] = strconv.FormatBool(*patch.TTSEnabled) + } + if patch.DetailLevel != nil { + fields["config.detail_level"] = *patch.DetailLevel + } + if patch.Language != nil { + fields["config.language"] = *patch.Language + } + + if err := m.rdb.HSet(ctx, metaKey(sessionID), fields).Err(); err != nil { + return fmt.Errorf("redis update config: %w", err) + } + + // 刷新 TTL + m.rdb.Expire(ctx, metaKey(sessionID), m.ttl) + logger.Log.Debugw("redis session config updated", "session", sessionID) + return nil +} + +// GetHistory 获取最近 N 轮对话历史。 +func (m *RedisManager) GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error) { + // 检查会话是否存在 + exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result() + if err != nil { + return nil, fmt.Errorf("redis check session: %w", err) + } + if exists == 0 { + return nil, ErrSessionNotFound + } + + if limit <= 0 { + limit = m.maxHistory + } + + // LRANGE 0 {limit-1}，最新在前（LPUSH），需要反转为时间顺序 + raws, err := m.rdb.LRange(ctx, histKey(sessionID), 0, int64(limit)).Result() + if err != nil { + return nil, fmt.Errorf("redis get history: %w", err) + } + + var msgs []models.Message + for _, raw := range raws { + if raw == placeholderHistoryMark { + continue + } + var msg models.Message + if err := json.Unmarshal([]byte(raw), &msg); err != nil { + logger.Log.Warnw("invalid history entry", "session", sessionID, "raw", raw) + continue + } + msgs = append(msgs, msg) + } + + // 反转为时间顺序（LPUSH 最新在前 → 需要最旧在前） + for i, j := 0, len(msgs)-1; i < j; i, j = i+1, j-1 { + msgs[i], msgs[j] = msgs[j], msgs[i] + } + + return msgs, nil +} + +// AppendMessage 追加一条对话消息，同时刷新 TTL。 +func (m *RedisManager) AppendMessage(ctx context.Context, sessionID string, msg models.Message) error { + // 检查会话是否存在 + exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result() + if err != nil { + return fmt.Errorf("redis check session: %w", err) + } + if exists == 0 { + return ErrSessionNotFound + } + + data, err := json.Marshal(msg) + if err != nil { + return fmt.Errorf("marshal message: %w", err) + } + + pipe := m.rdb.Pipeline() + // LPUSH 新消息到左头（最新在前） + pipe.LPush(ctx, histKey(sessionID), string(data)) + // LTRIM 保留最近 maxHistory 条（+1 是因为有占位符） + pipe.LTrim(ctx, histKey(sessionID), 0, int64(m.maxHistory)) + // 刷新 TTL + pipe.Expire(ctx, histKey(sessionID), m.ttl) + pipe.Expire(ctx, metaKey(sessionID), m.ttl) + // 更新 last_active + pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339)) + + if _, err := pipe.Exec(ctx); err != nil { + return fmt.Errorf("redis append message: %w", err) + } + + return nil +} + +// SetActiveRequest 标记当前正在处理的请求 ID。 +func (m *RedisManager) SetActiveRequest(ctx context.Context, sessionID string, requestID string) error { + exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result() + if err != nil { + return fmt.Errorf("redis check session: %w", err) + } + if exists == 0 { + return ErrSessionNotFound + } + + pipe := m.rdb.Pipeline() + pipe.HSet(ctx, metaKey(sessionID), "active_request_id", requestID) + pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339)) + pipe.Expire(ctx, metaKey(sessionID), m.ttl) + + if _, err := pipe.Exec(ctx); err != nil { + return fmt.Errorf("redis set active request: %w", err) + } + return nil +} + +// GetActiveRequestID 获取当前活跃请求 ID。 +func (m *RedisManager) GetActiveRequestID(ctx context.Context, sessionID string) (string, error) { + val, err := m.rdb.HGet(ctx, metaKey(sessionID), "active_request_id").Result() + if err == redis.Nil { + return "", ErrSessionNotFound + } + if err != nil { + return "", fmt.Errorf("redis get active request: %w", err) + } + return val, nil +} + +// ClearActiveRequest 清除活跃请求标记。 +func (m *RedisManager) ClearActiveRequest(ctx context.Context, sessionID string) error { + exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result() + if err != nil { + return fmt.Errorf("redis check session: %w", err) + } + if exists == 0 { + return ErrSessionNotFound + } + + pipe := m.rdb.Pipeline() + pipe.HSet(ctx, metaKey(sessionID), "active_request_id", "") + pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339)) + pipe.Expire(ctx, metaKey(sessionID), m.ttl) + + if _, err := pipe.Exec(ctx); err != nil { + return fmt.Errorf("redis clear active request: %w", err) + } + return nil +} + +// Touch 刷新 TTL。 +func (m *RedisManager) Touch(ctx context.Context, sessionID string) error { + exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result() + if err != nil { + return fmt.Errorf("redis check session: %w", err) + } + if exists == 0 { + return ErrSessionNotFound + } + + pipe := m.rdb.Pipeline() + pipe.Expire(ctx, metaKey(sessionID), m.ttl) + pipe.Expire(ctx, histKey(sessionID), m.ttl) + pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339)) + + if _, err := pipe.Exec(ctx); err != nil { + return fmt.Errorf("redis touch: %w", err) + } + return nil +} + +// Destroy 显式销毁会话。 +func (m *RedisManager) Destroy(ctx context.Context, sessionID string) error { + deleted, err := m.rdb.Del(ctx, metaKey(sessionID), histKey(sessionID)).Result() + if err != nil { + return fmt.Errorf("redis destroy session: %w", err) + } + if deleted == 0 { + return ErrSessionNotFound + } + + logger.Log.Debugw("redis session destroyed", "session", sessionID) + return nil +} + +// ActiveCount 返回当前活跃会话数。 +// Redis 实现通过 SCAN 遍历 meta key，适用于中等规模。 +// 大规模部署建议维护独立的活跃会话集合。 +func (m *RedisManager) ActiveCount() int { + ctx := context.Background() + count := 0 + var cursor uint64 + for { + keys, nextCursor, err := m.rdb.Scan(ctx, cursor, "session:*:meta", 100).Result() + if err != nil { + break + } + for _, key := range keys { + exists, _ := m.rdb.Exists(ctx, key).Result() + if exists > 0 { + count++ + } + } + cursor = nextCursor + if cursor == 0 { + break + } + } + return count +} + +// uuidNew 生成 UUID，便于测试时 mock。 +var uuidNew = func() string { + return uuid.New().String() +} diff --git a/backend/internal/ws/handler.go b/backend/internal/ws/handler.go index 166a4f5..eb84af9 100644 --- a/backend/internal/ws/handler.go +++ b/backend/internal/ws/handler.go @@ -1,17 +1,20 @@ package ws import ( + "context" "encoding/json" - "log" "net/http" "sync" "time" "github.com/gin-gonic/gin" - "github.com/google/uuid" "github.com/gorilla/websocket" + "github.com/hhs/camtalk/internal/errors" + "github.com/hhs/camtalk/internal/logger" "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/orchestrator" + "github.com/hhs/camtalk/internal/session" ) var upgrader = websocket.Upgrader{ @@ -20,36 +23,94 @@ var upgrader = websocket.Upgrader{ // Client 代表一个 WebSocket 客户端连接。 type Client struct { - conn *websocket.Conn - sessionID string - mu sync.Mutex + conn *websocket.Conn + sessionID string + sessionMgr session.Manager + orchestrator orchestrator.Orchestrator + cancelFuncs map[string]context.CancelFunc // requestID → cancel func + mu sync.Mutex } -func (c *Client) sendJSON(v any) error { +// SendJSON 向客户端发送 JSON 消息（公开以便 errors 包调用）。 +func (c *Client) SendJSON(v any) error { c.mu.Lock() defer c.mu.Unlock() return c.conn.WriteJSON(v) } +// WSClient 实现 orchestrator.Sender 接口，将消息推送到 WebSocket 连接。 +type WSClient struct { + client *Client + requestID string +} + +// SendSTTResult 发送语音识别结果。 +func (w *WSClient) SendSTTResult(result models.WsSTTResult) error { + result.RequestID = w.requestID + return w.client.SendJSON(result) +} + +// SendLLMChunk 发送 LLM 流式文本增量。 +func (w *WSClient) SendLLMChunk(chunk models.WsLLMChunk) error { + chunk.RequestID = w.requestID + return w.client.SendJSON(chunk) +} + +// SendLLMDone 发送 LLM 流结束信号。 +func (w *WSClient) SendLLMDone(done models.WsLLMDone) error { + done.RequestID = w.requestID + return w.client.SendJSON(done) +} + +// SendTTSAudio 发送 TTS 音频数据。 +func (w *WSClient) SendTTSAudio(audio models.WsTTSAudio) error { + audio.RequestID = w.requestID + return w.client.SendJSON(audio) +} + +// SendError 发送错误消息。 +func (w *WSClient) SendError(err models.WsError) error { + err.RequestID = w.requestID + return w.client.SendJSON(err) +} + // ServeWS 处理 WebSocket 升级请求。 -func ServeWS(c *gin.Context) { +func ServeWS(sessionMgr session.Manager, orch orchestrator.Orchestrator) gin.HandlerFunc { + return func(c *gin.Context) { + serveWS(c, sessionMgr, orch) + } +} + +func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orchestrator) { conn, err := upgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { - log.Printf("websocket upgrade failed: %v", err) + logger.Log.Errorw("websocket upgrade failed", "error", err) return } defer conn.Close() - sessionID := uuid.New().String() - client := &Client{conn: conn, sessionID: sessionID} + // 创建会话 + sessionID, err := sessionMgr.Create(context.Background(), models.DefaultConfig()) + if err != nil { + logger.Log.Errorw("create session failed", "error", err) + return + } + + client := &Client{ + conn: conn, + sessionID: sessionID, + sessionMgr: sessionMgr, + orchestrator: orch, + cancelFuncs: make(map[string]context.CancelFunc), + } // 发送 connected 消息 - _ = client.sendJSON(models.WsConnected{ + _ = client.SendJSON(models.WsConnected{ Type: "connected", SessionID: sessionID, ServerVersion: "0.1.0", }) - log.Printf("client connected: session=%s", sessionID) + logger.Log.Infow("client connected", "session", sessionID) // 心跳检测 lastPong := time.Now() @@ -67,7 +128,7 @@ func ServeWS(c *gin.Context) { select { case <-ticker.C: if time.Since(lastPong) > 60*time.Second { - log.Printf("heartbeat timeout: session=%s", sessionID) + logger.Log.Warnw("heartbeat timeout", "session", sessionID) conn.Close() return } @@ -82,7 +143,7 @@ func ServeWS(c *gin.Context) { _, message, err := conn.ReadMessage() if err != nil { if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) { - log.Printf("ws read error: %v", err) + logger.Log.Warnw("ws read error", "error", err) } break } @@ -92,51 +153,96 @@ func ServeWS(c *gin.Context) { Type string `json:"type"` } if err := json.Unmarshal(message, &envelope); err != nil { - _ = client.sendJSON(models.WsError{ - Type: "error", - Code: "INVALID_MESSAGE", - Message: "invalid JSON", - }) + errors.SendWSError(client, errors.CodeInvalidMessage, "", err) continue } switch envelope.Type { case "ping": - _ = client.sendJSON(models.WsPong{Type: "pong"}) + _ = client.SendJSON(models.WsPong{Type: "pong"}) case "query": var msg models.WsQuery if err := json.Unmarshal(message, &msg); err != nil { - _ = client.sendJSON(models.WsError{ - Type: "error", - Code: "INVALID_MESSAGE", - Message: "invalid query message", - RequestID: msg.RequestID, - }) + errors.SendWSError(client, errors.CodeInvalidMessage, msg.RequestID, err) continue } - log.Printf("query received: session=%s request=%s", sessionID, msg.RequestID) - // TODO: 调用 AI 编排流程（STT → LLM → TTS） + logger.Log.Infow("query received", "session", sessionID, "request", msg.RequestID) + + // 刷新会话 TTL + if err := client.sessionMgr.Touch(context.Background(), sessionID); err != nil { + logger.Log.Warnw("touch session failed", "session", sessionID, "error", err) + } + + // 标记活跃请求 + if err := client.sessionMgr.SetActiveRequest(context.Background(), sessionID, msg.RequestID); err != nil { + logger.Log.Warnw("set active request failed", "session", sessionID, "error", err) + } + + // 获取对话历史 + history, _ := client.sessionMgr.GetHistory(context.Background(), sessionID, 20) + + // 创建可取消的 context + ctx, cancel := context.WithCancel(context.Background()) + client.mu.Lock() + client.cancelFuncs[msg.RequestID] = cancel + client.mu.Unlock() + + // 创建 sender + sender := &WSClient{client: client, requestID: msg.RequestID} + + // 启动 orchestrator 处理 goroutine + go func() { + defer func() { + // 清理 cancel func + client.mu.Lock() + delete(client.cancelFuncs, msg.RequestID) + client.mu.Unlock() + cancel() + // 清除活跃请求 + _ = client.sessionMgr.ClearActiveRequest(context.Background(), sessionID) + }() + + if err := client.orchestrator.ProcessQuery(ctx, sessionID, msg, history, sender); err != nil { + logger.Log.Errorw("process query failed", "session", sessionID, "request", msg.RequestID, "error", err) + } + }() case "config": var msg models.WsConfig if err := json.Unmarshal(message, &msg); err != nil { - _ = client.sendJSON(models.WsError{ - Type: "error", - Code: "INVALID_MESSAGE", - Message: "invalid config message", - }) + errors.SendWSError(client, errors.CodeInvalidMessage, "", err) continue } - log.Printf("config update: session=%s", sessionID) - // TODO: 更新会话配置 + + patch := models.SessionConfigPatch{ + TTSEnabled: msg.Payload.TTSEnabled, + DetailLevel: msg.Payload.DetailLevel, + Language: msg.Payload.Language, + } + if err := client.sessionMgr.UpdateConfig(context.Background(), sessionID, patch); err != nil { + errors.SendWSError(client, errors.CodeInternalError, "", err) + continue + } + logger.Log.Infow("config updated", "session", sessionID) case "interrupt": - log.Printf("interrupt received: session=%s", sessionID) - // TODO: 中断当前 AI 响应 + logger.Log.Infow("interrupt received", "session", sessionID) + + // 获取活跃请求 ID 并取消 + reqID, _ := client.sessionMgr.GetActiveRequestID(context.Background(), sessionID) + if reqID != "" { + client.mu.Lock() + if cancel, ok := client.cancelFuncs[reqID]; ok { + cancel() + delete(client.cancelFuncs, reqID) + } + client.mu.Unlock() + _ = client.sessionMgr.ClearActiveRequest(context.Background(), sessionID) + } default: - _ = client.sendJSON(models.WsError{ + _ = client.SendJSON(models.WsError{ Type: "error", Code: "INVALID_MESSAGE", Message: "unknown message type: " + envelope.Type, @@ -145,5 +251,16 @@ func ServeWS(c *gin.Context) { } close(done) - log.Printf("client disconnected: session=%s", sessionID) + + // 取消所有活跃请求 + client.mu.Lock() + for reqID, cancel := range client.cancelFuncs { + logger.Log.Infow("canceling active request on disconnect", "session", sessionID, "request", reqID) + cancel() + } + client.cancelFuncs = make(map[string]context.CancelFunc) + client.mu.Unlock() + + // 断开连接时不销毁会话，让其自然过期（支持重连恢复） + logger.Log.Infow("client disconnected", "session", sessionID) } diff --git a/backend/internal/ws/handler_test.go b/backend/internal/ws/handler_test.go new file mode 100644 index 0000000..97cf697 --- /dev/null +++ b/backend/internal/ws/handler_test.go @@ -0,0 +1,563 @@ +package ws + +import ( + "encoding/base64" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "context" + + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/orchestrator" + "github.com/hhs/camtalk/internal/session" +) + +func init() { + logger.Init("debug", "console") + gin.SetMode(gin.TestMode) +} + +// --- Mock Orchestrator --- + +// MockOrchestrator 实现 orchestrator.Orchestrator 接口， +// 模拟完整的 STT → LLM → TTS 管道，通过 sender 推送消息。 +type MockOrchestrator struct { + // STTResult 模拟的语音识别结果 + STTResult string + // LLMDeltas 模拟的 LLM 流式输出 + LLMDeltas []string + // TTSAudios 模拟的 TTS 音频数据（每项一个 base64 编码的 MP3 片段） + TTSAudios []string + // Err 如果非 nil，ProcessQuery 直接返回此错误 + Err error + // Delay 每个消息之间的延迟（用于 interrupt 测试） + Delay time.Duration +} + +func (m *MockOrchestrator) ProcessQuery( + ctx context.Context, + sessionID string, + req models.WsQuery, + history []models.Message, + sender orchestrator.Sender, +) error { + if m.Err != nil { + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "INTERNAL_ERROR", + Message: m.Err.Error(), + }) + return m.Err + } + + // Step 1: 发送 STT 结果 + if m.STTResult != "" { + _ = sender.SendSTTResult(models.WsSTTResult{ + Type: "stt_result", + RequestID: req.RequestID, + Text: m.STTResult, + IsFinal: true, + }) + } + if m.Delay > 0 { + select { + case <-ctx.Done(): + return nil // 中断视为正常完成 + case <-time.After(m.Delay): + } + } + + // Step 2: 发送 LLM chunks + var fullText strings.Builder + for _, delta := range m.LLMDeltas { + select { + case <-ctx.Done(): + return nil // 中断视为正常完成 + default: + } + fullText.WriteString(delta) + _ = sender.SendLLMChunk(models.WsLLMChunk{ + Type: "llm_chunk", + RequestID: req.RequestID, + Delta: delta, + Role: "assistant", + }) + if m.Delay > 0 { + select { + case <-ctx.Done(): + return nil // 中断视为正常完成 + case <-time.After(m.Delay): + } + } + } + + // Step 3: 发送 TTS 音频 + for i, audio := range m.TTSAudios { + select { + case <-ctx.Done(): + return nil // 中断视为正常完成 + default: + } + isLast := i == len(m.TTSAudios)-1 + _ = sender.SendTTSAudio(models.WsTTSAudio{ + Type: "tts_audio", + RequestID: req.RequestID, + Audio: audio, + MimeType: "audio/mp3", + IsLast: isLast, + }) + } + + // Step 4: 发送 llm_done + _ = sender.SendLLMDone(models.WsLLMDone{ + Type: "llm_done", + RequestID: req.RequestID, + FullText: fullText.String(), + Model: "gpt-4o", + LatencyMs: 100, + }) + + return nil +} + +// --- 测试辅助函数 --- + +// setupTestServer 创建测试用 Gin 服务器和 WebSocket URL。 +func setupTestServer(t *testing.T, orch orchestrator.Orchestrator) (*httptest.Server, string) { + t.Helper() + + sessionMgr := session.NewMemoryManager(5*time.Minute, 20) + t.Cleanup(func() { sessionMgr.Stop() }) + + r := gin.New() + r.GET("/ws", ServeWS(sessionMgr, orch)) + + srv := httptest.NewServer(r) + + // 构造 WebSocket URL + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + "/ws" + + return srv, wsURL +} + +// connectWS 建立 WebSocket 连接并返回 conn。 +func connectWS(t *testing.T, wsURL string) *websocket.Conn { + t.Helper() + + conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + require.NoError(t, err, "WebSocket 连接失败") + t.Cleanup(func() { conn.Close() }) + return conn +} + +// readJSON 从 WebSocket 读取一条 JSON 消息。 +func readJSON(t *testing.T, conn *websocket.Conn) map[string]any { + t.Helper() + + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + var msg map[string]any + err := conn.ReadJSON(&msg) + require.NoError(t, err, "读取 WebSocket 消息失败") + return msg +} + +// --- 测试用例 --- + +// TestWS_Connected 验证连接建立后收到 connected 消息。 +func TestWS_Connected(t *testing.T) { + srv, wsURL := setupTestServer(t, &MockOrchestrator{}) + defer srv.Close() + + conn := connectWS(t, wsURL) + + msg := readJSON(t, conn) + assert.Equal(t, "connected", msg["type"]) + assert.NotEmpty(t, msg["session_id"]) + assert.Equal(t, "0.1.0", msg["server_version"]) +} + +// TestWS_PingPong 验证 ping/pong 心跳。 +func TestWS_PingPong(t *testing.T) { + srv, wsURL := setupTestServer(t, &MockOrchestrator{}) + defer srv.Close() + + conn := connectWS(t, wsURL) + + // 读取 connected 消息 + _ = readJSON(t, conn) + + // 发送 ping + err := conn.WriteJSON(map[string]string{"type": "ping"}) + require.NoError(t, err) + + // 读取 pong + msg := readJSON(t, conn) + assert.Equal(t, "pong", msg["type"]) +} + +// TestWS_QueryFullFlow 验证完整的 query → stt_result → llm_chunk → tts_audio → llm_done 流程。 +func TestWS_QueryFullFlow(t *testing.T) { + audioB64 := base64.StdEncoding.EncodeToString([]byte("fake-audio-data")) + imageB64 := base64.StdEncoding.EncodeToString([]byte("fake-image-data")) + + mock := &MockOrchestrator{ + STTResult: "你好，世界", + LLMDeltas: []string{"你好", "，世界！"}, + TTSAudios: []string{base64.StdEncoding.EncodeToString([]byte("mp3-data-1")), base64.StdEncoding.EncodeToString([]byte("mp3-data-2"))}, + } + + srv, wsURL := setupTestServer(t, mock) + defer srv.Close() + + conn := connectWS(t, wsURL) + + // 1. 读取 connected + connected := readJSON(t, conn) + assert.Equal(t, "connected", connected["type"]) + sessionID := connected["session_id"].(string) + assert.NotEmpty(t, sessionID) + + // 2. 发送 query + queryMsg := models.WsQuery{ + Type: "query", + RequestID: "req-test-001", + Image: imageB64, + Audio: audioB64, + MimeType: "audio/pcm", + } + err := conn.WriteJSON(queryMsg) + require.NoError(t, err) + + // 3. 读取 stt_result + sttResult := readJSON(t, conn) + assert.Equal(t, "stt_result", sttResult["type"]) + assert.Equal(t, "req-test-001", sttResult["request_id"]) + assert.Equal(t, "你好，世界", sttResult["text"]) + assert.Equal(t, true, sttResult["is_final"]) + + // 4. 读取 llm_chunk 消息 + chunk1 := readJSON(t, conn) + assert.Equal(t, "llm_chunk", chunk1["type"]) + assert.Equal(t, "req-test-001", chunk1["request_id"]) + assert.Equal(t, "你好", chunk1["delta"]) + assert.Equal(t, "assistant", chunk1["role"]) + + chunk2 := readJSON(t, conn) + assert.Equal(t, "llm_chunk", chunk2["type"]) + assert.Equal(t, "，世界！", chunk2["delta"]) + + // 5. 读取 tts_audio 消息 + tts1 := readJSON(t, conn) + assert.Equal(t, "tts_audio", tts1["type"]) + assert.Equal(t, "req-test-001", tts1["request_id"]) + assert.NotEmpty(t, tts1["audio"]) + assert.Equal(t, "audio/mp3", tts1["mime_type"]) + assert.Equal(t, false, tts1["is_last"]) + + tts2 := readJSON(t, conn) + assert.Equal(t, "tts_audio", tts2["type"]) + assert.Equal(t, true, tts2["is_last"]) + + // 6. 读取 llm_done + llmDone := readJSON(t, conn) + assert.Equal(t, "llm_done", llmDone["type"]) + assert.Equal(t, "req-test-001", llmDone["request_id"]) + assert.Equal(t, "你好，世界！", llmDone["full_text"]) + assert.Equal(t, "gpt-4o", llmDone["model"]) + assert.NotNil(t, llmDone["latency_ms"]) +} + +// TestWS_QuerySTTOnly 验证只有 STT 结果、无 LLM 输出的场景。 +func TestWS_QuerySTTOnly(t *testing.T) { + audioB64 := base64.StdEncoding.EncodeToString([]byte("audio")) + + mock := &MockOrchestrator{ + STTResult: "测试语音", + // LLMDeltas 为空 → 不发送 llm_chunk + // TTSAudios 为空 → 不发送 tts_audio + } + + srv, wsURL := setupTestServer(t, mock) + defer srv.Close() + + conn := connectWS(t, wsURL) + _ = readJSON(t, conn) // connected + + err := conn.WriteJSON(models.WsQuery{ + Type: "query", + RequestID: "req-stt-only", + Audio: audioB64, + }) + require.NoError(t, err) + + // 应收到 stt_result + stt := readJSON(t, conn) + assert.Equal(t, "stt_result", stt["type"]) + assert.Equal(t, "测试语音", stt["text"]) + + // 应收到 llm_done（即使没有 chunk） + done := readJSON(t, conn) + assert.Equal(t, "llm_done", done["type"]) + assert.Equal(t, "", done["full_text"]) +} + +// TestWS_UnknownMessageType 验证未知消息类型返回 error。 +func TestWS_UnknownMessageType(t *testing.T) { + srv, wsURL := setupTestServer(t, &MockOrchestrator{}) + defer srv.Close() + + conn := connectWS(t, wsURL) + _ = readJSON(t, conn) // connected + + err := conn.WriteJSON(map[string]string{"type": "unknown_type"}) + require.NoError(t, err) + + errMsg := readJSON(t, conn) + assert.Equal(t, "error", errMsg["type"]) + assert.Equal(t, "INVALID_MESSAGE", errMsg["code"]) + assert.Contains(t, errMsg["message"], "unknown message type") +} + +// TestWS_InvalidJSON 验证无效 JSON 返回 error。 +func TestWS_InvalidJSON(t *testing.T) { + srv, wsURL := setupTestServer(t, &MockOrchestrator{}) + defer srv.Close() + + conn := connectWS(t, wsURL) + _ = readJSON(t, conn) // connected + + err := conn.WriteMessage(websocket.TextMessage, []byte("not-json")) + require.NoError(t, err) + + errMsg := readJSON(t, conn) + assert.Equal(t, "error", errMsg["type"]) + assert.Equal(t, "INVALID_MESSAGE", errMsg["code"]) +} + +// TestWS_MultipleQueries 验证同一连接上可以发送多次 query。 +func TestWS_MultipleQueries(t *testing.T) { + mock := &MockOrchestrator{ + STTResult: "识别结果", + LLMDeltas: []string{"回复"}, + } + + srv, wsURL := setupTestServer(t, mock) + defer srv.Close() + + conn := connectWS(t, wsURL) + _ = readJSON(t, conn) // connected + + audioB64 := base64.StdEncoding.EncodeToString([]byte("audio")) + + for i := 0; i < 3; i++ { + reqID := "req-multi-" + string(rune('0'+i)) + err := conn.WriteJSON(models.WsQuery{ + Type: "query", + RequestID: reqID, + Audio: audioB64, + }) + require.NoError(t, err) + + // 每次应收到完整的响应序列 + stt := readJSON(t, conn) + assert.Equal(t, "stt_result", stt["type"], "第 %d 次 query", i+1) + + chunk := readJSON(t, conn) + assert.Equal(t, "llm_chunk", chunk["type"], "第 %d 次 query", i+1) + + done := readJSON(t, conn) + assert.Equal(t, "llm_done", done["type"], "第 %d 次 query", i+1) + } +} + +// TestWS_Interrupt 验证 interrupt 取消正在进行的请求。 +func TestWS_Interrupt(t *testing.T) { + // 使用较长延迟模拟慢请求 + mock := &MockOrchestrator{ + STTResult: "识别文本", + LLMDeltas: []string{"第一句", "第二句", "第三句", "第四句", "第五句"}, + Delay: 200 * time.Millisecond, + } + + srv, wsURL := setupTestServer(t, mock) + defer srv.Close() + + conn := connectWS(t, wsURL) + _ = readJSON(t, conn) // connected + + audioB64 := base64.StdEncoding.EncodeToString([]byte("audio")) + + // 发送 query + err := conn.WriteJSON(models.WsQuery{ + Type: "query", + RequestID: "req-interrupt", + Audio: audioB64, + }) + require.NoError(t, err) + + // 收到 stt_result + stt := readJSON(t, conn) + assert.Equal(t, "stt_result", stt["type"]) + + // 收到第一个 llm_chunk + chunk1 := readJSON(t, conn) + assert.Equal(t, "llm_chunk", chunk1["type"]) + + // 发送 interrupt + err = conn.WriteJSON(map[string]string{"type": "interrupt"}) + require.NoError(t, err) + + // 等待 interrupt 生效 + time.Sleep(500 * time.Millisecond) + + // 验证连接仍然存活（可以发 ping 收 pong） + require.NoError(t, conn.WriteJSON(map[string]string{"type": "ping"})) + + var pong map[string]any + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + require.NoError(t, conn.ReadJSON(&pong), "interrupt 后连接应仍存活") + assert.Equal(t, "pong", pong["type"]) +} + +// TestWS_DisconnectCleanup 验证断开连接时清理资源。 +func TestWS_DisconnectCleanup(t *testing.T) { + // 使用较长延迟模拟慢请求 + mock := &MockOrchestrator{ + STTResult: "识别文本", + LLMDeltas: []string{"长回复第一部分", "长回复第二部分"}, + Delay: 500 * time.Millisecond, + } + + srv, wsURL := setupTestServer(t, mock) + defer srv.Close() + + conn := connectWS(t, wsURL) + _ = readJSON(t, conn) // connected + + audioB64 := base64.StdEncoding.EncodeToString([]byte("audio")) + + // 发送 query + err := conn.WriteJSON(models.WsQuery{ + Type: "query", + RequestID: "req-disconnect", + Audio: audioB64, + }) + require.NoError(t, err) + + // 收到 stt_result + stt := readJSON(t, conn) + assert.Equal(t, "stt_result", stt["type"]) + + // 关闭连接（模拟客户端断开） + conn.Close() + + // 等待一小段时间让服务器处理断开 + time.Sleep(300 * time.Millisecond) + + // 如果没有 panic 或 goroutine 泄漏，测试通过 + // （Go test 的 -race 检测器会捕获数据竞争） +} + +// TestWS_SessionCreated 验证每次连接都创建新会话。 +func TestWS_SessionCreated(t *testing.T) { + srv, wsURL := setupTestServer(t, &MockOrchestrator{}) + defer srv.Close() + + // 第一次连接 + conn1 := connectWS(t, wsURL) + msg1 := readJSON(t, conn1) + sid1 := msg1["session_id"].(string) + conn1.Close() + + time.Sleep(100 * time.Millisecond) + + // 第二次连接 + conn2 := connectWS(t, wsURL) + sid2 := readJSON(t, conn2)["session_id"].(string) + + assert.NotEmpty(t, sid1) + assert.NotEmpty(t, sid2) + assert.NotEqual(t, sid1, sid2, "两次连接应创建不同的会话") +} + +// TestWS_QueryWithoutImage 验证不带图片的 query。 +func TestWS_QueryWithoutImage(t *testing.T) { + mock := &MockOrchestrator{ + STTResult: "纯语音输入", + LLMDeltas: []string{"收到"}, + } + + srv, wsURL := setupTestServer(t, mock) + defer srv.Close() + + conn := connectWS(t, wsURL) + _ = readJSON(t, conn) // connected + + audioB64 := base64.StdEncoding.EncodeToString([]byte("audio")) + + err := conn.WriteJSON(models.WsQuery{ + Type: "query", + RequestID: "req-no-image", + Audio: audioB64, + // Image 为空 + }) + require.NoError(t, err) + + stt := readJSON(t, conn) + assert.Equal(t, "stt_result", stt["type"]) + assert.Equal(t, "纯语音输入", stt["text"]) + + chunk := readJSON(t, conn) + assert.Equal(t, "llm_chunk", chunk["type"]) + + done := readJSON(t, conn) + assert.Equal(t, "llm_done", done["type"]) +} + +// TestWS_QueryWithTTSDisabled 验证 TTS 未启用时不应收到 tts_audio。 +func TestWS_QueryWithTTSDisabled(t *testing.T) { + // MockOrchestrator 的 TTSAudios 为空 → 不发送 tts_audio + mock := &MockOrchestrator{ + STTResult: "语音", + LLMDeltas: []string{"回复"}, + // TTSAudios 留空 + } + + srv, wsURL := setupTestServer(t, mock) + defer srv.Close() + + conn := connectWS(t, wsURL) + _ = readJSON(t, conn) // connected + + audioB64 := base64.StdEncoding.EncodeToString([]byte("audio")) + + err := conn.WriteJSON(models.WsQuery{ + Type: "query", + RequestID: "req-no-tts", + Audio: audioB64, + }) + require.NoError(t, err) + + stt := readJSON(t, conn) + assert.Equal(t, "stt_result", stt["type"]) + + chunk := readJSON(t, conn) + assert.Equal(t, "llm_chunk", chunk["type"]) + + done := readJSON(t, conn) + assert.Equal(t, "llm_done", done["type"]) + + // 不应有 tts_audio 消息；设置短超时验证 + conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + var extra map[string]any + err = conn.ReadJSON(&extra) + assert.Error(t, err, "不应有额外消息") +} diff --git a/docs/02-系统架构.md b/docs/02-系统架构.md index 9dd6a5f..41f2a33 100644 --- a/docs/02-系统架构.md +++ b/docs/02-系统架构.md @@ -213,11 +213,73 @@ CREATE TABLE usage_daily ( ## 部署架构 ``` -CDN（静态资源） ← 用户浏览器 -Nginx 负载均衡（sticky session for WebSocket） - ├── Gateway-1 ──→ Redis - ├── Gateway-2 ──→ Redis - └── Gateway-N ──→ AI Services（外部 API） +用户浏览器 + ↓ +Nginx（同源反代 + 负载均衡） + ├── / → 前端静态资源（CDN 或本地 dist） + ├── /api/* → Go Gateway（REST API） + └── /ws → Go Gateway（WebSocket） + ├── Gateway-1 ──→ Redis + ├── Gateway-2 ──→ Redis + └── Gateway-N ──→ AI Services（外部 API） ``` -WebSocket 是长连接，Nginx 需要配置 `proxy_set_header Upgrade` 和 sticky session，确保同一用户的请求始终路由到同一个 Gateway 实例。 +**跨域策略**：Nginx 将前端和后端统一到同一域名下，浏览器无跨域问题。 + +### Nginx 配置 + +```nginx +server { + listen 80; + server_name camtalk.example.com; + + # 前端静态资源 + location / { + root /var/www/camtalk/dist; + try_files $uri $uri/ /index.html; + } + + # REST API 反代 + location /api/ { + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # WebSocket 反代 + location /ws { + proxy_pass http://127.0.0.1:8080; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_read_timeout 86400s; # 长连接超时 24h + proxy_send_timeout 86400s; + } +} +``` + +> WebSocket 是长连接，Nginx 必须配置 `Upgrade` 和 `Connection` 头。`proxy_read_timeout` 需要覆盖心跳间隔（客户端 30s ping），否则 Nginx 会主动断开空闲连接。 + +### 开发环境（Vite proxy） + +开发时前端（Vite :5173）和后端（Gin :8080）不同端口，用 Vite 内置代理解决跨域： + +```typescript +// frontend/vite.config.ts +export default defineConfig({ + plugins: [react()], + server: { + proxy: { + "/api": "http://localhost:8080", + "/ws": { + target: "ws://localhost:8080", + ws: true, + }, + }, + }, +}); +``` + +前端代码中 WebSocket 地址改为相对路径 `ws://localhost:5173/ws`，Vite 自动代理到后端。部署时 Nginx 同理，前端无需区分开发/生产地址。 diff --git a/docs/03-接口文档.md b/docs/03-接口文档.md index 32b6974..f1b7e1c 100644 --- a/docs/03-接口文档.md +++ b/docs/03-接口文档.md @@ -256,7 +256,6 @@ POST /api/sessions Content-Type: application/json { - "user_id": "optional-user-id", "config": { "tts_enabled": true, "detail_level": "low", @@ -301,28 +300,18 @@ Go 网关内部与外部 AI 服务（Deepgram STT、GPT-4o、OpenAI TTS）的调 语音识别：接收前端采集的音频，返回识别文本。 ```go -// STTService 语音识别服务契约。 -type STTService interface { +// Service 语音识别服务契约。 +type Service interface { // Recognize 识别一段完整音频，返回最终文本。 - Recognize(ctx context.Context, audio []byte, opts STTOptions) (string, error) - - // RecognizeStream 流式识别（边说边识别，可选实现）。 - // audioStream 持续接收音频片段，返回的 channel 持续输出中间结果。 - RecognizeStream(ctx context.Context, audioStream <-chan []byte, opts STTOptions) (<-chan STTPartial, error) + Recognize(ctx context.Context, audio []byte, opts Options) (string, error) } -// STTOptions 语音识别参数。 -type STTOptions struct { +// Options 语音识别参数。 +type Options struct { Encoding string // "pcm_s16le" — 前端 VAD 输出格式 SampleRate int // 16000 — 前端麦克风采样率 Language string // "zh-CN" } - -// STTPartial 流式识别的中间/最终结果。 -type STTPartial struct { - Text string - IsFinal bool -} ``` **Deepgram 接入约定**： @@ -336,23 +325,23 @@ type STTPartial struct { 多模态推理：接收图像 + 文本 + 对话历史，流式返回回复。 ```go -// LLMService 多模态大模型服务契约。 -type LLMService interface { +// Service 多模态大模型服务契约。 +type Service interface { // ChatStream 流式推理，返回增量文本的 channel。 // 调用方必须消费 channel 直到 Done=true，否则需 cancel ctx 以释放连接。 - ChatStream(ctx context.Context, req LLMRequest) (<-chan LLMChunk, error) + ChatStream(ctx context.Context, req Request) (<-chan Chunk, error) } -// LLMRequest 推理请求。 -type LLMRequest struct { - Image []byte // JPEG 图片（已从 Base64 解码） - Text string // 用户语音识别后的文本 - History []Message // 最近 N 轮对话历史 - Language string // "zh-CN" +// Request 推理请求。 +type Request struct { + Image []byte // JPEG 图片（已从 Base64 解码） + Text string // 用户语音识别后的文本 + History []models.Message // 最近 N 轮对话历史 + Language string // "zh-CN" } -// LLMChunk 流式推理的一个增量片段。 -type LLMChunk struct { +// Chunk 流式推理的一个增量片段。 +type Chunk struct { Delta string // 增量文本 Done bool // 是否结束 TokensUsed *TokenUsage // 仅 Done=true 时有值 @@ -388,24 +377,24 @@ user: [图片 + 用户语音文本] 语音合成：接收文本流，输出音频 chunk 流。 ```go -// TTSService 语音合成服务契约。 -type TTSService interface { +// Service 语音合成服务契约。 +type Service interface { // SynthesizeStream 流式合成。 // textStream 接收句子级文本（由 Orchestrator 的句子切分器产出）， // 返回的 channel 持续输出 MP3 音频 chunk。 - SynthesizeStream(ctx context.Context, textStream <-chan string, opts TTSOptions) (<-chan TTSChunk, error) + SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error) } -// TTSOptions 合成参数。 -type TTSOptions struct { +// Options 合成参数。 +type Options struct { Voice string // "alloy" | "nova" | "shimmer" | ... Speed float64 // 1.0 为正常语速 OutputFmt string // "mp3" — 固定使用 MP3，浏览器原生支持 SampleRate int // 24000 } -// TTSChunk 一个音频片段。 -type TTSChunk struct { +// Chunk 一个音频片段。 +type Chunk struct { Audio []byte // MP3 音频数据（未 Base64 编码，由发送层编码） IsLast bool // 是否为最后一片 } @@ -446,73 +435,32 @@ LLM 流式输出: "这" "是一" "朵红色" "的花。" "它看起" "来很美 ### Orchestrator 接口 ```go -// Orchestrator AI 编排器，协调 STT → LLM → TTS 全链路。 -type Orchestrator struct { - stt STTService - llm LLMService - tts TTSService +// Orchestrator AI 编排器接口。 +type Orchestrator interface { + // ProcessQuery 处理一次完整的视觉对话请求。 + // 通过 sender 向前端实时推送 stt_result、llm_chunk、llm_done、tts_audio 消息。 + ProcessQuery(ctx context.Context, sessionID string, req models.WsQuery, + history []models.Message, sender Sender) error } -// ProcessQuery 处理一次完整的视觉对话请求。 -// 通过 client 向前端实时推送 stt_result、llm_chunk、llm_done、tts_audio 消息。 -func (o *Orchestrator) ProcessQuery(ctx context.Context, client MessageSender, req *QueryRequest) { - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - // Step 1: STT — 识别用户语音 - text, err := o.stt.Recognize(ctx, req.Audio, STTOptions{ - Encoding: "pcm_s16le", SampleRate: 16000, Language: "zh-CN", - }) - if err != nil { - client.SendError(req.RequestID, "STT_ERROR", err.Error()) - return - } - client.SendSTTResult(req.RequestID, text, true) - - // Step 2: LLM 流式输出 + 句子切分 - llmStream, _ := o.llm.ChatStream(ctx, LLMRequest{ - Image: req.Image, Text: text, Language: "zh-CN", - }) - - sentenceCh := make(chan string, 4) - go func() { - defer close(sentenceCh) - var buf strings.Builder - var fullText strings.Builder - for chunk := range llmStream { - // 即时推送文字给客户端（逐 token 显示） - client.SendLLMChunk(req.RequestID, chunk.Delta) - fullText.WriteString(chunk.Delta) - buf.WriteString(chunk.Delta) - // 遇到句子边界就吐出 - if isSentenceEnd(chunk.Delta) { - sentenceCh <- buf.String() - buf.Reset() - } - } - // 最后一段不足一句的也吐出 - if buf.Len() > 0 { - sentenceCh <- buf.String() - } - // 推送 llm_done - client.SendLLMDone(req.RequestID, fullText.String(), chunk.TokensUsed, chunk.Model) - }() - - // Step 3: TTS 并行消费句子流 - ttsStream, _ := o.tts.SynthesizeStream(ctx, sentenceCh, TTSOptions{ - Voice: "alloy", OutputFmt: "mp3", SampleRate: 24000, - }) - for chunk := range ttsStream { - client.SendTTSAudio(req.RequestID, chunk.Audio, chunk.IsLast) - } -} - -// isSentenceEnd 判断 delta 中是否包含句子结束标志。 -func isSentenceEnd(delta string) bool { - return strings.ContainsAny(delta, "。！？\n.!?\n") +// Sender 抽象 WebSocket 消息推送能力，便于测试时 mock。 +type Sender interface { + SendSTTResult(result models.WsSTTResult) error + SendLLMChunk(chunk models.WsLLMChunk) error + SendLLMDone(done models.WsLLMDone) error + SendTTSAudio(audio models.WsTTSAudio) error + SendError(err models.WsError) error } ``` +**Pipeline 实现**（`internal/orchestrator/pipeline.go`）： +1. Base64 解码音频/图片 +2. 调用 `stt.Recognize()` → 发送 `stt_result` +3. 调用 `llm.ChatStream()` 获取流式输出，goroutine 消费 token → 发送 `llm_chunk` + 句子切分 +4. 另一 goroutine 从句子 channel 读取 → 调用 `tts.SynthesizeStream()` → 发送 `tts_audio` +5. 流结束 → 发送 `llm_done` +6. TTS 失败静默跳过，STT/LLM 失败发送对应 error 消息 + ### 并发控制 - 每个 `ProcessQuery` 调用在独立 goroutine 中运行 @@ -584,9 +532,9 @@ session:{id}:history → List （对话历史） ### 接口定义 ```go -// SessionManager 会话管理器。 -// WebSocket Handler 通过此接口操作会话，不直接接触 Redis。 -type SessionManager interface { +// Manager 会话管理器接口。 +// WebSocket Handler 通过此接口操作会话，不直接接触存储层。 +type Manager interface { // Create 创建新会话，返回 session ID。 Create(ctx context.Context, config models.SessionConfig) (string, error) @@ -605,14 +553,20 @@ type SessionManager interface { // SetActiveRequest 标记当前正在处理的请求 ID（interrupt 用）。 SetActiveRequest(ctx context.Context, sessionID string, requestID string) error + // GetActiveRequestID 获取当前活跃请求 ID。 + GetActiveRequestID(ctx context.Context, sessionID string) (string, error) + // ClearActiveRequest 清除活跃请求标记（请求完成或中断后）。 ClearActiveRequest(ctx context.Context, sessionID string) error // Touch 刷新 TTL（心跳时调用）。 Touch(ctx context.Context, sessionID string) error - // Destroy 显式销毁会话（REST API DELETE 或连接断开清理）。 + // Destroy 显式销毁会话（REST API DELETE）。 Destroy(ctx context.Context, sessionID string) error + + // ActiveCount 返回当前活跃会话数（健康检查用）。 + ActiveCount() int } ``` @@ -629,7 +583,8 @@ case "query": history, _ := sessionMgr.GetHistory(ctx, sessionID, 20) // 获取对话上下文 - go orchestrator.ProcessQuery(ctx, client, &msg, history) // 异步编排 + sender := &WSClient{client: client, requestID: msg.RequestID} + go orch.ProcessQuery(ctx, sessionID, msg, history, sender) // 异步编排 // interrupt 分支 case "interrupt": @@ -648,26 +603,30 @@ case "interrupt": 联调阶段无 Redis 时，用同一接口的内存实现： ```go -type InMemorySessionManager struct { - mu sync.RWMutex - sessions map[string]*sessionEntry +type MemoryManager struct { + mu sync.RWMutex + sessions map[string]*sessionEntry + ttl time.Duration + maxHistory int + stopCleaner chan struct{} } type sessionEntry struct { session models.Session history []models.Message activeReqID string + lastActive time.Time } ``` 注入时根据配置切换： ```go -var sessionMgr SessionManager +var sessionMgr session.Manager if cfg.Redis.Addr != "" { - sessionMgr = NewRedisSessionManager(redisClient, 30*time.Minute, 20) + sessionMgr = session.NewRedisManager(redisClient, 30*time.Minute, 20) } else { - sessionMgr = NewInMemorySessionManager() + sessionMgr = session.NewMemoryManager(30*time.Minute, 20) } ``` @@ -931,10 +890,8 @@ type QueryRequest struct { } type Message struct { - Role string `json:"role"` // "user" | "assistant" - Content string `json:"content"` - ImageURL string `json:"image_url,omitempty"` - TokensUsed int `json:"tokens_used,omitempty"` + Role string `json:"role"` // "user" | "assistant" + Content string `json:"content"` } ``` @@ -1085,3 +1042,25 @@ function reconnect(attempt: number) { } // attempt: 0 → 1s, 1 → 2s, 2 → 4s, 3 → 8s, ... 最大 30s ``` + +### 跨域处理 + +采用 **Nginx 同源反代**方案，前后端统一到同一域名，浏览器层面不存在跨域问题。 + +**生产环境**：Nginx 将 `/`（前端）、`/api/*`（REST）、`/ws`（WebSocket）统一反代到同一域名，详见 `02-系统架构.md` 部署架构章节。 + +**开发环境**：Vite 内置代理，前端 :5173 的 `/api` 和 `/ws` 请求代理到后端 :8080： + +```typescript +// frontend/vite.config.ts +server: { + proxy: { + "/api": "http://localhost:8080", + "/ws": { target: "ws://localhost:8080", ws: true }, + }, +}, +``` + +**Go 后端 WebSocket CheckOrigin**：生产环境 Nginx 同源，`CheckOrigin` 可保持默认（拒绝跨域）。开发环境由 Vite proxy 转发，不存在跨域。因此后端无需配置 CORS 中间件，`CheckOrigin` 保持 gorilla/websocket 默认值即可。 + +> 如果未来需要支持第三方客户端直连（如移动端），再按需添加 CORS 中间件和 `CheckOrigin` 白名单。 diff --git a/docs/PLAN_BACKEND.md b/docs/PLAN_BACKEND.md new file mode 100644 index 0000000..a069e09 --- /dev/null +++ b/docs/PLAN_BACKEND.md @@ -0,0 +1,221 @@ +# CamTalk 后端完善计划 + +## Context + +后端当前是一个骨架：`main.go` 启动 Gin 服务器，`ws/handler.go` 实现了 WebSocket 连接生命周期和消息分发，`models/models.go` 定义了所有协议消息类型，`config/config.go` 实现了 Viper 配置加载。但所有业务逻辑都是 TODO 桩——没有 Session Manager、没有 AI 服务客户端、没有编排层、没有日志/错误工具、没有测试。前端已基本完成，正在等待后端提供真实的 AI 管道。 + +**目标**：按设计文档（`docs/03-接口文档.md` 为最高依据）逐步填充所有业务模块，使端到端的 STT → LLM → TTS 流式管道可用。 + +--- + +## 分阶段实施 + +### Phase 1：基础设施（logger、errors、config 接入、graceful shutdown） + +**目标**：为后续模块提供日志、错误码、配置等基础能力，替换 `main.go` 中的硬编码值。 + +| # | 任务 | 文件 | 说明 | +|---|------|------|------| +| 1.1 | 实现 Zap 日志封装 | `internal/logger/logger.go` | 提供 `Init(level, format)` 和全局 `*zap.SugaredLogger`，替换所有 `log.Printf` | +| 1.2 | 实现错误码常量 + WS 错误发送工具 | `internal/errors/codes.go` | 10 个错误码常量 + `SendWSError(client, code, requestID, err)` | +| 1.3 | main.go 接入 config.Load() | `cmd/server/main.go` | 用 `cfg.Server.Host:Port` 替换硬编码 `:8080`，初始化 logger | +| 1.4 | 添加 graceful shutdown | `cmd/server/main.go` | `signal.NotifyContext` + `http.Server.Shutdown`，10s drain | +| 1.5 | 添加 .gitignore | `backend/.gitignore` | 排除 `server` 二进制、`.env`、`tmp/` | + +> **CORS**：不在此处实现，生产环境由 Nginx 反向代理统一处理跨域。 + +--- + +### Phase 2：Session Manager + +**目标**：实现会话生命周期管理，让 WS handler 能追踪会话、存储对话历史。 + +| # | 任务 | 文件 | 说明 | +|---|------|------|------| +| 2.1 | 定义 SessionManager 接口 | `internal/session/manager.go` | 方法：`Create`, `Get`, `UpdateConfig`, `GetHistory`, `AppendMessage`, `SetActiveRequest`, `ClearActiveRequest`, `Touch`, `Destroy` | +| 2.2 | 实现内存版 SessionManager | `internal/session/memory.go` | `sync.RWMutex` + `map[string]*sessionEntry`，TTL 30 分钟，历史上限 20 条 | +| 2.3 | 实现 Redis 版 SessionManager | `internal/session/redis.go` | `session:{id}:meta` Hash + `session:{id}:history` List，TTL 刷新，选配 | +| 2.4 | 编写 Session Manager 测试 | `internal/session/memory_test.go` | 覆盖 Create/Get/Expire/Destroy/AppendMessage/History 上限 | +| 2.5 | WS handler 接入 SessionManager | `internal/ws/handler.go` | `ServeWS` 接收 `session.Manager` 参数；`connected` 消息后创建会话；`query` 时 Touch + SetActiveRequest；`config` 时 UpdateConfig；断开时不销毁（自然过期） | + +--- + +### Phase 3：AI 服务层接口 + 实现 + +**目标**：定义并实现三个 AI 服务客户端，每个服务一个独立包。 + +| # | 任务 | 文件 | 说明 | +|---|------|------|------| +| **3a. STT** | | | | +| 3.1 | STT 接口定义 | `internal/ai/stt/stt.go` | `Service` 接口：`Recognize(ctx, audio []byte, opts Options) (string, error)`。`Options`: Encoding, SampleRate, Language | +| 3.2 | Deepgram 实现 | `internal/ai/stt/deepgram.go` | WebSocket 连接 `wss://api.deepgram.com/v1/listen`，发送 PCM 音频，接收转录结果，5s 超时 | +| 3.3 | STT 测试（mock） | `internal/ai/stt/deepgram_test.go` | httptest/WebSocket mock，验证连接、发送、超时 | +| **3b. LLM** | | | | +| 3.4 | LLM 接口定义 | `internal/ai/llm/llm.go` | `Service` 接口：`ChatStream(ctx, req Request) (<-chan Chunk, error)`。`Request`: Image, Text, History, Language。`Chunk`: Delta, Done, TokensUsed, Model | +| 3.5 | OpenAI 实现 | `internal/ai/llm/openai.go` | `POST /v1/chat/completions` + `stream: true`，SSE 解析，10s 超时，image 以 `data:image/jpeg;base64,...` 传入 | +| 3.6 | System Prompt 定义 | `internal/ai/llm/prompt.go` | 中文视觉助手提示词，根据 Language/DetailLevel 动态构建 | +| 3.7 | LLM 测试（mock） | `internal/ai/llm/openai_test.go` | httptest mock SSE 流，验证流式解析、超时、错误处理 | +| **3c. TTS** | | | | +| 3.8 | TTS 接口定义 | `internal/ai/tts/tts.go` | `Service` 接口：`SynthesizeStream(ctx, textStream <-chan string, opts Options) (<-chan Chunk, error)`。`Chunk`: Audio []byte, IsLast | +| 3.9 | OpenAI 实现 | `internal/ai/tts/openai.go` | `POST /v1/audio/speech` 模型 `tts-1`，逐句发送，返回 MP3 流，5s/句超时 | +| 3.10 | TTS 测试（mock） | `internal/ai/tts/openai_test.go` | httptest mock，验证逐句合成、超时 | + +--- + +### Phase 4：AI Orchestrator（核心编排） + +**目标**：实现 STT → LLM → TTS 流式并行管道，这是后端最关键的业务逻辑。 + +| # | 任务 | 文件 | 说明 | +|---|------|------|------| +| 4.1 | Orchestrator 接口 | `internal/orchestrator/orchestrator.go` | `ProcessQuery(ctx, sessionID, req, history, sender)` — 接收查询并执行管道 | +| 4.2 | Sender 接口 | `internal/orchestrator/sender.go` | 抽象 WS 推送：`SendSTTResult`, `SendLLMChunk`, `SendLLMDone`, `SendTTSAudio`, `SendError`，便于测试 | +| 4.3 | 管道实现 | `internal/orchestrator/pipeline.go` | ① `stt.Recognize()` → 发送 `stt_result` ② `llm.ChatStream()` 并行消费 token → 发送 `llm_chunk` + 句子切分 → channel ③ `tts.SynthesizeStream()` 从 channel 读取 → 发送 `tts_audio` ④ 流结束 → 发送 `llm_done` | +| 4.4 | 句子切分器 | `internal/orchestrator/splitter.go` | 按 `。！？\n.!?` 切分，buffer size 4 channel | +| 4.5 | 错误降级 | 同上文件 | STT 失败→STT_ERROR+abort；LLM 超时→LLM_TIMEOUT；TTS 失败→静默跳过 | +| 4.6 | Interrupt 支持 | 同上文件 | context cancel 触发所有流中止 | +| 4.7 | Orchestrator 测试 | `internal/orchestrator/pipeline_test.go` | mock 三个 AI service + mock sender，验证完整流程、中断、错误降级 | + +--- + +### Phase 5：WS Handler 完整接入 + +**目标**：将 Session Manager + Orchestrator 串入 WebSocket handler，实现端到端消息处理。 + +| # | 任务 | 文件 | 说明 | +|---|------|------|------| +| 5.1 | Client 扩展 | `internal/ws/handler.go` | 添加 `session.Manager`、`orchestrator.Orchestrator`、`context.CancelFunc`（用于 interrupt） | +| 5.2 | query 处理 | 同上 | 解码 audio Base64 → `stt.Recognize` 的输入；Touch 会话；设置 active request；启动 `orchestrator.ProcessQuery` goroutine | +| 5.3 | config 处理 | 同上 | 调用 `session.UpdateConfig()` | +| 5.4 | interrupt 处理 | 同上 | 查找 active request 的 cancel func，调用 `cancel()`，ClearActiveRequest | +| 5.5 | Disconnect 处理 | 同上 | 取消当前活跃请求（如有），不销毁会话 | + +--- + +### Phase 6：REST API 补全 + +**目标**：补全设计文档中的 REST 端点。 + +| # | 任务 | 文件 | 说明 | +|---|------|------|------| +| 6.1 | Session 路由 | `internal/api/session.go` | `POST /api/sessions` 创建会话，`DELETE /api/sessions/:id` 销毁会话 | +| 6.2 | Health 更新 | `cmd/server/main.go` | 从 SessionManager 获取 `active_sessions` 真实值 | +| 6.3 | 路由注册 | `cmd/server/main.go` | 统一注册 REST + WS 路由，注入依赖 | + +--- + +### Phase 7：Rate Limiter + Model Router（可选/MVP 后） + +**目标**：防止滥用 + 智能模型选择，MVP 可简化或跳过。 + +| # | 任务 | 文件 | 说明 | +|---|------|------|------| +| 7.1 | 令牌桶 Rate Limiter | `internal/middleware/ratelimit.go` | `golang.org/x/time/rate` 或自实现，按 session ID 限流 | +| 7.2 | Rate Limiter 中间件 | `internal/middleware/ratelimit.go` | 在 WS query 路径上检查，超限返回 `RATE_LIMITED` | +| 7.3 | Model Router | `internal/ai/router.go` | 规则引擎：简单识别→GPT-4o-mini，深度分析→GPT-4o，暂不实现 o1 | + +--- + +### Phase 8：集成测试 + 文档同步 + +| # | 任务 | 文件 | 说明 | +|---|------|------|------| +| 8.1 | WS 集成测试 | `internal/ws/handler_test.go` | 启动 Gin test server + gorilla websocket client，验证完整 query→stt_result→llm_chunk→llm_done→tts_audio 流程 | +| 8.2 | 文档同步 | `docs/03-接口文档.md` | 代码实现与文档有偏差时更新文档 | +| 8.3 | go.sum 清理 | `backend/` | `go mod tidy` 清理无用依赖 | + +--- + +## 关键文件清单 + +``` +backend/ + cmd/server/main.go ← Phase 1.3, 1.4, 1.5, 6.2, 6.3 + internal/ + config/config.go ← 已完成，Phase 1.3 接入 + logger/logger.go ← Phase 1.1（新建） + errors/codes.go ← Phase 1.2（新建） + models/models.go ← 已完成，可能小幅扩展 + session/ + manager.go ← Phase 2.1（新建） + memory.go ← Phase 2.2（新建） + redis.go ← Phase 2.3（新建） + memory_test.go ← Phase 2.4（新建） + ai/ + stt/ + stt.go ← Phase 3.1（新建） + deepgram.go ← Phase 3.2（新建） + deepgram_test.go ← Phase 3.3（新建） + llm/ + llm.go ← Phase 3.4（新建） + openai.go ← Phase 3.5（新建） + prompt.go ← Phase 3.6（新建） + openai_test.go ← Phase 3.7（新建） + tts/ + tts.go ← Phase 3.8（新建） + openai.go ← Phase 3.9（新建） + openai_test.go ← Phase 3.10（新建） + router.go ← Phase 7.3（新建） + orchestrator/ + orchestrator.go ← Phase 4.1（新建） + sender.go ← Phase 4.2（新建） + pipeline.go ← Phase 4.3, 4.4, 4.5, 4.6（新建） + pipeline_test.go ← Phase 4.7（新建） + api/ + session.go ← Phase 6.1（新建） + middleware/ + ratelimit.go ← Phase 7.1, 7.2（新建） + ws/ + handler.go ← Phase 5.1-5.5（修改） + handler_test.go ← Phase 8.1（新建） +``` + +--- + +## 新增依赖 + +| 包 | 用途 | Phase | +|----|------|-------| +| `go.uber.org/zap` | 结构化日志 | 1 | +| `github.com/redis/go-redis/v9` | Redis 客户端 | 2.3 | +| `github.com/gorilla/websocket` | 已有，Deepgram WS 也复用 | 3.2 | + +--- + +## 执行顺序与依赖关系 + +``` +Phase 1 (基础设施) + ↓ +Phase 2 (Session Manager) + ↓ +Phase 3 (AI 服务层) ← 可与 Phase 2 并行开发 + ↓ +Phase 4 (Orchestrator) ← 依赖 Phase 2 + 3 + ↓ +Phase 5 (WS Handler 接入) ← 依赖 Phase 4 + ↓ +Phase 6 (REST API) ← 依赖 Phase 2 + ↓ +Phase 7 (Rate Limiter + Router) ← 独立，可推后 + ↓ +Phase 8 (集成测试 + 文档) +``` + +--- + +## 验证方案 + +1. **单元测试**：每个模块独立测试，mock 外部依赖（AI API、Redis） +2. **集成测试**：`httptest` 启动 Gin server，用 gorilla/websocket 客户端模拟完整 query 流程 +3. **端到端手动测试**：启动后端 → 打开前端 → 摄像头+麦克风对话 → 验证 stt_result / llm_chunk / tts_audio 消息流 +4. **go vet + go test ./...** 通过 + +--- + +## 设计文档参考 + +- 接口规范（最高优先级）：`docs/03-接口文档.md` +- 系统架构：`docs/02-系统架构.md` +- 技术选型：`docs/04-技术选型.md` +- 成本控制：`docs/08-成本控制.md` diff --git a/frontend/src/App.css b/frontend/src/App.css index 4cf2215..330c48c 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -1,19 +1,36 @@ /* ============================================================ - CamTalk — 主应用样式 + CamTalk — Web 端应用样式 + 双栏布局：左侧视频面板 + 右侧聊天面板 ============================================================ */ :root { --color-primary: #2563eb; --color-primary-hover: #1d4ed8; - --color-bg: #0f172a; - --color-surface: #1e293b; - --color-text: #f1f5f9; - --color-text-muted: #94a3b8; - --color-border: #334155; + --color-bg: #0b0f1a; + --color-surface: #151b2b; + --color-surface-2: #1c2438; + --color-text: #e2e8f0; + --color-text-muted: #64748b; + --color-border: #1e293b; --color-success: #22c55e; --color-warning: #f59e0b; --color-error: #ef4444; - --radius: 8px; + --radius: 10px; + --radius-sm: 6px; +} + +[data-theme="light"] { + --color-primary: #2563eb; + --color-primary-hover: #1d4ed8; + --color-bg: #f1f5f9; + --color-surface: #ffffff; + --color-surface-2: #f8fafc; + --color-text: #0f172a; + --color-text-muted: #64748b; + --color-border: #e2e8f0; + --color-success: #16a34a; + --color-warning: #d97706; + --color-error: #dc2626; } * { @@ -22,205 +39,634 @@ box-sizing: border-box; } +html, body, #root { + height: 100%; +} + body { - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", sans-serif; background: var(--color-bg); color: var(--color-text); - min-height: 100vh; } +/* ---- App Shell ---- */ + .app { display: flex; flex-direction: column; - min-height: 100vh; - max-width: 900px; - margin: 0 auto; - padding: 0 16px; + height: 100vh; + overflow: hidden; } /* ---- Header ---- */ -.app-header { +.header { display: flex; align-items: center; justify-content: space-between; - padding: 16px 0; - border-bottom: 1px solid var(--color-border); -} - -.app-header h1 { - font-size: 1.5rem; - font-weight: 700; -} - -.status { - font-size: 0.85rem; - padding: 4px 12px; - border-radius: 12px; + padding: 0 20px; + height: 48px; background: var(--color-surface); -} - -.status--connected { - color: var(--color-success); - border: 1px solid var(--color-success); -} - -.status--connecting { - color: var(--color-warning); - border: 1px solid var(--color-warning); -} - -.status--disconnected { - color: var(--color-text-muted); - border: 1px solid var(--color-border); -} - -/* ---- Main ---- */ - -.app-main { - flex: 1; - display: flex; - flex-direction: column; - gap: 16px; - padding: 16px 0; - overflow: hidden; -} - -.video-section { - position: relative; + border-bottom: 1px solid var(--color-border); flex-shrink: 0; } -.vad-indicator { - position: absolute; - bottom: 12px; - left: 12px; - background: rgba(0, 0, 0, 0.7); - color: var(--color-success); - padding: 6px 12px; - border-radius: var(--radius); - font-size: 0.85rem; - animation: pulse 1.5s ease-in-out infinite; -} - -.vad-indicator--loading { - color: var(--color-warning); -} - -.vad-indicator--error { - color: var(--color-error); - animation: none; -} - -@keyframes pulse { - 0%, 100% { opacity: 1; } - 50% { opacity: 0.6; } -} - -.chat-section { - flex: 1; +.header__left { display: flex; - flex-direction: column; + align-items: baseline; gap: 12px; - overflow: hidden; } -/* ---- Video Preview ---- */ - -.video-preview { - position: relative; - width: 100%; - aspect-ratio: 4 / 3; - background: var(--color-surface); - border-radius: var(--radius); - overflow: hidden; +.header__title { + font-size: 1.2rem; + font-weight: 700; + color: var(--color-text); } -.video-preview__video { - width: 100%; - height: 100%; - object-fit: cover; -} - -.video-preview__placeholder { - position: absolute; - inset: 0; - display: flex; - align-items: center; - justify-content: center; +.header__subtitle { + font-size: 0.8rem; color: var(--color-text-muted); } -/* ---- Chat Panel ---- */ +.header__right { + display: flex; + align-items: center; + gap: 12px; +} -.chat-panel { +.header__stats { + font-size: 0.75rem; + color: var(--color-text-muted); + padding: 4px 10px; + background: var(--color-surface-2); + border-radius: 12px; +} + +.badge { + font-size: 0.75rem; + padding: 3px 10px; + border-radius: 12px; + font-weight: 500; +} + +.badge--connected { + color: var(--color-success); + background: rgba(34, 197, 94, 0.1); + border: 1px solid rgba(34, 197, 94, 0.3); +} + +.badge--connecting { + color: var(--color-warning); + background: rgba(245, 158, 11, 0.1); + border: 1px solid rgba(245, 158, 11, 0.3); +} + +.badge--disconnected { + color: var(--color-text-muted); + background: var(--color-surface-2); + border: 1px solid var(--color-border); +} + +.btn-icon { + background: none; + border: none; + font-size: 1.1rem; + cursor: pointer; + padding: 6px; + border-radius: var(--radius-sm); + transition: background 0.15s; + color: var(--color-text-muted); +} + +.btn-icon:hover { + background: var(--color-surface-2); + color: var(--color-text); +} + +.lang-group { + display: flex; + gap: 2px; + background: var(--color-surface-2); + border-radius: var(--radius-sm); + padding: 2px; +} + +.lang-btn { + background: none; + border: none; + color: var(--color-text-muted); + font-size: 0.72rem; + font-weight: 600; + padding: 4px 10px; + border-radius: 4px; + cursor: pointer; + transition: all 0.15s; +} + +.lang-btn:hover { + color: var(--color-text); +} + +.lang-btn--active { + background: var(--color-primary); + color: white; +} + +/* ---- Workspace (双栏) ---- */ + +.workspace { + flex: 1; + display: flex; + overflow: hidden; +} + +/* ---- 左侧视频面板 ---- */ + +.video-panel { + width: 420px; + flex-shrink: 0; display: flex; flex-direction: column; + padding: 16px; gap: 12px; - overflow-y: auto; +} + +.video-container { + position: relative; + aspect-ratio: 4 / 3; + background: var(--color-surface-2); + border-radius: var(--radius); + overflow: hidden; + border: 1px solid var(--color-border); +} + +.video-container .video-preview { + width: 100%; + height: 100%; + border-radius: 0; +} + +.video-container .video-preview__video { + width: 100%; + height: 100%; + object-fit: cover; + background: #1a1a2e; +} + +.video-placeholder { + position: absolute; + inset: 0; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 8px; + color: var(--color-text-muted); + font-size: 0.82rem; +} + +.video-placeholder__icon { + font-size: 2.4rem; + opacity: 0.2; + margin-bottom: 4px; +} + +.video-indicator { + position: absolute; + bottom: 16px; + left: 16px; + background: rgba(0, 0, 0, 0.75); + backdrop-filter: blur(8px); + color: var(--color-success); + padding: 6px 14px; + border-radius: var(--radius-sm); + font-size: 0.8rem; + font-weight: 500; + animation: pulse 1.5s ease-in-out infinite; +} + +.live-badge { + position: absolute; + top: 12px; + left: 12px; + display: flex; + align-items: center; + gap: 6px; + background: rgba(0, 0, 0, 0.75); + backdrop-filter: blur(8px); + color: white; + padding: 4px 12px; + border-radius: var(--radius-sm); + font-size: 0.75rem; + font-weight: 600; + letter-spacing: 0.5px; +} + +.live-badge__dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--color-success); + animation: pulse 1.5s ease-in-out infinite; +} + +.video-indicator--loading { color: var(--color-warning); } +.video-indicator--audio { left: auto; right: 16px; color: var(--color-primary); } +.video-indicator--error { color: var(--color-error); animation: none; } + +.detail-badge { + position: absolute; + top: 12px; + right: 12px; + background: rgba(37, 99, 235, 0.85); + backdrop-filter: blur(8px); + color: white; + padding: 3px 10px; + border-radius: var(--radius-sm); + font-size: 0.7rem; + font-weight: 700; + letter-spacing: 0.5px; +} + +.observation-badge { + position: absolute; + top: 12px; + left: 12px; + background: rgba(34, 197, 94, 0.85); + backdrop-filter: blur(8px); + color: white; + padding: 3px 10px; + border-radius: var(--radius-sm); + font-size: 0.7rem; + font-weight: 700; + animation: pulse 2s ease-in-out infinite; +} + +.video-overlay { + position: absolute; + inset: 0; + background: rgba(0, 0, 0, 0.35); + display: flex; + align-items: center; + justify-content: center; + backdrop-filter: blur(2px); +} + +.video-overlay__spinner { + width: 36px; + height: 36px; + border: 3px solid rgba(255, 255, 255, 0.2); + border-top-color: white; + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +/* ---- 视频控制栏 ---- */ + +.video-controls { + flex-shrink: 0; + display: flex; + justify-content: center; + padding: 4px 0; +} + +.video-controls__row { + display: flex; + gap: 6px; + flex-wrap: wrap; + justify-content: center; +} + +.btn--ctrl { + padding: 6px 12px; + font-size: 0.78rem; + border-radius: var(--radius-sm); + border: 1px solid var(--color-border); + background: var(--color-surface-2); + color: var(--color-text-muted); + cursor: pointer; + transition: all 0.15s; + white-space: nowrap; +} + +.btn--ctrl:hover { + background: var(--color-border); + color: var(--color-text); +} + +.btn--ctrl-on { + background: rgba(34, 197, 94, 0.15); + color: var(--color-success); + border-color: rgba(34, 197, 94, 0.3); +} + +.btn--ctrl-off { + background: rgba(239, 68, 68, 0.1); + color: var(--color-error); + border-color: rgba(239, 68, 68, 0.2); +} + +.btn--speaking { + animation: micPulse 0.8s ease-in-out infinite; + box-shadow: 0 0 12px rgba(34, 197, 94, 0.5); +} + +@keyframes micPulse { + 0%, 100% { box-shadow: 0 0 8px rgba(34, 197, 94, 0.4); transform: scale(1); } + 50% { box-shadow: 0 0 16px rgba(34, 197, 94, 0.8); transform: scale(1.05); } +} + +/* ---- 右侧聊天面板 ---- */ + +.chat-panel-wrapper { flex: 1; + display: flex; + flex-direction: column; + background: var(--color-surface); + border-left: 1px solid var(--color-border); + min-width: 0; +} + +.chat-panel-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + font-weight: 600; + font-size: 0.85rem; + border-bottom: 1px solid var(--color-border); + flex-shrink: 0; + color: var(--color-text-muted); +} + +.chat-panel-header__mode { + font-size: 0.7rem; + font-weight: 500; + color: var(--color-success); + background: rgba(34, 197, 94, 0.1); + padding: 2px 8px; + border-radius: 10px; +} + +.chat-panel-body { + flex: 1; + overflow: hidden; + display: flex; + flex-direction: column; +} + +/* ---- Chat Panel (覆盖子组件样式) ---- */ + +.chat-panel { + flex: 1; + overflow-y: auto; + padding: 12px 16px; + display: flex; + flex-direction: column; + gap: 10px; } .chat-panel--empty { display: flex; align-items: center; justify-content: center; - padding: 48px 0; + flex: 1; color: var(--color-text-muted); + font-size: 0.85rem; } .chat-message { - padding: 12px 16px; - border-radius: var(--radius); - background: var(--color-surface); + padding: 10px 14px; + border-radius: var(--radius-sm); + background: var(--color-surface-2); + max-width: 92%; } .chat-message--user { - border-left: 3px solid var(--color-primary); + border-left: 2px solid var(--color-primary); + align-self: flex-end; + background: rgba(37, 99, 235, 0.08); } .chat-message--assistant { - border-left: 3px solid var(--color-success); + border-left: 2px solid var(--color-success); } .chat-message--streaming { - opacity: 0.8; + opacity: 0.85; } .chat-message__role { - font-size: 0.75rem; + font-size: 0.65rem; font-weight: 600; text-transform: uppercase; + letter-spacing: 0.5px; color: var(--color-text-muted); - margin-bottom: 4px; + margin-bottom: 3px; } .chat-message__content { - font-size: 0.95rem; - line-height: 1.5; + font-size: 0.82rem; + line-height: 1.55; white-space: pre-wrap; } -/* ---- Footer ---- */ - -.app-footer { - display: flex; - gap: 12px; - justify-content: center; - padding: 16px 0; - border-top: 1px solid var(--color-border); +.chat-message__meta { + margin-top: 6px; + font-size: 0.65rem; + color: var(--color-text-muted); } +/* ---- Streaming Cursor ---- */ + +.cursor { + display: inline; + animation: blink 0.8s step-end infinite; + color: var(--color-success); +} + +/* ---- System Message ---- */ + +.system-message { + text-align: center; + padding: 8px 16px; + margin: 0 20px; + border-radius: var(--radius-sm); + font-size: 0.8rem; +} + +.system-message--warning { + background: rgba(245, 158, 11, 0.1); + color: var(--color-warning); + border: 1px solid rgba(245, 158, 11, 0.2); +} + +/* ---- Drawer (Config Panel) ---- */ + +.drawer-overlay { + position: fixed; + inset: 0; + z-index: 100; + background: rgba(0, 0, 0, 0.4); + backdrop-filter: blur(2px); + animation: fadeIn 0.2s ease; +} + +.drawer { + position: absolute; + top: 0; + right: 0; + bottom: 0; + width: 320px; + background: var(--color-surface); + border-left: 1px solid var(--color-border); + display: flex; + flex-direction: column; + animation: slideInRight 0.25s ease; + box-shadow: -8px 0 24px rgba(0, 0, 0, 0.3); +} + +.drawer__header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 14px 16px; + border-bottom: 1px solid var(--color-border); + flex-shrink: 0; +} + +.drawer__title { + font-weight: 600; + font-size: 0.9rem; +} + +.drawer__close { + background: none; + border: none; + color: var(--color-text-muted); + cursor: pointer; + font-size: 1rem; + padding: 4px 8px; + border-radius: var(--radius-sm); + transition: all 0.15s; +} + +.drawer__close:hover { + background: var(--color-surface-2); + color: var(--color-text); +} + +.drawer__body { + flex: 1; + overflow-y: auto; + padding: 16px; +} + +/* ---- Config Group ---- */ + +.config-group { + margin-bottom: 20px; +} + +.config-group__title { + font-size: 0.7rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.8px; + color: var(--color-text-muted); + margin-bottom: 12px; + padding-bottom: 8px; + border-bottom: 1px solid var(--color-border); +} + +.config-row { + display: flex; + align-items: center; + justify-content: space-between; + padding: 10px 0; + font-size: 0.82rem; + color: var(--color-text); + cursor: pointer; +} + +.config-row__info { + display: flex; + flex-direction: column; + gap: 2px; +} + +.config-row__label { + font-weight: 500; + color: var(--color-text); +} + +.config-row__desc { + font-size: 0.72rem; + color: var(--color-text-muted); +} + +.config-row select, +.config-row input[type="checkbox"] { + background: var(--color-surface-2); + color: var(--color-text); + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + padding: 5px 10px; + font-size: 0.8rem; + cursor: pointer; +} + +.config-row input[type="checkbox"] { + width: 18px; + height: 18px; + cursor: pointer; +} + +/* ---- Toast ---- */ + +.toast-container { + position: fixed; + top: 68px; + right: 20px; + z-index: 1000; + display: flex; + flex-direction: column; + gap: 8px; +} + +.toast { + padding: 10px 16px; + border-radius: var(--radius-sm); + font-size: 0.82rem; + cursor: pointer; + animation: slideIn 0.3s ease-out; + max-width: 320px; + backdrop-filter: blur(8px); +} + +.toast--error { background: rgba(239, 68, 68, 0.9); color: white; } +.toast--warning { background: rgba(245, 158, 11, 0.9); color: #000; } +.toast--info { background: rgba(37, 99, 235, 0.9); color: white; } + /* ---- Buttons ---- */ .btn { - padding: 10px 24px; + padding: 8px 20px; border: none; - border-radius: var(--radius); - font-size: 0.95rem; + border-radius: var(--radius-sm); + font-size: 0.85rem; font-weight: 600; cursor: pointer; - transition: background 0.2s; + transition: all 0.15s; + display: inline-flex; + align-items: center; + gap: 6px; +} + +.btn--lg { + padding: 12px 32px; + font-size: 0.95rem; + border-radius: var(--radius); } .btn--primary { @@ -233,7 +679,7 @@ body { } .btn--secondary { - background: var(--color-surface); + background: var(--color-surface-2); color: var(--color-text); border: 1px solid var(--color-border); } @@ -247,16 +693,35 @@ body { color: #000; } -.btn--warning:hover { - opacity: 0.9; +.btn--warning:hover { opacity: 0.9; } + +.btn--danger { + background: rgba(239, 68, 68, 0.15); + color: var(--color-error); + border: 1px solid rgba(239, 68, 68, 0.3); } -/* ---- Streaming Cursor ---- */ +.btn--danger:hover { + background: rgba(239, 68, 68, 0.25); +} -.cursor { - display: inline; - animation: blink 0.8s step-end infinite; - color: var(--color-success); +.btn--active { + background: var(--color-primary); + color: white; + box-shadow: 0 0 12px rgba(37, 99, 235, 0.4); + animation: pulseBtn 2s ease-in-out infinite; +} + +/* ---- Animations ---- */ + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.6; } +} + +@keyframes pulseBtn { + 0%, 100% { box-shadow: 0 0 12px rgba(37, 99, 235, 0.4); } + 50% { box-shadow: 0 0 20px rgba(37, 99, 235, 0.7); } } @keyframes blink { @@ -264,97 +729,40 @@ body { 50% { opacity: 0; } } -/* ---- Message Meta ---- */ - -.chat-message__meta { - margin-top: 6px; - font-size: 0.75rem; - color: var(--color-text-muted); -} - -/* ---- System Message ---- */ - -.system-message { - text-align: center; - padding: 8px 16px; - border-radius: var(--radius); - font-size: 0.85rem; -} - -.system-message--warning { - background: rgba(245, 158, 11, 0.15); - color: var(--color-warning); - border: 1px solid rgba(245, 158, 11, 0.3); -} - -/* ---- Toast ---- */ - -.toast-container { - position: fixed; - top: 16px; - right: 16px; - z-index: 1000; - display: flex; - flex-direction: column; - gap: 8px; -} - -.toast { - padding: 10px 16px; - border-radius: var(--radius); - font-size: 0.85rem; - cursor: pointer; - animation: slideIn 0.3s ease-out; - max-width: 320px; -} - -.toast--error { - background: rgba(239, 68, 68, 0.9); - color: white; -} - -.toast--warning { - background: rgba(245, 158, 11, 0.9); - color: #000; -} - -.toast--info { - background: rgba(37, 99, 235, 0.9); - color: white; -} - -@keyframes slideIn { - from { - opacity: 0; - transform: translateX(20px); - } - to { - opacity: 1; - transform: translateX(0); - } -} - -/* ---- Video Overlay ---- */ - -.video-overlay { - position: absolute; - inset: 0; - background: rgba(0, 0, 0, 0.3); - display: flex; - align-items: center; - justify-content: center; - border-radius: var(--radius); -} - -.video-overlay__spinner { - width: 32px; - height: 32px; - border: 3px solid rgba(255, 255, 255, 0.3); - border-top-color: white; - border-radius: 50%; - animation: spin 0.8s linear infinite; -} - @keyframes spin { to { transform: rotate(360deg); } } + +@keyframes slideIn { + from { opacity: 0; transform: translateX(20px); } + to { opacity: 1; transform: translateX(0); } +} + +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +@keyframes slideInRight { + from { transform: translateX(100%); } + to { transform: translateX(0); } +} + +/* ---- Scrollbar ---- */ + +::-webkit-scrollbar { + width: 6px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: var(--color-border); + border-radius: 3px; +} + +::-webkit-scrollbar-thumb:hover { + background: var(--color-text-muted); +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 9bdb7b4..ffd567b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,97 +1,235 @@ // ============================================================ -// CamTalk — 主应用组件 +// CamTalk — 主应用组件（Web 端双栏布局） // ============================================================ +import { useCallback, useEffect, useRef, useState } from "react"; import { useVisionSession } from "./hooks/useVisionSession"; import { VideoPreview } from "./components/VideoPreview"; import { ChatPanel } from "./components/ChatPanel"; +import { ConfigPanel } from "./components/ConfigPanel"; import { ToastContainer } from "./components/Toast"; +import { loadTheme, saveTheme } from "./lib/storage"; +import type { Theme } from "./types"; import "./App.css"; function App() { + const [showConfig, setShowConfig] = useState(false); + const [theme, setTheme] = useState(loadTheme); + const [elapsed, setElapsed] = useState(0); + const timerRef = useRef | null>(null); + + // 切换主题时更新 的 data-theme 属性 + useEffect(() => { + document.documentElement.setAttribute("data-theme", theme); + }, [theme]); + + const handleThemeChange = useCallback((t: Theme) => { + setTheme(t); + saveTheme(t); + }, []); + + const formatTime = (s: number) => { + const m = Math.floor(s / 60); + const sec = s % 60; + return `${m.toString().padStart(2, "0")}:${sec.toString().padStart(2, "0")}`; + }; + const { messages, currentReply, isProcessing, + isAudioPlaying, isSpeaking, isVADReady, vadError, connectionStatus, videoRef, stream, + config, + updateConfig, + stats, + mode, + isObserving, + toggleMode, startSession, stopSession, interrupt, + isCameraOn, + isMicOn, + toggleCamera, + toggleMic, } = useVisionSession(); const isConnected = connectionStatus === "connected"; + // 连接计时器 + const elapsedRef = useRef(0); + useEffect(() => { + if (isConnected) { + elapsedRef.current = 0; + setElapsed(0); // eslint-disable-line react-hooks/set-state-in-effect -- 连接时重置计时器 + } + }, [isConnected]); + + useEffect(() => { + if (!isConnected) { + if (timerRef.current) { + clearInterval(timerRef.current); + timerRef.current = null; + } + return; + } + const id = setInterval(() => { + elapsedRef.current += 1; + setElapsed(elapsedRef.current); + }, 1000); + timerRef.current = id; + return () => clearInterval(id); + }, [isConnected]); + return (
-
-

CamTalk

- - {isConnected - ? "已连接" - : connectionStatus === "connecting" - ? "连接中..." - : "未连接"} - + {/* ---- 顶部导航栏 ---- */} +
+
+

CamTalk

+ AI 视觉对话助手 +
+
+ {isConnected && (stats.queryCount > 0 || stats.totalTokens > 0) && ( + + {stats.queryCount} 次请求 · {stats.totalTokens} tokens + + )} + + {isConnected ? "已连接" : connectionStatus === "connecting" ? "连接中..." : "未连接"} + + +
-
-
- - {isSpeaking &&
🎤 正在聆听...
} - {isConnected && !isVADReady && !vadError && ( -
- 正在初始化语音检测... -
- )} - {vadError && ( -
- ⚠️ {vadError} -
- )} - {isProcessing && ( -
-
-
- )} -
+ {showConfig && ( + setShowConfig(false)} + /> + )} -
- {connectionStatus === "disconnected" && messages.length > 0 && ( -
- 连接已断开，正在重连... -
- )} - -
-
- -
- {!isConnected ? ( - - ) : ( - <> - - {isProcessing && ( - + {/* ---- 主体：左侧视频 + 右侧聊天 ---- */} +
+ {/* 左侧：视频预览 + 控制栏 */} +
+
+ + {isConnected && ( +
+ + LIVE {formatTime(elapsed)} +
)} - - )} -
+ {config.detailLevel === "high" && ( +
HD
+ )} + {isObserving && ( +
👁️ 观察中
+ )} + {isSpeaking && ( +
🎤 正在聆听...
+ )} + {isAudioPlaying && config.ttsEnabled && ( +
🔊 正在播放...
+ )} + {isConnected && !isVADReady && !vadError && ( +
正在初始化语音检测...
+ )} + {vadError && ( +
⚠️ {vadError}
+ )} + {isProcessing && ( +
+
+
+ )} + {!isConnected && !stream && ( +
+ 📷 + 点击右侧按钮开始对话 +
+ )} +
+ + {/* 视频下方控制栏 */} +
+ {!isConnected ? ( + + ) : ( +
+ + + + {isProcessing && ( + + )} + +
+ )} +
+
+ + {/* 右侧：聊天面板 */} +
+
+ 对话 + {isConnected && mode === "observation" && ( + 观察模式 + )} +
+
+ {connectionStatus === "disconnected" && messages.length > 0 && ( +
+ 连接已断开，正在重连... +
+ )} + +
+
+ diff --git a/frontend/src/components/ConfigPanel/index.tsx b/frontend/src/components/ConfigPanel/index.tsx new file mode 100644 index 0000000..3844e6d --- /dev/null +++ b/frontend/src/components/ConfigPanel/index.tsx @@ -0,0 +1,92 @@ +// ============================================================ +// ConfigPanel — 右侧抽屉式配置面板 +// 职责：主题切换、TTS 开关、detail level 切换、语言选择 +// ============================================================ + +import type { SessionConfig, Theme } from "../../types"; + +interface ConfigPanelProps { + config: SessionConfig; + theme: Theme; + onUpdate: (partial: Partial) => void; + onThemeChange: (theme: Theme) => void; + onClose: () => void; +} + +export function ConfigPanel({ config, theme, onUpdate, onThemeChange, onClose }: ConfigPanelProps) { + return ( +
+
e.stopPropagation()}> +
+ 设置 + +
+ +
+
+
外观
+ + +
+ +
+
会话
+ + + + + + +
+
+
+
+ ); +} diff --git a/frontend/src/components/EdgeProcessor/index.tsx b/frontend/src/components/EdgeProcessor/index.tsx index 36978b7..a7c83a6 100644 --- a/frontend/src/components/EdgeProcessor/index.tsx +++ b/frontend/src/components/EdgeProcessor/index.tsx @@ -103,19 +103,61 @@ export function useVAD(options?: VADOptions) { return { isSpeaking, isReady, error, start, stop }; } -// ---- 关键帧检测（ONNX Runtime Web）---- +// ---- 关键帧检测 ---- -export function useKeyframeDetection() { - // TODO: 加载 ONNX 模型后设为 true - const isReady = false; +const DETECT_WIDTH = 160; +const DETECT_HEIGHT = 120; +const DIFF_THRESHOLD = 30; - const isKeyframe = useCallback( - (_currentFrame: ImageData, _previousFrame: ImageData): boolean => { - // TODO: 实现像素差异对比 - return true; // 暂时所有帧都视为关键帧 - }, - [], - ); +/** 离屏 canvas，用于降采样 */ +let offscreen: HTMLCanvasElement | null = null; - return { isReady, isKeyframe }; +function getOffscreen(): HTMLCanvasElement { + if (!offscreen) { + offscreen = document.createElement("canvas"); + offscreen.width = DETECT_WIDTH; + offscreen.height = DETECT_HEIGHT; + } + return offscreen; +} + +/** + * 将 video 元素缩放采样为 Uint8ClampedArray（RGBA） + * 返回 null 如果 video 未就绪 + */ +export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null { + if (video.readyState < 2) return null; + const canvas = getOffscreen(); + const ctx = canvas.getContext("2d"); + if (!ctx) return null; + ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT); + return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data; +} + +/** + * 对比两帧像素差异 + * @returns { isKeyframe: boolean, similarity: number } + */ +export function compareFrames( + prev: Uint8ClampedArray, + curr: Uint8ClampedArray, +): { isKeyframe: boolean; similarity: number } { + let diffSum = 0; + const len = Math.min(prev.length, curr.length); + const pixelCount = len / 4; + + for (let i = 0; i < len; i += 4) { + // 只比较 RGB，跳过 Alpha + diffSum += Math.abs(prev[i] - curr[i]); + diffSum += Math.abs(prev[i + 1] - curr[i + 1]); + diffSum += Math.abs(prev[i + 2] - curr[i + 2]); + } + + const avgDiff = diffSum / (pixelCount * 3); + const similarity = 1 - avgDiff / 255; + + return { + isKeyframe: avgDiff > DIFF_THRESHOLD, + similarity, + }; } diff --git a/frontend/src/hooks/useObservationMode.ts b/frontend/src/hooks/useObservationMode.ts new file mode 100644 index 0000000..e49eb62 --- /dev/null +++ b/frontend/src/hooks/useObservationMode.ts @@ -0,0 +1,88 @@ +// ============================================================ +// useObservationMode — 观察模式 Hook +// 职责：定时采帧 → 关键帧检测 → 画面变化时触发回调 +// 来源：docs/05-用户故事.md US-05（持续场景监控） +// ============================================================ + +import { useCallback, useEffect, useRef, useState } from "react"; +import { sampleFrame, compareFrames } from "../components/EdgeProcessor"; + +/** 画面变化显著阈值 */ +const CHANGE_THRESHOLD = 0.85; +/** 采样间隔（ms） */ +const SAMPLE_INTERVAL = 5000; + +export interface ObservationOptions { + /** 画面变化回调，携带当前帧的 DataURL */ + onChange?: (frameDataUrl: string) => void; +} + +export function useObservationMode(options?: ObservationOptions) { + const [isObserving, setIsObserving] = useState(false); + const intervalRef = useRef | null>(null); + const prevFrameRef = useRef(null); + const optionsRef = useRef(options); + + useEffect(() => { + optionsRef.current = options; + }, [options]); + + /** + * 启动观察模式 + * @param video 摄像头 video 元素 + * @param captureFrame 从 video 捕获 DataURL 的函数 + */ + const startObserving = useCallback( + (video: HTMLVideoElement | null, captureFrame: () => string | null) => { + if (!video) return; + + // 立即采一帧作为基准 + prevFrameRef.current = sampleFrame(video); + + intervalRef.current = setInterval(() => { + const current = sampleFrame(video); + if (!current) return; + + if (prevFrameRef.current) { + const { similarity } = compareFrames(prevFrameRef.current, current); + + if (similarity < CHANGE_THRESHOLD) { + console.log( + `[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`, + ); + const frameDataUrl = captureFrame(); + if (frameDataUrl) { + optionsRef.current?.onChange?.(frameDataUrl); + } + } + } + + prevFrameRef.current = current; + }, SAMPLE_INTERVAL); + + setIsObserving(true); + }, + [], + ); + + /** 停止观察模式 */ + const stopObserving = useCallback(() => { + if (intervalRef.current) { + clearInterval(intervalRef.current); + intervalRef.current = null; + } + prevFrameRef.current = null; + setIsObserving(false); + }, []); + + // 组件卸载时清理 + useEffect(() => { + return () => { + if (intervalRef.current) { + clearInterval(intervalRef.current); + } + }; + }, []); + + return { isObserving, startObserving, stopObserving }; +} diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts index a9112f6..0bdab7c 100644 --- a/frontend/src/hooks/useVisionSession.ts +++ b/frontend/src/hooks/useVisionSession.ts @@ -9,17 +9,52 @@ import { v4 as uuidv4 } from "uuid"; import { wsClient } from "../lib/websocket"; import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio"; import { getErrorMessage } from "../lib/errors"; +import { TTSPlayer } from "../lib/ttsPlayer"; import { showToast } from "../lib/toast"; +import { loadConfig, saveConfig } from "../lib/storage"; import { useCamera } from "../components/CameraManager"; import { useMicrophone } from "../components/MicManager"; -import { useVAD } from "../components/EdgeProcessor"; +import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor"; import { useWebSocketManager } from "../components/WebSocketManager"; -import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types"; +import { useObservationMode } from "./useObservationMode"; +import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types"; + +export type SessionMode = "dialogue" | "observation"; + +const MAX_HISTORY_ROUNDS = 10; + +export interface SessionStats { + queryCount: number; + totalTokens: number; +} export function useVisionSession() { const [messages, setMessages] = useState([]); const [currentReply, setCurrentReply] = useState(""); const [isProcessing, setIsProcessing] = useState(false); + const [isAudioPlaying, setIsAudioPlaying] = useState(false); + const [config, setConfig] = useState(loadConfig); + const [stats, setStats] = useState({ queryCount: 0, totalTokens: 0 }); + const [mode, setMode] = useState("dialogue"); + const [isCameraOn, setIsCameraOn] = useState(false); + const [isMicOn, setIsMicOn] = useState(false); + + // 上一帧采样数据（用于关键帧检测） + const prevFrameRef = useRef(null); + + // 对话历史（role + content），用于多轮上下文 + const historyRef = useRef>([]); + + // TTS 播放器 + const ttsPlayerRef = useRef(null); + const getTTSPlayer = useCallback(() => { + if (!ttsPlayerRef.current) { + const player = new TTSPlayer(); + player.onEnd(() => setIsAudioPlaying(false)); + ttsPlayerRef.current = player; + } + return ttsPlayerRef.current; + }, []); const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); const { startMic, stopMic } = useMicrophone(); @@ -31,6 +66,82 @@ export function useVisionSession() { isProcessingRef.current = isProcessing; }, [isProcessing]); + // 观察模式：画面变化时自动发送 query + const { isObserving, startObserving, stopObserving } = useObservationMode({ + onChange: useCallback( + (frameDataUrl: string) => { + if (isProcessingRef.current) return; + + const requestId = uuidv4(); + send({ + type: "query", + request_id: requestId, + image: dataUrlToBase64(frameDataUrl), + audio: "", // 观察模式无音频 + }); + + setMessages((prev) => [ + ...prev, + { + role: "user", + content: "👁️ 画面变化检测", + timestamp: Date.now(), + }, + ]); + setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 })); + setIsProcessing(true); + }, + [send], + ), + }); + + /** 切换对话/观察模式 */ + const toggleMode = useCallback(() => { + setMode((prev) => { + const next = prev === "dialogue" ? "observation" : "dialogue"; + if (next === "observation") { + startObserving(videoRef.current, captureFrame); + } else { + stopObserving(); + } + return next; + }); + }, [videoRef, captureFrame, startObserving, stopObserving]); + + // WebSocket 连接成功后发送 config + useEffect(() => { + if (status === "connected") { + send({ + type: "config", + payload: { + tts_enabled: config.ttsEnabled, + detail_level: config.detailLevel, + language: config.language, + }, + }); + } + }, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送 + + /** 更新会话配置 */ + const updateConfig = useCallback((partial: Partial) => { + setConfig((prev) => { + const next = { ...prev, ...partial }; + saveConfig(next); + // 如果已连接，立即发送更新 + if (status === "connected") { + send({ + type: "config", + payload: { + tts_enabled: next.ttsEnabled, + detail_level: next.detailLevel, + language: next.language, + }, + }); + } + return next; + }); + }, [status, send]); + // VAD：语音结束时自动发送 query const { isSpeaking, @@ -53,6 +164,23 @@ export function useVisionSession() { return; } + // 关键帧检测：与上一帧对比，相似度过高则跳过 + const video = videoRef.current; + if (video) { + const currentSample = sampleFrame(video); + if (currentSample && prevFrameRef.current) { + const { similarity } = compareFrames(prevFrameRef.current, currentSample); + if (similarity > 0.9) { + console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)})，跳过`); + prevFrameRef.current = currentSample; + return; + } + } + if (currentSample) { + prevFrameRef.current = currentSample; + } + } + const requestId = uuidv4(); send({ type: "query", @@ -61,6 +189,9 @@ export function useVisionSession() { audio: encodeAudioToBase64(audio), }); + // 更新请求统计 + setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 })); + // 添加用户消息（STT 流式结果会逐步更新文本） setMessages((prev) => [ ...prev, @@ -68,7 +199,7 @@ export function useVisionSession() { ]); setIsProcessing(true); }, - [captureFrame, send] + [captureFrame, send, videoRef] ), }); @@ -98,6 +229,21 @@ export function useVisionSession() { case "llm_done": { const done = msg as LLMDoneMessage; + // 记录到对话历史 + historyRef.current.push({ role: "assistant", content: done.full_text }); + // 裁剪历史到最近 N 轮 + if (historyRef.current.length > MAX_HISTORY_ROUNDS * 2) { + historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2); + } + + // 累计 token 统计 + if (done.tokens_used?.total) { + setStats((prev) => ({ + ...prev, + totalTokens: prev.totalTokens + done.tokens_used.total, + })); + } + setMessages((prev) => [ ...prev, { @@ -115,7 +261,10 @@ export function useVisionSession() { } case "tts_audio": - // TODO: 阶段 5 音频流播放 + getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last); + if (!msg.is_last) { + setIsAudioPlaying(true); + } break; case "error": @@ -127,25 +276,19 @@ export function useVisionSession() { }); return unsub; - }, []); - - // 连接断开时显示提示 - useEffect(() => { - if (status === "disconnected") { - // 只在非主动断开时提示（通过检查是否有活跃会话判断） - // 这里简单处理，由 App 层根据状态显示 - } - }, [status]); + }, [getTTSPlayer]); /** 启动会话 */ const startSession = useCallback(async () => { // 1. 获取摄像头和麦克风 await startCamera(); + setIsCameraOn(true); const micStream = await startMic(); if (!micStream) { showToast("无法获取麦克风权限", "error"); return; } + setIsMicOn(true); // 2. 连接 WebSocket connect(); @@ -156,24 +299,60 @@ export function useVisionSession() { /** 结束会话 */ const stopSession = useCallback(async () => { + stopObserving(); + setMode("dialogue"); await stopVAD(); stopMic(); stopCamera(); disconnect(); - // 清理所有对话状态 + // 停止 TTS 并清理状态 + ttsPlayerRef.current?.stop(); + setIsAudioPlaying(false); setMessages([]); setCurrentReply(""); setIsProcessing(false); - }, [stopVAD, stopMic, stopCamera, disconnect]); + setStats({ queryCount: 0, totalTokens: 0 }); + historyRef.current = []; + prevFrameRef.current = null; + setIsCameraOn(false); + setIsMicOn(false); + }, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]); + + /** 摄像头开关 */ + const toggleCamera = useCallback(async () => { + if (isCameraOn) { + stopCamera(); + setIsCameraOn(false); + } else { + await startCamera(); + setIsCameraOn(true); + } + }, [isCameraOn, startCamera, stopCamera]); + + /** 麦克风开关 */ + const toggleMic = useCallback(async () => { + if (isMicOn) { + stopMic(); + setIsMicOn(false); + } else { + const micStream = await startMic(); + setIsMicOn(!!micStream); + } + }, [isMicOn, startMic, stopMic]); /** 打断当前回复 */ const interrupt = useCallback(() => { send({ type: "interrupt" }); + // 停止 TTS 播放 + ttsPlayerRef.current?.stop(); + setIsAudioPlaying(false); // 将未完成的流式内容保存为最终消息 if (currentReply) { + const interrupted = currentReply + "（已打断）"; + historyRef.current.push({ role: "assistant", content: interrupted }); setMessages((prev) => [ ...prev, - { role: "assistant", content: currentReply + "（已打断）", timestamp: Date.now() }, + { role: "assistant", content: interrupted, timestamp: Date.now() }, ]); } setCurrentReply(""); @@ -184,14 +363,25 @@ export function useVisionSession() { messages, currentReply, isProcessing, + isAudioPlaying, isSpeaking, isVADReady, vadError, connectionStatus: status, videoRef, stream, + config, + updateConfig, + stats, + mode, + isObserving, + toggleMode, startSession, stopSession, interrupt, + isCameraOn, + isMicOn, + toggleCamera, + toggleMic, }; } diff --git a/frontend/src/index.css b/frontend/src/index.css index 7b0cf71..c509944 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1,7 +1 @@ /* 全局重置 — 详细样式在 App.css 中定义 */ - -#root { - min-height: 100vh; - display: flex; - flex-direction: column; -} diff --git a/frontend/src/lib/sampling.ts b/frontend/src/lib/sampling.ts new file mode 100644 index 0000000..235ec66 --- /dev/null +++ b/frontend/src/lib/sampling.ts @@ -0,0 +1,43 @@ +// ============================================================ +// Sampling — 混合采样策略 +// 来源：docs/08-成本控制.md — "定时低频 + 事件高频" +// ============================================================ + +/** 静默时采样间隔（ms） */ +const IDLE_INTERVAL = 5000; +/** 用户说话时采样间隔（ms） */ +const ACTIVE_INTERVAL = 1000; + +export class SamplingController { + private lastSampleTime = 0; + private _isUserSpeaking = false; + + /** 设置用户是否正在说话 */ + set speaking(value: boolean) { + this._isUserSpeaking = value; + } + + get speaking(): boolean { + return this._isUserSpeaking; + } + + /** 当前采样间隔 */ + get interval(): number { + return this._isUserSpeaking ? ACTIVE_INTERVAL : IDLE_INTERVAL; + } + + /** 是否应该采样（基于时间间隔） */ + shouldSample(): boolean { + const now = Date.now(); + if (now - this.lastSampleTime < this.interval) { + return false; + } + this.lastSampleTime = now; + return true; + } + + /** 强制允许下次采样（如用户刚说完话时） */ + resetTimer(): void { + this.lastSampleTime = 0; + } +} diff --git a/frontend/src/lib/storage.ts b/frontend/src/lib/storage.ts new file mode 100644 index 0000000..fb12324 --- /dev/null +++ b/frontend/src/lib/storage.ts @@ -0,0 +1,52 @@ +// ============================================================ +// Storage — localStorage 封装 +// 职责：会话配置持久化 +// ============================================================ + +import type { SessionConfig, Theme } from "../types"; + +const CONFIG_KEY = "camtalk:config"; +const THEME_KEY = "camtalk:theme"; + +const DEFAULT_CONFIG: SessionConfig = { + ttsEnabled: true, + detailLevel: "low", + language: "zh-CN", +}; + +/** 加载配置，无存储时返回默认值 */ +export function loadConfig(): SessionConfig { + try { + const raw = localStorage.getItem(CONFIG_KEY); + if (!raw) return DEFAULT_CONFIG; + const parsed = JSON.parse(raw) as Partial; + return { ...DEFAULT_CONFIG, ...parsed }; + } catch { + return DEFAULT_CONFIG; + } +} + +/** 保存配置到 localStorage */ +export function saveConfig(config: SessionConfig): void { + try { + localStorage.setItem(CONFIG_KEY, JSON.stringify(config)); + } catch { + // localStorage 不可用时静默失败 + } +} + +/** 加载主题偏好 */ +export function loadTheme(): Theme { + try { + const raw = localStorage.getItem(THEME_KEY); + if (raw === "light" || raw === "dark") return raw; + } catch { /* ignore */ } + return "dark"; +} + +/** 保存主题偏好 */ +export function saveTheme(theme: Theme): void { + try { + localStorage.setItem(THEME_KEY, theme); + } catch { /* ignore */ } +} diff --git a/frontend/src/lib/ttsPlayer.ts b/frontend/src/lib/ttsPlayer.ts new file mode 100644 index 0000000..a7f283f --- /dev/null +++ b/frontend/src/lib/ttsPlayer.ts @@ -0,0 +1,104 @@ +// ============================================================ +// TTS Player — 语音播放器 +// 职责：收集后端流式 tts_audio 片段，拼接后播放 +// 格式：MVP 仅支持 audio/mp3，pcm 为 TODO +// ============================================================ + +type OnEndCallback = () => void; + +export class TTSPlayer { + private chunks: string[] = []; + private audio: HTMLAudioElement | null = null; + private _isPlaying = false; + private onEndCallback: OnEndCallback | null = null; + + /** 注册播放完成回调 */ + onEnd(cb: OnEndCallback): void { + this.onEndCallback = cb; + } + + /** 当前是否正在播放 */ + get isPlaying(): boolean { + return this._isPlaying; + } + + /** + * 入队一个 TTS 音频片段 + * @param base64 Base64 编码的音频数据 + * @param mimeType 音频格式（"audio/mp3" 或 "audio/pcm"） + * @param isLast 是否为最后一个片段 + */ + enqueue(base64: string, mimeType: string, isLast: boolean): void { + this.chunks.push(base64); + + if (isLast) { + this.play(mimeType); + } + } + + /** 停止播放并清空缓冲区 */ + stop(): void { + if (this.audio) { + this.audio.pause(); + this.audio.removeAttribute("src"); + this.audio = null; + } + this.chunks = []; + this._isPlaying = false; + } + + /** 暂停播放 */ + pause(): void { + this.audio?.pause(); + } + + /** 恢复播放 */ + resume(): void { + this.audio?.play(); + } + + /** 拼接所有片段并播放 */ + private play(mimeType: string): void { + if (this.chunks.length === 0) return; + + // 拼接所有 Base64 片段 + const combined = this.chunks.join(""); + this.chunks = []; + + // Base64 → Uint8Array → Blob + const binary = atob(combined); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + const blob = new Blob([bytes], { type: mimeType }); + const url = URL.createObjectURL(blob); + + // 播放 + const audio = new Audio(url); + this.audio = audio; + this._isPlaying = true; + + audio.onended = () => { + URL.revokeObjectURL(url); + this._isPlaying = false; + this.audio = null; + this.onEndCallback?.(); + }; + + audio.onerror = () => { + console.error("[TTS] 播放失败"); + URL.revokeObjectURL(url); + this._isPlaying = false; + this.audio = null; + this.onEndCallback?.(); + }; + + audio.play().catch((err) => { + console.error("[TTS] play() 被拒绝:", err); + this._isPlaying = false; + this.audio = null; + this.onEndCallback?.(); + }); + } +} diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 4cad08a..0f29ec5 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -11,6 +11,8 @@ export interface SessionConfig { language: string; } +export type Theme = "dark" | "light"; + export interface Session { sessionId: string; createdAt: string;