Merge pull request '添加计时器,麦克风,摄像头开关' (#45) from develop-frontend8 into develop 33分钟前 #46

Merged
huanghaosheng merged 60 commits from develop into main 2026-06-13 20:43:07 +08:00
47 changed files with 6615 additions and 521 deletions

21
.env.example Normal file
View File

@@ -0,0 +1,21 @@
# CamTalk 环境变量模板
# 复制为 .env 并填入实际值cp .env.example .env
# .env 已在 .gitignore 中,不会提交到版本控制
# ---- AI 服务 API Key ----
CAMTALK_AI_LLM_API_KEY=sk-xxx
CAMTALK_AI_STT_API_KEY=
CAMTALK_AI_TTS_API_KEY=
# ---- 可选覆盖(默认值见 config.yaml----
# CAMTALK_AI_LLM_MODEL=gpt-4o
# CAMTALK_AI_LLM_ENDPOINT=https://api.openai.com/v1
# CAMTALK_AI_LLM_TIMEOUT=10
# CAMTALK_AI_STT_ENDPOINT=wss://api.deepgram.com/v1/listen
# CAMTALK_AI_TTS_ENDPOINT=https://api.openai.com/v1
# CAMTALK_AI_TTS_VOICE=alloy
# CAMTALK_AI_TTS_SPEED=1.0
# CAMTALK_AI_TTS_TIMEOUT=5
# ---- 应用 ----
# APP_ENV=dev

7
.gitignore vendored
View File

@@ -4,6 +4,7 @@ frontend/dist/
# ---- 后端 ----
backend/bin/
backend/server
# ---- 环境变量 ----
.env
@@ -18,5 +19,11 @@ Thumbs.db
.idea/
.vscode/
# ---- Playwright MCP ----
.playwright-mcp/
# ---- 截图 ----
*.png
# ---- Obsidian ----
.obsidian/

17
backend/.gitignore vendored Normal file
View File

@@ -0,0 +1,17 @@
# 编译产物
/server
bin/
# 环境配置
.env
config.dev.yaml
config.prod.yaml
# 临时文件
tmp/
# IDE
.idea/
.vscode/
*.swp
*.swo

View File

@@ -1,40 +1,118 @@
package main
import (
"log"
"context"
"errors"
"net/http"
"os/signal"
"syscall"
"time"
"github.com/gin-gonic/gin"
"github.com/hhs/camtalk/internal/api"
"github.com/hhs/camtalk/internal/ai/llm"
"github.com/hhs/camtalk/internal/ai/stt"
"github.com/hhs/camtalk/internal/ai/tts"
"github.com/hhs/camtalk/internal/config"
"github.com/hhs/camtalk/internal/logger"
"github.com/hhs/camtalk/internal/orchestrator"
"github.com/hhs/camtalk/internal/session"
"github.com/hhs/camtalk/internal/ws"
)
var startTime = time.Now()
func main() {
r := gin.Default()
// 加载配置
cfg, err := config.Load()
if err != nil {
panic("failed to load config: " + err.Error())
}
// 初始化日志
logger.Init(cfg.Log.Level, cfg.Log.Format)
defer logger.Sync()
logger.Log.Infow("config loaded",
"env", cfg.App.Env,
"addr", cfg.Server.Addr(),
)
// 初始化 Session ManagerMVP 默认内存实现)
var sessionMgr session.Manager
// TODO: 当 Redis 配置非空时切换为 RedisManager
sessionMgr = session.NewMemoryManager(30*time.Minute, 20)
defer sessionMgr.(*session.MemoryManager).Stop()
// 初始化 AI 服务
sttService := stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, logger.Log)
llmService := llm.NewOpenAIService(cfg.AI.LLM.APIKey, cfg.AI.LLM.Model, cfg.AI.LLM.Endpoint, cfg.AI.LLM.Timeout, logger.Log)
ttsService := tts.NewOpenAIService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Speed, cfg.AI.TTS.Timeout, logger.Log)
// 初始化 Orchestrator
orch := orchestrator.New(sttService, llmService, ttsService, sessionMgr, cfg.AI.LLM.Model)
// Gin 模式
if cfg.App.Env == "prod" {
gin.SetMode(gin.ReleaseMode)
}
r := gin.New()
r.Use(gin.Recovery())
// REST API
api := r.Group("/api")
apiGroup := r.Group("/api")
{
api.GET("/health", healthHandler)
apiGroup.GET("/health", healthHandler(sessionMgr))
}
// Session REST 端点
sessionHandler := api.NewSessionHandler(sessionMgr)
sessionHandler.RegisterRoutes(apiGroup)
// WebSocket
r.GET("/ws", ws.ServeWS)
r.GET("/ws", ws.ServeWS(sessionMgr, orch))
log.Println("CamTalk gateway starting on :8080")
if err := r.Run(":8080"); err != nil {
log.Fatalf("failed to start server: %v", err)
// HTTP Server
srv := &http.Server{
Addr: cfg.Server.Addr(),
Handler: r,
ReadTimeout: time.Duration(cfg.Server.ReadTimeout) * time.Second,
WriteTimeout: time.Duration(cfg.Server.WriteTimeout) * time.Second,
}
// Graceful shutdown
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
go func() {
logger.Log.Infow("server starting", "addr", srv.Addr)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
logger.Log.Fatalw("listen failed", "error", err)
}
}()
<-ctx.Done()
logger.Log.Info("shutting down...")
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
logger.Log.Errorw("shutdown error", "error", err)
}
logger.Log.Info("server stopped")
}
// healthHandler 健康检查。
func healthHandler(c *gin.Context) {
c.JSON(200, gin.H{
"status": "ok",
"version": "0.1.0",
"uptime": time.Since(startTime).String(),
"active_sessions": 0, // TODO: 接入 Session Manager
})
func healthHandler(sessionMgr session.Manager) gin.HandlerFunc {
return func(c *gin.Context) {
c.JSON(200, gin.H{
"status": "ok",
"version": "0.1.0",
"uptime_seconds": int(time.Since(startTime).Seconds()),
"active_sessions": sessionMgr.ActiveCount(),
})
}
}

39
backend/config.yaml Normal file
View File

@@ -0,0 +1,39 @@
# config.yaml — 默认配置
app:
env: dev
server:
host: "0.0.0.0"
port: 8080
read_timeout: 30
write_timeout: 30
redis:
addr: "localhost:6379"
password: ""
db: 0
ai:
stt:
provider: Xiaomi MiMo
model: mimo-v2.5
endpoint: "https://token-plan-cn.xiaomimimo.com/v1"
llm:
provider: dashscope
model: qwen3-vl-plus
endpoint: "https://dashscope.aliyuncs.com/compatible-mode/v1"
timeout: 30
tts:
provider: Xiaomi MiMo
model: mimo-v2.5
voice: alloy
speed: 1.0
endpoint: "https://token-plan-cn.xiaomimimo.com/v1"
timeout: 5
storage:
driver: memory
log:
level: info
format: console

View File

@@ -1,38 +1,58 @@
module github.com/hhs/camtalk
go 1.23
go 1.24
require (
github.com/gin-gonic/gin v1.10.0
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/redis/go-redis/v9 v9.20.1
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
go.uber.org/zap v1.28.0
)
require (
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/text v0.28.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@@ -1,7 +1,13 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
@@ -9,6 +15,10 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -23,21 +33,29 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -47,26 +65,56 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.20.1 h1:sfCU6A8P3dXbKyWes02uxA2baehGux9dZHfEKtsTB1w=
github.com/redis/go-redis/v9 v9.20.1/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
@@ -74,18 +122,16 @@ golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,37 @@
package llm
import (
"context"
"github.com/hhs/camtalk/internal/models"
)
// Service 多模态大模型服务契约。
type Service interface {
// ChatStream 流式推理,返回增量文本的 channel。
// 调用方必须消费 channel 直到 Done=true否则需 cancel ctx 以释放连接。
ChatStream(ctx context.Context, req Request) (<-chan Chunk, error)
}
// Request 推理请求。
type Request struct {
Image []byte // JPEG 图片(已从 Base64 解码)
Text string // 用户语音识别后的文本
History []models.Message // 最近 N 轮对话历史
Language string // 语言,如 "zh-CN"
}
// Chunk 流式推理的一个增量片段。
type Chunk struct {
Delta string // 增量文本
Done bool // 是否结束
TokensUsed *TokenUsage // 仅 Done=true 时有值
Model string // 实际使用的模型名
}
// TokenUsage 用量统计。
type TokenUsage struct {
Prompt int
Completion int
Total int
}

View File

@@ -0,0 +1,237 @@
package llm
import (
"bufio"
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"go.uber.org/zap"
)
// OpenAIService 基于 OpenAI Chat Completions API 的 LLM 实现。
type OpenAIService struct {
apiKey string
model string
endpoint string
timeout time.Duration
logger *zap.SugaredLogger
client *http.Client
}
// NewOpenAIService 创建 OpenAI LLM 服务。
func NewOpenAIService(apiKey, model, endpoint string, timeoutSec int, logger *zap.SugaredLogger) *OpenAIService {
if model == "" {
model = "gpt-4o"
}
if endpoint == "" {
endpoint = "https://api.openai.com/v1"
}
timeout := time.Duration(timeoutSec) * time.Second
if timeout <= 0 {
timeout = 10 * time.Second
}
return &OpenAIService{
apiKey: apiKey,
model: model,
endpoint: endpoint,
timeout: timeout,
logger: logger,
client: &http.Client{Timeout: 60 * time.Second}, // HTTP client timeout > LLM timeout
}
}
// --- OpenAI API 请求/响应结构 ---
type chatRequest struct {
Model string `json:"model"`
Messages []chatMessage `json:"messages"`
Stream bool `json:"stream"`
}
type chatMessage struct {
Role string `json:"role"`
Content []contentPart `json:"content"`
}
type contentPart struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *imageURL `json:"image_url,omitempty"`
}
type imageURL struct {
URL string `json:"url"`
}
// streamDelta SSE 流式响应的单个 delta。
type streamDelta struct {
Choices []struct {
Delta struct {
Content string `json:"content"`
} `json:"delta"`
FinishReason *string `json:"finish_reason"`
} `json:"choices"`
Usage *struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
} `json:"usage"`
Model string `json:"model"`
}
// ChatStream 实现 llm.Service。调用 OpenAI Chat Completions API 流式推理。
func (o *OpenAIService) ChatStream(ctx context.Context, req Request) (<-chan Chunk, error) {
// 构建请求
messages := o.buildMessages(req)
body := chatRequest{
Model: o.model,
Messages: messages,
Stream: true,
}
payload, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("llm: marshal request: %w", err)
}
// 创建带超时的 context
ctx, cancel := context.WithTimeout(ctx, o.timeout)
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, o.endpoint+"/chat/completions", bytes.NewReader(payload))
if err != nil {
cancel()
return nil, fmt.Errorf("llm: create request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Authorization", "Bearer "+o.apiKey)
resp, err := o.client.Do(httpReq)
if err != nil {
cancel()
return nil, fmt.Errorf("llm: send request: %w", err)
}
if resp.StatusCode != http.StatusOK {
cancel()
bodyBytes, _ := io.ReadAll(resp.Body)
resp.Body.Close()
return nil, fmt.Errorf("llm: api error (status %d): %s", resp.StatusCode, string(bodyBytes))
}
// 启动 goroutine 解析 SSE 流
ch := make(chan Chunk, 64)
go func() {
defer close(ch)
defer cancel()
defer resp.Body.Close()
o.parseSSEStream(resp.Body, ch)
}()
return ch, nil
}
// parseSSEStream 解析 SSE 流,将 delta 发送到 channel。
func (o *OpenAIService) parseSSEStream(body io.Reader, ch chan<- Chunk) {
scanner := bufio.NewScanner(body)
scanner.Buffer(make([]byte, 0, 64*1024), 256*1024)
var fullText strings.Builder
var lastModel string
for scanner.Scan() {
line := scanner.Text()
// SSE 格式data: {...}
if !strings.HasPrefix(line, "data: ") {
continue
}
data := strings.TrimPrefix(line, "data: ")
if data == "[DONE]" {
// 流结束,发送最终 chunk
ch <- Chunk{Delta: "", Done: true, Model: lastModel}
return
}
var delta streamDelta
if err := json.Unmarshal([]byte(data), &delta); err != nil {
o.logger.Warnw("llm: unmarshal delta failed", "error", err, "data", data)
continue
}
if delta.Model != "" {
lastModel = delta.Model
}
// 提取增量文本
if len(delta.Choices) > 0 {
content := delta.Choices[0].Delta.Content
if content != "" {
fullText.WriteString(content)
ch <- Chunk{Delta: content, Done: false, Model: lastModel}
}
// 某些模型在最后一个 choice 中携带 usage
if delta.Choices[0].FinishReason != nil && delta.Usage != nil {
ch <- Chunk{
Delta: "",
Done: true,
Model: lastModel,
TokensUsed: &TokenUsage{
Prompt: delta.Usage.PromptTokens,
Completion: delta.Usage.CompletionTokens,
Total: delta.Usage.TotalTokens,
},
}
return
}
}
}
// scanner 结束但没收到 [DONE]
if err := scanner.Err(); err != nil {
o.logger.Warnw("llm: scan error", "error", err)
}
ch <- Chunk{Delta: "", Done: true, Model: lastModel}
}
// buildMessages 构建 OpenAI Chat API 的 messages 数组。
func (o *OpenAIService) buildMessages(req Request) []chatMessage {
var messages []chatMessage
// System prompt
messages = append(messages, chatMessage{
Role: "system",
Content: []contentPart{{Type: "text", Text: BuildSystemPrompt(req.Language, "")}},
})
// 历史消息
for _, msg := range req.History {
messages = append(messages, chatMessage{
Role: msg.Role,
Content: []contentPart{{Type: "text", Text: msg.Content}},
})
}
// 当前用户消息(图像 + 文本)
var parts []contentPart
if len(req.Image) > 0 {
b64 := base64.StdEncoding.EncodeToString(req.Image)
parts = append(parts, contentPart{
Type: "image_url",
ImageURL: &imageURL{URL: "data:image/jpeg;base64," + b64},
})
}
parts = append(parts, contentPart{Type: "text", Text: req.Text})
messages = append(messages, chatMessage{Role: "user", Content: parts})
return messages
}

View File

@@ -0,0 +1,251 @@
package llm
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"go.uber.org/zap"
"github.com/hhs/camtalk/internal/models"
)
// mockLLMServer 创建模拟 OpenAI SSE 流式响应的 HTTP 服务器。
func mockLLMServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
t.Helper()
return httptest.NewServer(handler)
}
func TestOpenAIService_ChatStream_Success(t *testing.T) {
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
// 验证请求
if r.Method != http.MethodPost {
t.Errorf("method = %s, want POST", r.Method)
}
if !strings.Contains(r.URL.Path, "/chat/completions") {
t.Errorf("path = %s, should contain /chat/completions", r.URL.Path)
}
auth := r.Header.Get("Authorization")
if auth != "Bearer test-key" {
t.Errorf("Authorization = %q, want %q", auth, "Bearer test-key")
}
w.Header().Set("Content-Type", "text/event-stream")
flusher, ok := w.(http.Flusher)
if !ok {
t.Fatal("ResponseWriter does not support Flusher")
}
// 发送几个 delta
deltas := []string{"你好", "世界", ""}
for _, d := range deltas {
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"%s\"}}],\"model\":\"gpt-4o\"}\n\n", d)
flusher.Flush()
}
// 发送 [DONE]
fmt.Fprintf(w, "data: [DONE]\n\n")
flusher.Flush()
})
defer srv.Close()
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
ch, err := svc.ChatStream(context.Background(), Request{
Text: "这是什么?",
Language: "zh-CN",
})
if err != nil {
t.Fatalf("ChatStream() error: %v", err)
}
var chunks []Chunk
for c := range ch {
chunks = append(chunks, c)
}
// 应该有 3 个文本 chunk + 1 个 Done chunk
if len(chunks) != 4 {
t.Fatalf("got %d chunks, want 4", len(chunks))
}
// 验证文本内容
if chunks[0].Delta != "你好" {
t.Errorf("chunk[0].Delta = %q, want %q", chunks[0].Delta, "你好")
}
if chunks[1].Delta != "世界" {
t.Errorf("chunk[1].Delta = %q, want %q", chunks[1].Delta, "世界")
}
// 验证最后一个 chunk 是 Done
last := chunks[len(chunks)-1]
if !last.Done {
t.Error("last chunk should be Done")
}
if last.Model != "gpt-4o" {
t.Errorf("last chunk Model = %q, want %q", last.Model, "gpt-4o")
}
}
func TestOpenAIService_ChatStream_WithImage(t *testing.T) {
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}],\"model\":\"gpt-4o\"}\n\n")
fmt.Fprintf(w, "data: [DONE]\n\n")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
ch, err := svc.ChatStream(context.Background(), Request{
Image: []byte("fake-jpeg-data"),
Text: "描述图片",
Language: "zh-CN",
})
if err != nil {
t.Fatalf("ChatStream() error: %v", err)
}
// 消费 channel
for range ch {
}
}
func TestOpenAIService_ChatStream_WithHistory(t *testing.T) {
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}],\"model\":\"gpt-4o\"}\n\n")
fmt.Fprintf(w, "data: [DONE]\n\n")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
ch, err := svc.ChatStream(context.Background(), Request{
Text: "继续",
Language: "zh-CN",
History: []models.Message{
{Role: "user", Content: "你好"},
{Role: "assistant", Content: "你好!有什么可以帮助你的吗?"},
},
})
if err != nil {
t.Fatalf("ChatStream() error: %v", err)
}
for range ch {
}
}
func TestOpenAIService_ChatStream_APIError(t *testing.T) {
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
fmt.Fprintf(w, `{"error":{"message":"Invalid API key"}}`)
})
defer srv.Close()
svc := NewOpenAIService("bad-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
_, err := svc.ChatStream(context.Background(), Request{
Text: "test",
})
if err == nil {
t.Fatal("ChatStream() should return error for 401")
}
if !strings.Contains(err.Error(), "401") {
t.Errorf("error should mention 401, got: %v", err)
}
}
func TestOpenAIService_ChatStream_Timeout(t *testing.T) {
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
// 模拟慢响应
time.Sleep(5 * time.Second)
w.Header().Set("Content-Type", "text/event-stream")
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"late\"}}]}\n\n")
fmt.Fprintf(w, "data: [DONE]\n\n")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 1, zap.NewNop().Sugar()) // 1s timeout
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
ch, err := svc.ChatStream(ctx, Request{Text: "test"})
if err != nil {
// 超时可能在建立连接时或读取时发生
return
}
// 如果连接成功,消费 channel 应该超时
var gotContent bool
for c := range ch {
if c.Delta != "" {
gotContent = true
}
}
if gotContent {
t.Error("should not receive content before timeout")
}
}
func TestOpenAIService_ChatStream_UsageInResponse(t *testing.T) {
srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
// 带 usage 的最后一个 chunk
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"},\"finish_reason\":\"stop\"}],\"model\":\"gpt-4o\",\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":5,\"total_tokens\":15}}\n\n")
fmt.Fprintf(w, "data: [DONE]\n\n")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, zap.NewNop().Sugar())
ch, err := svc.ChatStream(context.Background(), Request{Text: "test"})
if err != nil {
t.Fatalf("ChatStream() error: %v", err)
}
var last Chunk
for c := range ch {
last = c
}
if !last.Done {
t.Error("last chunk should be Done")
}
if last.TokensUsed == nil {
t.Fatal("last chunk should have TokensUsed")
}
if last.TokensUsed.Total != 15 {
t.Errorf("TokensUsed.Total = %d, want 15", last.TokensUsed.Total)
}
}
func TestBuildSystemPrompt(t *testing.T) {
tests := []struct {
name string
language string
detailLevel string
wantContain string
}{
{"chinese default", "zh-CN", "", "视觉助手"},
{"chinese high", "zh-CN", "high", "更详细"},
{"english default", "en", "", "visual assistant"},
{"english high", "en", "high", "detailed"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := BuildSystemPrompt(tt.language, tt.detailLevel)
if !strings.Contains(got, tt.wantContain) {
t.Errorf("BuildSystemPrompt(%q, %q) should contain %q", tt.language, tt.detailLevel, tt.wantContain)
}
})
}
}

View File

@@ -0,0 +1,25 @@
package llm
import "strings"
// BuildSystemPrompt 根据语言和细节级别构建系统提示词。
func BuildSystemPrompt(language, detailLevel string) string {
isChinese := strings.HasPrefix(language, "zh")
var prompt strings.Builder
if isChinese {
prompt.WriteString("你是一个视觉助手。用户通过摄像头看到一个场景,并用语音向你提问。请用简洁自然的中文回答。如果涉及视觉描述,先说\"我看到……\"。回答控制在3-5句话以内除非用户要求详细说明。")
} else {
prompt.WriteString("You are a visual assistant. The user sees a scene through their camera and asks questions by voice. Answer concisely and naturally. If describing visual content, start with 'I see...'. Keep answers to 3-5 sentences unless the user asks for detail.")
}
if detailLevel == "high" {
if isChinese {
prompt.WriteString("请提供更详细的视觉描述,包括颜色、位置、数量等细节。")
} else {
prompt.WriteString(" Provide detailed visual descriptions including colors, positions, quantities, and other details.")
}
}
return prompt.String()
}

View File

@@ -0,0 +1,141 @@
package stt
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/gorilla/websocket"
"go.uber.org/zap"
)
// DeepgramService 基于 Deepgram WebSocket API 的语音识别实现。
type DeepgramService struct {
apiKey string
model string
endpoint string
logger *zap.SugaredLogger
}
// NewDeepgramService 创建 Deepgram STT 服务。
func NewDeepgramService(apiKey, model, endpoint string, logger *zap.SugaredLogger) *DeepgramService {
if model == "" {
model = "nova-2"
}
if endpoint == "" {
endpoint = "wss://api.deepgram.com/v1/listen"
}
return &DeepgramService{
apiKey: apiKey,
model: model,
endpoint: endpoint,
logger: logger,
}
}
// deepgramResponse Deepgram WebSocket 响应。
type deepgramResponse struct {
Channel struct {
Alternatives []struct {
Transcript string `json:"transcript"`
Confidence float64 `json:"confidence"`
} `json:"alternatives"`
} `json:"channel"`
IsFinal bool `json:"is_final"`
}
// Recognize 实现 stt.Service。通过 WebSocket 发送音频到 Deepgram返回最终识别文本。
func (d *DeepgramService) Recognize(ctx context.Context, audio []byte, opts Options) (string, error) {
if len(audio) == 0 {
return "", fmt.Errorf("stt: empty audio")
}
// 构建 WebSocket URL附带查询参数
wsURL := d.buildURL(opts)
// 5 秒总超时
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
// 建立 WebSocket 连接
conn, _, err := websocket.DefaultDialer.DialContext(ctx, wsURL, http.Header{
"Authorization": []string{"Token " + d.apiKey},
})
if err != nil {
return "", fmt.Errorf("stt: connect deepgram: %w", err)
}
defer conn.Close()
// 发送音频数据(一次性)
if err := conn.WriteMessage(websocket.BinaryMessage, audio); err != nil {
return "", fmt.Errorf("stt: send audio: %w", err)
}
// 发送 Close 消息通知服务端音频已发送完毕
closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")
_ = conn.WriteMessage(websocket.CloseMessage, closeMsg)
// 读取识别结果
var transcript strings.Builder
for {
_, message, err := conn.ReadMessage()
if err != nil {
// Close 帧是正常的结束信号
if websocket.IsCloseError(err, websocket.CloseNormalClosure) {
break
}
if websocket.IsUnexpectedCloseError(err, websocket.CloseNormalClosure) {
break
}
return "", fmt.Errorf("stt: read response: %w", err)
}
var resp deepgramResponse
if err := json.Unmarshal(message, &resp); err != nil {
d.logger.Warnw("stt: unmarshal response failed", "error", err)
continue
}
// 只累积 final 结果,跳过中间结果
if resp.IsFinal && len(resp.Channel.Alternatives) > 0 {
text := strings.TrimSpace(resp.Channel.Alternatives[0].Transcript)
if text != "" {
transcript.WriteString(text)
}
}
}
return strings.TrimSpace(transcript.String()), nil
}
// buildURL 构建 Deepgram WebSocket URL包含音频格式参数。
func (d *DeepgramService) buildURL(opts Options) string {
u, _ := url.Parse(d.endpoint)
encoding := opts.Encoding
if encoding == "" {
encoding = "pcm_s16le"
}
sampleRate := opts.SampleRate
if sampleRate == 0 {
sampleRate = 16000
}
language := opts.Language
if language == "" {
language = "zh-CN"
}
q := u.Query()
q.Set("encoding", encoding)
q.Set("sample_rate", fmt.Sprintf("%d", sampleRate))
q.Set("language", language)
q.Set("model", d.model)
q.Set("punctuate", "true")
u.RawQuery = q.Encode()
return u.String()
}

View File

@@ -0,0 +1,191 @@
package stt
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
"go.uber.org/zap"
)
var upgrader = websocket.Upgrader{
CheckOrigin: func(r *http.Request) bool { return true },
}
// newMockDeepgram 创建模拟 Deepgram WebSocket 服务。
// 返回 httptest.Server 和对应的 ws:// URL。
func newMockDeepgram(t *testing.T, handler func(conn *websocket.Conn)) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
t.Logf("upgrade error: %v", err)
return
}
defer conn.Close()
handler(conn)
}))
return srv
}
// wsToWss 将 http:// 转换为 ws://。
func wsToWss(httpURL string) string {
return "ws" + strings.TrimPrefix(httpURL, "http")
}
func TestDeepgramService_Recognize_Success(t *testing.T) {
srv := newMockDeepgram(t, func(conn *websocket.Conn) {
// 读取音频数据
_, _, err := conn.ReadMessage()
if err != nil {
t.Errorf("read audio: %v", err)
return
}
// 发送中间结果(非 final
intermediate := deepgramResponse{
IsFinal: false,
}
intermediate.Channel.Alternatives = []struct {
Transcript string `json:"transcript"`
Confidence float64 `json:"confidence"`
}{{Transcript: "你好", Confidence: 0.9}}
data, _ := json.Marshal(intermediate)
_ = conn.WriteMessage(websocket.TextMessage, data)
// 发送最终结果
final := deepgramResponse{
IsFinal: true,
}
final.Channel.Alternatives = []struct {
Transcript string `json:"transcript"`
Confidence float64 `json:"confidence"`
}{{Transcript: "你好世界", Confidence: 0.95}}
data, _ = json.Marshal(final)
_ = conn.WriteMessage(websocket.TextMessage, data)
// 等待客户端关闭
_, _, _ = conn.ReadMessage()
})
defer srv.Close()
svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar())
text, err := svc.Recognize(context.Background(), []byte("fake-pcm-audio"), Options{
Encoding: "pcm_s16le",
SampleRate: 16000,
Language: "zh-CN",
})
if err != nil {
t.Fatalf("Recognize() error: %v", err)
}
if text != "你好世界" {
t.Errorf("Recognize() = %q, want %q", text, "你好世界")
}
}
func TestDeepgramService_Recognize_EmptyAudio(t *testing.T) {
svc := NewDeepgramService("test-key", "", "ws://localhost", zap.NewNop().Sugar())
_, err := svc.Recognize(context.Background(), nil, Options{})
if err == nil {
t.Fatal("Recognize() with empty audio should return error")
}
}
func TestDeepgramService_Recognize_ConnectError(t *testing.T) {
svc := NewDeepgramService("test-key", "", "ws://localhost:1", zap.NewNop().Sugar())
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_, err := svc.Recognize(ctx, []byte("audio"), Options{})
if err == nil {
t.Fatal("Recognize() with bad endpoint should return error")
}
}
func TestDeepgramService_Recognize_Timeout(t *testing.T) {
// 模拟一个永不响应的服务端
srv := newMockDeepgram(t, func(conn *websocket.Conn) {
// 读取音频但不发送任何结果,让客户端超时
_, _, _ = conn.ReadMessage()
time.Sleep(10 * time.Second)
})
defer srv.Close()
svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar())
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
_, err := svc.Recognize(ctx, []byte("audio"), Options{})
if err == nil {
t.Fatal("Recognize() should timeout")
}
}
func TestDeepgramService_Recognize_MultipleFinals(t *testing.T) {
srv := newMockDeepgram(t, func(conn *websocket.Conn) {
_, _, _ = conn.ReadMessage()
// 发送多个 final 结果(多句话场景)
for _, text := range []string{"你好", "世界"} {
resp := deepgramResponse{IsFinal: true}
resp.Channel.Alternatives = []struct {
Transcript string `json:"transcript"`
Confidence float64 `json:"confidence"`
}{{Transcript: text, Confidence: 0.9}}
data, _ := json.Marshal(resp)
_ = conn.WriteMessage(websocket.TextMessage, data)
}
_, _, _ = conn.ReadMessage()
})
defer srv.Close()
svc := NewDeepgramService("test-key", "", wsToWss(srv.URL)+"/v1/listen", zap.NewNop().Sugar())
text, err := svc.Recognize(context.Background(), []byte("audio"), Options{})
if err != nil {
t.Fatalf("Recognize() error: %v", err)
}
if text != "你好世界" {
t.Errorf("Recognize() = %q, want %q", text, "你好世界")
}
}
func TestDeepgramService_buildURL(t *testing.T) {
svc := NewDeepgramService("key", "", "wss://api.deepgram.com/v1/listen", zap.NewNop().Sugar())
tests := []struct {
name string
opts Options
want []string // URL 中应包含的参数
}{
{
name: "defaults",
opts: Options{},
want: []string{"encoding=pcm_s16le", "sample_rate=16000", "language=zh-CN"},
},
{
name: "custom",
opts: Options{Encoding: "wav", SampleRate: 44100, Language: "en"},
want: []string{"encoding=wav", "sample_rate=44100", "language=en"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
u := svc.buildURL(tt.opts)
for _, param := range tt.want {
if !strings.Contains(u, param) {
t.Errorf("buildURL() = %q, should contain %q", u, param)
}
}
})
}
}

View File

@@ -0,0 +1,16 @@
package stt
import "context"
// Service 语音识别服务契约。
type Service interface {
// Recognize 识别一段完整音频,返回最终文本。
Recognize(ctx context.Context, audio []byte, opts Options) (string, error)
}
// Options 语音识别参数。
type Options struct {
Encoding string // 音频编码,如 "pcm_s16le"
SampleRate int // 采样率,如 16000
Language string // 语言,如 "zh-CN"
}

View File

@@ -0,0 +1,154 @@
package tts
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"go.uber.org/zap"
)
// OpenAIService 基于 OpenAI TTS API 的语音合成实现。
type OpenAIService struct {
apiKey string
model string
voice string
speed float64
endpoint string
timeout time.Duration
logger *zap.SugaredLogger
client *http.Client
}
// NewOpenAIService 创建 OpenAI TTS 服务。
func NewOpenAIService(apiKey, model, voice, endpoint string, speed float64, timeoutSec int, logger *zap.SugaredLogger) *OpenAIService {
if model == "" {
model = "tts-1"
}
if voice == "" {
voice = "alloy"
}
if speed <= 0 {
speed = 1.0
}
if endpoint == "" {
endpoint = "https://api.openai.com/v1"
}
timeout := time.Duration(timeoutSec) * time.Second
if timeout <= 0 {
timeout = 5 * time.Second
}
return &OpenAIService{
apiKey: apiKey,
model: model,
voice: voice,
speed: speed,
endpoint: endpoint,
timeout: timeout,
logger: logger,
client: &http.Client{Timeout: 30 * time.Second},
}
}
// ttsRequest OpenAI TTS API 请求。
type ttsRequest struct {
Model string `json:"model"`
Input string `json:"input"`
Voice string `json:"voice"`
ResponseFormat string `json:"response_format"`
Speed float64 `json:"speed"`
}
// SynthesizeStream 实现 tts.Service。从 textStream 读取句子,逐句调用 OpenAI TTS API。
func (o *OpenAIService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error) {
voice := opts.Voice
if voice == "" {
voice = o.voice
}
speed := opts.Speed
if speed <= 0 {
speed = o.speed
}
ch := make(chan Chunk, 4)
go func() {
defer close(ch)
for text := range textStream {
if text == "" {
continue
}
audio, err := o.synthesize(ctx, text, voice, speed)
if err != nil {
o.logger.Warnw("tts: synthesize failed", "error", err, "text", text)
// 静默跳过,不中断整个流
continue
}
select {
case ch <- Chunk{Audio: audio, IsLast: false}:
case <-ctx.Done():
return
}
}
// textStream 关闭,发送 IsLast 标记
select {
case ch <- Chunk{Audio: nil, IsLast: true}:
case <-ctx.Done():
}
}()
return ch, nil
}
// synthesize 调用 OpenAI TTS API 合成单个句子。
func (o *OpenAIService) synthesize(ctx context.Context, text, voice string, speed float64) ([]byte, error) {
// 单句超时
ctx, cancel := context.WithTimeout(ctx, o.timeout)
defer cancel()
body := ttsRequest{
Model: o.model,
Input: text,
Voice: voice,
ResponseFormat: "mp3",
Speed: speed,
}
payload, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("tts: marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, o.endpoint+"/audio/speech", bytes.NewReader(payload))
if err != nil {
return nil, fmt.Errorf("tts: create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+o.apiKey)
resp, err := o.client.Do(req)
if err != nil {
return nil, fmt.Errorf("tts: send request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
errBody, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("tts: api error (status %d): %s", resp.StatusCode, string(errBody))
}
// 读取整个 MP3 响应
audio, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("tts: read response: %w", err)
}
return audio, nil
}

View File

@@ -0,0 +1,300 @@
package tts
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"go.uber.org/zap"
)
// mockTTSServer 创建模拟 OpenAI TTS API 的 HTTP 服务器。
func mockTTSServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
t.Helper()
return httptest.NewServer(handler)
}
// sendSentences 向 channel 发送句子并关闭。
func sendSentences(sentences ...string) <-chan string {
ch := make(chan string, len(sentences))
for _, s := range sentences {
ch <- s
}
close(ch)
return ch
}
func TestOpenAIService_SynthesizeStream_Success(t *testing.T) {
var callCount int32
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&callCount, 1)
if r.Method != http.MethodPost {
t.Errorf("method = %s, want POST", r.Method)
}
if !strings.Contains(r.URL.Path, "/audio/speech") {
t.Errorf("path = %s, should contain /audio/speech", r.URL.Path)
}
auth := r.Header.Get("Authorization")
if auth != "Bearer test-key" {
t.Errorf("Authorization = %q, want %q", auth, "Bearer test-key")
}
// 验证请求体
body, _ := io.ReadAll(r.Body)
if !strings.Contains(string(body), "tts-1") {
t.Errorf("request body should contain model tts-1")
}
// 返回假 MP3 数据
w.Header().Set("Content-Type", "audio/mpeg")
fmt.Fprintf(w, "fake-mp3-data")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
textStream := sendSentences("你好", "世界", "")
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{
Voice: "alloy", Speed: 1.0, OutputFmt: "mp3", SampleRate: 24000,
})
if err != nil {
t.Fatalf("SynthesizeStream() error: %v", err)
}
var chunks []Chunk
for c := range ch {
chunks = append(chunks, c)
}
// 应该有 3 个音频 chunk + 1 个 IsLast 标记
if len(chunks) != 4 {
t.Fatalf("got %d chunks, want 4", len(chunks))
}
// 验证前 3 个有音频数据
for i := 0; i < 3; i++ {
if string(chunks[i].Audio) != "fake-mp3-data" {
t.Errorf("chunk[%d].Audio = %q, want %q", i, string(chunks[i].Audio), "fake-mp3-data")
}
if chunks[i].IsLast {
t.Errorf("chunk[%d].IsLast should be false", i)
}
}
// 验证最后一个是 IsLast
if !chunks[3].IsLast {
t.Error("last chunk should be IsLast")
}
if chunks[3].Audio != nil {
t.Error("last chunk Audio should be nil")
}
// 验证调用了 3 次 API3 个句子)
if atomic.LoadInt32(&callCount) != 3 {
t.Errorf("API called %d times, want 3", callCount)
}
}
func TestOpenAIService_SynthesizeStream_APIError(t *testing.T) {
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "internal error")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
textStream := sendSentences("你好")
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{})
if err != nil {
t.Fatalf("SynthesizeStream() error: %v", err)
}
// 应该只有一个 IsLast chunk音频被跳过
var chunks []Chunk
for c := range ch {
chunks = append(chunks, c)
}
if len(chunks) != 1 {
t.Fatalf("got %d chunks, want 1 (IsLast only)", len(chunks))
}
if !chunks[0].IsLast {
t.Error("chunk should be IsLast")
}
}
func TestOpenAIService_SynthesizeStream_Timeout(t *testing.T) {
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
time.Sleep(3 * time.Second)
w.Header().Set("Content-Type", "audio/mpeg")
fmt.Fprintf(w, "late-mp3")
})
defer srv.Close()
// 1 秒超时
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 1, zap.NewNop().Sugar())
textStream := sendSentences("很长的句子")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
ch, err := svc.SynthesizeStream(ctx, textStream, Options{})
if err != nil {
t.Fatalf("SynthesizeStream() error: %v", err)
}
var chunks []Chunk
for c := range ch {
chunks = append(chunks, c)
}
// 超时后音频被跳过,只有 IsLast
if len(chunks) != 1 {
t.Fatalf("got %d chunks, want 1", len(chunks))
}
if !chunks[0].IsLast {
t.Error("chunk should be IsLast")
}
}
func TestOpenAIService_SynthesizeStream_EmptyText(t *testing.T) {
var callCount int32
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&callCount, 1)
w.Header().Set("Content-Type", "audio/mpeg")
fmt.Fprintf(w, "mp3")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
// 空句子应该被跳过
textStream := sendSentences("", "你好", "")
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{})
if err != nil {
t.Fatalf("SynthesizeStream() error: %v", err)
}
var chunks []Chunk
for c := range ch {
chunks = append(chunks, c)
}
// 只有 "你好" 应该被合成
if atomic.LoadInt32(&callCount) != 1 {
t.Errorf("API called %d times, want 1", callCount)
}
// 1 个音频 + 1 个 IsLast
if len(chunks) != 2 {
t.Fatalf("got %d chunks, want 2", len(chunks))
}
}
func TestOpenAIService_SynthesizeStream_ContextCancelled(t *testing.T) {
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "audio/mpeg")
fmt.Fprintf(w, "mp3")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
// 发送多个句子,但在第一个后取消
textStream := make(chan string, 3)
textStream <- "第一句"
textStream <- "第二句"
textStream <- "第三句"
close(textStream)
ctx, cancel := context.WithCancel(context.Background())
// 立即取消
cancel()
ch, err := svc.SynthesizeStream(ctx, textStream, Options{})
if err != nil {
t.Fatalf("SynthesizeStream() error: %v", err)
}
// 消费 channel应该很快结束
var count int
for range ch {
count++
}
// 可能收到 0 个或 1 个 chunk取决于时序
t.Logf("received %d chunks after context cancel", count)
}
func TestOpenAIService_SynthesizeStream_PartialFailure(t *testing.T) {
var callCount int32
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(&callCount, 1)
if n == 2 {
// 第二个句子失败
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "error")
return
}
w.Header().Set("Content-Type", "audio/mpeg")
fmt.Fprintf(w, "mp3-%d", n)
})
defer srv.Close()
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
textStream := sendSentences("第一句", "第二句", "第三句")
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{})
if err != nil {
t.Fatalf("SynthesizeStream() error: %v", err)
}
var chunks []Chunk
for c := range ch {
chunks = append(chunks, c)
}
// 2 个成功音频 + 1 个 IsLast第二句被跳过
if len(chunks) != 3 {
t.Fatalf("got %d chunks, want 3", len(chunks))
}
if !chunks[len(chunks)-1].IsLast {
t.Error("last chunk should be IsLast")
}
}
func TestOpenAIService_SynthesizeStream_CustomVoice(t *testing.T) {
srv := mockTTSServer(t, func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
if !strings.Contains(string(body), "nova") {
t.Errorf("request body should contain voice 'nova', got: %s", string(body))
}
w.Header().Set("Content-Type", "audio/mpeg")
fmt.Fprintf(w, "mp3")
})
defer srv.Close()
svc := NewOpenAIService("test-key", "", "alloy", srv.URL, 1.0, 5, zap.NewNop().Sugar())
textStream := sendSentences("你好")
ch, err := svc.SynthesizeStream(context.Background(), textStream, Options{Voice: "nova"})
if err != nil {
t.Fatalf("SynthesizeStream() error: %v", err)
}
for range ch {
}
}

View File

@@ -0,0 +1,25 @@
package tts
import "context"
// Service 语音合成服务契约。
type Service interface {
// SynthesizeStream 流式合成。
// textStream 接收句子级文本(由 Orchestrator 的句子切分器产出),
// 返回的 channel 持续输出 MP3 音频 chunk。
SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error)
}
// Options 合成参数。
type Options struct {
Voice string // "alloy" | "nova" | "shimmer" 等
Speed float64 // 1.0 为正常语速
OutputFmt string // "mp3" — 固定使用 MP3
SampleRate int // 24000
}
// Chunk 一个音频片段。
type Chunk struct {
Audio []byte // MP3 音频数据(未 Base64 编码)
IsLast bool // 是否为最后一片
}

View File

@@ -0,0 +1,91 @@
// Package api 提供 REST API 处理函数。
package api
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/hhs/camtalk/internal/models"
"github.com/hhs/camtalk/internal/session"
)
// SessionHandler 提供会话相关的 REST 端点。
type SessionHandler struct {
sessionMgr session.Manager
}
// NewSessionHandler 创建 SessionHandler。
func NewSessionHandler(sessionMgr session.Manager) *SessionHandler {
return &SessionHandler{sessionMgr: sessionMgr}
}
// CreateSessionRequest POST /api/sessions 请求体(所有字段可选)。
type CreateSessionRequest struct {
Config *models.SessionConfig `json:"config,omitempty"`
}
// CreateSession POST /api/sessions — 创建新会话。
func (h *SessionHandler) CreateSession(c *gin.Context) {
var req CreateSessionRequest
// 请求体可选,解析失败不报错(使用默认配置)
_ = c.ShouldBindJSON(&req)
cfg := models.DefaultConfig()
if req.Config != nil {
cfg = *req.Config
}
sessionID, err := h.sessionMgr.Create(c.Request.Context(), cfg)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"code": "INTERNAL_ERROR",
"message": "failed to create session",
})
return
}
// 获取创建后的会话以返回 created_at
sess, err := h.sessionMgr.Get(c.Request.Context(), sessionID)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"code": "INTERNAL_ERROR",
"message": "failed to retrieve created session",
})
return
}
c.JSON(http.StatusCreated, gin.H{
"session_id": sess.ID,
"created_at": sess.CreatedAt,
})
}
// DestroySession DELETE /api/sessions/:id — 销毁会话。
func (h *SessionHandler) DestroySession(c *gin.Context) {
sessionID := c.Param("id")
err := h.sessionMgr.Destroy(c.Request.Context(), sessionID)
if err != nil {
if err == session.ErrSessionNotFound {
c.JSON(http.StatusNotFound, gin.H{
"code": "SESSION_NOT_FOUND",
"message": "session not found or already expired",
})
return
}
c.JSON(http.StatusInternalServerError, gin.H{
"code": "INTERNAL_ERROR",
"message": "failed to destroy session",
})
return
}
c.Status(http.StatusNoContent)
}
// RegisterRoutes 注册会话相关路由到给定的路由组。
func (h *SessionHandler) RegisterRoutes(rg *gin.RouterGroup) {
rg.POST("/sessions", h.CreateSession)
rg.DELETE("/sessions/:id", h.DestroySession)
}

View File

@@ -0,0 +1,160 @@
package config
import (
"fmt"
"os"
"strings"
"github.com/joho/godotenv"
"github.com/spf13/viper"
)
// Config 应用配置。
type Config struct {
App AppConfig `mapstructure:"app"`
Server ServerConfig `mapstructure:"server"`
Redis RedisConfig `mapstructure:"redis"`
AI AIConfig `mapstructure:"ai"`
Storage StorageConfig `mapstructure:"storage"`
Log LogConfig `mapstructure:"log"`
}
type AppConfig struct {
Env string `mapstructure:"env"`
Version string `mapstructure:"version"`
}
type ServerConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
ReadTimeout int `mapstructure:"read_timeout"`
WriteTimeout int `mapstructure:"write_timeout"`
}
// Addr 返回 host:port 地址。
func (s ServerConfig) Addr() string {
return fmt.Sprintf("%s:%d", s.Host, s.Port)
}
type RedisConfig struct {
Addr string `mapstructure:"addr"`
Password string `mapstructure:"password"`
DB int `mapstructure:"db"`
}
type AIConfig struct {
STT STTConfig `mapstructure:"stt"`
LLM LLMConfig `mapstructure:"llm"`
TTS TTSConfig `mapstructure:"tts"`
}
type STTConfig struct {
Provider string `mapstructure:"provider"`
APIKey string `mapstructure:"api_key"`
Model string `mapstructure:"model"`
Endpoint string `mapstructure:"endpoint"`
}
type LLMConfig struct {
Provider string `mapstructure:"provider"`
APIKey string `mapstructure:"api_key"`
Model string `mapstructure:"model"`
Endpoint string `mapstructure:"endpoint"`
Timeout int `mapstructure:"timeout"`
}
type TTSConfig struct {
Provider string `mapstructure:"provider"`
APIKey string `mapstructure:"api_key"`
Model string `mapstructure:"model"`
Voice string `mapstructure:"voice"`
Speed float64 `mapstructure:"speed"`
Endpoint string `mapstructure:"endpoint"`
Timeout int `mapstructure:"timeout"`
}
type StorageConfig struct {
Driver string `mapstructure:"driver"`
DSN string `mapstructure:"dsn"`
}
type LogConfig struct {
Level string `mapstructure:"level"`
Format string `mapstructure:"format"`
}
// Load 加载配置。优先级:环境变量 > config.{env}.yaml > config.yaml。
func Load() (*Config, error) {
v := viper.New()
v.SetConfigName("config")
v.SetConfigType("yaml")
v.AddConfigPath(".")
v.AddConfigPath("./config")
v.AddConfigPath("./backend")
// 默认值
v.SetDefault("app.env", "dev")
v.SetDefault("server.host", "0.0.0.0")
v.SetDefault("server.port", 8080)
v.SetDefault("server.read_timeout", 30)
v.SetDefault("server.write_timeout", 30)
v.SetDefault("redis.addr", "localhost:6379")
v.SetDefault("redis.db", 0)
v.SetDefault("ai.stt.provider", "deepgram")
v.SetDefault("ai.stt.model", "nova-2")
v.SetDefault("ai.stt.endpoint", "wss://api.deepgram.com/v1/listen")
v.SetDefault("ai.llm.provider", "openai")
v.SetDefault("ai.llm.model", "gpt-4o")
v.SetDefault("ai.llm.endpoint", "https://api.openai.com/v1")
v.SetDefault("ai.llm.timeout", 10)
v.SetDefault("ai.tts.provider", "openai")
v.SetDefault("ai.tts.model", "tts-1")
v.SetDefault("ai.tts.voice", "alloy")
v.SetDefault("ai.tts.speed", 1.0)
v.SetDefault("ai.tts.endpoint", "https://api.openai.com/v1")
v.SetDefault("ai.tts.timeout", 5)
v.SetDefault("storage.driver", "memory")
v.SetDefault("log.level", "info")
v.SetDefault("log.format", "console")
// 读取基础配置文件
_ = v.ReadInConfig() // 文件不存在不报错
// 根据 APP_ENV 覆盖
env := os.Getenv("APP_ENV")
if env == "" {
env = v.GetString("app.env")
}
if env != "" {
v.SetConfigName("config." + env)
_ = v.MergeInConfig()
}
// 加载 .env 文件(不覆盖已有环境变量)
// 按优先级尝试:当前目录、上级目录(兼容从 backend/ 或项目根目录启动)
_ = godotenv.Load()
_ = godotenv.Load("../.env")
// 环境变量覆盖
v.SetEnvPrefix("CAMTALK")
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.AutomaticEnv()
var cfg Config
if err := v.Unmarshal(&cfg); err != nil {
return nil, fmt.Errorf("config unmarshal: %w", err)
}
// 填充默认值
if cfg.Server.Host == "" {
cfg.Server.Host = "0.0.0.0"
}
if cfg.Server.Port == 0 {
cfg.Server.Port = 8080
}
if cfg.App.Env == "" {
cfg.App.Env = "dev"
}
return &cfg, nil
}

View File

@@ -0,0 +1,33 @@
package errors
import "github.com/hhs/camtalk/internal/models"
// 错误码常量,与 docs/03-接口文档.md 保持一致。
const (
CodeInvalidMessage = "INVALID_MESSAGE"
CodeSessionNotFound = "SESSION_NOT_FOUND"
CodeRateLimited = "RATE_LIMITED"
CodeImageTooLarge = "IMAGE_TOO_LARGE"
CodeAudioTooShort = "AUDIO_TOO_SHORT"
CodeLLMTimeout = "LLM_TIMEOUT"
CodeLLMError = "LLM_ERROR"
CodeSTTError = "STT_ERROR"
CodeTTSError = "TTS_ERROR"
CodeInternalError = "INTERNAL_ERROR"
)
// Sender 定义发送 WS 错误消息的接口,便于测试 mock。
type Sender interface {
SendError(code, requestID, message string)
}
// SendWSError 向客户端发送 error 消息。
// sender 是一个具有 sendJSON 方法的对象,这里用接口抽象。
func SendWSError(sender interface{ SendJSON(v any) error }, code, requestID string, err error) {
_ = sender.SendJSON(models.WsError{
Type: "error",
Code: code,
RequestID: requestID,
Message: err.Error(),
})
}

View File

@@ -0,0 +1,57 @@
package logger
import (
"os"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
// Log 是全局 SugaredLogger由 Init 初始化。
var Log *zap.SugaredLogger
// Init 初始化全局日志器。
// level: "debug", "info", "warn", "error"
// format: "json" 或 "console"
func Init(level, format string) {
var lvl zapcore.Level
switch level {
case "debug":
lvl = zapcore.DebugLevel
case "warn":
lvl = zapcore.WarnLevel
case "error":
lvl = zapcore.ErrorLevel
default:
lvl = zapcore.InfoLevel
}
encoderCfg := zap.NewProductionEncoderConfig()
encoderCfg.TimeKey = "ts"
encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder
var core zapcore.Core
if format == "console" {
core = zapcore.NewCore(
zapcore.NewConsoleEncoder(encoderCfg),
zapcore.AddSync(os.Stdout),
lvl,
)
} else {
core = zapcore.NewCore(
zapcore.NewJSONEncoder(encoderCfg),
zapcore.AddSync(os.Stdout),
lvl,
)
}
logger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel))
Log = logger.Sugar()
}
// Sync 刷新缓冲区,退出前调用。
func Sync() {
if Log != nil {
_ = Log.Sync()
}
}

View File

@@ -21,6 +21,26 @@ func DefaultConfig() SessionConfig {
return SessionConfig{TTSEnabled: true, DetailLevel: "low", Language: "zh-CN"}
}
// SessionConfigPatch 会话配置增量更新(指针字段表示"未传则不更新")。
type SessionConfigPatch struct {
TTSEnabled *bool `json:"tts_enabled,omitempty"`
DetailLevel *string `json:"detail_level,omitempty"`
Language *string `json:"language,omitempty"`
}
// Apply 将 patch 中的非 nil 字段覆盖到 cfg。
func (p SessionConfigPatch) Apply(cfg *SessionConfig) {
if p.TTSEnabled != nil {
cfg.TTSEnabled = *p.TTSEnabled
}
if p.DetailLevel != nil {
cfg.DetailLevel = *p.DetailLevel
}
if p.Language != nil {
cfg.Language = *p.Language
}
}
// Message 对话消息。
type Message struct {
Role string `json:"role"` // "user" | "assistant"

View File

@@ -0,0 +1,33 @@
// Package orchestrator 实现 STT → LLM → TTS 流式并行管道。
package orchestrator
import (
"context"
"github.com/hhs/camtalk/internal/models"
)
// Orchestrator AI 编排器接口。
// 接收查询并执行完整的 STT → LLM → TTS 管道。
type Orchestrator interface {
// ProcessQuery 处理一次用户查询。
// ctx 用于整体超时和中断控制。
// sessionID 用于会话管理和历史获取。
// req 包含图像和音频数据。
// history 是最近的对话历史。
// sender 用于向客户端推送消息。
ProcessQuery(
ctx context.Context,
sessionID string,
req models.WsQuery,
history []models.Message,
sender Sender,
) error
}
// QueryRequest 查询请求(内部使用)。
type QueryRequest struct {
Image []byte // JPEG 图片(已从 Base64 解码)
Audio []byte // 音频数据(已从 Base64 解码)
Language string // 语言,如 "zh-CN"
}

View File

@@ -0,0 +1,328 @@
package orchestrator
import (
"context"
"encoding/base64"
"strings"
"sync"
"time"
"unicode/utf8"
"github.com/hhs/camtalk/internal/ai/llm"
"github.com/hhs/camtalk/internal/ai/stt"
"github.com/hhs/camtalk/internal/ai/tts"
"github.com/hhs/camtalk/internal/logger"
"github.com/hhs/camtalk/internal/models"
"github.com/hhs/camtalk/internal/session"
)
// Pipeline 实现 Orchestrator 接口,管理 STT → LLM → TTS 流式管道。
type Pipeline struct {
sttService stt.Service
llmService llm.Service
ttsService tts.Service
sessionMgr session.Manager
model string // LLM 模型名,用于 llm_done 上报
}
// New 创建 Pipeline 实例。
func New(
sttService stt.Service,
llmService llm.Service,
ttsService tts.Service,
sessionMgr session.Manager,
model string,
) *Pipeline {
return &Pipeline{
sttService: sttService,
llmService: llmService,
ttsService: ttsService,
sessionMgr: sessionMgr,
model: model,
}
}
// ProcessQuery 实现 Orchestrator 接口。
func (p *Pipeline) ProcessQuery(
ctx context.Context,
sessionID string,
req models.WsQuery,
history []models.Message,
sender Sender,
) error {
log := logger.Log
startTime := time.Now()
// 解码音频数据
audio, err := base64.StdEncoding.DecodeString(req.Audio)
if err != nil {
log.Errorw("音频解码失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "INVALID_MESSAGE",
Message: "音频数据解码失败",
})
return err
}
// 解码图片数据(可选)
var image []byte
if req.Image != "" {
image, err = base64.StdEncoding.DecodeString(req.Image)
if err != nil {
log.Errorw("图片解码失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "INVALID_MESSAGE",
Message: "图片数据解码失败",
})
return err
}
}
// 设置活跃请求
if err := p.sessionMgr.SetActiveRequest(ctx, sessionID, req.RequestID); err != nil {
log.Errorw("设置活跃请求失败", "error", err)
}
defer p.sessionMgr.ClearActiveRequest(ctx, sessionID)
// 获取会话配置
sess, err := p.sessionMgr.Get(ctx, sessionID)
if err != nil {
log.Errorw("获取会话失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "SESSION_NOT_FOUND",
Message: "会话不存在",
})
return err
}
// Step 1: STT 语音识别
log.Infow("开始语音识别", "request_id", req.RequestID)
sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{
Encoding: "pcm_s16le",
SampleRate: 16000,
Language: sess.Config.Language,
})
if err != nil {
log.Errorw("语音识别失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "STT_ERROR",
Message: "语音识别失败",
})
return err
}
// 发送 STT 结果
if err := sender.SendSTTResult(models.WsSTTResult{
Type: "stt_result",
RequestID: req.RequestID,
Text: sttResult,
IsFinal: true,
}); err != nil {
log.Errorw("发送 STT 结果失败", "error", err)
}
// 追加用户消息到历史
p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{
Role: "user",
Content: sttResult,
})
// Step 2+3: LLM 流式推理 + TTS 并行合成
log.Infow("开始 LLM 推理", "request_id", req.RequestID)
llmReq := llm.Request{
Image: image,
Text: sttResult,
History: history,
Language: sess.Config.Language,
}
llmStream, err := p.llmService.ChatStream(ctx, llmReq)
if err != nil {
log.Errorw("LLM 流式推理启动失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "LLM_ERROR",
Message: "LLM 推理失败",
})
return err
}
// 创建句子切分器
sentenceCh := make(chan string, 4)
splitter := NewSplitter(sentenceCh)
// 并行LLM 消费 + TTS 合成
var wg sync.WaitGroup
var fullText string
var ttsErr error
// goroutine 1: 消费 LLM token + 句子切分
wg.Add(1)
go func() {
defer wg.Done()
defer close(sentenceCh)
fullText = p.consumeLLMStream(ctx, llmStream, req.RequestID, sender, splitter)
}()
// goroutine 2: TTS 合成(如果启用)
if sess.Config.TTSEnabled {
wg.Add(1)
go func() {
defer wg.Done()
log.Infow("开始 TTS 合成", "request_id", req.RequestID)
ttsErr = p.synthesizeTTS(ctx, sentenceCh, req.RequestID, sender)
}()
} else {
// 如果 TTS 未启用,需要消费 sentenceCh 防止阻塞
go func() {
for range sentenceCh {
}
}()
}
// 等待所有 goroutine 完成
wg.Wait()
// TTS 失败静默跳过
if ttsErr != nil {
log.Warnw("TTS 合成失败(已跳过)", "error", ttsErr)
}
// 追加助手消息到历史
p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{
Role: "assistant",
Content: fullText,
})
// 发送 llm_done
latency := time.Since(startTime).Milliseconds()
if err := sender.SendLLMDone(models.WsLLMDone{
Type: "llm_done",
RequestID: req.RequestID,
FullText: fullText,
Model: p.model,
LatencyMs: latency,
}); err != nil {
log.Errorw("发送 llm_done 失败", "error", err)
}
log.Infow("查询处理完成",
"request_id", req.RequestID,
"latency_ms", latency,
"text_length", utf8.RuneCountInString(fullText),
)
return nil
}
// consumeLLMStream 消费 LLM 流式输出,发送 llm_chunk 并进行句子切分。
func (p *Pipeline) consumeLLMStream(
ctx context.Context,
stream <-chan llm.Chunk,
requestID string,
sender Sender,
splitter *Splitter,
) string {
log := logger.Log
var fullText strings.Builder
for chunk := range stream {
// 检查上下文是否已取消
select {
case <-ctx.Done():
log.Infow("LLM 流被中断", "request_id", requestID)
return fullText.String()
default:
}
if chunk.Done {
// 流结束
if chunk.TokensUsed != nil {
log.Infow("LLM 用量统计",
"request_id", requestID,
"prompt_tokens", chunk.TokensUsed.Prompt,
"completion_tokens", chunk.TokensUsed.Completion,
"total_tokens", chunk.TokensUsed.Total,
)
}
break
}
// 累积全文
fullText.WriteString(chunk.Delta)
// 发送 llm_chunk
if err := sender.SendLLMChunk(models.WsLLMChunk{
Type: "llm_chunk",
RequestID: requestID,
Delta: chunk.Delta,
Role: "assistant",
}); err != nil {
log.Errorw("发送 llm_chunk 失败", "error", err)
}
// 句子切分
splitter.Feed(chunk.Delta)
}
// 刷新切分器中的剩余文本
splitter.Flush()
return fullText.String()
}
// synthesizeTTS 从句子 channel 读取文本,进行 TTS 合成并发送音频。
func (p *Pipeline) synthesizeTTS(
ctx context.Context,
sentenceCh <-chan string,
requestID string,
sender Sender,
) error {
log := logger.Log
ttsStream, err := p.ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{
Voice: "alloy",
Speed: 1.0,
OutputFmt: "mp3",
SampleRate: 24000,
})
if err != nil {
log.Errorw("TTS 合成启动失败", "error", err)
return err
}
// 消费 TTS 音频流
for chunk := range ttsStream {
// 检查上下文是否已取消
select {
case <-ctx.Done():
log.Infow("TTS 流被中断", "request_id", requestID)
return ctx.Err()
default:
}
// Base64 编码音频数据
audioBase64 := base64.StdEncoding.EncodeToString(chunk.Audio)
if err := sender.SendTTSAudio(models.WsTTSAudio{
Type: "tts_audio",
RequestID: requestID,
Audio: audioBase64,
MimeType: "audio/mp3",
IsLast: chunk.IsLast,
}); err != nil {
log.Errorw("发送 tts_audio 失败", "error", err)
}
}
return nil
}

View File

@@ -0,0 +1,661 @@
package orchestrator
import (
"context"
"encoding/base64"
"errors"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/hhs/camtalk/internal/ai/llm"
"github.com/hhs/camtalk/internal/ai/stt"
"github.com/hhs/camtalk/internal/ai/tts"
"github.com/hhs/camtalk/internal/logger"
"github.com/hhs/camtalk/internal/models"
)
func init() {
logger.Init("debug", "console")
}
// MockSTTService mock STT 服务
type MockSTTService struct {
mock.Mock
}
func (m *MockSTTService) Recognize(ctx context.Context, audio []byte, opts stt.Options) (string, error) {
args := m.Called(ctx, audio, opts)
return args.String(0), args.Error(1)
}
// MockLLMService mock LLM 服务
type MockLLMService struct {
mock.Mock
}
func (m *MockLLMService) ChatStream(ctx context.Context, req llm.Request) (<-chan llm.Chunk, error) {
args := m.Called(ctx, req)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(<-chan llm.Chunk), args.Error(1)
}
// MockTTSService mock TTS 服务
type MockTTSService struct {
mock.Mock
}
func (m *MockTTSService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts tts.Options) (<-chan tts.Chunk, error) {
args := m.Called(ctx, textStream, opts)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(<-chan tts.Chunk), args.Error(1)
}
// MockSessionManager mock 会话管理器
type MockSessionManager struct {
mock.Mock
}
func (m *MockSessionManager) Create(ctx context.Context, config models.SessionConfig) (string, error) {
args := m.Called(ctx, config)
return args.String(0), args.Error(1)
}
func (m *MockSessionManager) Get(ctx context.Context, sessionID string) (*models.Session, error) {
args := m.Called(ctx, sessionID)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*models.Session), args.Error(1)
}
func (m *MockSessionManager) UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error {
args := m.Called(ctx, sessionID, patch)
return args.Error(0)
}
func (m *MockSessionManager) GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error) {
args := m.Called(ctx, sessionID, limit)
return args.Get(0).([]models.Message), args.Error(1)
}
func (m *MockSessionManager) AppendMessage(ctx context.Context, sessionID string, msg models.Message) error {
args := m.Called(ctx, sessionID, msg)
return args.Error(0)
}
func (m *MockSessionManager) SetActiveRequest(ctx context.Context, sessionID string, requestID string) error {
args := m.Called(ctx, sessionID, requestID)
return args.Error(0)
}
func (m *MockSessionManager) GetActiveRequestID(ctx context.Context, sessionID string) (string, error) {
args := m.Called(ctx, sessionID)
return args.String(0), args.Error(1)
}
func (m *MockSessionManager) ClearActiveRequest(ctx context.Context, sessionID string) error {
args := m.Called(ctx, sessionID)
return args.Error(0)
}
func (m *MockSessionManager) Touch(ctx context.Context, sessionID string) error {
args := m.Called(ctx, sessionID)
return args.Error(0)
}
func (m *MockSessionManager) Destroy(ctx context.Context, sessionID string) error {
args := m.Called(ctx, sessionID)
return args.Error(0)
}
func (m *MockSessionManager) ActiveCount() int {
args := m.Called()
return args.Int(0)
}
// MockSender mock WebSocket 发送器
type MockSender struct {
mock.Mock
STTResults []models.WsSTTResult
LLMChunks []models.WsLLMChunk
LLMDones []models.WsLLMDone
TTSAudios []models.WsTTSAudio
Errors []models.WsError
}
func NewMockSender() *MockSender {
return &MockSender{
STTResults: make([]models.WsSTTResult, 0),
LLMChunks: make([]models.WsLLMChunk, 0),
LLMDones: make([]models.WsLLMDone, 0),
TTSAudios: make([]models.WsTTSAudio, 0),
Errors: make([]models.WsError, 0),
}
}
func (m *MockSender) SendSTTResult(result models.WsSTTResult) error {
m.STTResults = append(m.STTResults, result)
args := m.Called(result)
return args.Error(0)
}
func (m *MockSender) SendLLMChunk(chunk models.WsLLMChunk) error {
m.LLMChunks = append(m.LLMChunks, chunk)
args := m.Called(chunk)
return args.Error(0)
}
func (m *MockSender) SendLLMDone(done models.WsLLMDone) error {
m.LLMDones = append(m.LLMDones, done)
args := m.Called(done)
return args.Error(0)
}
func (m *MockSender) SendTTSAudio(audio models.WsTTSAudio) error {
m.TTSAudios = append(m.TTSAudios, audio)
args := m.Called(audio)
return args.Error(0)
}
func (m *MockSender) SendError(err models.WsError) error {
m.Errors = append(m.Errors, err)
args := m.Called(err)
return args.Error(0)
}
// 辅助函数:创建 LLM 流式响应
func createLLMStream(chunks []llm.Chunk) <-chan llm.Chunk {
ch := make(chan llm.Chunk, len(chunks))
for _, chunk := range chunks {
ch <- chunk
}
close(ch)
return ch
}
// 辅助函数:创建 TTS 流式响应
func createTTSStream(chunks []tts.Chunk) <-chan tts.Chunk {
ch := make(chan tts.Chunk, len(chunks))
for _, chunk := range chunks {
ch <- chunk
}
close(ch)
return ch
}
// TestProcessQuery_Success 测试完整流程
func TestProcessQuery_Success(t *testing.T) {
// 准备测试数据
audioData := []byte("test audio")
imageData := []byte("test image")
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
imageBase64 := base64.StdEncoding.EncodeToString(imageData)
req := models.WsQuery{
Type: "query",
RequestID: "req-123",
Image: imageBase64,
Audio: audioBase64,
}
session := &models.Session{
ID: "session-123",
Config: models.SessionConfig{
TTSEnabled: true,
DetailLevel: "low",
Language: "zh-CN",
},
}
// 创建 mock
mockSTT := new(MockSTTService)
mockLLM := new(MockLLMService)
mockTTS := new(MockTTSService)
mockSession := new(MockSessionManager)
mockSender := NewMockSender()
// 设置 mock 期望
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
mockSTT.On("Recognize", mock.Anything, audioData, stt.Options{
Encoding: "pcm_s16le",
SampleRate: 16000,
Language: "zh-CN",
}).Return("你好,世界", nil)
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
llmChunks := []llm.Chunk{
{Delta: "你好"},
{Delta: ",世界!"},
{Done: true, TokensUsed: &llm.TokenUsage{Prompt: 10, Completion: 5, Total: 15}},
}
mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil)
mockSender.On("SendLLMChunk", mock.Anything).Return(nil)
mockSender.On("SendLLMDone", mock.Anything).Return(nil)
ttsChunks := []tts.Chunk{
{Audio: []byte("audio1"), IsLast: false},
{Audio: []byte("audio2"), IsLast: true},
}
mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return(createTTSStream(ttsChunks), nil)
mockSender.On("SendTTSAudio", mock.Anything).Return(nil)
// 创建 Pipeline
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
// 执行
ctx := context.Background()
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
// 验证
assert.NoError(t, err)
assert.Len(t, mockSender.STTResults, 1)
assert.Equal(t, "你好,世界", mockSender.STTResults[0].Text)
assert.Len(t, mockSender.LLMChunks, 2)
assert.Len(t, mockSender.LLMDones, 1)
assert.Len(t, mockSender.TTSAudios, 2)
mockSTT.AssertExpectations(t)
mockLLM.AssertExpectations(t)
mockTTS.AssertExpectations(t)
mockSession.AssertExpectations(t)
}
// TestProcessQuery_STTError 测试 STT 失败降级
func TestProcessQuery_STTError(t *testing.T) {
audioData := []byte("test audio")
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
req := models.WsQuery{
Type: "query",
RequestID: "req-123",
Audio: audioBase64,
}
mockSTT := new(MockSTTService)
mockLLM := new(MockLLMService)
mockTTS := new(MockTTSService)
mockSession := new(MockSessionManager)
mockSender := NewMockSender()
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
mockSession.On("Get", mock.Anything, "session-123").Return(&models.Session{
ID: "session-123",
Config: models.SessionConfig{
Language: "zh-CN",
},
}, nil)
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).
Return("", errors.New("STT service unavailable"))
mockSender.On("SendError", mock.Anything).Return(nil)
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
ctx := context.Background()
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
assert.Error(t, err)
assert.Len(t, mockSender.Errors, 1)
assert.Equal(t, "STT_ERROR", mockSender.Errors[0].Code)
mockSTT.AssertExpectations(t)
mockLLM.AssertNotCalled(t, "ChatStream")
mockTTS.AssertNotCalled(t, "SynthesizeStream")
}
// TestProcessQuery_LLMError 测试 LLM 失败降级
func TestProcessQuery_LLMError(t *testing.T) {
audioData := []byte("test audio")
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
req := models.WsQuery{
Type: "query",
RequestID: "req-123",
Audio: audioBase64,
}
session := &models.Session{
ID: "session-123",
Config: models.SessionConfig{
TTSEnabled: true,
Language: "zh-CN",
},
}
mockSTT := new(MockSTTService)
mockLLM := new(MockLLMService)
mockTTS := new(MockTTSService)
mockSession := new(MockSessionManager)
mockSender := NewMockSender()
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil)
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
mockLLM.On("ChatStream", mock.Anything, mock.Anything).
Return(nil, errors.New("LLM service unavailable"))
mockSender.On("SendError", mock.Anything).Return(nil)
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
ctx := context.Background()
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
assert.Error(t, err)
assert.Len(t, mockSender.Errors, 1)
assert.Equal(t, "LLM_ERROR", mockSender.Errors[0].Code)
mockSTT.AssertExpectations(t)
mockLLM.AssertExpectations(t)
mockTTS.AssertNotCalled(t, "SynthesizeStream")
}
// TestProcessQuery_TTSError 测试 TTS 失败静默跳过
func TestProcessQuery_TTSError(t *testing.T) {
audioData := []byte("test audio")
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
req := models.WsQuery{
Type: "query",
RequestID: "req-123",
Audio: audioBase64,
}
session := &models.Session{
ID: "session-123",
Config: models.SessionConfig{
TTSEnabled: true,
Language: "zh-CN",
},
}
mockSTT := new(MockSTTService)
mockLLM := new(MockLLMService)
mockTTS := new(MockTTSService)
mockSession := new(MockSessionManager)
mockSender := NewMockSender()
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil)
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
llmChunks := []llm.Chunk{
{Delta: "你好"},
{Done: true},
}
mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil)
mockSender.On("SendLLMChunk", mock.Anything).Return(nil)
mockSender.On("SendLLMDone", mock.Anything).Return(nil)
mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).
Return(nil, errors.New("TTS service unavailable"))
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
ctx := context.Background()
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
// TTS 失败应该静默跳过,不返回错误
assert.NoError(t, err)
assert.Len(t, mockSender.LLMDones, 1)
assert.Len(t, mockSender.TTSAudios, 0)
mockSTT.AssertExpectations(t)
mockLLM.AssertExpectations(t)
mockTTS.AssertExpectations(t)
}
// TestProcessQuery_ContextCancelled 测试上下文取消Interrupt
func TestProcessQuery_ContextCancelled(t *testing.T) {
audioData := []byte("test audio")
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
req := models.WsQuery{
Type: "query",
RequestID: "req-123",
Audio: audioBase64,
}
session := &models.Session{
ID: "session-123",
Config: models.SessionConfig{
TTSEnabled: true,
Language: "zh-CN",
},
}
mockSTT := new(MockSTTService)
mockLLM := new(MockLLMService)
mockTTS := new(MockTTSService)
mockSession := new(MockSessionManager)
mockSender := NewMockSender()
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil)
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
// 创建一个会延迟的 LLM 流,以便我们可以取消上下文
llmCh := make(chan llm.Chunk)
go func() {
time.Sleep(100 * time.Millisecond)
llmCh <- llm.Chunk{Delta: "你"}
time.Sleep(100 * time.Millisecond)
llmCh <- llm.Chunk{Delta: "好"}
close(llmCh)
}()
mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return((<-chan llm.Chunk)(llmCh), nil)
mockSender.On("SendLLMChunk", mock.Anything).Return(nil)
mockSender.On("SendLLMDone", mock.Anything).Return(nil)
// 创建一个会延迟的 TTS 流
ttsCh := make(chan tts.Chunk)
go func() {
time.Sleep(200 * time.Millisecond)
close(ttsCh)
}()
mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return((<-chan tts.Chunk)(ttsCh), nil)
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
// 创建可取消的上下文
ctx, cancel := context.WithCancel(context.Background())
// 在 50ms 后取消
go func() {
time.Sleep(50 * time.Millisecond)
cancel()
}()
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
// 上下文取消后,流程应该正常完成(中断流但不返回错误)
assert.NoError(t, err)
mockSTT.AssertExpectations(t)
}
// TestProcessQuery_DisabledTTS 测试 TTS 未启用的情况
func TestProcessQuery_DisabledTTS(t *testing.T) {
audioData := []byte("test audio")
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
req := models.WsQuery{
Type: "query",
RequestID: "req-123",
Audio: audioBase64,
}
session := &models.Session{
ID: "session-123",
Config: models.SessionConfig{
TTSEnabled: false, // TTS 未启用
Language: "zh-CN",
},
}
mockSTT := new(MockSTTService)
mockLLM := new(MockLLMService)
mockTTS := new(MockTTSService)
mockSession := new(MockSessionManager)
mockSender := NewMockSender()
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
mockSession.On("Get", mock.Anything, "session-123").Return(session, nil)
mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil)
mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil)
mockSender.On("SendSTTResult", mock.Anything).Return(nil)
llmChunks := []llm.Chunk{
{Delta: "你好"},
{Done: true},
}
mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil)
mockSender.On("SendLLMChunk", mock.Anything).Return(nil)
mockSender.On("SendLLMDone", mock.Anything).Return(nil)
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
ctx := context.Background()
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
assert.NoError(t, err)
assert.Len(t, mockSender.LLMDones, 1)
assert.Len(t, mockSender.TTSAudios, 0)
// TTS 不应该被调用
mockTTS.AssertNotCalled(t, "SynthesizeStream")
}
// TestSplitter 测试句子切分器
func TestSplitter(t *testing.T) {
ch := make(chan string, 10)
splitter := NewSplitter(ch)
// 输入包含多个句子的文本
splitter.Feed("你好。")
splitter.Feed("世界!")
splitter.Feed("这是")
splitter.Feed("一个测试。")
splitter.Flush()
// 应该有 3 个句子
assert.Equal(t, 3, len(ch))
assert.Equal(t, "你好。", <-ch)
assert.Equal(t, "世界!", <-ch)
assert.Equal(t, "这是一个测试。", <-ch)
}
// TestSplitter_NoDelimiter 测试没有分隔符的情况
func TestSplitter_NoDelimiter(t *testing.T) {
ch := make(chan string, 10)
splitter := NewSplitter(ch)
splitter.Feed("没有分隔符的文本")
splitter.Flush()
// 应该有 1 个句子Flush 会发送剩余内容)
assert.Equal(t, 1, len(ch))
assert.Equal(t, "没有分隔符的文本", <-ch)
}
// TestSplitter_Empty 测试空输入
func TestSplitter_Empty(t *testing.T) {
ch := make(chan string, 10)
splitter := NewSplitter(ch)
splitter.Flush()
// 应该没有句子
assert.Equal(t, 0, len(ch))
}
// TestProcessQuery_InvalidAudio 测试无效音频数据
func TestProcessQuery_InvalidAudio(t *testing.T) {
req := models.WsQuery{
Type: "query",
RequestID: "req-123",
Audio: "invalid-base64!!!",
}
mockSTT := new(MockSTTService)
mockLLM := new(MockLLMService)
mockTTS := new(MockTTSService)
mockSession := new(MockSessionManager)
mockSender := NewMockSender()
mockSender.On("SendError", mock.Anything).Return(nil)
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
ctx := context.Background()
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
assert.Error(t, err)
assert.Len(t, mockSender.Errors, 1)
assert.Equal(t, "INVALID_MESSAGE", mockSender.Errors[0].Code)
}
// TestProcessQuery_SessionNotFound 测试会话不存在
func TestProcessQuery_SessionNotFound(t *testing.T) {
audioData := []byte("test audio")
audioBase64 := base64.StdEncoding.EncodeToString(audioData)
req := models.WsQuery{
Type: "query",
RequestID: "req-123",
Audio: audioBase64,
}
mockSTT := new(MockSTTService)
mockLLM := new(MockLLMService)
mockTTS := new(MockTTSService)
mockSession := new(MockSessionManager)
mockSender := NewMockSender()
mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil)
mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil)
mockSession.On("Get", mock.Anything, "session-123").Return(nil, errors.New("session not found"))
mockSender.On("SendError", mock.Anything).Return(nil)
pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, "gpt-4o")
ctx := context.Background()
err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender)
assert.Error(t, err)
assert.Len(t, mockSender.Errors, 1)
assert.Equal(t, "SESSION_NOT_FOUND", mockSender.Errors[0].Code)
}

View File

@@ -0,0 +1,22 @@
package orchestrator
import "github.com/hhs/camtalk/internal/models"
// Sender 抽象 WebSocket 消息推送能力。
// 便于测试时 mock避免依赖真实 WebSocket 连接。
type Sender interface {
// SendSTTResult 发送语音识别结果。
SendSTTResult(result models.WsSTTResult) error
// SendLLMChunk 发送 LLM 流式文本增量。
SendLLMChunk(chunk models.WsLLMChunk) error
// SendLLMDone 发送 LLM 流结束信号。
SendLLMDone(done models.WsLLMDone) error
// SendTTSAudio 发送 TTS 音频数据。
SendTTSAudio(audio models.WsTTSAudio) error
// SendError 发送错误消息。
SendError(err models.WsError) error
}

View File

@@ -0,0 +1,55 @@
package orchestrator
import "strings"
// sentenceDelimiters 句子分隔符集合。
var sentenceDelimiters = map[rune]bool{
'。': true,
'': true,
'': true,
'\n': true,
'.': true,
'!': true,
'?': true,
}
// Splitter 句子切分器。
// 将流式文本按句子边界切分,发送到 channel 供 TTS 合成。
type Splitter struct {
ch chan<- string
buffer strings.Builder
}
// NewSplitter 创建句子切分器。
// ch 用于接收切分后的句子文本。
func NewSplitter(ch chan<- string) *Splitter {
return &Splitter{
ch: ch,
}
}
// Feed 输入增量文本,遇到句子分隔符时发送完整句子。
func (s *Splitter) Feed(delta string) {
for _, r := range delta {
s.buffer.WriteRune(r)
if sentenceDelimiters[r] {
s.flushBuffer()
}
}
}
// Flush 刷新缓冲区中的剩余文本(即使没有句子分隔符)。
func (s *Splitter) Flush() {
if s.buffer.Len() > 0 {
s.flushBuffer()
}
}
// flushBuffer 将缓冲区内容发送到 channel 并清空。
func (s *Splitter) flushBuffer() {
text := strings.TrimSpace(s.buffer.String())
if text != "" {
s.ch <- text
}
s.buffer.Reset()
}

View File

@@ -0,0 +1,49 @@
// Package session 提供会话生命周期管理能力。
package session
import (
"context"
"errors"
"github.com/hhs/camtalk/internal/models"
)
// ErrSessionNotFound 会话不存在或已过期。
var ErrSessionNotFound = errors.New("session not found")
// Manager 会话管理器接口。
// WebSocket Handler 通过此接口操作会话,不直接接触存储层。
type Manager interface {
// Create 创建新会话,返回 session ID。
Create(ctx context.Context, config models.SessionConfig) (string, error)
// Get 获取会话(含 config。不存在返回 ErrSessionNotFound。
Get(ctx context.Context, sessionID string) (*models.Session, error)
// UpdateConfig 更新会话配置config 消息触发)。
UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error
// GetHistory 获取最近 N 轮对话历史(供 Orchestrator 构建 LLM 上下文)。
GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error)
// AppendMessage 追加一条对话消息,同时刷新 TTL。
AppendMessage(ctx context.Context, sessionID string, msg models.Message) error
// SetActiveRequest 标记当前正在处理的请求 IDinterrupt 用)。
SetActiveRequest(ctx context.Context, sessionID string, requestID string) error
// GetActiveRequestID 获取当前活跃请求 ID。
GetActiveRequestID(ctx context.Context, sessionID string) (string, error)
// ClearActiveRequest 清除活跃请求标记(请求完成或中断后)。
ClearActiveRequest(ctx context.Context, sessionID string) error
// Touch 刷新 TTL心跳时调用
Touch(ctx context.Context, sessionID string) error
// Destroy 显式销毁会话REST API DELETE 或连接断开清理)。
Destroy(ctx context.Context, sessionID string) error
// ActiveCount 返回当前活跃会话数(健康检查用)。
ActiveCount() int
}

View File

@@ -0,0 +1,275 @@
package session
import (
"context"
"sync"
"time"
"github.com/google/uuid"
"github.com/hhs/camtalk/internal/logger"
"github.com/hhs/camtalk/internal/models"
)
const (
defaultTTL = 30 * time.Minute
defaultHistorySize = 20
)
// sessionEntry 内部会话条目。
type sessionEntry struct {
session models.Session
history []models.Message
activeReqID string
lastActive time.Time
}
// MemoryManager 基于内存的 SessionManager 实现。
// 适用于 MVP 和无 Redis 的开发环境。
type MemoryManager struct {
mu sync.RWMutex
sessions map[string]*sessionEntry
ttl time.Duration
maxHistory int
stopCleaner chan struct{}
}
// NewMemoryManager 创建内存版 SessionManager。
// ttl 为会话过期时间maxHistory 为对话历史上限0 表示使用默认值 20
func NewMemoryManager(ttl time.Duration, maxHistory int) *MemoryManager {
if ttl <= 0 {
ttl = defaultTTL
}
if maxHistory <= 0 {
maxHistory = defaultHistorySize
}
m := &MemoryManager{
sessions: make(map[string]*sessionEntry),
ttl: ttl,
maxHistory: maxHistory,
stopCleaner: make(chan struct{}),
}
// 启动后台清理 goroutine每分钟清除过期会话。
go m.cleanLoop()
return m
}
// cleanLoop 后台定期清理过期会话。
func (m *MemoryManager) cleanLoop() {
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
for {
select {
case <-ticker.C:
m.cleanExpired()
case <-m.stopCleaner:
return
}
}
}
// cleanExpired 清除所有过期会话。
func (m *MemoryManager) cleanExpired() {
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now()
for id, entry := range m.sessions {
if now.Sub(entry.lastActive) > m.ttl {
delete(m.sessions, id)
logger.Log.Debugw("session expired (cleaner)", "session", id)
}
}
}
// Stop 停止后台清理 goroutine。应用退出前调用。
func (m *MemoryManager) Stop() {
close(m.stopCleaner)
}
// isExpired 检查会话是否过期(调用方需持锁或在已知 entry 存在时调用)。
func (m *MemoryManager) isExpired(entry *sessionEntry) bool {
return time.Since(entry.lastActive) > m.ttl
}
// Create 创建新会话。
func (m *MemoryManager) Create(_ context.Context, config models.SessionConfig) (string, error) {
m.mu.Lock()
defer m.mu.Unlock()
id := uuid.New().String()
now := time.Now()
m.sessions[id] = &sessionEntry{
session: models.Session{
ID: id,
CreatedAt: now,
Config: config,
},
history: make([]models.Message, 0),
lastActive: now,
}
logger.Log.Debugw("session created", "session", id)
return id, nil
}
// Get 获取会话。
func (m *MemoryManager) Get(_ context.Context, sessionID string) (*models.Session, error) {
m.mu.RLock()
defer m.mu.RUnlock()
entry, ok := m.sessions[sessionID]
if !ok || m.isExpired(entry) {
return nil, ErrSessionNotFound
}
sess := entry.session // 复制一份返回
return &sess, nil
}
// UpdateConfig 更新会话配置。
func (m *MemoryManager) UpdateConfig(_ context.Context, sessionID string, patch models.SessionConfigPatch) error {
m.mu.Lock()
defer m.mu.Unlock()
entry, ok := m.sessions[sessionID]
if !ok || m.isExpired(entry) {
return ErrSessionNotFound
}
patch.Apply(&entry.session.Config)
entry.lastActive = time.Now()
logger.Log.Debugw("session config updated", "session", sessionID)
return nil
}
// GetHistory 获取最近 N 轮对话历史。
func (m *MemoryManager) GetHistory(_ context.Context, sessionID string, limit int) ([]models.Message, error) {
m.mu.RLock()
defer m.mu.RUnlock()
entry, ok := m.sessions[sessionID]
if !ok || m.isExpired(entry) {
return nil, ErrSessionNotFound
}
if limit <= 0 || limit > len(entry.history) {
limit = len(entry.history)
}
// 返回最近 limit 条的副本
result := make([]models.Message, limit)
copy(result, entry.history[len(entry.history)-limit:])
return result, nil
}
// AppendMessage 追加一条对话消息,同时刷新 TTL。
func (m *MemoryManager) AppendMessage(_ context.Context, sessionID string, msg models.Message) error {
m.mu.Lock()
defer m.mu.Unlock()
entry, ok := m.sessions[sessionID]
if !ok || m.isExpired(entry) {
return ErrSessionNotFound
}
entry.history = append(entry.history, msg)
// 超过上限时裁剪,保留最新的 maxHistory 条
if len(entry.history) > m.maxHistory {
entry.history = entry.history[len(entry.history)-m.maxHistory:]
}
entry.lastActive = time.Now()
return nil
}
// SetActiveRequest 标记当前正在处理的请求 ID。
func (m *MemoryManager) SetActiveRequest(_ context.Context, sessionID string, requestID string) error {
m.mu.Lock()
defer m.mu.Unlock()
entry, ok := m.sessions[sessionID]
if !ok || m.isExpired(entry) {
return ErrSessionNotFound
}
entry.activeReqID = requestID
entry.lastActive = time.Now()
return nil
}
// GetActiveRequestID 获取当前活跃请求 ID。
func (m *MemoryManager) GetActiveRequestID(_ context.Context, sessionID string) (string, error) {
m.mu.RLock()
defer m.mu.RUnlock()
entry, ok := m.sessions[sessionID]
if !ok || m.isExpired(entry) {
return "", ErrSessionNotFound
}
return entry.activeReqID, nil
}
// ClearActiveRequest 清除活跃请求标记。
func (m *MemoryManager) ClearActiveRequest(_ context.Context, sessionID string) error {
m.mu.Lock()
defer m.mu.Unlock()
entry, ok := m.sessions[sessionID]
if !ok || m.isExpired(entry) {
return ErrSessionNotFound
}
entry.activeReqID = ""
entry.lastActive = time.Now()
return nil
}
// Touch 刷新 TTL。
func (m *MemoryManager) Touch(_ context.Context, sessionID string) error {
m.mu.Lock()
defer m.mu.Unlock()
entry, ok := m.sessions[sessionID]
if !ok || m.isExpired(entry) {
return ErrSessionNotFound
}
entry.lastActive = time.Now()
return nil
}
// Destroy 显式销毁会话。
func (m *MemoryManager) Destroy(_ context.Context, sessionID string) error {
m.mu.Lock()
defer m.mu.Unlock()
if _, ok := m.sessions[sessionID]; !ok {
return ErrSessionNotFound
}
delete(m.sessions, sessionID)
logger.Log.Debugw("session destroyed", "session", sessionID)
return nil
}
// ActiveCount 返回当前活跃会话数。
func (m *MemoryManager) ActiveCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
now := time.Now()
count := 0
for _, entry := range m.sessions {
if now.Sub(entry.lastActive) <= m.ttl {
count++
}
}
return count
}

View File

@@ -0,0 +1,286 @@
package session
import (
"context"
"testing"
"time"
"github.com/hhs/camtalk/internal/logger"
"github.com/hhs/camtalk/internal/models"
)
func init() {
logger.Init("debug", "console")
}
func TestCreateAndGet(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
config := models.DefaultConfig()
id, err := m.Create(ctx, config)
if err != nil {
t.Fatalf("Create: %v", err)
}
if id == "" {
t.Fatal("Create returned empty ID")
}
sess, err := m.Get(ctx, id)
if err != nil {
t.Fatalf("Get: %v", err)
}
if sess.ID != id {
t.Errorf("ID = %q, want %q", sess.ID, id)
}
if sess.Config.Language != "zh-CN" {
t.Errorf("Language = %q, want %q", sess.Config.Language, "zh-CN")
}
}
func TestGetNotFound(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
_, err := m.Get(ctx, "nonexistent")
if err != ErrSessionNotFound {
t.Errorf("Get nonexistent: err = %v, want ErrSessionNotFound", err)
}
}
func TestExpire(t *testing.T) {
// 使用极短 TTL 测试过期
m := NewMemoryManager(50*time.Millisecond, 20)
defer m.Stop()
ctx := context.Background()
id, _ := m.Create(ctx, models.DefaultConfig())
// 未过期时应能获取
_, err := m.Get(ctx, id)
if err != nil {
t.Fatalf("Get before expire: %v", err)
}
// 等待过期
time.Sleep(80 * time.Millisecond)
_, err = m.Get(ctx, id)
if err != ErrSessionNotFound {
t.Errorf("Get after expire: err = %v, want ErrSessionNotFound", err)
}
}
func TestDestroy(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
id, _ := m.Create(ctx, models.DefaultConfig())
if err := m.Destroy(ctx, id); err != nil {
t.Fatalf("Destroy: %v", err)
}
_, err := m.Get(ctx, id)
if err != ErrSessionNotFound {
t.Errorf("Get after Destroy: err = %v, want ErrSessionNotFound", err)
}
}
func TestDestroyNotFound(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
err := m.Destroy(ctx, "nonexistent")
if err != ErrSessionNotFound {
t.Errorf("Destroy nonexistent: err = %v, want ErrSessionNotFound", err)
}
}
func TestAppendMessageAndGetHistory(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
id, _ := m.Create(ctx, models.DefaultConfig())
msgs := []models.Message{
{Role: "user", Content: "你好"},
{Role: "assistant", Content: "你好!有什么可以帮你的吗?"},
{Role: "user", Content: "这是什么?"},
{Role: "assistant", Content: "这是一朵花。"},
}
for _, msg := range msgs {
if err := m.AppendMessage(ctx, id, msg); err != nil {
t.Fatalf("AppendMessage: %v", err)
}
}
history, err := m.GetHistory(ctx, id, 0)
if err != nil {
t.Fatalf("GetHistory: %v", err)
}
if len(history) != 4 {
t.Fatalf("GetHistory len = %d, want 4", len(history))
}
if history[0].Content != "你好" {
t.Errorf("history[0] = %q, want %q", history[0].Content, "你好")
}
}
func TestGetHistoryLimit(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
id, _ := m.Create(ctx, models.DefaultConfig())
for i := 0; i < 10; i++ {
m.AppendMessage(ctx, id, models.Message{Role: "user", Content: "msg"})
}
history, err := m.GetHistory(ctx, id, 3)
if err != nil {
t.Fatalf("GetHistory: %v", err)
}
if len(history) != 3 {
t.Fatalf("GetHistory limit=3: len = %d, want 3", len(history))
}
}
func TestHistoryLimit(t *testing.T) {
const maxHistory = 5
m := NewMemoryManager(30*time.Minute, maxHistory)
defer m.Stop()
ctx := context.Background()
id, _ := m.Create(ctx, models.DefaultConfig())
// 插入超过上限的消息
for i := 0; i < 10; i++ {
m.AppendMessage(ctx, id, models.Message{Role: "user", Content: "msg"})
}
history, err := m.GetHistory(ctx, id, 0)
if err != nil {
t.Fatalf("GetHistory: %v", err)
}
if len(history) != maxHistory {
t.Fatalf("GetHistory after overflow: len = %d, want %d", len(history), maxHistory)
}
}
func TestUpdateConfig(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
id, _ := m.Create(ctx, models.DefaultConfig())
ttsEnabled := false
detailLevel := "high"
patch := models.SessionConfigPatch{
TTSEnabled: &ttsEnabled,
DetailLevel: &detailLevel,
}
if err := m.UpdateConfig(ctx, id, patch); err != nil {
t.Fatalf("UpdateConfig: %v", err)
}
sess, _ := m.Get(ctx, id)
if sess.Config.TTSEnabled != false {
t.Errorf("TTSEnabled = %v, want false", sess.Config.TTSEnabled)
}
if sess.Config.DetailLevel != "high" {
t.Errorf("DetailLevel = %q, want %q", sess.Config.DetailLevel, "high")
}
// Language 未传,应保持原值
if sess.Config.Language != "zh-CN" {
t.Errorf("Language = %q, want %q", sess.Config.Language, "zh-CN")
}
}
func TestActiveRequest(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
id, _ := m.Create(ctx, models.DefaultConfig())
// 初始应为空
reqID, err := m.GetActiveRequestID(ctx, id)
if err != nil {
t.Fatalf("GetActiveRequestID: %v", err)
}
if reqID != "" {
t.Errorf("initial active request = %q, want empty", reqID)
}
// 设置
if err := m.SetActiveRequest(ctx, id, "req-123"); err != nil {
t.Fatalf("SetActiveRequest: %v", err)
}
reqID, _ = m.GetActiveRequestID(ctx, id)
if reqID != "req-123" {
t.Errorf("active request = %q, want %q", reqID, "req-123")
}
// 清除
if err := m.ClearActiveRequest(ctx, id); err != nil {
t.Fatalf("ClearActiveRequest: %v", err)
}
reqID, _ = m.GetActiveRequestID(ctx, id)
if reqID != "" {
t.Errorf("active request after clear = %q, want empty", reqID)
}
}
func TestTouchRefreshesTTL(t *testing.T) {
m := NewMemoryManager(100*time.Millisecond, 20)
defer m.Stop()
ctx := context.Background()
id, _ := m.Create(ctx, models.DefaultConfig())
// 50ms 后 Touch应重置 TTL
time.Sleep(50 * time.Millisecond)
if err := m.Touch(ctx, id); err != nil {
t.Fatalf("Touch: %v", err)
}
// 再等 70ms距创建 120ms但距 Touch 只有 70ms不应过期
time.Sleep(70 * time.Millisecond)
_, err := m.Get(ctx, id)
if err != nil {
t.Errorf("Get after Touch: %v, want nil (should not expire yet)", err)
}
// 再等 50ms距 Touch 120ms应过期
time.Sleep(50 * time.Millisecond)
_, err = m.Get(ctx, id)
if err != ErrSessionNotFound {
t.Errorf("Get after TTL: err = %v, want ErrSessionNotFound", err)
}
}
func TestActiveCount(t *testing.T) {
m := NewMemoryManager(30*time.Minute, 20)
defer m.Stop()
ctx := context.Background()
if m.ActiveCount() != 0 {
t.Errorf("initial ActiveCount = %d, want 0", m.ActiveCount())
}
m.Create(ctx, models.DefaultConfig())
m.Create(ctx, models.DefaultConfig())
if m.ActiveCount() != 2 {
t.Errorf("ActiveCount = %d, want 2", m.ActiveCount())
}
}

View File

@@ -0,0 +1,324 @@
package session
import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"github.com/hhs/camtalk/internal/logger"
"github.com/hhs/camtalk/internal/models"
)
// RedisManager 基于 Redis 的 SessionManager 实现。
// 数据结构:
// - session:{id}:meta → Hash会话元数据
// - session:{id}:history → List对话历史
type RedisManager struct {
rdb *redis.Client
ttl time.Duration
maxHistory int
}
// NewRedisManager 创建 Redis 版 SessionManager。
func NewRedisManager(rdb *redis.Client, ttl time.Duration, maxHistory int) *RedisManager {
if ttl <= 0 {
ttl = defaultTTL
}
if maxHistory <= 0 {
maxHistory = defaultHistorySize
}
return &RedisManager{rdb: rdb, ttl: ttl, maxHistory: maxHistory}
}
func metaKey(id string) string { return fmt.Sprintf("session:%s:meta", id) }
func histKey(id string) string { return fmt.Sprintf("session:%s:history", id) }
// Create 创建新会话。
func (m *RedisManager) Create(ctx context.Context, config models.SessionConfig) (string, error) {
id := uuidNew()
now := time.Now().UTC()
pipe := m.rdb.Pipeline()
// 写入 meta Hash
pipe.HSet(ctx, metaKey(id), map[string]interface{}{
"session_id": id,
"config.tts_enabled": strconv.FormatBool(config.TTSEnabled),
"config.detail_level": config.DetailLevel,
"config.language": config.Language,
"created_at": now.Format(time.RFC3339),
"last_active": now.Format(time.RFC3339),
"active_request_id": "",
})
pipe.Expire(ctx, metaKey(id), m.ttl)
// 初始化空 history List
pipe.RPush(ctx, histKey(id), placeholderHistoryMark)
pipe.Expire(ctx, histKey(id), m.ttl)
if _, err := pipe.Exec(ctx); err != nil {
return "", fmt.Errorf("redis create session: %w", err)
}
logger.Log.Debugw("redis session created", "session", id)
return id, nil
}
// placeholderHistoryMark 占位符,避免 Redis 对空 key 的特殊行为。
const placeholderHistoryMark = "__placeholder__"
// Get 获取会话。
func (m *RedisManager) Get(ctx context.Context, sessionID string) (*models.Session, error) {
vals, err := m.rdb.HGetAll(ctx, metaKey(sessionID)).Result()
if err != nil {
return nil, fmt.Errorf("redis get session: %w", err)
}
if len(vals) == 0 {
return nil, ErrSessionNotFound
}
sess := &models.Session{
ID: vals["session_id"],
}
sess.CreatedAt, _ = time.Parse(time.RFC3339, vals["created_at"])
sess.Config.TTSEnabled, _ = strconv.ParseBool(vals["config.tts_enabled"])
sess.Config.DetailLevel = vals["config.detail_level"]
sess.Config.Language = vals["config.language"]
return sess, nil
}
// UpdateConfig 更新会话配置。
func (m *RedisManager) UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error {
// 先检查会话是否存在
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
if err != nil {
return fmt.Errorf("redis check session: %w", err)
}
if exists == 0 {
return ErrSessionNotFound
}
fields := map[string]interface{}{
"last_active": time.Now().UTC().Format(time.RFC3339),
}
if patch.TTSEnabled != nil {
fields["config.tts_enabled"] = strconv.FormatBool(*patch.TTSEnabled)
}
if patch.DetailLevel != nil {
fields["config.detail_level"] = *patch.DetailLevel
}
if patch.Language != nil {
fields["config.language"] = *patch.Language
}
if err := m.rdb.HSet(ctx, metaKey(sessionID), fields).Err(); err != nil {
return fmt.Errorf("redis update config: %w", err)
}
// 刷新 TTL
m.rdb.Expire(ctx, metaKey(sessionID), m.ttl)
logger.Log.Debugw("redis session config updated", "session", sessionID)
return nil
}
// GetHistory 获取最近 N 轮对话历史。
func (m *RedisManager) GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error) {
// 检查会话是否存在
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
if err != nil {
return nil, fmt.Errorf("redis check session: %w", err)
}
if exists == 0 {
return nil, ErrSessionNotFound
}
if limit <= 0 {
limit = m.maxHistory
}
// LRANGE 0 {limit-1}最新在前LPUSH需要反转为时间顺序
raws, err := m.rdb.LRange(ctx, histKey(sessionID), 0, int64(limit)).Result()
if err != nil {
return nil, fmt.Errorf("redis get history: %w", err)
}
var msgs []models.Message
for _, raw := range raws {
if raw == placeholderHistoryMark {
continue
}
var msg models.Message
if err := json.Unmarshal([]byte(raw), &msg); err != nil {
logger.Log.Warnw("invalid history entry", "session", sessionID, "raw", raw)
continue
}
msgs = append(msgs, msg)
}
// 反转为时间顺序LPUSH 最新在前 → 需要最旧在前)
for i, j := 0, len(msgs)-1; i < j; i, j = i+1, j-1 {
msgs[i], msgs[j] = msgs[j], msgs[i]
}
return msgs, nil
}
// AppendMessage 追加一条对话消息,同时刷新 TTL。
func (m *RedisManager) AppendMessage(ctx context.Context, sessionID string, msg models.Message) error {
// 检查会话是否存在
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
if err != nil {
return fmt.Errorf("redis check session: %w", err)
}
if exists == 0 {
return ErrSessionNotFound
}
data, err := json.Marshal(msg)
if err != nil {
return fmt.Errorf("marshal message: %w", err)
}
pipe := m.rdb.Pipeline()
// LPUSH 新消息到左头(最新在前)
pipe.LPush(ctx, histKey(sessionID), string(data))
// LTRIM 保留最近 maxHistory 条(+1 是因为有占位符)
pipe.LTrim(ctx, histKey(sessionID), 0, int64(m.maxHistory))
// 刷新 TTL
pipe.Expire(ctx, histKey(sessionID), m.ttl)
pipe.Expire(ctx, metaKey(sessionID), m.ttl)
// 更新 last_active
pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339))
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("redis append message: %w", err)
}
return nil
}
// SetActiveRequest 标记当前正在处理的请求 ID。
func (m *RedisManager) SetActiveRequest(ctx context.Context, sessionID string, requestID string) error {
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
if err != nil {
return fmt.Errorf("redis check session: %w", err)
}
if exists == 0 {
return ErrSessionNotFound
}
pipe := m.rdb.Pipeline()
pipe.HSet(ctx, metaKey(sessionID), "active_request_id", requestID)
pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339))
pipe.Expire(ctx, metaKey(sessionID), m.ttl)
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("redis set active request: %w", err)
}
return nil
}
// GetActiveRequestID 获取当前活跃请求 ID。
func (m *RedisManager) GetActiveRequestID(ctx context.Context, sessionID string) (string, error) {
val, err := m.rdb.HGet(ctx, metaKey(sessionID), "active_request_id").Result()
if err == redis.Nil {
return "", ErrSessionNotFound
}
if err != nil {
return "", fmt.Errorf("redis get active request: %w", err)
}
return val, nil
}
// ClearActiveRequest 清除活跃请求标记。
func (m *RedisManager) ClearActiveRequest(ctx context.Context, sessionID string) error {
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
if err != nil {
return fmt.Errorf("redis check session: %w", err)
}
if exists == 0 {
return ErrSessionNotFound
}
pipe := m.rdb.Pipeline()
pipe.HSet(ctx, metaKey(sessionID), "active_request_id", "")
pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339))
pipe.Expire(ctx, metaKey(sessionID), m.ttl)
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("redis clear active request: %w", err)
}
return nil
}
// Touch 刷新 TTL。
func (m *RedisManager) Touch(ctx context.Context, sessionID string) error {
exists, err := m.rdb.Exists(ctx, metaKey(sessionID)).Result()
if err != nil {
return fmt.Errorf("redis check session: %w", err)
}
if exists == 0 {
return ErrSessionNotFound
}
pipe := m.rdb.Pipeline()
pipe.Expire(ctx, metaKey(sessionID), m.ttl)
pipe.Expire(ctx, histKey(sessionID), m.ttl)
pipe.HSet(ctx, metaKey(sessionID), "last_active", time.Now().UTC().Format(time.RFC3339))
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("redis touch: %w", err)
}
return nil
}
// Destroy 显式销毁会话。
func (m *RedisManager) Destroy(ctx context.Context, sessionID string) error {
deleted, err := m.rdb.Del(ctx, metaKey(sessionID), histKey(sessionID)).Result()
if err != nil {
return fmt.Errorf("redis destroy session: %w", err)
}
if deleted == 0 {
return ErrSessionNotFound
}
logger.Log.Debugw("redis session destroyed", "session", sessionID)
return nil
}
// ActiveCount 返回当前活跃会话数。
// Redis 实现通过 SCAN 遍历 meta key适用于中等规模。
// 大规模部署建议维护独立的活跃会话集合。
func (m *RedisManager) ActiveCount() int {
ctx := context.Background()
count := 0
var cursor uint64
for {
keys, nextCursor, err := m.rdb.Scan(ctx, cursor, "session:*:meta", 100).Result()
if err != nil {
break
}
for _, key := range keys {
exists, _ := m.rdb.Exists(ctx, key).Result()
if exists > 0 {
count++
}
}
cursor = nextCursor
if cursor == 0 {
break
}
}
return count
}
// uuidNew 生成 UUID便于测试时 mock。
var uuidNew = func() string {
return uuid.New().String()
}

View File

@@ -1,17 +1,20 @@
package ws
import (
"context"
"encoding/json"
"log"
"net/http"
"sync"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/gorilla/websocket"
"github.com/hhs/camtalk/internal/errors"
"github.com/hhs/camtalk/internal/logger"
"github.com/hhs/camtalk/internal/models"
"github.com/hhs/camtalk/internal/orchestrator"
"github.com/hhs/camtalk/internal/session"
)
var upgrader = websocket.Upgrader{
@@ -20,36 +23,94 @@ var upgrader = websocket.Upgrader{
// Client 代表一个 WebSocket 客户端连接。
type Client struct {
conn *websocket.Conn
sessionID string
mu sync.Mutex
conn *websocket.Conn
sessionID string
sessionMgr session.Manager
orchestrator orchestrator.Orchestrator
cancelFuncs map[string]context.CancelFunc // requestID → cancel func
mu sync.Mutex
}
func (c *Client) sendJSON(v any) error {
// SendJSON 向客户端发送 JSON 消息(公开以便 errors 包调用)。
func (c *Client) SendJSON(v any) error {
c.mu.Lock()
defer c.mu.Unlock()
return c.conn.WriteJSON(v)
}
// WSClient 实现 orchestrator.Sender 接口,将消息推送到 WebSocket 连接。
type WSClient struct {
client *Client
requestID string
}
// SendSTTResult 发送语音识别结果。
func (w *WSClient) SendSTTResult(result models.WsSTTResult) error {
result.RequestID = w.requestID
return w.client.SendJSON(result)
}
// SendLLMChunk 发送 LLM 流式文本增量。
func (w *WSClient) SendLLMChunk(chunk models.WsLLMChunk) error {
chunk.RequestID = w.requestID
return w.client.SendJSON(chunk)
}
// SendLLMDone 发送 LLM 流结束信号。
func (w *WSClient) SendLLMDone(done models.WsLLMDone) error {
done.RequestID = w.requestID
return w.client.SendJSON(done)
}
// SendTTSAudio 发送 TTS 音频数据。
func (w *WSClient) SendTTSAudio(audio models.WsTTSAudio) error {
audio.RequestID = w.requestID
return w.client.SendJSON(audio)
}
// SendError 发送错误消息。
func (w *WSClient) SendError(err models.WsError) error {
err.RequestID = w.requestID
return w.client.SendJSON(err)
}
// ServeWS 处理 WebSocket 升级请求。
func ServeWS(c *gin.Context) {
func ServeWS(sessionMgr session.Manager, orch orchestrator.Orchestrator) gin.HandlerFunc {
return func(c *gin.Context) {
serveWS(c, sessionMgr, orch)
}
}
func serveWS(c *gin.Context, sessionMgr session.Manager, orch orchestrator.Orchestrator) {
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
log.Printf("websocket upgrade failed: %v", err)
logger.Log.Errorw("websocket upgrade failed", "error", err)
return
}
defer conn.Close()
sessionID := uuid.New().String()
client := &Client{conn: conn, sessionID: sessionID}
// 创建会话
sessionID, err := sessionMgr.Create(context.Background(), models.DefaultConfig())
if err != nil {
logger.Log.Errorw("create session failed", "error", err)
return
}
client := &Client{
conn: conn,
sessionID: sessionID,
sessionMgr: sessionMgr,
orchestrator: orch,
cancelFuncs: make(map[string]context.CancelFunc),
}
// 发送 connected 消息
_ = client.sendJSON(models.WsConnected{
_ = client.SendJSON(models.WsConnected{
Type: "connected",
SessionID: sessionID,
ServerVersion: "0.1.0",
})
log.Printf("client connected: session=%s", sessionID)
logger.Log.Infow("client connected", "session", sessionID)
// 心跳检测
lastPong := time.Now()
@@ -67,7 +128,7 @@ func ServeWS(c *gin.Context) {
select {
case <-ticker.C:
if time.Since(lastPong) > 60*time.Second {
log.Printf("heartbeat timeout: session=%s", sessionID)
logger.Log.Warnw("heartbeat timeout", "session", sessionID)
conn.Close()
return
}
@@ -82,7 +143,7 @@ func ServeWS(c *gin.Context) {
_, message, err := conn.ReadMessage()
if err != nil {
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) {
log.Printf("ws read error: %v", err)
logger.Log.Warnw("ws read error", "error", err)
}
break
}
@@ -92,51 +153,96 @@ func ServeWS(c *gin.Context) {
Type string `json:"type"`
}
if err := json.Unmarshal(message, &envelope); err != nil {
_ = client.sendJSON(models.WsError{
Type: "error",
Code: "INVALID_MESSAGE",
Message: "invalid JSON",
})
errors.SendWSError(client, errors.CodeInvalidMessage, "", err)
continue
}
switch envelope.Type {
case "ping":
_ = client.sendJSON(models.WsPong{Type: "pong"})
_ = client.SendJSON(models.WsPong{Type: "pong"})
case "query":
var msg models.WsQuery
if err := json.Unmarshal(message, &msg); err != nil {
_ = client.sendJSON(models.WsError{
Type: "error",
Code: "INVALID_MESSAGE",
Message: "invalid query message",
RequestID: msg.RequestID,
})
errors.SendWSError(client, errors.CodeInvalidMessage, msg.RequestID, err)
continue
}
log.Printf("query received: session=%s request=%s", sessionID, msg.RequestID)
// TODO: 调用 AI 编排流程STT → LLM → TTS
logger.Log.Infow("query received", "session", sessionID, "request", msg.RequestID)
// 刷新会话 TTL
if err := client.sessionMgr.Touch(context.Background(), sessionID); err != nil {
logger.Log.Warnw("touch session failed", "session", sessionID, "error", err)
}
// 标记活跃请求
if err := client.sessionMgr.SetActiveRequest(context.Background(), sessionID, msg.RequestID); err != nil {
logger.Log.Warnw("set active request failed", "session", sessionID, "error", err)
}
// 获取对话历史
history, _ := client.sessionMgr.GetHistory(context.Background(), sessionID, 20)
// 创建可取消的 context
ctx, cancel := context.WithCancel(context.Background())
client.mu.Lock()
client.cancelFuncs[msg.RequestID] = cancel
client.mu.Unlock()
// 创建 sender
sender := &WSClient{client: client, requestID: msg.RequestID}
// 启动 orchestrator 处理 goroutine
go func() {
defer func() {
// 清理 cancel func
client.mu.Lock()
delete(client.cancelFuncs, msg.RequestID)
client.mu.Unlock()
cancel()
// 清除活跃请求
_ = client.sessionMgr.ClearActiveRequest(context.Background(), sessionID)
}()
if err := client.orchestrator.ProcessQuery(ctx, sessionID, msg, history, sender); err != nil {
logger.Log.Errorw("process query failed", "session", sessionID, "request", msg.RequestID, "error", err)
}
}()
case "config":
var msg models.WsConfig
if err := json.Unmarshal(message, &msg); err != nil {
_ = client.sendJSON(models.WsError{
Type: "error",
Code: "INVALID_MESSAGE",
Message: "invalid config message",
})
errors.SendWSError(client, errors.CodeInvalidMessage, "", err)
continue
}
log.Printf("config update: session=%s", sessionID)
// TODO: 更新会话配置
patch := models.SessionConfigPatch{
TTSEnabled: msg.Payload.TTSEnabled,
DetailLevel: msg.Payload.DetailLevel,
Language: msg.Payload.Language,
}
if err := client.sessionMgr.UpdateConfig(context.Background(), sessionID, patch); err != nil {
errors.SendWSError(client, errors.CodeInternalError, "", err)
continue
}
logger.Log.Infow("config updated", "session", sessionID)
case "interrupt":
log.Printf("interrupt received: session=%s", sessionID)
// TODO: 中断当前 AI 响应
logger.Log.Infow("interrupt received", "session", sessionID)
// 获取活跃请求 ID 并取消
reqID, _ := client.sessionMgr.GetActiveRequestID(context.Background(), sessionID)
if reqID != "" {
client.mu.Lock()
if cancel, ok := client.cancelFuncs[reqID]; ok {
cancel()
delete(client.cancelFuncs, reqID)
}
client.mu.Unlock()
_ = client.sessionMgr.ClearActiveRequest(context.Background(), sessionID)
}
default:
_ = client.sendJSON(models.WsError{
_ = client.SendJSON(models.WsError{
Type: "error",
Code: "INVALID_MESSAGE",
Message: "unknown message type: " + envelope.Type,
@@ -145,5 +251,16 @@ func ServeWS(c *gin.Context) {
}
close(done)
log.Printf("client disconnected: session=%s", sessionID)
// 取消所有活跃请求
client.mu.Lock()
for reqID, cancel := range client.cancelFuncs {
logger.Log.Infow("canceling active request on disconnect", "session", sessionID, "request", reqID)
cancel()
}
client.cancelFuncs = make(map[string]context.CancelFunc)
client.mu.Unlock()
// 断开连接时不销毁会话,让其自然过期(支持重连恢复)
logger.Log.Infow("client disconnected", "session", sessionID)
}

View File

@@ -0,0 +1,563 @@
package ws
import (
"encoding/base64"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"context"
"github.com/hhs/camtalk/internal/logger"
"github.com/hhs/camtalk/internal/models"
"github.com/hhs/camtalk/internal/orchestrator"
"github.com/hhs/camtalk/internal/session"
)
func init() {
logger.Init("debug", "console")
gin.SetMode(gin.TestMode)
}
// --- Mock Orchestrator ---
// MockOrchestrator 实现 orchestrator.Orchestrator 接口,
// 模拟完整的 STT → LLM → TTS 管道,通过 sender 推送消息。
type MockOrchestrator struct {
// STTResult 模拟的语音识别结果
STTResult string
// LLMDeltas 模拟的 LLM 流式输出
LLMDeltas []string
// TTSAudios 模拟的 TTS 音频数据(每项一个 base64 编码的 MP3 片段)
TTSAudios []string
// Err 如果非 nilProcessQuery 直接返回此错误
Err error
// Delay 每个消息之间的延迟(用于 interrupt 测试)
Delay time.Duration
}
func (m *MockOrchestrator) ProcessQuery(
ctx context.Context,
sessionID string,
req models.WsQuery,
history []models.Message,
sender orchestrator.Sender,
) error {
if m.Err != nil {
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "INTERNAL_ERROR",
Message: m.Err.Error(),
})
return m.Err
}
// Step 1: 发送 STT 结果
if m.STTResult != "" {
_ = sender.SendSTTResult(models.WsSTTResult{
Type: "stt_result",
RequestID: req.RequestID,
Text: m.STTResult,
IsFinal: true,
})
}
if m.Delay > 0 {
select {
case <-ctx.Done():
return nil // 中断视为正常完成
case <-time.After(m.Delay):
}
}
// Step 2: 发送 LLM chunks
var fullText strings.Builder
for _, delta := range m.LLMDeltas {
select {
case <-ctx.Done():
return nil // 中断视为正常完成
default:
}
fullText.WriteString(delta)
_ = sender.SendLLMChunk(models.WsLLMChunk{
Type: "llm_chunk",
RequestID: req.RequestID,
Delta: delta,
Role: "assistant",
})
if m.Delay > 0 {
select {
case <-ctx.Done():
return nil // 中断视为正常完成
case <-time.After(m.Delay):
}
}
}
// Step 3: 发送 TTS 音频
for i, audio := range m.TTSAudios {
select {
case <-ctx.Done():
return nil // 中断视为正常完成
default:
}
isLast := i == len(m.TTSAudios)-1
_ = sender.SendTTSAudio(models.WsTTSAudio{
Type: "tts_audio",
RequestID: req.RequestID,
Audio: audio,
MimeType: "audio/mp3",
IsLast: isLast,
})
}
// Step 4: 发送 llm_done
_ = sender.SendLLMDone(models.WsLLMDone{
Type: "llm_done",
RequestID: req.RequestID,
FullText: fullText.String(),
Model: "gpt-4o",
LatencyMs: 100,
})
return nil
}
// --- 测试辅助函数 ---
// setupTestServer 创建测试用 Gin 服务器和 WebSocket URL。
func setupTestServer(t *testing.T, orch orchestrator.Orchestrator) (*httptest.Server, string) {
t.Helper()
sessionMgr := session.NewMemoryManager(5*time.Minute, 20)
t.Cleanup(func() { sessionMgr.Stop() })
r := gin.New()
r.GET("/ws", ServeWS(sessionMgr, orch))
srv := httptest.NewServer(r)
// 构造 WebSocket URL
wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + "/ws"
return srv, wsURL
}
// connectWS 建立 WebSocket 连接并返回 conn。
func connectWS(t *testing.T, wsURL string) *websocket.Conn {
t.Helper()
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
require.NoError(t, err, "WebSocket 连接失败")
t.Cleanup(func() { conn.Close() })
return conn
}
// readJSON 从 WebSocket 读取一条 JSON 消息。
func readJSON(t *testing.T, conn *websocket.Conn) map[string]any {
t.Helper()
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
var msg map[string]any
err := conn.ReadJSON(&msg)
require.NoError(t, err, "读取 WebSocket 消息失败")
return msg
}
// --- 测试用例 ---
// TestWS_Connected 验证连接建立后收到 connected 消息。
func TestWS_Connected(t *testing.T) {
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
defer srv.Close()
conn := connectWS(t, wsURL)
msg := readJSON(t, conn)
assert.Equal(t, "connected", msg["type"])
assert.NotEmpty(t, msg["session_id"])
assert.Equal(t, "0.1.0", msg["server_version"])
}
// TestWS_PingPong 验证 ping/pong 心跳。
func TestWS_PingPong(t *testing.T) {
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
defer srv.Close()
conn := connectWS(t, wsURL)
// 读取 connected 消息
_ = readJSON(t, conn)
// 发送 ping
err := conn.WriteJSON(map[string]string{"type": "ping"})
require.NoError(t, err)
// 读取 pong
msg := readJSON(t, conn)
assert.Equal(t, "pong", msg["type"])
}
// TestWS_QueryFullFlow 验证完整的 query → stt_result → llm_chunk → tts_audio → llm_done 流程。
func TestWS_QueryFullFlow(t *testing.T) {
audioB64 := base64.StdEncoding.EncodeToString([]byte("fake-audio-data"))
imageB64 := base64.StdEncoding.EncodeToString([]byte("fake-image-data"))
mock := &MockOrchestrator{
STTResult: "你好,世界",
LLMDeltas: []string{"你好", ",世界!"},
TTSAudios: []string{base64.StdEncoding.EncodeToString([]byte("mp3-data-1")), base64.StdEncoding.EncodeToString([]byte("mp3-data-2"))},
}
srv, wsURL := setupTestServer(t, mock)
defer srv.Close()
conn := connectWS(t, wsURL)
// 1. 读取 connected
connected := readJSON(t, conn)
assert.Equal(t, "connected", connected["type"])
sessionID := connected["session_id"].(string)
assert.NotEmpty(t, sessionID)
// 2. 发送 query
queryMsg := models.WsQuery{
Type: "query",
RequestID: "req-test-001",
Image: imageB64,
Audio: audioB64,
MimeType: "audio/pcm",
}
err := conn.WriteJSON(queryMsg)
require.NoError(t, err)
// 3. 读取 stt_result
sttResult := readJSON(t, conn)
assert.Equal(t, "stt_result", sttResult["type"])
assert.Equal(t, "req-test-001", sttResult["request_id"])
assert.Equal(t, "你好,世界", sttResult["text"])
assert.Equal(t, true, sttResult["is_final"])
// 4. 读取 llm_chunk 消息
chunk1 := readJSON(t, conn)
assert.Equal(t, "llm_chunk", chunk1["type"])
assert.Equal(t, "req-test-001", chunk1["request_id"])
assert.Equal(t, "你好", chunk1["delta"])
assert.Equal(t, "assistant", chunk1["role"])
chunk2 := readJSON(t, conn)
assert.Equal(t, "llm_chunk", chunk2["type"])
assert.Equal(t, ",世界!", chunk2["delta"])
// 5. 读取 tts_audio 消息
tts1 := readJSON(t, conn)
assert.Equal(t, "tts_audio", tts1["type"])
assert.Equal(t, "req-test-001", tts1["request_id"])
assert.NotEmpty(t, tts1["audio"])
assert.Equal(t, "audio/mp3", tts1["mime_type"])
assert.Equal(t, false, tts1["is_last"])
tts2 := readJSON(t, conn)
assert.Equal(t, "tts_audio", tts2["type"])
assert.Equal(t, true, tts2["is_last"])
// 6. 读取 llm_done
llmDone := readJSON(t, conn)
assert.Equal(t, "llm_done", llmDone["type"])
assert.Equal(t, "req-test-001", llmDone["request_id"])
assert.Equal(t, "你好,世界!", llmDone["full_text"])
assert.Equal(t, "gpt-4o", llmDone["model"])
assert.NotNil(t, llmDone["latency_ms"])
}
// TestWS_QuerySTTOnly 验证只有 STT 结果、无 LLM 输出的场景。
func TestWS_QuerySTTOnly(t *testing.T) {
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
mock := &MockOrchestrator{
STTResult: "测试语音",
// LLMDeltas 为空 → 不发送 llm_chunk
// TTSAudios 为空 → 不发送 tts_audio
}
srv, wsURL := setupTestServer(t, mock)
defer srv.Close()
conn := connectWS(t, wsURL)
_ = readJSON(t, conn) // connected
err := conn.WriteJSON(models.WsQuery{
Type: "query",
RequestID: "req-stt-only",
Audio: audioB64,
})
require.NoError(t, err)
// 应收到 stt_result
stt := readJSON(t, conn)
assert.Equal(t, "stt_result", stt["type"])
assert.Equal(t, "测试语音", stt["text"])
// 应收到 llm_done即使没有 chunk
done := readJSON(t, conn)
assert.Equal(t, "llm_done", done["type"])
assert.Equal(t, "", done["full_text"])
}
// TestWS_UnknownMessageType 验证未知消息类型返回 error。
func TestWS_UnknownMessageType(t *testing.T) {
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
defer srv.Close()
conn := connectWS(t, wsURL)
_ = readJSON(t, conn) // connected
err := conn.WriteJSON(map[string]string{"type": "unknown_type"})
require.NoError(t, err)
errMsg := readJSON(t, conn)
assert.Equal(t, "error", errMsg["type"])
assert.Equal(t, "INVALID_MESSAGE", errMsg["code"])
assert.Contains(t, errMsg["message"], "unknown message type")
}
// TestWS_InvalidJSON 验证无效 JSON 返回 error。
func TestWS_InvalidJSON(t *testing.T) {
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
defer srv.Close()
conn := connectWS(t, wsURL)
_ = readJSON(t, conn) // connected
err := conn.WriteMessage(websocket.TextMessage, []byte("not-json"))
require.NoError(t, err)
errMsg := readJSON(t, conn)
assert.Equal(t, "error", errMsg["type"])
assert.Equal(t, "INVALID_MESSAGE", errMsg["code"])
}
// TestWS_MultipleQueries 验证同一连接上可以发送多次 query。
func TestWS_MultipleQueries(t *testing.T) {
mock := &MockOrchestrator{
STTResult: "识别结果",
LLMDeltas: []string{"回复"},
}
srv, wsURL := setupTestServer(t, mock)
defer srv.Close()
conn := connectWS(t, wsURL)
_ = readJSON(t, conn) // connected
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
for i := 0; i < 3; i++ {
reqID := "req-multi-" + string(rune('0'+i))
err := conn.WriteJSON(models.WsQuery{
Type: "query",
RequestID: reqID,
Audio: audioB64,
})
require.NoError(t, err)
// 每次应收到完整的响应序列
stt := readJSON(t, conn)
assert.Equal(t, "stt_result", stt["type"], "第 %d 次 query", i+1)
chunk := readJSON(t, conn)
assert.Equal(t, "llm_chunk", chunk["type"], "第 %d 次 query", i+1)
done := readJSON(t, conn)
assert.Equal(t, "llm_done", done["type"], "第 %d 次 query", i+1)
}
}
// TestWS_Interrupt 验证 interrupt 取消正在进行的请求。
func TestWS_Interrupt(t *testing.T) {
// 使用较长延迟模拟慢请求
mock := &MockOrchestrator{
STTResult: "识别文本",
LLMDeltas: []string{"第一句", "第二句", "第三句", "第四句", "第五句"},
Delay: 200 * time.Millisecond,
}
srv, wsURL := setupTestServer(t, mock)
defer srv.Close()
conn := connectWS(t, wsURL)
_ = readJSON(t, conn) // connected
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
// 发送 query
err := conn.WriteJSON(models.WsQuery{
Type: "query",
RequestID: "req-interrupt",
Audio: audioB64,
})
require.NoError(t, err)
// 收到 stt_result
stt := readJSON(t, conn)
assert.Equal(t, "stt_result", stt["type"])
// 收到第一个 llm_chunk
chunk1 := readJSON(t, conn)
assert.Equal(t, "llm_chunk", chunk1["type"])
// 发送 interrupt
err = conn.WriteJSON(map[string]string{"type": "interrupt"})
require.NoError(t, err)
// 等待 interrupt 生效
time.Sleep(500 * time.Millisecond)
// 验证连接仍然存活(可以发 ping 收 pong
require.NoError(t, conn.WriteJSON(map[string]string{"type": "ping"}))
var pong map[string]any
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
require.NoError(t, conn.ReadJSON(&pong), "interrupt 后连接应仍存活")
assert.Equal(t, "pong", pong["type"])
}
// TestWS_DisconnectCleanup 验证断开连接时清理资源。
func TestWS_DisconnectCleanup(t *testing.T) {
// 使用较长延迟模拟慢请求
mock := &MockOrchestrator{
STTResult: "识别文本",
LLMDeltas: []string{"长回复第一部分", "长回复第二部分"},
Delay: 500 * time.Millisecond,
}
srv, wsURL := setupTestServer(t, mock)
defer srv.Close()
conn := connectWS(t, wsURL)
_ = readJSON(t, conn) // connected
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
// 发送 query
err := conn.WriteJSON(models.WsQuery{
Type: "query",
RequestID: "req-disconnect",
Audio: audioB64,
})
require.NoError(t, err)
// 收到 stt_result
stt := readJSON(t, conn)
assert.Equal(t, "stt_result", stt["type"])
// 关闭连接(模拟客户端断开)
conn.Close()
// 等待一小段时间让服务器处理断开
time.Sleep(300 * time.Millisecond)
// 如果没有 panic 或 goroutine 泄漏,测试通过
// Go test 的 -race 检测器会捕获数据竞争)
}
// TestWS_SessionCreated 验证每次连接都创建新会话。
func TestWS_SessionCreated(t *testing.T) {
srv, wsURL := setupTestServer(t, &MockOrchestrator{})
defer srv.Close()
// 第一次连接
conn1 := connectWS(t, wsURL)
msg1 := readJSON(t, conn1)
sid1 := msg1["session_id"].(string)
conn1.Close()
time.Sleep(100 * time.Millisecond)
// 第二次连接
conn2 := connectWS(t, wsURL)
sid2 := readJSON(t, conn2)["session_id"].(string)
assert.NotEmpty(t, sid1)
assert.NotEmpty(t, sid2)
assert.NotEqual(t, sid1, sid2, "两次连接应创建不同的会话")
}
// TestWS_QueryWithoutImage 验证不带图片的 query。
func TestWS_QueryWithoutImage(t *testing.T) {
mock := &MockOrchestrator{
STTResult: "纯语音输入",
LLMDeltas: []string{"收到"},
}
srv, wsURL := setupTestServer(t, mock)
defer srv.Close()
conn := connectWS(t, wsURL)
_ = readJSON(t, conn) // connected
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
err := conn.WriteJSON(models.WsQuery{
Type: "query",
RequestID: "req-no-image",
Audio: audioB64,
// Image 为空
})
require.NoError(t, err)
stt := readJSON(t, conn)
assert.Equal(t, "stt_result", stt["type"])
assert.Equal(t, "纯语音输入", stt["text"])
chunk := readJSON(t, conn)
assert.Equal(t, "llm_chunk", chunk["type"])
done := readJSON(t, conn)
assert.Equal(t, "llm_done", done["type"])
}
// TestWS_QueryWithTTSDisabled 验证 TTS 未启用时不应收到 tts_audio。
func TestWS_QueryWithTTSDisabled(t *testing.T) {
// MockOrchestrator 的 TTSAudios 为空 → 不发送 tts_audio
mock := &MockOrchestrator{
STTResult: "语音",
LLMDeltas: []string{"回复"},
// TTSAudios 留空
}
srv, wsURL := setupTestServer(t, mock)
defer srv.Close()
conn := connectWS(t, wsURL)
_ = readJSON(t, conn) // connected
audioB64 := base64.StdEncoding.EncodeToString([]byte("audio"))
err := conn.WriteJSON(models.WsQuery{
Type: "query",
RequestID: "req-no-tts",
Audio: audioB64,
})
require.NoError(t, err)
stt := readJSON(t, conn)
assert.Equal(t, "stt_result", stt["type"])
chunk := readJSON(t, conn)
assert.Equal(t, "llm_chunk", chunk["type"])
done := readJSON(t, conn)
assert.Equal(t, "llm_done", done["type"])
// 不应有 tts_audio 消息;设置短超时验证
conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond))
var extra map[string]any
err = conn.ReadJSON(&extra)
assert.Error(t, err, "不应有额外消息")
}

View File

@@ -213,11 +213,73 @@ CREATE TABLE usage_daily (
## 部署架构
```
CDN静态资源用户浏览器
Nginx 负载均衡sticky session for WebSocket
├── Gateway-1 ──→ Redis
├── Gateway-2 ──→ Redis
── Gateway-N ──→ AI Services外部 API
用户浏览器
Nginx同源反代 + 负载均衡)
├── / → 前端静态资源CDN 或本地 dist
── /api/* → Go GatewayREST API
└── /ws → Go GatewayWebSocket
├── Gateway-1 ──→ Redis
├── Gateway-2 ──→ Redis
└── Gateway-N ──→ AI Services外部 API
```
WebSocket 是长连接Nginx 需要配置 `proxy_set_header Upgrade` 和 sticky session确保同一用户的请求始终路由到同一个 Gateway 实例。
**跨域策略**Nginx 将前端和后端统一到同一域名下,浏览器无跨域问题。
### Nginx 配置
```nginx
server {
listen 80;
server_name camtalk.example.com;
# 前端静态资源
location / {
root /var/www/camtalk/dist;
try_files $uri $uri/ /index.html;
}
# REST API 反代
location /api/ {
proxy_pass http://127.0.0.1:8080;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
}
# WebSocket 反代
location /ws {
proxy_pass http://127.0.0.1:8080;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_read_timeout 86400s; # 长连接超时 24h
proxy_send_timeout 86400s;
}
}
```
> WebSocket 是长连接Nginx 必须配置 `Upgrade` 和 `Connection` 头。`proxy_read_timeout` 需要覆盖心跳间隔(客户端 30s ping否则 Nginx 会主动断开空闲连接。
### 开发环境Vite proxy
开发时前端Vite :5173和后端Gin :8080不同端口用 Vite 内置代理解决跨域:
```typescript
// frontend/vite.config.ts
export default defineConfig({
plugins: [react()],
server: {
proxy: {
"/api": "http://localhost:8080",
"/ws": {
target: "ws://localhost:8080",
ws: true,
},
},
},
});
```
前端代码中 WebSocket 地址改为相对路径 `ws://localhost:5173/ws`Vite 自动代理到后端。部署时 Nginx 同理,前端无需区分开发/生产地址。

View File

@@ -256,7 +256,6 @@ POST /api/sessions
Content-Type: application/json
{
"user_id": "optional-user-id",
"config": {
"tts_enabled": true,
"detail_level": "low",
@@ -301,28 +300,18 @@ Go 网关内部与外部 AI 服务Deepgram STT、GPT-4o、OpenAI TTS的调
语音识别:接收前端采集的音频,返回识别文本。
```go
// STTService 语音识别服务契约。
type STTService interface {
// Service 语音识别服务契约。
type Service interface {
// Recognize 识别一段完整音频,返回最终文本。
Recognize(ctx context.Context, audio []byte, opts STTOptions) (string, error)
// RecognizeStream 流式识别(边说边识别,可选实现)。
// audioStream 持续接收音频片段,返回的 channel 持续输出中间结果。
RecognizeStream(ctx context.Context, audioStream <-chan []byte, opts STTOptions) (<-chan STTPartial, error)
Recognize(ctx context.Context, audio []byte, opts Options) (string, error)
}
// STTOptions 语音识别参数。
type STTOptions struct {
// Options 语音识别参数。
type Options struct {
Encoding string // "pcm_s16le" — 前端 VAD 输出格式
SampleRate int // 16000 — 前端麦克风采样率
Language string // "zh-CN"
}
// STTPartial 流式识别的中间/最终结果。
type STTPartial struct {
Text string
IsFinal bool
}
```
**Deepgram 接入约定**
@@ -336,23 +325,23 @@ type STTPartial struct {
多模态推理:接收图像 + 文本 + 对话历史,流式返回回复。
```go
// LLMService 多模态大模型服务契约。
type LLMService interface {
// Service 多模态大模型服务契约。
type Service interface {
// ChatStream 流式推理,返回增量文本的 channel。
// 调用方必须消费 channel 直到 Done=true否则需 cancel ctx 以释放连接。
ChatStream(ctx context.Context, req LLMRequest) (<-chan LLMChunk, error)
ChatStream(ctx context.Context, req Request) (<-chan Chunk, error)
}
// LLMRequest 推理请求。
type LLMRequest struct {
Image []byte // JPEG 图片(已从 Base64 解码)
Text string // 用户语音识别后的文本
History []Message // 最近 N 轮对话历史
Language string // "zh-CN"
// Request 推理请求。
type Request struct {
Image []byte // JPEG 图片(已从 Base64 解码)
Text string // 用户语音识别后的文本
History []models.Message // 最近 N 轮对话历史
Language string // "zh-CN"
}
// LLMChunk 流式推理的一个增量片段。
type LLMChunk struct {
// Chunk 流式推理的一个增量片段。
type Chunk struct {
Delta string // 增量文本
Done bool // 是否结束
TokensUsed *TokenUsage // 仅 Done=true 时有值
@@ -388,24 +377,24 @@ user: [图片 + 用户语音文本]
语音合成:接收文本流,输出音频 chunk 流。
```go
// TTSService 语音合成服务契约。
type TTSService interface {
// Service 语音合成服务契约。
type Service interface {
// SynthesizeStream 流式合成。
// textStream 接收句子级文本(由 Orchestrator 的句子切分器产出),
// 返回的 channel 持续输出 MP3 音频 chunk。
SynthesizeStream(ctx context.Context, textStream <-chan string, opts TTSOptions) (<-chan TTSChunk, error)
SynthesizeStream(ctx context.Context, textStream <-chan string, opts Options) (<-chan Chunk, error)
}
// TTSOptions 合成参数。
type TTSOptions struct {
// Options 合成参数。
type Options struct {
Voice string // "alloy" | "nova" | "shimmer" | ...
Speed float64 // 1.0 为正常语速
OutputFmt string // "mp3" — 固定使用 MP3浏览器原生支持
SampleRate int // 24000
}
// TTSChunk 一个音频片段。
type TTSChunk struct {
// Chunk 一个音频片段。
type Chunk struct {
Audio []byte // MP3 音频数据(未 Base64 编码,由发送层编码)
IsLast bool // 是否为最后一片
}
@@ -446,73 +435,32 @@ LLM 流式输出: "这" "是一" "朵红色" "的花。" "它看起" "来很美
### Orchestrator 接口
```go
// Orchestrator AI 编排器,协调 STT → LLM → TTS 全链路
type Orchestrator struct {
stt STTService
llm LLMService
tts TTSService
// Orchestrator AI 编排器接口
type Orchestrator interface {
// ProcessQuery 处理一次完整的视觉对话请求。
// 通过 sender 向前端实时推送 stt_result、llm_chunk、llm_done、tts_audio 消息。
ProcessQuery(ctx context.Context, sessionID string, req models.WsQuery,
history []models.Message, sender Sender) error
}
// ProcessQuery 处理一次完整的视觉对话请求
// 通过 client 向前端实时推送 stt_result、llm_chunk、llm_done、tts_audio 消息。
func (o *Orchestrator) ProcessQuery(ctx context.Context, client MessageSender, req *QueryRequest) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
// Step 1: STT — 识别用户语音
text, err := o.stt.Recognize(ctx, req.Audio, STTOptions{
Encoding: "pcm_s16le", SampleRate: 16000, Language: "zh-CN",
})
if err != nil {
client.SendError(req.RequestID, "STT_ERROR", err.Error())
return
}
client.SendSTTResult(req.RequestID, text, true)
// Step 2: LLM 流式输出 + 句子切分
llmStream, _ := o.llm.ChatStream(ctx, LLMRequest{
Image: req.Image, Text: text, Language: "zh-CN",
})
sentenceCh := make(chan string, 4)
go func() {
defer close(sentenceCh)
var buf strings.Builder
var fullText strings.Builder
for chunk := range llmStream {
// 即时推送文字给客户端(逐 token 显示)
client.SendLLMChunk(req.RequestID, chunk.Delta)
fullText.WriteString(chunk.Delta)
buf.WriteString(chunk.Delta)
// 遇到句子边界就吐出
if isSentenceEnd(chunk.Delta) {
sentenceCh <- buf.String()
buf.Reset()
}
}
// 最后一段不足一句的也吐出
if buf.Len() > 0 {
sentenceCh <- buf.String()
}
// 推送 llm_done
client.SendLLMDone(req.RequestID, fullText.String(), chunk.TokensUsed, chunk.Model)
}()
// Step 3: TTS 并行消费句子流
ttsStream, _ := o.tts.SynthesizeStream(ctx, sentenceCh, TTSOptions{
Voice: "alloy", OutputFmt: "mp3", SampleRate: 24000,
})
for chunk := range ttsStream {
client.SendTTSAudio(req.RequestID, chunk.Audio, chunk.IsLast)
}
}
// isSentenceEnd 判断 delta 中是否包含句子结束标志。
func isSentenceEnd(delta string) bool {
return strings.ContainsAny(delta, "。!?\n.!?\n")
// Sender 抽象 WebSocket 消息推送能力,便于测试时 mock
type Sender interface {
SendSTTResult(result models.WsSTTResult) error
SendLLMChunk(chunk models.WsLLMChunk) error
SendLLMDone(done models.WsLLMDone) error
SendTTSAudio(audio models.WsTTSAudio) error
SendError(err models.WsError) error
}
```
**Pipeline 实现**`internal/orchestrator/pipeline.go`
1. Base64 解码音频/图片
2. 调用 `stt.Recognize()` → 发送 `stt_result`
3. 调用 `llm.ChatStream()` 获取流式输出goroutine 消费 token → 发送 `llm_chunk` + 句子切分
4. 另一 goroutine 从句子 channel 读取 → 调用 `tts.SynthesizeStream()` → 发送 `tts_audio`
5. 流结束 → 发送 `llm_done`
6. TTS 失败静默跳过STT/LLM 失败发送对应 error 消息
### 并发控制
- 每个 `ProcessQuery` 调用在独立 goroutine 中运行
@@ -584,9 +532,9 @@ session:{id}:history → List (对话历史)
### 接口定义
```go
// SessionManager 会话管理器。
// WebSocket Handler 通过此接口操作会话,不直接接触 Redis
type SessionManager interface {
// Manager 会话管理器接口
// WebSocket Handler 通过此接口操作会话,不直接接触存储层
type Manager interface {
// Create 创建新会话,返回 session ID。
Create(ctx context.Context, config models.SessionConfig) (string, error)
@@ -605,14 +553,20 @@ type SessionManager interface {
// SetActiveRequest 标记当前正在处理的请求 IDinterrupt 用)。
SetActiveRequest(ctx context.Context, sessionID string, requestID string) error
// GetActiveRequestID 获取当前活跃请求 ID。
GetActiveRequestID(ctx context.Context, sessionID string) (string, error)
// ClearActiveRequest 清除活跃请求标记(请求完成或中断后)。
ClearActiveRequest(ctx context.Context, sessionID string) error
// Touch 刷新 TTL心跳时调用
Touch(ctx context.Context, sessionID string) error
// Destroy 显式销毁会话REST API DELETE 或连接断开清理)。
// Destroy 显式销毁会话REST API DELETE
Destroy(ctx context.Context, sessionID string) error
// ActiveCount 返回当前活跃会话数(健康检查用)。
ActiveCount() int
}
```
@@ -629,7 +583,8 @@ case "query":
history, _ := sessionMgr.GetHistory(ctx, sessionID, 20) // 获取对话上下文
go orchestrator.ProcessQuery(ctx, client, &msg, history) // 异步编排
sender := &WSClient{client: client, requestID: msg.RequestID}
go orch.ProcessQuery(ctx, sessionID, msg, history, sender) // 异步编排
// interrupt 分支
case "interrupt":
@@ -648,26 +603,30 @@ case "interrupt":
联调阶段无 Redis 时,用同一接口的内存实现:
```go
type InMemorySessionManager struct {
mu sync.RWMutex
sessions map[string]*sessionEntry
type MemoryManager struct {
mu sync.RWMutex
sessions map[string]*sessionEntry
ttl time.Duration
maxHistory int
stopCleaner chan struct{}
}
type sessionEntry struct {
session models.Session
history []models.Message
activeReqID string
lastActive time.Time
}
```
注入时根据配置切换:
```go
var sessionMgr SessionManager
var sessionMgr session.Manager
if cfg.Redis.Addr != "" {
sessionMgr = NewRedisSessionManager(redisClient, 30*time.Minute, 20)
sessionMgr = session.NewRedisManager(redisClient, 30*time.Minute, 20)
} else {
sessionMgr = NewInMemorySessionManager()
sessionMgr = session.NewMemoryManager(30*time.Minute, 20)
}
```
@@ -931,10 +890,8 @@ type QueryRequest struct {
}
type Message struct {
Role string `json:"role"` // "user" | "assistant"
Content string `json:"content"`
ImageURL string `json:"image_url,omitempty"`
TokensUsed int `json:"tokens_used,omitempty"`
Role string `json:"role"` // "user" | "assistant"
Content string `json:"content"`
}
```
@@ -1085,3 +1042,25 @@ function reconnect(attempt: number) {
}
// attempt: 0 → 1s, 1 → 2s, 2 → 4s, 3 → 8s, ... 最大 30s
```
### 跨域处理
采用 **Nginx 同源反代**方案,前后端统一到同一域名,浏览器层面不存在跨域问题。
**生产环境**Nginx 将 `/`(前端)、`/api/*`REST`/ws`WebSocket统一反代到同一域名详见 `02-系统架构.md` 部署架构章节。
**开发环境**Vite 内置代理,前端 :5173 的 `/api``/ws` 请求代理到后端 :8080
```typescript
// frontend/vite.config.ts
server: {
proxy: {
"/api": "http://localhost:8080",
"/ws": { target: "ws://localhost:8080", ws: true },
},
},
```
**Go 后端 WebSocket CheckOrigin**:生产环境 Nginx 同源,`CheckOrigin` 可保持默认(拒绝跨域)。开发环境由 Vite proxy 转发,不存在跨域。因此后端无需配置 CORS 中间件,`CheckOrigin` 保持 gorilla/websocket 默认值即可。
> 如果未来需要支持第三方客户端直连(如移动端),再按需添加 CORS 中间件和 `CheckOrigin` 白名单。

221
docs/PLAN_BACKEND.md Normal file
View File

@@ -0,0 +1,221 @@
# CamTalk 后端完善计划
## Context
后端当前是一个骨架:`main.go` 启动 Gin 服务器,`ws/handler.go` 实现了 WebSocket 连接生命周期和消息分发,`models/models.go` 定义了所有协议消息类型,`config/config.go` 实现了 Viper 配置加载。但所有业务逻辑都是 TODO 桩——没有 Session Manager、没有 AI 服务客户端、没有编排层、没有日志/错误工具、没有测试。前端已基本完成,正在等待后端提供真实的 AI 管道。
**目标**:按设计文档(`docs/03-接口文档.md` 为最高依据)逐步填充所有业务模块,使端到端的 STT → LLM → TTS 流式管道可用。
---
## 分阶段实施
### Phase 1基础设施logger、errors、config 接入、graceful shutdown
**目标**:为后续模块提供日志、错误码、配置等基础能力,替换 `main.go` 中的硬编码值。
| # | 任务 | 文件 | 说明 |
|---|------|------|------|
| 1.1 | 实现 Zap 日志封装 | `internal/logger/logger.go` | 提供 `Init(level, format)` 和全局 `*zap.SugaredLogger`,替换所有 `log.Printf` |
| 1.2 | 实现错误码常量 + WS 错误发送工具 | `internal/errors/codes.go` | 10 个错误码常量 + `SendWSError(client, code, requestID, err)` |
| 1.3 | main.go 接入 config.Load() | `cmd/server/main.go` | 用 `cfg.Server.Host:Port` 替换硬编码 `:8080`,初始化 logger |
| 1.4 | 添加 graceful shutdown | `cmd/server/main.go` | `signal.NotifyContext` + `http.Server.Shutdown`10s drain |
| 1.5 | 添加 .gitignore | `backend/.gitignore` | 排除 `server` 二进制、`.env``tmp/` |
> **CORS**:不在此处实现,生产环境由 Nginx 反向代理统一处理跨域。
---
### Phase 2Session Manager
**目标**:实现会话生命周期管理,让 WS handler 能追踪会话、存储对话历史。
| # | 任务 | 文件 | 说明 |
|---|------|------|------|
| 2.1 | 定义 SessionManager 接口 | `internal/session/manager.go` | 方法:`Create`, `Get`, `UpdateConfig`, `GetHistory`, `AppendMessage`, `SetActiveRequest`, `ClearActiveRequest`, `Touch`, `Destroy` |
| 2.2 | 实现内存版 SessionManager | `internal/session/memory.go` | `sync.RWMutex` + `map[string]*sessionEntry`TTL 30 分钟,历史上限 20 条 |
| 2.3 | 实现 Redis 版 SessionManager | `internal/session/redis.go` | `session:{id}:meta` Hash + `session:{id}:history` ListTTL 刷新,选配 |
| 2.4 | 编写 Session Manager 测试 | `internal/session/memory_test.go` | 覆盖 Create/Get/Expire/Destroy/AppendMessage/History 上限 |
| 2.5 | WS handler 接入 SessionManager | `internal/ws/handler.go` | `ServeWS` 接收 `session.Manager` 参数;`connected` 消息后创建会话;`query` 时 Touch + SetActiveRequest`config` 时 UpdateConfig断开时不销毁自然过期 |
---
### Phase 3AI 服务层接口 + 实现
**目标**:定义并实现三个 AI 服务客户端,每个服务一个独立包。
| # | 任务 | 文件 | 说明 |
|---|------|------|------|
| **3a. STT** | | | |
| 3.1 | STT 接口定义 | `internal/ai/stt/stt.go` | `Service` 接口:`Recognize(ctx, audio []byte, opts Options) (string, error)``Options`: Encoding, SampleRate, Language |
| 3.2 | Deepgram 实现 | `internal/ai/stt/deepgram.go` | WebSocket 连接 `wss://api.deepgram.com/v1/listen`,发送 PCM 音频接收转录结果5s 超时 |
| 3.3 | STT 测试mock | `internal/ai/stt/deepgram_test.go` | httptest/WebSocket mock验证连接、发送、超时 |
| **3b. LLM** | | | |
| 3.4 | LLM 接口定义 | `internal/ai/llm/llm.go` | `Service` 接口:`ChatStream(ctx, req Request) (<-chan Chunk, error)``Request`: Image, Text, History, Language。`Chunk`: Delta, Done, TokensUsed, Model |
| 3.5 | OpenAI 实现 | `internal/ai/llm/openai.go` | `POST /v1/chat/completions` + `stream: true`SSE 解析10s 超时image 以 `data:image/jpeg;base64,...` 传入 |
| 3.6 | System Prompt 定义 | `internal/ai/llm/prompt.go` | 中文视觉助手提示词,根据 Language/DetailLevel 动态构建 |
| 3.7 | LLM 测试mock | `internal/ai/llm/openai_test.go` | httptest mock SSE 流,验证流式解析、超时、错误处理 |
| **3c. TTS** | | | |
| 3.8 | TTS 接口定义 | `internal/ai/tts/tts.go` | `Service` 接口:`SynthesizeStream(ctx, textStream <-chan string, opts Options) (<-chan Chunk, error)``Chunk`: Audio []byte, IsLast |
| 3.9 | OpenAI 实现 | `internal/ai/tts/openai.go` | `POST /v1/audio/speech` 模型 `tts-1`,逐句发送,返回 MP3 流5s/句超时 |
| 3.10 | TTS 测试mock | `internal/ai/tts/openai_test.go` | httptest mock验证逐句合成、超时 |
---
### Phase 4AI Orchestrator核心编排
**目标**:实现 STT → LLM → TTS 流式并行管道,这是后端最关键的业务逻辑。
| # | 任务 | 文件 | 说明 |
|---|------|------|------|
| 4.1 | Orchestrator 接口 | `internal/orchestrator/orchestrator.go` | `ProcessQuery(ctx, sessionID, req, history, sender)` — 接收查询并执行管道 |
| 4.2 | Sender 接口 | `internal/orchestrator/sender.go` | 抽象 WS 推送:`SendSTTResult`, `SendLLMChunk`, `SendLLMDone`, `SendTTSAudio`, `SendError`,便于测试 |
| 4.3 | 管道实现 | `internal/orchestrator/pipeline.go` | ① `stt.Recognize()` → 发送 `stt_result``llm.ChatStream()` 并行消费 token → 发送 `llm_chunk` + 句子切分 → channel ③ `tts.SynthesizeStream()` 从 channel 读取 → 发送 `tts_audio` ④ 流结束 → 发送 `llm_done` |
| 4.4 | 句子切分器 | `internal/orchestrator/splitter.go` | 按 `。!?\n.!?` 切分buffer size 4 channel |
| 4.5 | 错误降级 | 同上文件 | STT 失败→STT_ERROR+abortLLM 超时→LLM_TIMEOUTTTS 失败→静默跳过 |
| 4.6 | Interrupt 支持 | 同上文件 | context cancel 触发所有流中止 |
| 4.7 | Orchestrator 测试 | `internal/orchestrator/pipeline_test.go` | mock 三个 AI service + mock sender验证完整流程、中断、错误降级 |
---
### Phase 5WS Handler 完整接入
**目标**:将 Session Manager + Orchestrator 串入 WebSocket handler实现端到端消息处理。
| # | 任务 | 文件 | 说明 |
|---|------|------|------|
| 5.1 | Client 扩展 | `internal/ws/handler.go` | 添加 `session.Manager``orchestrator.Orchestrator``context.CancelFunc`(用于 interrupt |
| 5.2 | query 处理 | 同上 | 解码 audio Base64 → `stt.Recognize` 的输入Touch 会话;设置 active request启动 `orchestrator.ProcessQuery` goroutine |
| 5.3 | config 处理 | 同上 | 调用 `session.UpdateConfig()` |
| 5.4 | interrupt 处理 | 同上 | 查找 active request 的 cancel func调用 `cancel()`ClearActiveRequest |
| 5.5 | Disconnect 处理 | 同上 | 取消当前活跃请求(如有),不销毁会话 |
---
### Phase 6REST API 补全
**目标**:补全设计文档中的 REST 端点。
| # | 任务 | 文件 | 说明 |
|---|------|------|------|
| 6.1 | Session 路由 | `internal/api/session.go` | `POST /api/sessions` 创建会话,`DELETE /api/sessions/:id` 销毁会话 |
| 6.2 | Health 更新 | `cmd/server/main.go` | 从 SessionManager 获取 `active_sessions` 真实值 |
| 6.3 | 路由注册 | `cmd/server/main.go` | 统一注册 REST + WS 路由,注入依赖 |
---
### Phase 7Rate Limiter + Model Router可选/MVP 后)
**目标**:防止滥用 + 智能模型选择MVP 可简化或跳过。
| # | 任务 | 文件 | 说明 |
|---|------|------|------|
| 7.1 | 令牌桶 Rate Limiter | `internal/middleware/ratelimit.go` | `golang.org/x/time/rate` 或自实现,按 session ID 限流 |
| 7.2 | Rate Limiter 中间件 | `internal/middleware/ratelimit.go` | 在 WS query 路径上检查,超限返回 `RATE_LIMITED` |
| 7.3 | Model Router | `internal/ai/router.go` | 规则引擎简单识别→GPT-4o-mini深度分析→GPT-4o暂不实现 o1 |
---
### Phase 8集成测试 + 文档同步
| # | 任务 | 文件 | 说明 |
|---|------|------|------|
| 8.1 | WS 集成测试 | `internal/ws/handler_test.go` | 启动 Gin test server + gorilla websocket client验证完整 query→stt_result→llm_chunk→llm_done→tts_audio 流程 |
| 8.2 | 文档同步 | `docs/03-接口文档.md` | 代码实现与文档有偏差时更新文档 |
| 8.3 | go.sum 清理 | `backend/` | `go mod tidy` 清理无用依赖 |
---
## 关键文件清单
```
backend/
cmd/server/main.go ← Phase 1.3, 1.4, 1.5, 6.2, 6.3
internal/
config/config.go ← 已完成Phase 1.3 接入
logger/logger.go ← Phase 1.1(新建)
errors/codes.go ← Phase 1.2(新建)
models/models.go ← 已完成,可能小幅扩展
session/
manager.go ← Phase 2.1(新建)
memory.go ← Phase 2.2(新建)
redis.go ← Phase 2.3(新建)
memory_test.go ← Phase 2.4(新建)
ai/
stt/
stt.go ← Phase 3.1(新建)
deepgram.go ← Phase 3.2(新建)
deepgram_test.go ← Phase 3.3(新建)
llm/
llm.go ← Phase 3.4(新建)
openai.go ← Phase 3.5(新建)
prompt.go ← Phase 3.6(新建)
openai_test.go ← Phase 3.7(新建)
tts/
tts.go ← Phase 3.8(新建)
openai.go ← Phase 3.9(新建)
openai_test.go ← Phase 3.10(新建)
router.go ← Phase 7.3(新建)
orchestrator/
orchestrator.go ← Phase 4.1(新建)
sender.go ← Phase 4.2(新建)
pipeline.go ← Phase 4.3, 4.4, 4.5, 4.6(新建)
pipeline_test.go ← Phase 4.7(新建)
api/
session.go ← Phase 6.1(新建)
middleware/
ratelimit.go ← Phase 7.1, 7.2(新建)
ws/
handler.go ← Phase 5.1-5.5(修改)
handler_test.go ← Phase 8.1(新建)
```
---
## 新增依赖
| 包 | 用途 | Phase |
|----|------|-------|
| `go.uber.org/zap` | 结构化日志 | 1 |
| `github.com/redis/go-redis/v9` | Redis 客户端 | 2.3 |
| `github.com/gorilla/websocket` | 已有Deepgram WS 也复用 | 3.2 |
---
## 执行顺序与依赖关系
```
Phase 1 (基础设施)
Phase 2 (Session Manager)
Phase 3 (AI 服务层) ← 可与 Phase 2 并行开发
Phase 4 (Orchestrator) ← 依赖 Phase 2 + 3
Phase 5 (WS Handler 接入) ← 依赖 Phase 4
Phase 6 (REST API) ← 依赖 Phase 2
Phase 7 (Rate Limiter + Router) ← 独立,可推后
Phase 8 (集成测试 + 文档)
```
---
## 验证方案
1. **单元测试**每个模块独立测试mock 外部依赖AI API、Redis
2. **集成测试**`httptest` 启动 Gin server用 gorilla/websocket 客户端模拟完整 query 流程
3. **端到端手动测试**:启动后端 → 打开前端 → 摄像头+麦克风对话 → 验证 stt_result / llm_chunk / tts_audio 消息流
4. **go vet + go test ./...** 通过
---
## 设计文档参考
- 接口规范(最高优先级):`docs/03-接口文档.md`
- 系统架构:`docs/02-系统架构.md`
- 技术选型:`docs/04-技术选型.md`
- 成本控制:`docs/08-成本控制.md`

View File

@@ -1,19 +1,36 @@
/* ============================================================
CamTalk — 应用样式
CamTalk — Web 端应用样式
双栏布局:左侧视频面板 + 右侧聊天面板
============================================================ */
:root {
--color-primary: #2563eb;
--color-primary-hover: #1d4ed8;
--color-bg: #0f172a;
--color-surface: #1e293b;
--color-text: #f1f5f9;
--color-text-muted: #94a3b8;
--color-border: #334155;
--color-bg: #0b0f1a;
--color-surface: #151b2b;
--color-surface-2: #1c2438;
--color-text: #e2e8f0;
--color-text-muted: #64748b;
--color-border: #1e293b;
--color-success: #22c55e;
--color-warning: #f59e0b;
--color-error: #ef4444;
--radius: 8px;
--radius: 10px;
--radius-sm: 6px;
}
[data-theme="light"] {
--color-primary: #2563eb;
--color-primary-hover: #1d4ed8;
--color-bg: #f1f5f9;
--color-surface: #ffffff;
--color-surface-2: #f8fafc;
--color-text: #0f172a;
--color-text-muted: #64748b;
--color-border: #e2e8f0;
--color-success: #16a34a;
--color-warning: #d97706;
--color-error: #dc2626;
}
* {
@@ -22,205 +39,634 @@
box-sizing: border-box;
}
html, body, #root {
height: 100%;
}
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", sans-serif;
background: var(--color-bg);
color: var(--color-text);
min-height: 100vh;
}
/* ---- App Shell ---- */
.app {
display: flex;
flex-direction: column;
min-height: 100vh;
max-width: 900px;
margin: 0 auto;
padding: 0 16px;
height: 100vh;
overflow: hidden;
}
/* ---- Header ---- */
.app-header {
.header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 16px 0;
border-bottom: 1px solid var(--color-border);
}
.app-header h1 {
font-size: 1.5rem;
font-weight: 700;
}
.status {
font-size: 0.85rem;
padding: 4px 12px;
border-radius: 12px;
padding: 0 20px;
height: 48px;
background: var(--color-surface);
}
.status--connected {
color: var(--color-success);
border: 1px solid var(--color-success);
}
.status--connecting {
color: var(--color-warning);
border: 1px solid var(--color-warning);
}
.status--disconnected {
color: var(--color-text-muted);
border: 1px solid var(--color-border);
}
/* ---- Main ---- */
.app-main {
flex: 1;
display: flex;
flex-direction: column;
gap: 16px;
padding: 16px 0;
overflow: hidden;
}
.video-section {
position: relative;
border-bottom: 1px solid var(--color-border);
flex-shrink: 0;
}
.vad-indicator {
position: absolute;
bottom: 12px;
left: 12px;
background: rgba(0, 0, 0, 0.7);
color: var(--color-success);
padding: 6px 12px;
border-radius: var(--radius);
font-size: 0.85rem;
animation: pulse 1.5s ease-in-out infinite;
}
.vad-indicator--loading {
color: var(--color-warning);
}
.vad-indicator--error {
color: var(--color-error);
animation: none;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.6; }
}
.chat-section {
flex: 1;
.header__left {
display: flex;
flex-direction: column;
align-items: baseline;
gap: 12px;
overflow: hidden;
}
/* ---- Video Preview ---- */
.video-preview {
position: relative;
width: 100%;
aspect-ratio: 4 / 3;
background: var(--color-surface);
border-radius: var(--radius);
overflow: hidden;
.header__title {
font-size: 1.2rem;
font-weight: 700;
color: var(--color-text);
}
.video-preview__video {
width: 100%;
height: 100%;
object-fit: cover;
}
.video-preview__placeholder {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
.header__subtitle {
font-size: 0.8rem;
color: var(--color-text-muted);
}
/* ---- Chat Panel ---- */
.header__right {
display: flex;
align-items: center;
gap: 12px;
}
.chat-panel {
.header__stats {
font-size: 0.75rem;
color: var(--color-text-muted);
padding: 4px 10px;
background: var(--color-surface-2);
border-radius: 12px;
}
.badge {
font-size: 0.75rem;
padding: 3px 10px;
border-radius: 12px;
font-weight: 500;
}
.badge--connected {
color: var(--color-success);
background: rgba(34, 197, 94, 0.1);
border: 1px solid rgba(34, 197, 94, 0.3);
}
.badge--connecting {
color: var(--color-warning);
background: rgba(245, 158, 11, 0.1);
border: 1px solid rgba(245, 158, 11, 0.3);
}
.badge--disconnected {
color: var(--color-text-muted);
background: var(--color-surface-2);
border: 1px solid var(--color-border);
}
.btn-icon {
background: none;
border: none;
font-size: 1.1rem;
cursor: pointer;
padding: 6px;
border-radius: var(--radius-sm);
transition: background 0.15s;
color: var(--color-text-muted);
}
.btn-icon:hover {
background: var(--color-surface-2);
color: var(--color-text);
}
.lang-group {
display: flex;
gap: 2px;
background: var(--color-surface-2);
border-radius: var(--radius-sm);
padding: 2px;
}
.lang-btn {
background: none;
border: none;
color: var(--color-text-muted);
font-size: 0.72rem;
font-weight: 600;
padding: 4px 10px;
border-radius: 4px;
cursor: pointer;
transition: all 0.15s;
}
.lang-btn:hover {
color: var(--color-text);
}
.lang-btn--active {
background: var(--color-primary);
color: white;
}
/* ---- Workspace (双栏) ---- */
.workspace {
flex: 1;
display: flex;
overflow: hidden;
}
/* ---- 左侧视频面板 ---- */
.video-panel {
width: 420px;
flex-shrink: 0;
display: flex;
flex-direction: column;
padding: 16px;
gap: 12px;
overflow-y: auto;
}
.video-container {
position: relative;
aspect-ratio: 4 / 3;
background: var(--color-surface-2);
border-radius: var(--radius);
overflow: hidden;
border: 1px solid var(--color-border);
}
.video-container .video-preview {
width: 100%;
height: 100%;
border-radius: 0;
}
.video-container .video-preview__video {
width: 100%;
height: 100%;
object-fit: cover;
background: #1a1a2e;
}
.video-placeholder {
position: absolute;
inset: 0;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 8px;
color: var(--color-text-muted);
font-size: 0.82rem;
}
.video-placeholder__icon {
font-size: 2.4rem;
opacity: 0.2;
margin-bottom: 4px;
}
.video-indicator {
position: absolute;
bottom: 16px;
left: 16px;
background: rgba(0, 0, 0, 0.75);
backdrop-filter: blur(8px);
color: var(--color-success);
padding: 6px 14px;
border-radius: var(--radius-sm);
font-size: 0.8rem;
font-weight: 500;
animation: pulse 1.5s ease-in-out infinite;
}
.live-badge {
position: absolute;
top: 12px;
left: 12px;
display: flex;
align-items: center;
gap: 6px;
background: rgba(0, 0, 0, 0.75);
backdrop-filter: blur(8px);
color: white;
padding: 4px 12px;
border-radius: var(--radius-sm);
font-size: 0.75rem;
font-weight: 600;
letter-spacing: 0.5px;
}
.live-badge__dot {
width: 8px;
height: 8px;
border-radius: 50%;
background: var(--color-success);
animation: pulse 1.5s ease-in-out infinite;
}
.video-indicator--loading { color: var(--color-warning); }
.video-indicator--audio { left: auto; right: 16px; color: var(--color-primary); }
.video-indicator--error { color: var(--color-error); animation: none; }
.detail-badge {
position: absolute;
top: 12px;
right: 12px;
background: rgba(37, 99, 235, 0.85);
backdrop-filter: blur(8px);
color: white;
padding: 3px 10px;
border-radius: var(--radius-sm);
font-size: 0.7rem;
font-weight: 700;
letter-spacing: 0.5px;
}
.observation-badge {
position: absolute;
top: 12px;
left: 12px;
background: rgba(34, 197, 94, 0.85);
backdrop-filter: blur(8px);
color: white;
padding: 3px 10px;
border-radius: var(--radius-sm);
font-size: 0.7rem;
font-weight: 700;
animation: pulse 2s ease-in-out infinite;
}
.video-overlay {
position: absolute;
inset: 0;
background: rgba(0, 0, 0, 0.35);
display: flex;
align-items: center;
justify-content: center;
backdrop-filter: blur(2px);
}
.video-overlay__spinner {
width: 36px;
height: 36px;
border: 3px solid rgba(255, 255, 255, 0.2);
border-top-color: white;
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
/* ---- 视频控制栏 ---- */
.video-controls {
flex-shrink: 0;
display: flex;
justify-content: center;
padding: 4px 0;
}
.video-controls__row {
display: flex;
gap: 6px;
flex-wrap: wrap;
justify-content: center;
}
.btn--ctrl {
padding: 6px 12px;
font-size: 0.78rem;
border-radius: var(--radius-sm);
border: 1px solid var(--color-border);
background: var(--color-surface-2);
color: var(--color-text-muted);
cursor: pointer;
transition: all 0.15s;
white-space: nowrap;
}
.btn--ctrl:hover {
background: var(--color-border);
color: var(--color-text);
}
.btn--ctrl-on {
background: rgba(34, 197, 94, 0.15);
color: var(--color-success);
border-color: rgba(34, 197, 94, 0.3);
}
.btn--ctrl-off {
background: rgba(239, 68, 68, 0.1);
color: var(--color-error);
border-color: rgba(239, 68, 68, 0.2);
}
.btn--speaking {
animation: micPulse 0.8s ease-in-out infinite;
box-shadow: 0 0 12px rgba(34, 197, 94, 0.5);
}
@keyframes micPulse {
0%, 100% { box-shadow: 0 0 8px rgba(34, 197, 94, 0.4); transform: scale(1); }
50% { box-shadow: 0 0 16px rgba(34, 197, 94, 0.8); transform: scale(1.05); }
}
/* ---- 右侧聊天面板 ---- */
.chat-panel-wrapper {
flex: 1;
display: flex;
flex-direction: column;
background: var(--color-surface);
border-left: 1px solid var(--color-border);
min-width: 0;
}
.chat-panel-header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 12px 16px;
font-weight: 600;
font-size: 0.85rem;
border-bottom: 1px solid var(--color-border);
flex-shrink: 0;
color: var(--color-text-muted);
}
.chat-panel-header__mode {
font-size: 0.7rem;
font-weight: 500;
color: var(--color-success);
background: rgba(34, 197, 94, 0.1);
padding: 2px 8px;
border-radius: 10px;
}
.chat-panel-body {
flex: 1;
overflow: hidden;
display: flex;
flex-direction: column;
}
/* ---- Chat Panel (覆盖子组件样式) ---- */
.chat-panel {
flex: 1;
overflow-y: auto;
padding: 12px 16px;
display: flex;
flex-direction: column;
gap: 10px;
}
.chat-panel--empty {
display: flex;
align-items: center;
justify-content: center;
padding: 48px 0;
flex: 1;
color: var(--color-text-muted);
font-size: 0.85rem;
}
.chat-message {
padding: 12px 16px;
border-radius: var(--radius);
background: var(--color-surface);
padding: 10px 14px;
border-radius: var(--radius-sm);
background: var(--color-surface-2);
max-width: 92%;
}
.chat-message--user {
border-left: 3px solid var(--color-primary);
border-left: 2px solid var(--color-primary);
align-self: flex-end;
background: rgba(37, 99, 235, 0.08);
}
.chat-message--assistant {
border-left: 3px solid var(--color-success);
border-left: 2px solid var(--color-success);
}
.chat-message--streaming {
opacity: 0.8;
opacity: 0.85;
}
.chat-message__role {
font-size: 0.75rem;
font-size: 0.65rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.5px;
color: var(--color-text-muted);
margin-bottom: 4px;
margin-bottom: 3px;
}
.chat-message__content {
font-size: 0.95rem;
line-height: 1.5;
font-size: 0.82rem;
line-height: 1.55;
white-space: pre-wrap;
}
/* ---- Footer ---- */
.app-footer {
display: flex;
gap: 12px;
justify-content: center;
padding: 16px 0;
border-top: 1px solid var(--color-border);
.chat-message__meta {
margin-top: 6px;
font-size: 0.65rem;
color: var(--color-text-muted);
}
/* ---- Streaming Cursor ---- */
.cursor {
display: inline;
animation: blink 0.8s step-end infinite;
color: var(--color-success);
}
/* ---- System Message ---- */
.system-message {
text-align: center;
padding: 8px 16px;
margin: 0 20px;
border-radius: var(--radius-sm);
font-size: 0.8rem;
}
.system-message--warning {
background: rgba(245, 158, 11, 0.1);
color: var(--color-warning);
border: 1px solid rgba(245, 158, 11, 0.2);
}
/* ---- Drawer (Config Panel) ---- */
.drawer-overlay {
position: fixed;
inset: 0;
z-index: 100;
background: rgba(0, 0, 0, 0.4);
backdrop-filter: blur(2px);
animation: fadeIn 0.2s ease;
}
.drawer {
position: absolute;
top: 0;
right: 0;
bottom: 0;
width: 320px;
background: var(--color-surface);
border-left: 1px solid var(--color-border);
display: flex;
flex-direction: column;
animation: slideInRight 0.25s ease;
box-shadow: -8px 0 24px rgba(0, 0, 0, 0.3);
}
.drawer__header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 14px 16px;
border-bottom: 1px solid var(--color-border);
flex-shrink: 0;
}
.drawer__title {
font-weight: 600;
font-size: 0.9rem;
}
.drawer__close {
background: none;
border: none;
color: var(--color-text-muted);
cursor: pointer;
font-size: 1rem;
padding: 4px 8px;
border-radius: var(--radius-sm);
transition: all 0.15s;
}
.drawer__close:hover {
background: var(--color-surface-2);
color: var(--color-text);
}
.drawer__body {
flex: 1;
overflow-y: auto;
padding: 16px;
}
/* ---- Config Group ---- */
.config-group {
margin-bottom: 20px;
}
.config-group__title {
font-size: 0.7rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.8px;
color: var(--color-text-muted);
margin-bottom: 12px;
padding-bottom: 8px;
border-bottom: 1px solid var(--color-border);
}
.config-row {
display: flex;
align-items: center;
justify-content: space-between;
padding: 10px 0;
font-size: 0.82rem;
color: var(--color-text);
cursor: pointer;
}
.config-row__info {
display: flex;
flex-direction: column;
gap: 2px;
}
.config-row__label {
font-weight: 500;
color: var(--color-text);
}
.config-row__desc {
font-size: 0.72rem;
color: var(--color-text-muted);
}
.config-row select,
.config-row input[type="checkbox"] {
background: var(--color-surface-2);
color: var(--color-text);
border: 1px solid var(--color-border);
border-radius: var(--radius-sm);
padding: 5px 10px;
font-size: 0.8rem;
cursor: pointer;
}
.config-row input[type="checkbox"] {
width: 18px;
height: 18px;
cursor: pointer;
}
/* ---- Toast ---- */
.toast-container {
position: fixed;
top: 68px;
right: 20px;
z-index: 1000;
display: flex;
flex-direction: column;
gap: 8px;
}
.toast {
padding: 10px 16px;
border-radius: var(--radius-sm);
font-size: 0.82rem;
cursor: pointer;
animation: slideIn 0.3s ease-out;
max-width: 320px;
backdrop-filter: blur(8px);
}
.toast--error { background: rgba(239, 68, 68, 0.9); color: white; }
.toast--warning { background: rgba(245, 158, 11, 0.9); color: #000; }
.toast--info { background: rgba(37, 99, 235, 0.9); color: white; }
/* ---- Buttons ---- */
.btn {
padding: 10px 24px;
padding: 8px 20px;
border: none;
border-radius: var(--radius);
font-size: 0.95rem;
border-radius: var(--radius-sm);
font-size: 0.85rem;
font-weight: 600;
cursor: pointer;
transition: background 0.2s;
transition: all 0.15s;
display: inline-flex;
align-items: center;
gap: 6px;
}
.btn--lg {
padding: 12px 32px;
font-size: 0.95rem;
border-radius: var(--radius);
}
.btn--primary {
@@ -233,7 +679,7 @@ body {
}
.btn--secondary {
background: var(--color-surface);
background: var(--color-surface-2);
color: var(--color-text);
border: 1px solid var(--color-border);
}
@@ -247,16 +693,35 @@ body {
color: #000;
}
.btn--warning:hover {
opacity: 0.9;
.btn--warning:hover { opacity: 0.9; }
.btn--danger {
background: rgba(239, 68, 68, 0.15);
color: var(--color-error);
border: 1px solid rgba(239, 68, 68, 0.3);
}
/* ---- Streaming Cursor ---- */
.btn--danger:hover {
background: rgba(239, 68, 68, 0.25);
}
.cursor {
display: inline;
animation: blink 0.8s step-end infinite;
color: var(--color-success);
.btn--active {
background: var(--color-primary);
color: white;
box-shadow: 0 0 12px rgba(37, 99, 235, 0.4);
animation: pulseBtn 2s ease-in-out infinite;
}
/* ---- Animations ---- */
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.6; }
}
@keyframes pulseBtn {
0%, 100% { box-shadow: 0 0 12px rgba(37, 99, 235, 0.4); }
50% { box-shadow: 0 0 20px rgba(37, 99, 235, 0.7); }
}
@keyframes blink {
@@ -264,97 +729,40 @@ body {
50% { opacity: 0; }
}
/* ---- Message Meta ---- */
.chat-message__meta {
margin-top: 6px;
font-size: 0.75rem;
color: var(--color-text-muted);
}
/* ---- System Message ---- */
.system-message {
text-align: center;
padding: 8px 16px;
border-radius: var(--radius);
font-size: 0.85rem;
}
.system-message--warning {
background: rgba(245, 158, 11, 0.15);
color: var(--color-warning);
border: 1px solid rgba(245, 158, 11, 0.3);
}
/* ---- Toast ---- */
.toast-container {
position: fixed;
top: 16px;
right: 16px;
z-index: 1000;
display: flex;
flex-direction: column;
gap: 8px;
}
.toast {
padding: 10px 16px;
border-radius: var(--radius);
font-size: 0.85rem;
cursor: pointer;
animation: slideIn 0.3s ease-out;
max-width: 320px;
}
.toast--error {
background: rgba(239, 68, 68, 0.9);
color: white;
}
.toast--warning {
background: rgba(245, 158, 11, 0.9);
color: #000;
}
.toast--info {
background: rgba(37, 99, 235, 0.9);
color: white;
}
@keyframes slideIn {
from {
opacity: 0;
transform: translateX(20px);
}
to {
opacity: 1;
transform: translateX(0);
}
}
/* ---- Video Overlay ---- */
.video-overlay {
position: absolute;
inset: 0;
background: rgba(0, 0, 0, 0.3);
display: flex;
align-items: center;
justify-content: center;
border-radius: var(--radius);
}
.video-overlay__spinner {
width: 32px;
height: 32px;
border: 3px solid rgba(255, 255, 255, 0.3);
border-top-color: white;
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg); }
}
@keyframes slideIn {
from { opacity: 0; transform: translateX(20px); }
to { opacity: 1; transform: translateX(0); }
}
@keyframes fadeIn {
from { opacity: 0; }
to { opacity: 1; }
}
@keyframes slideInRight {
from { transform: translateX(100%); }
to { transform: translateX(0); }
}
/* ---- Scrollbar ---- */
::-webkit-scrollbar {
width: 6px;
}
::-webkit-scrollbar-track {
background: transparent;
}
::-webkit-scrollbar-thumb {
background: var(--color-border);
border-radius: 3px;
}
::-webkit-scrollbar-thumb:hover {
background: var(--color-text-muted);
}

View File

@@ -1,97 +1,235 @@
// ============================================================
// CamTalk — 主应用组件
// CamTalk — 主应用组件Web 端双栏布局)
// ============================================================
import { useCallback, useEffect, useRef, useState } from "react";
import { useVisionSession } from "./hooks/useVisionSession";
import { VideoPreview } from "./components/VideoPreview";
import { ChatPanel } from "./components/ChatPanel";
import { ConfigPanel } from "./components/ConfigPanel";
import { ToastContainer } from "./components/Toast";
import { loadTheme, saveTheme } from "./lib/storage";
import type { Theme } from "./types";
import "./App.css";
function App() {
const [showConfig, setShowConfig] = useState(false);
const [theme, setTheme] = useState<Theme>(loadTheme);
const [elapsed, setElapsed] = useState(0);
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
// 切换主题时更新 <html> 的 data-theme 属性
useEffect(() => {
document.documentElement.setAttribute("data-theme", theme);
}, [theme]);
const handleThemeChange = useCallback((t: Theme) => {
setTheme(t);
saveTheme(t);
}, []);
const formatTime = (s: number) => {
const m = Math.floor(s / 60);
const sec = s % 60;
return `${m.toString().padStart(2, "0")}:${sec.toString().padStart(2, "0")}`;
};
const {
messages,
currentReply,
isProcessing,
isAudioPlaying,
isSpeaking,
isVADReady,
vadError,
connectionStatus,
videoRef,
stream,
config,
updateConfig,
stats,
mode,
isObserving,
toggleMode,
startSession,
stopSession,
interrupt,
isCameraOn,
isMicOn,
toggleCamera,
toggleMic,
} = useVisionSession();
const isConnected = connectionStatus === "connected";
// 连接计时器
const elapsedRef = useRef(0);
useEffect(() => {
if (isConnected) {
elapsedRef.current = 0;
setElapsed(0); // eslint-disable-line react-hooks/set-state-in-effect -- 连接时重置计时器
}
}, [isConnected]);
useEffect(() => {
if (!isConnected) {
if (timerRef.current) {
clearInterval(timerRef.current);
timerRef.current = null;
}
return;
}
const id = setInterval(() => {
elapsedRef.current += 1;
setElapsed(elapsedRef.current);
}, 1000);
timerRef.current = id;
return () => clearInterval(id);
}, [isConnected]);
return (
<div className="app">
<header className="app-header">
<h1>CamTalk</h1>
<span className={`status status--${connectionStatus}`}>
{isConnected
? "已连接"
: connectionStatus === "connecting"
? "连接中..."
: "未连接"}
</span>
{/* ---- 顶部导航栏 ---- */}
<header className="header">
<div className="header__left">
<h1 className="header__title">CamTalk</h1>
<span className="header__subtitle">AI </span>
</div>
<div className="header__right">
{isConnected && (stats.queryCount > 0 || stats.totalTokens > 0) && (
<span className="header__stats">
{stats.queryCount} · {stats.totalTokens} tokens
</span>
)}
<span className={`badge badge--${connectionStatus}`}>
{isConnected ? "已连接" : connectionStatus === "connecting" ? "连接中..." : "未连接"}
</span>
<button
className="btn-icon"
onClick={() => setShowConfig((v) => !v)}
title="设置"
>
</button>
</div>
</header>
<main className="app-main">
<div className="video-section">
<VideoPreview ref={videoRef} isStreaming={!!stream} />
{isSpeaking && <div className="vad-indicator">🎤 ...</div>}
{isConnected && !isVADReady && !vadError && (
<div className="vad-indicator vad-indicator--loading">
...
</div>
)}
{vadError && (
<div className="vad-indicator vad-indicator--error">
{vadError}
</div>
)}
{isProcessing && (
<div className="video-overlay">
<div className="video-overlay__spinner" />
</div>
)}
</div>
{showConfig && (
<ConfigPanel
config={config}
theme={theme}
onUpdate={updateConfig}
onThemeChange={handleThemeChange}
onClose={() => setShowConfig(false)}
/>
)}
<div className="chat-section">
{connectionStatus === "disconnected" && messages.length > 0 && (
<div className="system-message system-message--warning">
...
</div>
)}
<ChatPanel
messages={messages}
currentReply={currentReply}
connectionStatus={connectionStatus}
/>
</div>
</main>
<footer className="app-footer">
{!isConnected ? (
<button className="btn btn--primary" onClick={startSession}>
{connectionStatus === "connecting" ? "连接中..." : "开始对话"}
</button>
) : (
<>
<button className="btn btn--secondary" onClick={stopSession}>
</button>
{isProcessing && (
<button className="btn btn--warning" onClick={interrupt}>
</button>
{/* ---- 主体:左侧视频 + 右侧聊天 ---- */}
<div className="workspace">
{/* 左侧:视频预览 + 控制栏 */}
<div className="video-panel">
<div className="video-container">
<VideoPreview ref={videoRef} isStreaming={!!stream} />
{isConnected && (
<div className="live-badge">
<span className="live-badge__dot" />
LIVE {formatTime(elapsed)}
</div>
)}
</>
)}
</footer>
{config.detailLevel === "high" && (
<div className="detail-badge">HD</div>
)}
{isObserving && (
<div className="observation-badge">👁 </div>
)}
{isSpeaking && (
<div className="video-indicator">🎤 ...</div>
)}
{isAudioPlaying && config.ttsEnabled && (
<div className="video-indicator video-indicator--audio">🔊 ...</div>
)}
{isConnected && !isVADReady && !vadError && (
<div className="video-indicator video-indicator--loading">...</div>
)}
{vadError && (
<div className="video-indicator video-indicator--error"> {vadError}</div>
)}
{isProcessing && (
<div className="video-overlay">
<div className="video-overlay__spinner" />
</div>
)}
{!isConnected && !stream && (
<div className="video-placeholder">
<span className="video-placeholder__icon">📷</span>
<span></span>
</div>
)}
</div>
{/* 视频下方控制栏 */}
<div className="video-controls">
{!isConnected ? (
<button className="btn btn--primary btn--lg" onClick={startSession}>
{connectionStatus === "connecting" ? "连接中..." : "🎙️ 开始对话"}
</button>
) : (
<div className="video-controls__row">
<button
className={`btn btn--ctrl ${isCameraOn ? "btn--ctrl-on" : "btn--ctrl-off"}`}
onClick={toggleCamera}
title={isCameraOn ? "关闭摄像头" : "开启摄像头"}
>
📷
</button>
<button
className={`btn btn--ctrl ${isMicOn ? "btn--ctrl-on" : "btn--ctrl-off"} ${isSpeaking ? "btn--speaking" : ""}`}
onClick={toggleMic}
title={isMicOn ? "关闭麦克风" : "开启麦克风"}
>
🎤
</button>
<button
className={`btn ${mode === "observation" ? "btn--active" : "btn--secondary"}`}
onClick={toggleMode}
>
{mode === "observation" ? "👁️ 观察中" : "👁️ 观察模式"}
</button>
{isProcessing && (
<button className="btn btn--warning" onClick={interrupt}>
</button>
)}
<button className="btn btn--danger" onClick={stopSession}>
</button>
</div>
)}
</div>
</div>
{/* 右侧:聊天面板 */}
<div className="chat-panel-wrapper">
<div className="chat-panel-header">
<span></span>
{isConnected && mode === "observation" && (
<span className="chat-panel-header__mode"></span>
)}
</div>
<div className="chat-panel-body">
{connectionStatus === "disconnected" && messages.length > 0 && (
<div className="system-message system-message--warning">
...
</div>
)}
<ChatPanel
messages={messages}
currentReply={currentReply}
connectionStatus={connectionStatus}
/>
</div>
</div>
</div>
<ToastContainer />
</div>

View File

@@ -0,0 +1,92 @@
// ============================================================
// ConfigPanel — 右侧抽屉式配置面板
// 职责主题切换、TTS 开关、detail level 切换、语言选择
// ============================================================
import type { SessionConfig, Theme } from "../../types";
interface ConfigPanelProps {
config: SessionConfig;
theme: Theme;
onUpdate: (partial: Partial<SessionConfig>) => void;
onThemeChange: (theme: Theme) => void;
onClose: () => void;
}
export function ConfigPanel({ config, theme, onUpdate, onThemeChange, onClose }: ConfigPanelProps) {
return (
<div className="drawer-overlay" onClick={onClose}>
<div className="drawer" onClick={(e) => e.stopPropagation()}>
<div className="drawer__header">
<span className="drawer__title"></span>
<button className="drawer__close" onClick={onClose}></button>
</div>
<div className="drawer__body">
<div className="config-group">
<div className="config-group__title"></div>
<label className="config-row">
<div className="config-row__info">
<span className="config-row__label"></span>
<span className="config-row__desc"></span>
</div>
<select
value={theme}
onChange={(e) => onThemeChange(e.target.value as Theme)}
>
<option value="dark"></option>
<option value="light"></option>
</select>
</label>
</div>
<div className="config-group">
<div className="config-group__title"></div>
<label className="config-row">
<div className="config-row__info">
<span className="config-row__label"></span>
<span className="config-row__desc">AI </span>
</div>
<input
type="checkbox"
checked={config.ttsEnabled}
onChange={(e) => onUpdate({ ttsEnabled: e.target.checked })}
/>
</label>
<label className="config-row">
<div className="config-row__info">
<span className="config-row__label"></span>
<span className="config-row__desc"></span>
</div>
<select
value={config.detailLevel}
onChange={(e) => onUpdate({ detailLevel: e.target.value as "low" | "high" })}
>
<option value="low"></option>
<option value="high"></option>
</select>
</label>
<label className="config-row">
<div className="config-row__info">
<span className="config-row__label"></span>
<span className="config-row__desc"></span>
</div>
<select
value={config.language}
onChange={(e) => onUpdate({ language: e.target.value })}
>
<option value="zh-CN"></option>
<option value="en-US">English</option>
<option value="ja-JP"></option>
</select>
</label>
</div>
</div>
</div>
</div>
);
}

View File

@@ -103,19 +103,61 @@ export function useVAD(options?: VADOptions) {
return { isSpeaking, isReady, error, start, stop };
}
// ---- 关键帧检测ONNX Runtime Web----
// ---- 关键帧检测 ----
export function useKeyframeDetection() {
// TODO: 加载 ONNX 模型后设为 true
const isReady = false;
const DETECT_WIDTH = 160;
const DETECT_HEIGHT = 120;
const DIFF_THRESHOLD = 30;
const isKeyframe = useCallback(
(_currentFrame: ImageData, _previousFrame: ImageData): boolean => {
// TODO: 实现像素差异对比
return true; // 暂时所有帧都视为关键帧
},
[],
);
/** 离屏 canvas用于降采样 */
let offscreen: HTMLCanvasElement | null = null;
return { isReady, isKeyframe };
function getOffscreen(): HTMLCanvasElement {
if (!offscreen) {
offscreen = document.createElement("canvas");
offscreen.width = DETECT_WIDTH;
offscreen.height = DETECT_HEIGHT;
}
return offscreen;
}
/**
* 将 video 元素缩放采样为 Uint8ClampedArrayRGBA
* 返回 null 如果 video 未就绪
*/
export function sampleFrame(video: HTMLVideoElement): Uint8ClampedArray | null {
if (video.readyState < 2) return null;
const canvas = getOffscreen();
const ctx = canvas.getContext("2d");
if (!ctx) return null;
ctx.drawImage(video, 0, 0, DETECT_WIDTH, DETECT_HEIGHT);
return ctx.getImageData(0, 0, DETECT_WIDTH, DETECT_HEIGHT).data;
}
/**
* 对比两帧像素差异
* @returns { isKeyframe: boolean, similarity: number }
*/
export function compareFrames(
prev: Uint8ClampedArray,
curr: Uint8ClampedArray,
): { isKeyframe: boolean; similarity: number } {
let diffSum = 0;
const len = Math.min(prev.length, curr.length);
const pixelCount = len / 4;
for (let i = 0; i < len; i += 4) {
// 只比较 RGB跳过 Alpha
diffSum += Math.abs(prev[i] - curr[i]);
diffSum += Math.abs(prev[i + 1] - curr[i + 1]);
diffSum += Math.abs(prev[i + 2] - curr[i + 2]);
}
const avgDiff = diffSum / (pixelCount * 3);
const similarity = 1 - avgDiff / 255;
return {
isKeyframe: avgDiff > DIFF_THRESHOLD,
similarity,
};
}

View File

@@ -0,0 +1,88 @@
// ============================================================
// useObservationMode — 观察模式 Hook
// 职责:定时采帧 → 关键帧检测 → 画面变化时触发回调
// 来源docs/05-用户故事.md US-05持续场景监控
// ============================================================
import { useCallback, useEffect, useRef, useState } from "react";
import { sampleFrame, compareFrames } from "../components/EdgeProcessor";
/** 画面变化显著阈值 */
const CHANGE_THRESHOLD = 0.85;
/** 采样间隔ms */
const SAMPLE_INTERVAL = 5000;
export interface ObservationOptions {
/** 画面变化回调,携带当前帧的 DataURL */
onChange?: (frameDataUrl: string) => void;
}
export function useObservationMode(options?: ObservationOptions) {
const [isObserving, setIsObserving] = useState(false);
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
const optionsRef = useRef(options);
useEffect(() => {
optionsRef.current = options;
}, [options]);
/**
* 启动观察模式
* @param video 摄像头 video 元素
* @param captureFrame 从 video 捕获 DataURL 的函数
*/
const startObserving = useCallback(
(video: HTMLVideoElement | null, captureFrame: () => string | null) => {
if (!video) return;
// 立即采一帧作为基准
prevFrameRef.current = sampleFrame(video);
intervalRef.current = setInterval(() => {
const current = sampleFrame(video);
if (!current) return;
if (prevFrameRef.current) {
const { similarity } = compareFrames(prevFrameRef.current, current);
if (similarity < CHANGE_THRESHOLD) {
console.log(
`[Observation] 画面变化 (similarity=${similarity.toFixed(2)})`,
);
const frameDataUrl = captureFrame();
if (frameDataUrl) {
optionsRef.current?.onChange?.(frameDataUrl);
}
}
}
prevFrameRef.current = current;
}, SAMPLE_INTERVAL);
setIsObserving(true);
},
[],
);
/** 停止观察模式 */
const stopObserving = useCallback(() => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
prevFrameRef.current = null;
setIsObserving(false);
}, []);
// 组件卸载时清理
useEffect(() => {
return () => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
}
};
}, []);
return { isObserving, startObserving, stopObserving };
}

View File

@@ -9,17 +9,52 @@ import { v4 as uuidv4 } from "uuid";
import { wsClient } from "../lib/websocket";
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
import { getErrorMessage } from "../lib/errors";
import { TTSPlayer } from "../lib/ttsPlayer";
import { showToast } from "../lib/toast";
import { loadConfig, saveConfig } from "../lib/storage";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
import { useVAD } from "../components/EdgeProcessor";
import { useVAD, sampleFrame, compareFrames } from "../components/EdgeProcessor";
import { useWebSocketManager } from "../components/WebSocketManager";
import type { ChatMessage, ServerMessage, LLMDoneMessage } from "../types";
import { useObservationMode } from "./useObservationMode";
import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";
export type SessionMode = "dialogue" | "observation";
const MAX_HISTORY_ROUNDS = 10;
export interface SessionStats {
queryCount: number;
totalTokens: number;
}
export function useVisionSession() {
const [messages, setMessages] = useState<ChatMessage[]>([]);
const [currentReply, setCurrentReply] = useState<string>("");
const [isProcessing, setIsProcessing] = useState(false);
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
const [config, setConfig] = useState<SessionConfig>(loadConfig);
const [stats, setStats] = useState<SessionStats>({ queryCount: 0, totalTokens: 0 });
const [mode, setMode] = useState<SessionMode>("dialogue");
const [isCameraOn, setIsCameraOn] = useState(false);
const [isMicOn, setIsMicOn] = useState(false);
// 上一帧采样数据(用于关键帧检测)
const prevFrameRef = useRef<Uint8ClampedArray | null>(null);
// 对话历史role + content用于多轮上下文
const historyRef = useRef<Array<{ role: string; content: string }>>([]);
// TTS 播放器
const ttsPlayerRef = useRef<TTSPlayer | null>(null);
const getTTSPlayer = useCallback(() => {
if (!ttsPlayerRef.current) {
const player = new TTSPlayer();
player.onEnd(() => setIsAudioPlaying(false));
ttsPlayerRef.current = player;
}
return ttsPlayerRef.current;
}, []);
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
const { startMic, stopMic } = useMicrophone();
@@ -31,6 +66,82 @@ export function useVisionSession() {
isProcessingRef.current = isProcessing;
}, [isProcessing]);
// 观察模式:画面变化时自动发送 query
const { isObserving, startObserving, stopObserving } = useObservationMode({
onChange: useCallback(
(frameDataUrl: string) => {
if (isProcessingRef.current) return;
const requestId = uuidv4();
send({
type: "query",
request_id: requestId,
image: dataUrlToBase64(frameDataUrl),
audio: "", // 观察模式无音频
});
setMessages((prev) => [
...prev,
{
role: "user",
content: "👁️ 画面变化检测",
timestamp: Date.now(),
},
]);
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
setIsProcessing(true);
},
[send],
),
});
/** 切换对话/观察模式 */
const toggleMode = useCallback(() => {
setMode((prev) => {
const next = prev === "dialogue" ? "observation" : "dialogue";
if (next === "observation") {
startObserving(videoRef.current, captureFrame);
} else {
stopObserving();
}
return next;
});
}, [videoRef, captureFrame, startObserving, stopObserving]);
// WebSocket 连接成功后发送 config
useEffect(() => {
if (status === "connected") {
send({
type: "config",
payload: {
tts_enabled: config.ttsEnabled,
detail_level: config.detailLevel,
language: config.language,
},
});
}
}, [status]); // eslint-disable-line react-hooks/exhaustive-deps -- 仅在连接状态变化时发送
/** 更新会话配置 */
const updateConfig = useCallback((partial: Partial<SessionConfig>) => {
setConfig((prev) => {
const next = { ...prev, ...partial };
saveConfig(next);
// 如果已连接,立即发送更新
if (status === "connected") {
send({
type: "config",
payload: {
tts_enabled: next.ttsEnabled,
detail_level: next.detailLevel,
language: next.language,
},
});
}
return next;
});
}, [status, send]);
// VAD语音结束时自动发送 query
const {
isSpeaking,
@@ -53,6 +164,23 @@ export function useVisionSession() {
return;
}
// 关键帧检测:与上一帧对比,相似度过高则跳过
const video = videoRef.current;
if (video) {
const currentSample = sampleFrame(video);
if (currentSample && prevFrameRef.current) {
const { similarity } = compareFrames(prevFrameRef.current, currentSample);
if (similarity > 0.9) {
console.log(`[Session] 画面无变化 (similarity=${similarity.toFixed(2)}),跳过`);
prevFrameRef.current = currentSample;
return;
}
}
if (currentSample) {
prevFrameRef.current = currentSample;
}
}
const requestId = uuidv4();
send({
type: "query",
@@ -61,6 +189,9 @@ export function useVisionSession() {
audio: encodeAudioToBase64(audio),
});
// 更新请求统计
setStats((prev) => ({ ...prev, queryCount: prev.queryCount + 1 }));
// 添加用户消息STT 流式结果会逐步更新文本)
setMessages((prev) => [
...prev,
@@ -68,7 +199,7 @@ export function useVisionSession() {
]);
setIsProcessing(true);
},
[captureFrame, send]
[captureFrame, send, videoRef]
),
});
@@ -98,6 +229,21 @@ export function useVisionSession() {
case "llm_done": {
const done = msg as LLMDoneMessage;
// 记录到对话历史
historyRef.current.push({ role: "assistant", content: done.full_text });
// 裁剪历史到最近 N 轮
if (historyRef.current.length > MAX_HISTORY_ROUNDS * 2) {
historyRef.current = historyRef.current.slice(-MAX_HISTORY_ROUNDS * 2);
}
// 累计 token 统计
if (done.tokens_used?.total) {
setStats((prev) => ({
...prev,
totalTokens: prev.totalTokens + done.tokens_used.total,
}));
}
setMessages((prev) => [
...prev,
{
@@ -115,7 +261,10 @@ export function useVisionSession() {
}
case "tts_audio":
// TODO: 阶段 5 音频流播放
getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
if (!msg.is_last) {
setIsAudioPlaying(true);
}
break;
case "error":
@@ -127,25 +276,19 @@ export function useVisionSession() {
});
return unsub;
}, []);
// 连接断开时显示提示
useEffect(() => {
if (status === "disconnected") {
// 只在非主动断开时提示(通过检查是否有活跃会话判断)
// 这里简单处理,由 App 层根据状态显示
}
}, [status]);
}, [getTTSPlayer]);
/** 启动会话 */
const startSession = useCallback(async () => {
// 1. 获取摄像头和麦克风
await startCamera();
setIsCameraOn(true);
const micStream = await startMic();
if (!micStream) {
showToast("无法获取麦克风权限", "error");
return;
}
setIsMicOn(true);
// 2. 连接 WebSocket
connect();
@@ -156,24 +299,60 @@ export function useVisionSession() {
/** 结束会话 */
const stopSession = useCallback(async () => {
stopObserving();
setMode("dialogue");
await stopVAD();
stopMic();
stopCamera();
disconnect();
// 清理所有对话状态
// 停止 TTS 并清理状态
ttsPlayerRef.current?.stop();
setIsAudioPlaying(false);
setMessages([]);
setCurrentReply("");
setIsProcessing(false);
}, [stopVAD, stopMic, stopCamera, disconnect]);
setStats({ queryCount: 0, totalTokens: 0 });
historyRef.current = [];
prevFrameRef.current = null;
setIsCameraOn(false);
setIsMicOn(false);
}, [stopObserving, stopVAD, stopMic, stopCamera, disconnect]);
/** 摄像头开关 */
const toggleCamera = useCallback(async () => {
if (isCameraOn) {
stopCamera();
setIsCameraOn(false);
} else {
await startCamera();
setIsCameraOn(true);
}
}, [isCameraOn, startCamera, stopCamera]);
/** 麦克风开关 */
const toggleMic = useCallback(async () => {
if (isMicOn) {
stopMic();
setIsMicOn(false);
} else {
const micStream = await startMic();
setIsMicOn(!!micStream);
}
}, [isMicOn, startMic, stopMic]);
/** 打断当前回复 */
const interrupt = useCallback(() => {
send({ type: "interrupt" });
// 停止 TTS 播放
ttsPlayerRef.current?.stop();
setIsAudioPlaying(false);
// 将未完成的流式内容保存为最终消息
if (currentReply) {
const interrupted = currentReply + "(已打断)";
historyRef.current.push({ role: "assistant", content: interrupted });
setMessages((prev) => [
...prev,
{ role: "assistant", content: currentReply + "(已打断)", timestamp: Date.now() },
{ role: "assistant", content: interrupted, timestamp: Date.now() },
]);
}
setCurrentReply("");
@@ -184,14 +363,25 @@ export function useVisionSession() {
messages,
currentReply,
isProcessing,
isAudioPlaying,
isSpeaking,
isVADReady,
vadError,
connectionStatus: status,
videoRef,
stream,
config,
updateConfig,
stats,
mode,
isObserving,
toggleMode,
startSession,
stopSession,
interrupt,
isCameraOn,
isMicOn,
toggleCamera,
toggleMic,
};
}

View File

@@ -1,7 +1 @@
/* 全局重置 — 详细样式在 App.css 中定义 */
#root {
min-height: 100vh;
display: flex;
flex-direction: column;
}

View File

@@ -0,0 +1,43 @@
// ============================================================
// Sampling — 混合采样策略
// 来源docs/08-成本控制.md — "定时低频 + 事件高频"
// ============================================================
/** 静默时采样间隔ms */
const IDLE_INTERVAL = 5000;
/** 用户说话时采样间隔ms */
const ACTIVE_INTERVAL = 1000;
export class SamplingController {
private lastSampleTime = 0;
private _isUserSpeaking = false;
/** 设置用户是否正在说话 */
set speaking(value: boolean) {
this._isUserSpeaking = value;
}
get speaking(): boolean {
return this._isUserSpeaking;
}
/** 当前采样间隔 */
get interval(): number {
return this._isUserSpeaking ? ACTIVE_INTERVAL : IDLE_INTERVAL;
}
/** 是否应该采样(基于时间间隔) */
shouldSample(): boolean {
const now = Date.now();
if (now - this.lastSampleTime < this.interval) {
return false;
}
this.lastSampleTime = now;
return true;
}
/** 强制允许下次采样(如用户刚说完话时) */
resetTimer(): void {
this.lastSampleTime = 0;
}
}

View File

@@ -0,0 +1,52 @@
// ============================================================
// Storage — localStorage 封装
// 职责:会话配置持久化
// ============================================================
import type { SessionConfig, Theme } from "../types";
const CONFIG_KEY = "camtalk:config";
const THEME_KEY = "camtalk:theme";
const DEFAULT_CONFIG: SessionConfig = {
ttsEnabled: true,
detailLevel: "low",
language: "zh-CN",
};
/** 加载配置,无存储时返回默认值 */
export function loadConfig(): SessionConfig {
try {
const raw = localStorage.getItem(CONFIG_KEY);
if (!raw) return DEFAULT_CONFIG;
const parsed = JSON.parse(raw) as Partial<SessionConfig>;
return { ...DEFAULT_CONFIG, ...parsed };
} catch {
return DEFAULT_CONFIG;
}
}
/** 保存配置到 localStorage */
export function saveConfig(config: SessionConfig): void {
try {
localStorage.setItem(CONFIG_KEY, JSON.stringify(config));
} catch {
// localStorage 不可用时静默失败
}
}
/** 加载主题偏好 */
export function loadTheme(): Theme {
try {
const raw = localStorage.getItem(THEME_KEY);
if (raw === "light" || raw === "dark") return raw;
} catch { /* ignore */ }
return "dark";
}
/** 保存主题偏好 */
export function saveTheme(theme: Theme): void {
try {
localStorage.setItem(THEME_KEY, theme);
} catch { /* ignore */ }
}

View File

@@ -0,0 +1,104 @@
// ============================================================
// TTS Player — 语音播放器
// 职责:收集后端流式 tts_audio 片段,拼接后播放
// 格式MVP 仅支持 audio/mp3pcm 为 TODO
// ============================================================
type OnEndCallback = () => void;
export class TTSPlayer {
private chunks: string[] = [];
private audio: HTMLAudioElement | null = null;
private _isPlaying = false;
private onEndCallback: OnEndCallback | null = null;
/** 注册播放完成回调 */
onEnd(cb: OnEndCallback): void {
this.onEndCallback = cb;
}
/** 当前是否正在播放 */
get isPlaying(): boolean {
return this._isPlaying;
}
/**
* 入队一个 TTS 音频片段
* @param base64 Base64 编码的音频数据
* @param mimeType 音频格式("audio/mp3" 或 "audio/pcm"
* @param isLast 是否为最后一个片段
*/
enqueue(base64: string, mimeType: string, isLast: boolean): void {
this.chunks.push(base64);
if (isLast) {
this.play(mimeType);
}
}
/** 停止播放并清空缓冲区 */
stop(): void {
if (this.audio) {
this.audio.pause();
this.audio.removeAttribute("src");
this.audio = null;
}
this.chunks = [];
this._isPlaying = false;
}
/** 暂停播放 */
pause(): void {
this.audio?.pause();
}
/** 恢复播放 */
resume(): void {
this.audio?.play();
}
/** 拼接所有片段并播放 */
private play(mimeType: string): void {
if (this.chunks.length === 0) return;
// 拼接所有 Base64 片段
const combined = this.chunks.join("");
this.chunks = [];
// Base64 → Uint8Array → Blob
const binary = atob(combined);
const bytes = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i++) {
bytes[i] = binary.charCodeAt(i);
}
const blob = new Blob([bytes], { type: mimeType });
const url = URL.createObjectURL(blob);
// 播放
const audio = new Audio(url);
this.audio = audio;
this._isPlaying = true;
audio.onended = () => {
URL.revokeObjectURL(url);
this._isPlaying = false;
this.audio = null;
this.onEndCallback?.();
};
audio.onerror = () => {
console.error("[TTS] 播放失败");
URL.revokeObjectURL(url);
this._isPlaying = false;
this.audio = null;
this.onEndCallback?.();
};
audio.play().catch((err) => {
console.error("[TTS] play() 被拒绝:", err);
this._isPlaying = false;
this.audio = null;
this.onEndCallback?.();
});
}
}

View File

@@ -11,6 +11,8 @@ export interface SessionConfig {
language: string;
}
export type Theme = "dark" | "light";
export interface Session {
sessionId: string;
createdAt: string;