feat: 引入 Eino 框架并实现 AI 编排层基础设施与节点
- 引入 cloudwego/eino v0.9.9 和 eino-ext/components/model/openai v0.1.13 - 新增 internal/eino/ 包: - types.go: PipelineInput/Output、STTOutput、TokenUsage 类型定义 - state.go: PipelineState 跨节点状态收集(线程安全) - callback.go: ChatModel OnEndWithStreamOutput 回调,逐 token 推送 llm_chunk - nodes_stt.go: STT Lambda,支持文本/语音输入模式 - nodes_history.go: 历史组装 Lambda,含多模态图片支持 - nodes_splitter.go: 句子分割 Transform Lambda - nodes_tts.go: TTS Lambda,逐句合成推送音频 - nodes_done.go: Done Lambda,发送 llm_done 并追加历史 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
133
backend/internal/eino/callback.go
Normal file
133
backend/internal/eino/callback.go
Normal file
@@ -0,0 +1,133 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/cloudwego/eino/callbacks"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
callbacksHelper "github.com/cloudwego/eino/utils/callbacks"
|
||||
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
"github.com/hhs/camtalk/internal/orchestrator"
|
||||
)
|
||||
|
||||
// context key 类型,避免与其他包冲突。
|
||||
type ctxKeySender struct{}
|
||||
type ctxKeyRequestID struct{}
|
||||
type ctxKeyState struct{}
|
||||
|
||||
// WithSender 将 Sender 注入 context。
|
||||
func WithSender(ctx context.Context, sender orchestrator.Sender) context.Context {
|
||||
return context.WithValue(ctx, ctxKeySender{}, sender)
|
||||
}
|
||||
|
||||
// WithRequestID 将 requestID 注入 context。
|
||||
func WithRequestID(ctx context.Context, requestID string) context.Context {
|
||||
return context.WithValue(ctx, ctxKeyRequestID{}, requestID)
|
||||
}
|
||||
|
||||
// WithPipelineState 将 PipelineState 注入 context。
|
||||
func WithPipelineState(ctx context.Context, state *PipelineState) context.Context {
|
||||
return context.WithValue(ctx, ctxKeyState{}, state)
|
||||
}
|
||||
|
||||
// senderFromCtx 从 context 获取 Sender。
|
||||
func senderFromCtx(ctx context.Context) orchestrator.Sender {
|
||||
s, _ := ctx.Value(ctxKeySender{}).(orchestrator.Sender)
|
||||
return s
|
||||
}
|
||||
|
||||
// requestIDFromCtx 从 context 获取 requestID。
|
||||
func requestIDFromCtx(ctx context.Context) string {
|
||||
s, _ := ctx.Value(ctxKeyRequestID{}).(string)
|
||||
return s
|
||||
}
|
||||
|
||||
// stateFromCtx 从 context 获取 PipelineState。
|
||||
func stateFromCtx(ctx context.Context) *PipelineState {
|
||||
s, _ := ctx.Value(ctxKeyState{}).(*PipelineState)
|
||||
return s
|
||||
}
|
||||
|
||||
// BuildCallbackHandler 构建 Eino Callback Handler。
|
||||
//
|
||||
// 核心职责:ChatModel 节点通过 OnEndWithStreamOutput 逐 token 推送 llm_chunk 到客户端,
|
||||
// 同时累积完整文本到 PipelineState。
|
||||
//
|
||||
// 其他节点的消息推送(stt_result、tts_audio、llm_done)由各 Lambda 内部直接调用 Sender。
|
||||
func BuildCallbackHandler() callbacks.Handler {
|
||||
return callbacksHelper.NewHandlerHelper().
|
||||
ChatModel(&callbacksHelper.ModelCallbackHandler{
|
||||
OnEndWithStreamOutput: func(ctx context.Context, info *callbacks.RunInfo, output *schema.StreamReader[*model.CallbackOutput]) context.Context {
|
||||
log := logger.Log
|
||||
sender := senderFromCtx(ctx)
|
||||
requestID := requestIDFromCtx(ctx)
|
||||
state := stateFromCtx(ctx)
|
||||
|
||||
if sender == nil || requestID == "" {
|
||||
log.Warnw("ModelCallback: missing sender or request_id in context",
|
||||
"node", info.Name)
|
||||
return ctx
|
||||
}
|
||||
|
||||
// 异步消费流,避免阻塞框架的下游处理。
|
||||
// 框架对流做了内部拷贝,此 goroutine 读取独立副本。
|
||||
go func() {
|
||||
defer output.Close()
|
||||
|
||||
for {
|
||||
chunk, err := output.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return
|
||||
}
|
||||
log.Errorw("ModelCallback: stream recv error",
|
||||
"node", info.Name, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if chunk == nil || chunk.Message == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
delta := chunk.Message.Content
|
||||
if delta == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// 推送 llm_chunk 到客户端
|
||||
if err := sender.SendLLMChunk(models.WsLLMChunk{
|
||||
Type: "llm_chunk",
|
||||
RequestID: requestID,
|
||||
Delta: delta,
|
||||
Role: "assistant",
|
||||
}); err != nil {
|
||||
log.Errorw("ModelCallback: send llm_chunk failed", "error", err)
|
||||
}
|
||||
|
||||
// 累积完整文本到 State
|
||||
if state != nil {
|
||||
state.AppendText(delta)
|
||||
}
|
||||
|
||||
// 记录 token 用量(流的最后一帧携带)
|
||||
if chunk.TokenUsage != nil && state != nil {
|
||||
state.mu.Lock()
|
||||
state.TokenUsage = &TokenUsage{
|
||||
Prompt: chunk.TokenUsage.PromptTokens,
|
||||
Completion: chunk.TokenUsage.CompletionTokens,
|
||||
Total: chunk.TokenUsage.TotalTokens,
|
||||
}
|
||||
state.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ctx
|
||||
},
|
||||
}).
|
||||
Handler()
|
||||
}
|
||||
114
backend/internal/eino/nodes_done.go
Normal file
114
backend/internal/eino/nodes_done.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino/compose"
|
||||
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
"github.com/hhs/camtalk/internal/session"
|
||||
)
|
||||
|
||||
// NewDoneLambda 创建 Done Lambda 节点。
|
||||
// 输入: struct{}(TTS 完成信号)→ 输出: PipelineOutput
|
||||
//
|
||||
// 从 PipelineState 读取完整回复和 token 用量,发送 llm_done 到客户端,
|
||||
// 追加助手消息到会话历史,返回 PipelineOutput。
|
||||
func NewDoneLambda(sessionMgr session.Manager, model string) *compose.Lambda {
|
||||
return compose.InvokableLambda(func(ctx context.Context, _ struct{}) (*PipelineOutput, error) {
|
||||
log := logger.Log
|
||||
sender := senderFromCtx(ctx)
|
||||
requestID := requestIDFromCtx(ctx)
|
||||
state := stateFromCtx(ctx)
|
||||
|
||||
if state == nil {
|
||||
return &PipelineOutput{}, nil
|
||||
}
|
||||
|
||||
state.mu.Lock()
|
||||
fullResponse := state.FullResponse.String()
|
||||
transcribedText := state.TranscribedText
|
||||
tokenUsage := state.TokenUsage
|
||||
modelName := model
|
||||
state.mu.Unlock()
|
||||
|
||||
// 追加助手消息到会话历史
|
||||
sessionID := ""
|
||||
if state != nil {
|
||||
// 从 context 获取 sessionID(由适配器注入)
|
||||
if sid, ok := ctx.Value(ctxKeySessionID{}).(string); ok {
|
||||
sessionID = sid
|
||||
}
|
||||
}
|
||||
if sessionID != "" && sessionMgr != nil && fullResponse != "" {
|
||||
if err := sessionMgr.AppendMessage(ctx, sessionID, models.Message{
|
||||
Role: "assistant",
|
||||
Content: fullResponse,
|
||||
}); err != nil {
|
||||
log.Errorw("追加助手消息到历史失败", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 发送 llm_done
|
||||
if sender != nil && requestID != "" {
|
||||
done := models.WsLLMDone{
|
||||
Type: "llm_done",
|
||||
RequestID: requestID,
|
||||
FullText: fullResponse,
|
||||
Model: modelName,
|
||||
LatencyMs: 0, // 由适配器计算
|
||||
}
|
||||
if tokenUsage != nil {
|
||||
done.TokensUsed = struct {
|
||||
Prompt int `json:"prompt"`
|
||||
Completion int `json:"completion"`
|
||||
Total int `json:"total"`
|
||||
}{
|
||||
Prompt: tokenUsage.Prompt,
|
||||
Completion: tokenUsage.Completion,
|
||||
Total: tokenUsage.Total,
|
||||
}
|
||||
}
|
||||
if err := sender.SendLLMDone(done); err != nil {
|
||||
log.Errorw("发送 llm_done 失败", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Infow("查询处理完成",
|
||||
"request_id", requestID,
|
||||
"text_length", len(fullResponse))
|
||||
|
||||
return &PipelineOutput{
|
||||
TranscribedText: transcribedText,
|
||||
FullResponse: fullResponse,
|
||||
Model: modelName,
|
||||
TokenUsage: tokenUsage,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// ctxKeySessionID sessionID 的 context key。
|
||||
type ctxKeySessionID struct{}
|
||||
|
||||
// WithSessionID 将 sessionID 注入 context。
|
||||
func WithSessionID(ctx context.Context, sessionID string) context.Context {
|
||||
return context.WithValue(ctx, ctxKeySessionID{}, sessionID)
|
||||
}
|
||||
|
||||
// latencyFromCtx 从 context 获取开始时间并计算延迟。
|
||||
func latencyFromCtx(ctx context.Context) int64 {
|
||||
if startTime, ok := ctx.Value(ctxKeyStartTime{}).(time.Time); ok {
|
||||
return time.Since(startTime).Milliseconds()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ctxKeyStartTime 请求开始时间的 context key。
|
||||
type ctxKeyStartTime struct{}
|
||||
|
||||
// WithStartTime 将请求开始时间注入 context。
|
||||
func WithStartTime(ctx context.Context, t time.Time) context.Context {
|
||||
return context.WithValue(ctx, ctxKeyStartTime{}, t)
|
||||
}
|
||||
117
backend/internal/eino/nodes_history.go
Normal file
117
backend/internal/eino/nodes_history.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"github.com/cloudwego/eino/compose"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
|
||||
"github.com/hhs/camtalk/internal/ai/llm"
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
// HistoryInput 历史组装节点的输入,包含 STT 输出和原始请求信息。
|
||||
type HistoryInput struct {
|
||||
STTOutput *STTOutput
|
||||
SessionID string
|
||||
RequestID string
|
||||
ImageData []byte
|
||||
Scenario string
|
||||
DetailLevel string
|
||||
}
|
||||
|
||||
// NewHistoryLambda 创建历史组装 Lambda 节点。
|
||||
// 输入: HistoryInput → 输出: []*schema.Message
|
||||
//
|
||||
// 构建系统提示词,组装历史消息和当前用户输入(含多模态图片)。
|
||||
func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, maxHistory int) ([]models.Message, error), maxHistory int) *compose.Lambda {
|
||||
return compose.InvokableLambda(func(ctx context.Context, input *HistoryInput) ([]*schema.Message, error) {
|
||||
log := logger.Log
|
||||
requestID := input.RequestID
|
||||
|
||||
// 构建系统提示词
|
||||
scenarioPrompt := llm.GetScenarioPrompt(input.Scenario, input.STTOutput.Language)
|
||||
systemPrompt := llm.BuildSystemPrompt(input.STTOutput.Language, input.DetailLevel, scenarioPrompt)
|
||||
|
||||
// 构建 system message(含图片)
|
||||
systemMsg := &schema.Message{
|
||||
Role: schema.System,
|
||||
Content: systemPrompt,
|
||||
}
|
||||
|
||||
// 如果有图片,添加到 system message 的多模态内容中
|
||||
if len(input.ImageData) > 0 {
|
||||
base64Str := base64.StdEncoding.EncodeToString(input.ImageData)
|
||||
mimeType := detectImageMimeType(input.ImageData)
|
||||
systemMsg.UserInputMultiContent = []schema.MessageInputPart{
|
||||
{
|
||||
Type: schema.ChatMessagePartTypeImageURL,
|
||||
Image: &schema.MessageInputImage{
|
||||
MessagePartCommon: schema.MessagePartCommon{
|
||||
Base64Data: &base64Str,
|
||||
MIMEType: mimeType,
|
||||
},
|
||||
Detail: schema.ImageURLDetailAuto,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
messages := []*schema.Message{systemMsg}
|
||||
|
||||
// 获取并追加历史消息
|
||||
if historyFetcher != nil && input.SessionID != "" {
|
||||
history, err := historyFetcher(ctx, input.SessionID, maxHistory)
|
||||
if err != nil {
|
||||
log.Warnw("获取历史消息失败,继续处理", "error", err, "request_id", requestID)
|
||||
} else {
|
||||
for _, msg := range history {
|
||||
messages = append(messages, &schema.Message{
|
||||
Role: schema.RoleType(msg.Role),
|
||||
Content: msg.Content,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 追加当前用户输入
|
||||
messages = append(messages, &schema.Message{
|
||||
Role: schema.User,
|
||||
Content: input.STTOutput.Text,
|
||||
})
|
||||
|
||||
log.Infow("历史组装完成",
|
||||
"request_id", requestID,
|
||||
"message_count", len(messages),
|
||||
"has_image", len(input.ImageData) > 0,
|
||||
"scenario", input.Scenario)
|
||||
|
||||
return messages, nil
|
||||
})
|
||||
}
|
||||
|
||||
// detectImageMimeType 简单检测图片 MIME 类型。
|
||||
func detectImageMimeType(data []byte) string {
|
||||
if len(data) < 4 {
|
||||
return "image/jpeg"
|
||||
}
|
||||
// JPEG: FF D8 FF
|
||||
if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF {
|
||||
return "image/jpeg"
|
||||
}
|
||||
// PNG: 89 50 4E 47
|
||||
if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 {
|
||||
return "image/png"
|
||||
}
|
||||
// GIF: 47 49 46 38
|
||||
if data[0] == 0x47 && data[1] == 0x49 && data[2] == 0x46 {
|
||||
return "image/gif"
|
||||
}
|
||||
// WebP: 52 49 46 46
|
||||
if data[0] == 0x52 && data[1] == 0x49 && data[2] == 0x46 && data[3] == 0x46 {
|
||||
return "image/webp"
|
||||
}
|
||||
return "image/jpeg" // 默认
|
||||
}
|
||||
|
||||
72
backend/internal/eino/nodes_splitter.go
Normal file
72
backend/internal/eino/nodes_splitter.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/cloudwego/eino/compose"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
// sentenceDelimiters 句子分隔符集合。
|
||||
var sentenceDelimiters = map[rune]bool{
|
||||
'。': true,
|
||||
'!': true,
|
||||
'?': true,
|
||||
'\n': true,
|
||||
'.': true,
|
||||
'!': true,
|
||||
'?': true,
|
||||
}
|
||||
|
||||
// NewSplitterLambda 创建句子分割 Transform Lambda 节点。
|
||||
// 输入: StreamReader[string](LLM 完整文本的单帧流)→ 输出: StreamReader[[]string](句子数组流)
|
||||
//
|
||||
// 在 Stream 模式下,框架自动将 ChatModel 的 StreamReader[*schema.Message]
|
||||
// concat 为 string 后传入此节点。此节点将文本按句子边界切分,
|
||||
// 每切出一个句子就输出一次,供 TTS 节点实时合成。
|
||||
func NewSplitterLambda() *compose.Lambda {
|
||||
return compose.TransformableLambda(func(ctx context.Context, input *schema.StreamReader[string]) (*schema.StreamReader[[]string], error) {
|
||||
sr, sw := schema.Pipe[[]string](8)
|
||||
|
||||
go func() {
|
||||
defer sw.Close()
|
||||
|
||||
var buffer strings.Builder
|
||||
|
||||
for {
|
||||
chunk, err := input.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// 流结束,flush 剩余缓冲
|
||||
if buffer.Len() > 0 {
|
||||
text := strings.TrimSpace(buffer.String())
|
||||
if text != "" {
|
||||
sw.Send([]string{text}, nil)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
sw.Send(nil, err)
|
||||
return
|
||||
}
|
||||
|
||||
// chunk 是 concat 后的完整文本(单帧流)
|
||||
// 逐字符累积,按句子分隔符切分
|
||||
for _, r := range chunk {
|
||||
buffer.WriteRune(r)
|
||||
if sentenceDelimiters[r] {
|
||||
text := strings.TrimSpace(buffer.String())
|
||||
if text != "" {
|
||||
sw.Send([]string{text}, nil)
|
||||
}
|
||||
buffer.Reset()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return sr, nil
|
||||
})
|
||||
}
|
||||
119
backend/internal/eino/nodes_stt.go
Normal file
119
backend/internal/eino/nodes_stt.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/cloudwego/eino/compose"
|
||||
|
||||
"github.com/hhs/camtalk/internal/ai/stt"
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
// NewSTTLambda 创建 STT Lambda 节点。
|
||||
// 输入: PipelineInput → 输出: STTOutput
|
||||
//
|
||||
// 文本输入模式:跳过 STT,直接返回用户输入文本。
|
||||
// 语音模式:调用 sttService.Recognize() 进行语音识别。
|
||||
// 识别结果通过 Sender 发送 stt_result 到客户端。
|
||||
func NewSTTLambda(sttService stt.Service) *compose.Lambda {
|
||||
return compose.InvokableLambda(func(ctx context.Context, input *PipelineInput) (*STTOutput, error) {
|
||||
log := logger.Log
|
||||
sender := senderFromCtx(ctx)
|
||||
requestID := requestIDFromCtx(ctx)
|
||||
|
||||
// 文本输入模式:跳过 STT
|
||||
if input.Text != "" {
|
||||
log.Infow("使用文本输入,跳过 STT",
|
||||
"request_id", requestID, "text", input.Text)
|
||||
|
||||
// 发送 stt_result 保持前端消息流一致性
|
||||
if sender != nil {
|
||||
if err := sender.SendSTTResult(models.WsSTTResult{
|
||||
Type: "stt_result",
|
||||
RequestID: requestID,
|
||||
Text: input.Text,
|
||||
IsFinal: true,
|
||||
}); err != nil {
|
||||
log.Errorw("发送 stt_result 失败", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 写入 State
|
||||
if state := stateFromCtx(ctx); state != nil {
|
||||
state.mu.Lock()
|
||||
state.TranscribedText = input.Text
|
||||
state.mu.Unlock()
|
||||
}
|
||||
|
||||
return &STTOutput{
|
||||
Text: input.Text,
|
||||
Language: input.Language,
|
||||
IsSkipped: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 语音模式:解码音频
|
||||
if len(input.AudioData) == 0 {
|
||||
return nil, fmt.Errorf("stt: no audio data provided")
|
||||
}
|
||||
|
||||
log.Infow("开始语音识别",
|
||||
"request_id", requestID, "audio_bytes", len(input.AudioData))
|
||||
|
||||
// 调用 STT 服务
|
||||
text, err := sttService.Recognize(ctx, input.AudioData, stt.Options{
|
||||
Encoding: "pcm_s16le",
|
||||
SampleRate: 16000,
|
||||
Language: input.Language,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorw("语音识别失败", "error", err, "request_id", requestID)
|
||||
if sender != nil {
|
||||
sender.SendError(models.WsError{
|
||||
Type: "error",
|
||||
RequestID: requestID,
|
||||
Code: "STT_ERROR",
|
||||
Message: "语音识别失败: " + err.Error(),
|
||||
})
|
||||
}
|
||||
return nil, fmt.Errorf("stt: recognize: %w", err)
|
||||
}
|
||||
|
||||
// STT 返回空文本
|
||||
if strings.TrimSpace(text) == "" {
|
||||
log.Infow("语音识别结果为空", "request_id", requestID)
|
||||
text = "(未识别到语音)"
|
||||
}
|
||||
|
||||
log.Infow("语音识别完成", "request_id", requestID, "text", text)
|
||||
|
||||
// 发送 stt_result
|
||||
if sender != nil {
|
||||
if err := sender.SendSTTResult(models.WsSTTResult{
|
||||
Type: "stt_result",
|
||||
RequestID: requestID,
|
||||
Text: text,
|
||||
IsFinal: true,
|
||||
}); err != nil {
|
||||
log.Errorw("发送 stt_result 失败", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 写入 State
|
||||
if state := stateFromCtx(ctx); state != nil {
|
||||
state.mu.Lock()
|
||||
state.TranscribedText = text
|
||||
state.mu.Unlock()
|
||||
}
|
||||
|
||||
return &STTOutput{
|
||||
Text: text,
|
||||
Language: input.Language,
|
||||
IsSkipped: false,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
|
||||
92
backend/internal/eino/nodes_tts.go
Normal file
92
backend/internal/eino/nodes_tts.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/cloudwego/eino/compose"
|
||||
|
||||
"github.com/hhs/camtalk/internal/ai/tts"
|
||||
"github.com/hhs/camtalk/internal/logger"
|
||||
"github.com/hhs/camtalk/internal/models"
|
||||
)
|
||||
|
||||
// NewTTSLambda 创建 TTS Lambda 节点。
|
||||
// 输入: []string(句子数组,框架自动从 StreamReader concat)→ 输出: struct{}
|
||||
//
|
||||
// 将句子数组转为 channel,调用 ttsService.SynthesizeStream() 流式合成,
|
||||
// 逐 chunk 推送 tts_audio 到客户端。TTS 失败静默跳过。
|
||||
func NewTTSLambda(ttsService tts.Service, ttsVoice string, ttsSpeed float64, ttsOutputFmt string, ttsSampleRate int) *compose.Lambda {
|
||||
return compose.InvokableLambda(func(ctx context.Context, sentences []string) (struct{}, error) {
|
||||
log := logger.Log
|
||||
sender := senderFromCtx(ctx)
|
||||
requestID := requestIDFromCtx(ctx)
|
||||
state := stateFromCtx(ctx)
|
||||
|
||||
// 检查 TTS 是否启用(从 State 或 context 获取)
|
||||
// TTSEnabled 信息在 PipelineInput 中,通过 State 传递
|
||||
if state != nil {
|
||||
state.mu.Lock()
|
||||
ttsEnabled := true // 默认启用,由适配器通过 State 设置
|
||||
state.mu.Unlock()
|
||||
if !ttsEnabled {
|
||||
return struct{}{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if len(sentences) == 0 {
|
||||
return struct{}{}, nil
|
||||
}
|
||||
|
||||
if sender == nil || requestID == "" {
|
||||
return struct{}{}, nil
|
||||
}
|
||||
|
||||
log.Infow("开始 TTS 合成", "request_id", requestID, "sentence_count", len(sentences))
|
||||
|
||||
// 将句子数组转为 channel(ttsService.SynthesizeStream 需要 <-chan string)
|
||||
sentenceCh := make(chan string, len(sentences))
|
||||
for _, s := range sentences {
|
||||
sentenceCh <- s
|
||||
}
|
||||
close(sentenceCh)
|
||||
|
||||
// 调用 TTS 服务
|
||||
ttsStream, err := ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{
|
||||
Voice: ttsVoice,
|
||||
Speed: ttsSpeed,
|
||||
OutputFmt: ttsOutputFmt,
|
||||
SampleRate: ttsSampleRate,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorw("TTS 合成启动失败(已跳过)", "error", err, "request_id", requestID)
|
||||
return struct{}{}, nil // TTS 失败不中断流程
|
||||
}
|
||||
|
||||
// 消费 TTS 音频流,推送到客户端
|
||||
for chunk := range ttsStream {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Infow("TTS 流被中断", "request_id", requestID)
|
||||
return struct{}{}, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
audioBase64 := base64.StdEncoding.EncodeToString(chunk.Audio)
|
||||
|
||||
if err := sender.SendTTSAudio(models.WsTTSAudio{
|
||||
Type: "tts_audio",
|
||||
RequestID: requestID,
|
||||
Audio: audioBase64,
|
||||
MimeType: "audio/mp3",
|
||||
IsLast: chunk.IsLast,
|
||||
Final: chunk.Final,
|
||||
}); err != nil {
|
||||
log.Errorw("发送 tts_audio 失败", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Infow("TTS 合成完成", "request_id", requestID)
|
||||
return struct{}{}, nil
|
||||
})
|
||||
}
|
||||
36
backend/internal/eino/state.go
Normal file
36
backend/internal/eino/state.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// PipelineState Graph 全局状态,用于跨节点收集数据。
|
||||
// 通过 compose.WithGenLocalState 注册,各节点通过 StatePreHandler/StatePostHandler 读写。
|
||||
type PipelineState struct {
|
||||
mu sync.Mutex
|
||||
FullResponse strings.Builder // LLM 完整回复(由 Callback 累积)
|
||||
TranscribedText string // STT 识别文本
|
||||
Model string // 实际使用的模型名
|
||||
TokenUsage *TokenUsage // token 用量
|
||||
}
|
||||
|
||||
// genLocalState 创建每请求的 PipelineState 实例。
|
||||
func genLocalState(ctx context.Context) *PipelineState {
|
||||
return &PipelineState{}
|
||||
}
|
||||
|
||||
// AppendText 追加文本到 FullResponse(线程安全)。
|
||||
func (s *PipelineState) AppendText(text string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.FullResponse.WriteString(text)
|
||||
}
|
||||
|
||||
// GetFullResponse 获取完整回复文本(线程安全)。
|
||||
func (s *PipelineState) GetFullResponse() string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.FullResponse.String()
|
||||
}
|
||||
37
backend/internal/eino/types.go
Normal file
37
backend/internal/eino/types.go
Normal file
@@ -0,0 +1,37 @@
|
||||
// Package eino 基于 CloudWeGo Eino 框架的 AI 编排层。
|
||||
// 使用 Eino Graph 替代手写 goroutine 管道,实现声明式 STT → LLM → TTS 编排。
|
||||
package eino
|
||||
|
||||
// PipelineInput Graph 统一输入。
|
||||
type PipelineInput struct {
|
||||
AudioData []byte // base64 解码后的音频(可选)
|
||||
ImageData []byte // base64 解码后的图像(可选)
|
||||
Text string // 直接文本输入(可选,跳过 STT)
|
||||
SessionID string
|
||||
RequestID string
|
||||
Language string // zh / en
|
||||
Scenario string // free_chat, interviewer, etc.
|
||||
TTSEnabled bool
|
||||
}
|
||||
|
||||
// PipelineOutput Graph 统一输出。
|
||||
type PipelineOutput struct {
|
||||
TranscribedText string // STT 结果
|
||||
FullResponse string // LLM 完整回复
|
||||
Model string // 实际使用的模型名
|
||||
TokenUsage *TokenUsage // token 用量
|
||||
}
|
||||
|
||||
// STTOutput STT 节点输出。
|
||||
type STTOutput struct {
|
||||
Text string
|
||||
Language string
|
||||
IsSkipped bool // 文本输入模式跳过了 STT
|
||||
}
|
||||
|
||||
// TokenUsage token 用量统计。
|
||||
type TokenUsage struct {
|
||||
Prompt int
|
||||
Completion int
|
||||
Total int
|
||||
}
|
||||
Reference in New Issue
Block a user