From fd5c7712f84c9d40b90d344376f6adbb1626e180 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Fri, 19 Jun 2026 21:49:28 +0800 Subject: [PATCH 1/7] =?UTF-8?q?feat:=20=E5=BC=95=E5=85=A5=20Eino=20?= =?UTF-8?q?=E6=A1=86=E6=9E=B6=E5=B9=B6=E5=AE=9E=E7=8E=B0=20AI=20=E7=BC=96?= =?UTF-8?q?=E6=8E=92=E5=B1=82=E5=9F=BA=E7=A1=80=E8=AE=BE=E6=96=BD=E4=B8=8E?= =?UTF-8?q?=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 引入 cloudwego/eino v0.9.9 和 eino-ext/components/model/openai v0.1.13 - 新增 internal/eino/ 包： - types.go: PipelineInput/Output、STTOutput、TokenUsage 类型定义 - state.go: PipelineState 跨节点状态收集（线程安全） - callback.go: ChatModel OnEndWithStreamOutput 回调，逐 token 推送 llm_chunk - nodes_stt.go: STT Lambda，支持文本/语音输入模式 - nodes_history.go: 历史组装 Lambda，含多模态图片支持 - nodes_splitter.go: 句子分割 Transform Lambda - nodes_tts.go: TTS Lambda，逐句合成推送音频 - nodes_done.go: Done Lambda，发送 llm_done 并追加历史 Co-Authored-By: Claude --- CamTalk-演讲稿.md | 116 +++++++++++++++++++++ backend/go.mod | 30 ++++-- backend/go.sum | 104 ++++++++++++++---- backend/internal/eino/callback.go | 133 ++++++++++++++++++++++++ backend/internal/eino/nodes_done.go | 114 ++++++++++++++++++++ backend/internal/eino/nodes_history.go | 117 +++++++++++++++++++++ backend/internal/eino/nodes_splitter.go | 72 +++++++++++++ backend/internal/eino/nodes_stt.go | 119 +++++++++++++++++++++ backend/internal/eino/nodes_tts.go | 92 ++++++++++++++++ backend/internal/eino/state.go | 36 +++++++ backend/internal/eino/types.go | 37 +++++++ 11 files changed, 946 insertions(+), 24 deletions(-) create mode 100644 CamTalk-演讲稿.md create mode 100644 backend/internal/eino/callback.go create mode 100644 backend/internal/eino/nodes_done.go create mode 100644 backend/internal/eino/nodes_history.go create mode 100644 backend/internal/eino/nodes_splitter.go create mode 100644 backend/internal/eino/nodes_stt.go create mode 100644 backend/internal/eino/nodes_tts.go create mode 100644 backend/internal/eino/state.go create mode 100644 backend/internal/eino/types.go diff --git a/CamTalk-演讲稿.md b/CamTalk-演讲稿.md new file mode 100644 index 0000000..1779f00 --- /dev/null +++ b/CamTalk-演讲稿.md @@ -0,0 +1,116 @@ +## CamTalk — 多模态实时 AI 视觉对话助手 · 演讲稿 + +> 面向面试官，预计 10-15 分钟。建议配合架构图或项目文档做演示。 + +--- + +### 开场（约 1 分钟） + +各位好，今天我想和大家分享一个我主导设计和开发的项目——**CamTalk**，一个多模态实时 AI 视觉对话助手。 + +简单来说，用户打开浏览器，对着摄像头，用语音提问，AI 就能同时"看到"画面、"听到"语音，然后用文字和语音自然地回应。整个过程不需要打字，就像一个面对面的助手。 + +做这个项目的初衷其实很直接——现在的大模型已经具备多模态能力，但大多数产品还是"上传一张图、输入一段文字"的交互方式。我认为真正的多模态交互应该是**无感的**——用户只需要说话，AI 自己去理解视觉场景，就像两个人面对面聊天一样。 + +--- + +### 系统架构（约 3 分钟） + +CamTalk 采用三层架构：**前端做轻量预处理，后端做智能编排，云端 AI 服务按需调用**。 + +**前端**是 React 18 加 TypeScript，跑在浏览器里。它负责三件事：摄像头和麦克风的采集，边缘侧的预处理——比如语音活动检测和关键帧过滤，以及 UI 渲染。通过 WebSocket 与后端通信。 + +**后端**是 Go 写的网关服务，用 Gin 框架做 HTTP 路由，gorilla/websocket 处理长连接。它是整个系统的"大脑"，负责会话管理、AI 编排，以及和各家 AI 服务的对接。每个 WebSocket 连接对应一个 goroutine，天然适合这种长连接场景。 + +**AI 服务层**是可插拔的。LLM 默认用 GPT-4o，通过 OpenAI 兼容接口可以随时切换成通义千问等国产模型。语音识别默认 Deepgram，语音合成默认 OpenAI TTS，同时也支持小米的 MiMo 系列作为备选。 + +有人可能会问：为什么不直接让前端调用 AI API？这里有三点考虑。第一是**安全性**，API Key 不应该暴露在客户端。第二是**统一管控**，速率限制、成本监控、模型路由这些逻辑集中在网关层更好维护。第三是**可扩展性**，未来加缓存、做负载均衡、多实例部署，都在网关层解决。 + +部署方面，我们设计了 Nginx 做同源反向代理，前端静态资源和后端 API 在同一个域名下，天然解决跨域问题。Go 网关可以水平扩展，通过 Redis 共享会话状态。目前也已经配置好了 Docker Compose 一键部署方案，包含前端、后端和 PostgreSQL 三个容器。 + +--- + +### 核心交互流程（约 3 分钟） + +我想重点讲一下一次完整的交互流程，因为它串起了整个系统最核心的技术挑战。 + +用户对着摄像头说了一句话，比如"这是什么花"。首先是**前端**的 VAD——语音活动检测模块——在浏览器端实时检测用户何时开始说话、何时说完。这一步完全在端侧完成，用的是 @ricky0123/vad-web，基于 WebRTC VAD 算法。好处是：用户不说话时不需要上传任何音频，节省约 70% 的无效带宽。 + +VAD 检测到语音结束后，前端会同时做两件事：把音频编码成 PCM 格式，以及从摄像头捕获当前画面，一起通过 WebSocket 发给后端。 + +后端收到后，启动一个 **AI 编排管道**（我们叫它 Orchestrator）。第一步，把音频发给 STT 服务做语音识别，拿到文字结果。第二步，把识别出的文字、摄像头画面以及对话历史，一起打包发给多模态 LLM 做推理。LLM 以流式方式逐 token 输出。第三步，也是最关键的优化——我们不等待 LLM 输出完再调用 TTS，而是做**句子级切分**：LLM 每输出一个完整句子，就立即送入 TTS 合成并推送给客户端。 + +所以客户端的体验是这样的：文字一个 token 一个 token 地出现，几乎同时语音就开始播放了。用户**先看到文字、紧接着听到语音**，感知延迟可以控制在 0.5 秒以内。整个端到端的目标延迟是 1.5 到 2 秒。 + +这个"LLM 文本流和 TTS 音频流并行推送"的设计，是我们降低感知延迟最关键的手段。 + +--- + +### 成本控制（约 2 分钟） + +做实时多模态应用，成本是最容易失控的地方。我在设计之初就把成本控制作为架构级别的考量。 + +最直观的例子是视觉链路：如果按 1fps 全量发送画面给 LLM，一个用户每天用 10 分钟，一天就是 60 万帧的 token 消耗，1000 个用户时成本完全不可控。 + +我们的核心策略叫**端云协同**——把适合的计算前置到客户端。 + +在视觉侧，我们做了三个优化：一是降低采样频率，空闲时 5 秒一帧，用户说话时 1 秒一帧；二是关键帧过滤，通过 Canvas 像素比较计算帧间相似度，画面没有显著变化就不发送；三是只在用户提问时捕获画面，而不是持续上传视频流。 + +在语音侧，VAD 在浏览器端检测，只上传有效语音片段，环境噪音和静默时段完全不消耗带宽。 + +在推理侧，我们规划了模型分级策略——简单识别类问题走 GPT-4o-mini，深度分析走 GPT-4o，复杂推理走 o1。同时对话历史做了裁剪，前端保留最近 10 轮，后端保留 20 轮，限制每轮的固定 token 开销。 + +这些策略综合下来，预估月成本可以从无优化的约 5000 美元降到 300 到 500 美元，降幅大约 90%。 + +--- + +### 工程设计与取舍（约 2 分钟） + +除了技术实现，我想分享几个设计上的取舍。 + +**存储方案的分阶段设计**。MVP 阶段我们用进程内存存会话状态，快速验证核心功能。但代码层面我们已经通过 Repository 接口模式做了抽象——HistoryRepository、UsageRepository 这些接口定义好了，底层实现可以是 Memory、Redis 或 PostgreSQL，通过配置切换。目前 Redis 实现已经就绪，PostgreSQL 的 schema 也设计好了，包括 sessions、messages、usage_daily 三张表。这种渐进式设计让我们既能快速交付，又为后续扩展留好了空间。 + +**文档驱动开发**。项目里有一套完整的设计文档，涵盖架构、接口协议、技术选型、成本控制等。我们遵循"文档优先"原则——实现功能前先写设计文档，实现和文档不一致时优先更新文档。这在团队协作中特别重要，接口契约清晰，前后端可以并行开发。 + +**WebSocket 协议的可靠性设计**。客户端每 30 秒发心跳，服务端 60 秒没收到心跳就断开。断线后用指数退避加抖动重连——1 秒、2 秒、4 秒、8 秒，最大 30 秒。消息用统一信封格式，所有消息都带 type 字段做类型分发。 + +--- + +### 用户故事与产品规划（约 2 分钟） + +最后讲一下产品层面的思考。用户故事我按 P0 到 P2 分了三个优先级。 + +P0 是 MVP 必做的四个场景：AI 识别画面中的物体、语音对话无需打字、AI 能看到摄像头画面、AI 用语音回答。这四个跑通了，核心价值就成立了。 + +P1 是体验增强：AI 主动观察画面变化并提示重要事件、识别画面中的文字做 OCR、以及多轮对话的上下文记忆。 + +P2 是进阶探索：比如视障用户的无障碍辅助——AI 实时描述周围环境并提示障碍物，画面中外语内容的实时翻译，以及"观察模式"和"对话模式"的切换。 + +优先级判断用两个维度交叉评估：用户价值和实现成本。P0 是高价值且成本合理的，P1 是高价值但成本较高的，P2 是探索性的，验证后再投入。 + +目前还有几个功能创意在规划中，包括视频录制、对话翻译、对话总结、手动对话输入，以及对话情景选择——比如面试官模式、英语老师模式、辩论赛模式等。 + +--- + +### 总结（约 1 分钟） + +总结一下，CamTalk 这个项目有几个我比较满意的设计点。 + +第一是**架构清晰**：三层分离，每层职责明确，前端做轻量预处理，后端做智能编排，AI 服务可插拔。 + +第二是**体验导向**：从用户感知延迟倒推技术方案，流式并行推送、句子级切分、端侧 VAD 这些手段都是围绕"让对话像真人一样自然"这个目标设计的。 + +第三是**成本意识**：从架构层面就融入了成本控制，端云协同、智能采样、模型分级，不是等功能做完再去优化成本。 + +第四是**工程成熟度**：接口抽象、文档驱动、渐进式存储升级，为项目的长期演进留好了空间。 + +以上就是 CamTalk 项目的整体介绍。谢谢大家，有什么问题我们可以一起讨论。 + +--- + +> **附：讲解提示** +> +> - 如果面试官追问技术深度，可以展开讲 Orchestrator 的管道实现细节（goroutine 并发、context 取消、句子切分算法）或 VAD 参数调优。 +> - 如果追问产品思维，可以展开讲用户故事的优先级判断逻辑，以及观察模式和对话模式的差异设计。 +> - 如果追问可扩展性，可以讲 Redis 共享会话、多 Gateway 水平扩展、模型路由器的规划。 +> - 如果追问成本数据，可以给出具体的 token 消耗计算过程和各种优化手段的量化效果。 diff --git a/backend/go.mod b/backend/go.mod index 755a60d..6a23eff 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -13,16 +13,24 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 go.uber.org/zap v1.28.0 - golang.org/x/crypto v0.23.0 + golang.org/x/crypto v0.31.0 ) require ( - github.com/bytedance/sonic v1.11.6 // indirect - github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cloudwego/eino v0.9.9 // indirect + github.com/cloudwego/eino-ext/components/model/openai v0.1.13 // indirect + github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eino-contrib/jsonschema v1.0.3 // indirect + github.com/evanphx/json-patch v0.5.2 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -31,18 +39,25 @@ require ( github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/goph/emperror v0.17.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/meguminnnnnnnnn/go-openai v0.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect @@ -51,10 +66,13 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/arch v0.8.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.30.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 18135ca..5c289db 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,26 +1,50 @@ +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= -github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= -github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cloudwego/eino v0.9.9 h1:x63hvRif6ANPh9YEPoTIrp1potEeoLQFAjOclKaX/Kg= +github.com/cloudwego/eino v0.9.9/go.mod h1:OBD1mrkfkt/pJa4rkg1P0VnaMeOVl7l8IAdEqY//3IQ= +github.com/cloudwego/eino-ext/components/model/openai v0.1.13 h1:5XHRTiTD5bt9KQrMHcfvuWNklEC3tpm3XHejdozt9vM= +github.com/cloudwego/eino-ext/components/model/openai v0.1.13/go.mod h1:mgIoqYYOc0eECCqvLbEYpOJrQNTNxkwXzSJzFU+v5sQ= +github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 h1:EeVcR1TslRA2IdNW1h/2LaGbPlffwGhQm99jM3zWZiI= +github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17/go.mod h1:Zkcx6DPTR2NfWmtSXbhItswGw6hqUezNPhNcke0pOG8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eino-contrib/jsonschema v1.0.3 h1:2Kfsm1xlMV0ssY2nuxshS4AwbLFuqmPmzIjLVJ1Fsp0= +github.com/eino-contrib/jsonschema v1.0.3/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= @@ -37,15 +61,20 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -54,37 +83,60 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/meguminnnnnnnnn/go-openai v0.1.2 h1:iXombGGjqjBrmE9WaSidUhhi3YQhf42QTHvHLMkgvCA= +github.com/meguminnnnnnnnn/go-openai v0.1.2/go.mod h1:qs96ysDmxhE4BZoU45I43zcyfnaYxU3X+aRzLko/htY= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.20.1 h1:sfCU6A8P3dXbKyWes02uxA2baehGux9dZHfEKtsTB1w= github.com/redis/go-redis/v9 v9.20.1/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -96,15 +148,18 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= @@ -113,6 +168,10 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -125,18 +184,26 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= -golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= @@ -144,8 +211,9 @@ google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/backend/internal/eino/callback.go b/backend/internal/eino/callback.go new file mode 100644 index 0000000..880c003 --- /dev/null +++ b/backend/internal/eino/callback.go @@ -0,0 +1,133 @@ +package eino + +import ( + "context" + "io" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + callbacksHelper "github.com/cloudwego/eino/utils/callbacks" + + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/orchestrator" +) + +// context key 类型，避免与其他包冲突。 +type ctxKeySender struct{} +type ctxKeyRequestID struct{} +type ctxKeyState struct{} + +// WithSender 将 Sender 注入 context。 +func WithSender(ctx context.Context, sender orchestrator.Sender) context.Context { + return context.WithValue(ctx, ctxKeySender{}, sender) +} + +// WithRequestID 将 requestID 注入 context。 +func WithRequestID(ctx context.Context, requestID string) context.Context { + return context.WithValue(ctx, ctxKeyRequestID{}, requestID) +} + +// WithPipelineState 将 PipelineState 注入 context。 +func WithPipelineState(ctx context.Context, state *PipelineState) context.Context { + return context.WithValue(ctx, ctxKeyState{}, state) +} + +// senderFromCtx 从 context 获取 Sender。 +func senderFromCtx(ctx context.Context) orchestrator.Sender { + s, _ := ctx.Value(ctxKeySender{}).(orchestrator.Sender) + return s +} + +// requestIDFromCtx 从 context 获取 requestID。 +func requestIDFromCtx(ctx context.Context) string { + s, _ := ctx.Value(ctxKeyRequestID{}).(string) + return s +} + +// stateFromCtx 从 context 获取 PipelineState。 +func stateFromCtx(ctx context.Context) *PipelineState { + s, _ := ctx.Value(ctxKeyState{}).(*PipelineState) + return s +} + +// BuildCallbackHandler 构建 Eino Callback Handler。 +// +// 核心职责：ChatModel 节点通过 OnEndWithStreamOutput 逐 token 推送 llm_chunk 到客户端， +// 同时累积完整文本到 PipelineState。 +// +// 其他节点的消息推送（stt_result、tts_audio、llm_done）由各 Lambda 内部直接调用 Sender。 +func BuildCallbackHandler() callbacks.Handler { + return callbacksHelper.NewHandlerHelper(). + ChatModel(&callbacksHelper.ModelCallbackHandler{ + OnEndWithStreamOutput: func(ctx context.Context, info *callbacks.RunInfo, output *schema.StreamReader[*model.CallbackOutput]) context.Context { + log := logger.Log + sender := senderFromCtx(ctx) + requestID := requestIDFromCtx(ctx) + state := stateFromCtx(ctx) + + if sender == nil || requestID == "" { + log.Warnw("ModelCallback: missing sender or request_id in context", + "node", info.Name) + return ctx + } + + // 异步消费流，避免阻塞框架的下游处理。 + // 框架对流做了内部拷贝，此 goroutine 读取独立副本。 + go func() { + defer output.Close() + + for { + chunk, err := output.Recv() + if err != nil { + if err == io.EOF { + return + } + log.Errorw("ModelCallback: stream recv error", + "node", info.Name, "error", err) + return + } + + if chunk == nil || chunk.Message == nil { + continue + } + + delta := chunk.Message.Content + if delta == "" { + continue + } + + // 推送 llm_chunk 到客户端 + if err := sender.SendLLMChunk(models.WsLLMChunk{ + Type: "llm_chunk", + RequestID: requestID, + Delta: delta, + Role: "assistant", + }); err != nil { + log.Errorw("ModelCallback: send llm_chunk failed", "error", err) + } + + // 累积完整文本到 State + if state != nil { + state.AppendText(delta) + } + + // 记录 token 用量（流的最后一帧携带） + if chunk.TokenUsage != nil && state != nil { + state.mu.Lock() + state.TokenUsage = &TokenUsage{ + Prompt: chunk.TokenUsage.PromptTokens, + Completion: chunk.TokenUsage.CompletionTokens, + Total: chunk.TokenUsage.TotalTokens, + } + state.mu.Unlock() + } + } + }() + + return ctx + }, + }). + Handler() +} diff --git a/backend/internal/eino/nodes_done.go b/backend/internal/eino/nodes_done.go new file mode 100644 index 0000000..90eb362 --- /dev/null +++ b/backend/internal/eino/nodes_done.go @@ -0,0 +1,114 @@ +package eino + +import ( + "context" + "time" + + "github.com/cloudwego/eino/compose" + + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/session" +) + +// NewDoneLambda 创建 Done Lambda 节点。 +// 输入: struct{}（TTS 完成信号）→ 输出: PipelineOutput +// +// 从 PipelineState 读取完整回复和 token 用量，发送 llm_done 到客户端， +// 追加助手消息到会话历史，返回 PipelineOutput。 +func NewDoneLambda(sessionMgr session.Manager, model string) *compose.Lambda { + return compose.InvokableLambda(func(ctx context.Context, _ struct{}) (*PipelineOutput, error) { + log := logger.Log + sender := senderFromCtx(ctx) + requestID := requestIDFromCtx(ctx) + state := stateFromCtx(ctx) + + if state == nil { + return &PipelineOutput{}, nil + } + + state.mu.Lock() + fullResponse := state.FullResponse.String() + transcribedText := state.TranscribedText + tokenUsage := state.TokenUsage + modelName := model + state.mu.Unlock() + + // 追加助手消息到会话历史 + sessionID := "" + if state != nil { + // 从 context 获取 sessionID（由适配器注入） + if sid, ok := ctx.Value(ctxKeySessionID{}).(string); ok { + sessionID = sid + } + } + if sessionID != "" && sessionMgr != nil && fullResponse != "" { + if err := sessionMgr.AppendMessage(ctx, sessionID, models.Message{ + Role: "assistant", + Content: fullResponse, + }); err != nil { + log.Errorw("追加助手消息到历史失败", "error", err) + } + } + + // 发送 llm_done + if sender != nil && requestID != "" { + done := models.WsLLMDone{ + Type: "llm_done", + RequestID: requestID, + FullText: fullResponse, + Model: modelName, + LatencyMs: 0, // 由适配器计算 + } + if tokenUsage != nil { + done.TokensUsed = struct { + Prompt int `json:"prompt"` + Completion int `json:"completion"` + Total int `json:"total"` + }{ + Prompt: tokenUsage.Prompt, + Completion: tokenUsage.Completion, + Total: tokenUsage.Total, + } + } + if err := sender.SendLLMDone(done); err != nil { + log.Errorw("发送 llm_done 失败", "error", err) + } + } + + log.Infow("查询处理完成", + "request_id", requestID, + "text_length", len(fullResponse)) + + return &PipelineOutput{ + TranscribedText: transcribedText, + FullResponse: fullResponse, + Model: modelName, + TokenUsage: tokenUsage, + }, nil + }) +} + +// ctxKeySessionID sessionID 的 context key。 +type ctxKeySessionID struct{} + +// WithSessionID 将 sessionID 注入 context。 +func WithSessionID(ctx context.Context, sessionID string) context.Context { + return context.WithValue(ctx, ctxKeySessionID{}, sessionID) +} + +// latencyFromCtx 从 context 获取开始时间并计算延迟。 +func latencyFromCtx(ctx context.Context) int64 { + if startTime, ok := ctx.Value(ctxKeyStartTime{}).(time.Time); ok { + return time.Since(startTime).Milliseconds() + } + return 0 +} + +// ctxKeyStartTime 请求开始时间的 context key。 +type ctxKeyStartTime struct{} + +// WithStartTime 将请求开始时间注入 context。 +func WithStartTime(ctx context.Context, t time.Time) context.Context { + return context.WithValue(ctx, ctxKeyStartTime{}, t) +} diff --git a/backend/internal/eino/nodes_history.go b/backend/internal/eino/nodes_history.go new file mode 100644 index 0000000..0858f24 --- /dev/null +++ b/backend/internal/eino/nodes_history.go @@ -0,0 +1,117 @@ +package eino + +import ( + "context" + "encoding/base64" + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" + + "github.com/hhs/camtalk/internal/ai/llm" + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" +) + +// HistoryInput 历史组装节点的输入，包含 STT 输出和原始请求信息。 +type HistoryInput struct { + STTOutput *STTOutput + SessionID string + RequestID string + ImageData []byte + Scenario string + DetailLevel string +} + +// NewHistoryLambda 创建历史组装 Lambda 节点。 +// 输入: HistoryInput → 输出: []*schema.Message +// +// 构建系统提示词，组装历史消息和当前用户输入（含多模态图片）。 +func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, maxHistory int) ([]models.Message, error), maxHistory int) *compose.Lambda { + return compose.InvokableLambda(func(ctx context.Context, input *HistoryInput) ([]*schema.Message, error) { + log := logger.Log + requestID := input.RequestID + + // 构建系统提示词 + scenarioPrompt := llm.GetScenarioPrompt(input.Scenario, input.STTOutput.Language) + systemPrompt := llm.BuildSystemPrompt(input.STTOutput.Language, input.DetailLevel, scenarioPrompt) + + // 构建 system message（含图片） + systemMsg := &schema.Message{ + Role: schema.System, + Content: systemPrompt, + } + + // 如果有图片，添加到 system message 的多模态内容中 + if len(input.ImageData) > 0 { + base64Str := base64.StdEncoding.EncodeToString(input.ImageData) + mimeType := detectImageMimeType(input.ImageData) + systemMsg.UserInputMultiContent = []schema.MessageInputPart{ + { + Type: schema.ChatMessagePartTypeImageURL, + Image: &schema.MessageInputImage{ + MessagePartCommon: schema.MessagePartCommon{ + Base64Data: &base64Str, + MIMEType: mimeType, + }, + Detail: schema.ImageURLDetailAuto, + }, + }, + } + } + + messages := []*schema.Message{systemMsg} + + // 获取并追加历史消息 + if historyFetcher != nil && input.SessionID != "" { + history, err := historyFetcher(ctx, input.SessionID, maxHistory) + if err != nil { + log.Warnw("获取历史消息失败，继续处理", "error", err, "request_id", requestID) + } else { + for _, msg := range history { + messages = append(messages, &schema.Message{ + Role: schema.RoleType(msg.Role), + Content: msg.Content, + }) + } + } + } + + // 追加当前用户输入 + messages = append(messages, &schema.Message{ + Role: schema.User, + Content: input.STTOutput.Text, + }) + + log.Infow("历史组装完成", + "request_id", requestID, + "message_count", len(messages), + "has_image", len(input.ImageData) > 0, + "scenario", input.Scenario) + + return messages, nil + }) +} + +// detectImageMimeType 简单检测图片 MIME 类型。 +func detectImageMimeType(data []byte) string { + if len(data) < 4 { + return "image/jpeg" + } + // JPEG: FF D8 FF + if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { + return "image/jpeg" + } + // PNG: 89 50 4E 47 + if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 { + return "image/png" + } + // GIF: 47 49 46 38 + if data[0] == 0x47 && data[1] == 0x49 && data[2] == 0x46 { + return "image/gif" + } + // WebP: 52 49 46 46 + if data[0] == 0x52 && data[1] == 0x49 && data[2] == 0x46 && data[3] == 0x46 { + return "image/webp" + } + return "image/jpeg" // 默认 +} + diff --git a/backend/internal/eino/nodes_splitter.go b/backend/internal/eino/nodes_splitter.go new file mode 100644 index 0000000..6f3de8b --- /dev/null +++ b/backend/internal/eino/nodes_splitter.go @@ -0,0 +1,72 @@ +package eino + +import ( + "context" + "io" + "strings" + + "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" +) + +// sentenceDelimiters 句子分隔符集合。 +var sentenceDelimiters = map[rune]bool{ + '。': true, + '！': true, + '？': true, + '\n': true, + '.': true, + '!': true, + '?': true, +} + +// NewSplitterLambda 创建句子分割 Transform Lambda 节点。 +// 输入: StreamReader[string]（LLM 完整文本的单帧流）→ 输出: StreamReader[[]string]（句子数组流） +// +// 在 Stream 模式下，框架自动将 ChatModel 的 StreamReader[*schema.Message] +// concat 为 string 后传入此节点。此节点将文本按句子边界切分， +// 每切出一个句子就输出一次，供 TTS 节点实时合成。 +func NewSplitterLambda() *compose.Lambda { + return compose.TransformableLambda(func(ctx context.Context, input *schema.StreamReader[string]) (*schema.StreamReader[[]string], error) { + sr, sw := schema.Pipe[[]string](8) + + go func() { + defer sw.Close() + + var buffer strings.Builder + + for { + chunk, err := input.Recv() + if err != nil { + if err == io.EOF { + // 流结束，flush 剩余缓冲 + if buffer.Len() > 0 { + text := strings.TrimSpace(buffer.String()) + if text != "" { + sw.Send([]string{text}, nil) + } + } + return + } + sw.Send(nil, err) + return + } + + // chunk 是 concat 后的完整文本（单帧流） + // 逐字符累积，按句子分隔符切分 + for _, r := range chunk { + buffer.WriteRune(r) + if sentenceDelimiters[r] { + text := strings.TrimSpace(buffer.String()) + if text != "" { + sw.Send([]string{text}, nil) + } + buffer.Reset() + } + } + } + }() + + return sr, nil + }) +} diff --git a/backend/internal/eino/nodes_stt.go b/backend/internal/eino/nodes_stt.go new file mode 100644 index 0000000..6659863 --- /dev/null +++ b/backend/internal/eino/nodes_stt.go @@ -0,0 +1,119 @@ +package eino + +import ( + "context" + "fmt" + "strings" + + "github.com/cloudwego/eino/compose" + + "github.com/hhs/camtalk/internal/ai/stt" + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" +) + +// NewSTTLambda 创建 STT Lambda 节点。 +// 输入: PipelineInput → 输出: STTOutput +// +// 文本输入模式：跳过 STT，直接返回用户输入文本。 +// 语音模式：调用 sttService.Recognize() 进行语音识别。 +// 识别结果通过 Sender 发送 stt_result 到客户端。 +func NewSTTLambda(sttService stt.Service) *compose.Lambda { + return compose.InvokableLambda(func(ctx context.Context, input *PipelineInput) (*STTOutput, error) { + log := logger.Log + sender := senderFromCtx(ctx) + requestID := requestIDFromCtx(ctx) + + // 文本输入模式：跳过 STT + if input.Text != "" { + log.Infow("使用文本输入，跳过 STT", + "request_id", requestID, "text", input.Text) + + // 发送 stt_result 保持前端消息流一致性 + if sender != nil { + if err := sender.SendSTTResult(models.WsSTTResult{ + Type: "stt_result", + RequestID: requestID, + Text: input.Text, + IsFinal: true, + }); err != nil { + log.Errorw("发送 stt_result 失败", "error", err) + } + } + + // 写入 State + if state := stateFromCtx(ctx); state != nil { + state.mu.Lock() + state.TranscribedText = input.Text + state.mu.Unlock() + } + + return &STTOutput{ + Text: input.Text, + Language: input.Language, + IsSkipped: true, + }, nil + } + + // 语音模式：解码音频 + if len(input.AudioData) == 0 { + return nil, fmt.Errorf("stt: no audio data provided") + } + + log.Infow("开始语音识别", + "request_id", requestID, "audio_bytes", len(input.AudioData)) + + // 调用 STT 服务 + text, err := sttService.Recognize(ctx, input.AudioData, stt.Options{ + Encoding: "pcm_s16le", + SampleRate: 16000, + Language: input.Language, + }) + if err != nil { + log.Errorw("语音识别失败", "error", err, "request_id", requestID) + if sender != nil { + sender.SendError(models.WsError{ + Type: "error", + RequestID: requestID, + Code: "STT_ERROR", + Message: "语音识别失败: " + err.Error(), + }) + } + return nil, fmt.Errorf("stt: recognize: %w", err) + } + + // STT 返回空文本 + if strings.TrimSpace(text) == "" { + log.Infow("语音识别结果为空", "request_id", requestID) + text = "（未识别到语音）" + } + + log.Infow("语音识别完成", "request_id", requestID, "text", text) + + // 发送 stt_result + if sender != nil { + if err := sender.SendSTTResult(models.WsSTTResult{ + Type: "stt_result", + RequestID: requestID, + Text: text, + IsFinal: true, + }); err != nil { + log.Errorw("发送 stt_result 失败", "error", err) + } + } + + // 写入 State + if state := stateFromCtx(ctx); state != nil { + state.mu.Lock() + state.TranscribedText = text + state.mu.Unlock() + } + + return &STTOutput{ + Text: text, + Language: input.Language, + IsSkipped: false, + }, nil + }) +} + diff --git a/backend/internal/eino/nodes_tts.go b/backend/internal/eino/nodes_tts.go new file mode 100644 index 0000000..5d4e7b7 --- /dev/null +++ b/backend/internal/eino/nodes_tts.go @@ -0,0 +1,92 @@ +package eino + +import ( + "context" + "encoding/base64" + + "github.com/cloudwego/eino/compose" + + "github.com/hhs/camtalk/internal/ai/tts" + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" +) + +// NewTTSLambda 创建 TTS Lambda 节点。 +// 输入: []string（句子数组，框架自动从 StreamReader concat）→ 输出: struct{} +// +// 将句子数组转为 channel，调用 ttsService.SynthesizeStream() 流式合成， +// 逐 chunk 推送 tts_audio 到客户端。TTS 失败静默跳过。 +func NewTTSLambda(ttsService tts.Service, ttsVoice string, ttsSpeed float64, ttsOutputFmt string, ttsSampleRate int) *compose.Lambda { + return compose.InvokableLambda(func(ctx context.Context, sentences []string) (struct{}, error) { + log := logger.Log + sender := senderFromCtx(ctx) + requestID := requestIDFromCtx(ctx) + state := stateFromCtx(ctx) + + // 检查 TTS 是否启用（从 State 或 context 获取） + // TTSEnabled 信息在 PipelineInput 中，通过 State 传递 + if state != nil { + state.mu.Lock() + ttsEnabled := true // 默认启用，由适配器通过 State 设置 + state.mu.Unlock() + if !ttsEnabled { + return struct{}{}, nil + } + } + + if len(sentences) == 0 { + return struct{}{}, nil + } + + if sender == nil || requestID == "" { + return struct{}{}, nil + } + + log.Infow("开始 TTS 合成", "request_id", requestID, "sentence_count", len(sentences)) + + // 将句子数组转为 channel（ttsService.SynthesizeStream 需要 <-chan string） + sentenceCh := make(chan string, len(sentences)) + for _, s := range sentences { + sentenceCh <- s + } + close(sentenceCh) + + // 调用 TTS 服务 + ttsStream, err := ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{ + Voice: ttsVoice, + Speed: ttsSpeed, + OutputFmt: ttsOutputFmt, + SampleRate: ttsSampleRate, + }) + if err != nil { + log.Errorw("TTS 合成启动失败（已跳过）", "error", err, "request_id", requestID) + return struct{}{}, nil // TTS 失败不中断流程 + } + + // 消费 TTS 音频流，推送到客户端 + for chunk := range ttsStream { + select { + case <-ctx.Done(): + log.Infow("TTS 流被中断", "request_id", requestID) + return struct{}{}, ctx.Err() + default: + } + + audioBase64 := base64.StdEncoding.EncodeToString(chunk.Audio) + + if err := sender.SendTTSAudio(models.WsTTSAudio{ + Type: "tts_audio", + RequestID: requestID, + Audio: audioBase64, + MimeType: "audio/mp3", + IsLast: chunk.IsLast, + Final: chunk.Final, + }); err != nil { + log.Errorw("发送 tts_audio 失败", "error", err) + } + } + + log.Infow("TTS 合成完成", "request_id", requestID) + return struct{}{}, nil + }) +} diff --git a/backend/internal/eino/state.go b/backend/internal/eino/state.go new file mode 100644 index 0000000..613cd9b --- /dev/null +++ b/backend/internal/eino/state.go @@ -0,0 +1,36 @@ +package eino + +import ( + "context" + "strings" + "sync" +) + +// PipelineState Graph 全局状态，用于跨节点收集数据。 +// 通过 compose.WithGenLocalState 注册，各节点通过 StatePreHandler/StatePostHandler 读写。 +type PipelineState struct { + mu sync.Mutex + FullResponse strings.Builder // LLM 完整回复（由 Callback 累积） + TranscribedText string // STT 识别文本 + Model string // 实际使用的模型名 + TokenUsage *TokenUsage // token 用量 +} + +// genLocalState 创建每请求的 PipelineState 实例。 +func genLocalState(ctx context.Context) *PipelineState { + return &PipelineState{} +} + +// AppendText 追加文本到 FullResponse（线程安全）。 +func (s *PipelineState) AppendText(text string) { + s.mu.Lock() + defer s.mu.Unlock() + s.FullResponse.WriteString(text) +} + +// GetFullResponse 获取完整回复文本（线程安全）。 +func (s *PipelineState) GetFullResponse() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.FullResponse.String() +} diff --git a/backend/internal/eino/types.go b/backend/internal/eino/types.go new file mode 100644 index 0000000..49f0b48 --- /dev/null +++ b/backend/internal/eino/types.go @@ -0,0 +1,37 @@ +// Package eino 基于 CloudWeGo Eino 框架的 AI 编排层。 +// 使用 Eino Graph 替代手写 goroutine 管道，实现声明式 STT → LLM → TTS 编排。 +package eino + +// PipelineInput Graph 统一输入。 +type PipelineInput struct { + AudioData []byte // base64 解码后的音频（可选） + ImageData []byte // base64 解码后的图像（可选） + Text string // 直接文本输入（可选，跳过 STT） + SessionID string + RequestID string + Language string // zh / en + Scenario string // free_chat, interviewer, etc. + TTSEnabled bool +} + +// PipelineOutput Graph 统一输出。 +type PipelineOutput struct { + TranscribedText string // STT 结果 + FullResponse string // LLM 完整回复 + Model string // 实际使用的模型名 + TokenUsage *TokenUsage // token 用量 +} + +// STTOutput STT 节点输出。 +type STTOutput struct { + Text string + Language string + IsSkipped bool // 文本输入模式跳过了 STT +} + +// TokenUsage token 用量统计。 +type TokenUsage struct { + Prompt int + Completion int + Total int +} -- 2.49.1 From 4b731b5ac0cba47fde794727713f493a3e757a33 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Fri, 19 Jun 2026 21:58:17 +0800 Subject: [PATCH 2/7] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=20Eino=20Graph?= =?UTF-8?q?=20=E6=9E=84=E5=BB=BA=E4=B8=8E=20Orchestrator=20=E9=80=82?= =?UTF-8?q?=E9=85=8D=E5=99=A8=EF=BC=8C=E5=88=87=E6=8D=A2=20main.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - graph.go: 构建 Graph 拓扑 START→STT→History→ChatModel→Splitter→TTS→Done→END - 创建 eino-ext ChatModel 对接 DashScope OpenAI 兼容接口 - 统一使用值类型（PipelineInput/PipelineOutput） - Callback 在运行时通过 Stream option 传入 - adapter.go: EinoOrchestrator 实现 orchestrator.Orchestrator 接口 - 解码 base64 音频/图片，注入 context 值 - 调用 Graph.Stream() 触发惰性执行并消费输出 - 追加用户/助手消息到历史 - main.go: 移除旧 llmService + orchestrator.New() 替换为 eino.NewPipelineGraph() + eino.NewEinoOrchestrator() - 各节点统一使用值类型，State 传递请求元数据 Co-Authored-By: Claude --- backend/cmd/server/main.go | 14 +-- backend/go.mod | 4 +- backend/go.sum | 22 ++++ backend/internal/eino/adapter.go | 165 +++++++++++++++++++++++++ backend/internal/eino/graph.go | 113 +++++++++++++++++ backend/internal/eino/nodes_done.go | 80 ++++-------- backend/internal/eino/nodes_history.go | 61 ++++----- backend/internal/eino/nodes_stt.go | 23 +++- backend/internal/eino/state.go | 11 +- 9 files changed, 395 insertions(+), 98 deletions(-) create mode 100644 backend/internal/eino/adapter.go create mode 100644 backend/internal/eino/graph.go diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index c266172..95fd1e4 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -13,12 +13,11 @@ import ( "github.com/hhs/camtalk/internal/api" "github.com/hhs/camtalk/internal/auth" - "github.com/hhs/camtalk/internal/ai/llm" "github.com/hhs/camtalk/internal/ai/stt" "github.com/hhs/camtalk/internal/ai/tts" "github.com/hhs/camtalk/internal/config" + eino "github.com/hhs/camtalk/internal/eino" "github.com/hhs/camtalk/internal/logger" - "github.com/hhs/camtalk/internal/orchestrator" "github.com/hhs/camtalk/internal/session" "github.com/hhs/camtalk/internal/store" "github.com/hhs/camtalk/internal/ws" @@ -116,9 +115,6 @@ func main() { sttService = stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, cfg.AI.STT.Timeout, logger.Log) logger.Log.Infow("STT service initialized", "provider", "deepgram", "model", cfg.AI.STT.Model) } - llmService := llm.NewOpenAIService(cfg.AI.LLM.APIKey, cfg.AI.LLM.Model, cfg.AI.LLM.Endpoint, cfg.AI.LLM.Timeout, cfg.AI.LLM.HTTPClientTimeout, logger.Log) - logger.Log.Infow("LLM service initialized", "provider", cfg.AI.LLM.Provider, "model", cfg.AI.LLM.Model, "endpoint", cfg.AI.LLM.Endpoint, "timeout", cfg.AI.LLM.Timeout) - var ttsService tts.Service switch strings.ToLower(cfg.AI.TTS.Provider) { case "mimo", "xiaomi": @@ -129,8 +125,12 @@ func main() { logger.Log.Infow("TTS service initialized", "provider", "openai", "model", cfg.AI.TTS.Model, "voice", cfg.AI.TTS.Voice, "speed", cfg.AI.TTS.Speed) } - // 初始化 Orchestrator - orch := orchestrator.New(sttService, llmService, ttsService, sessionMgr, cfg) + // 初始化 Eino Graph + Orchestrator + pipelineGraph, err := eino.NewPipelineGraph(ctx, cfg, sttService, ttsService, sessionMgr) + if err != nil { + logger.Log.Fatalw("failed to create eino pipeline graph", "error", err) + } + orch := eino.NewEinoOrchestrator(pipelineGraph, sessionMgr, cfg.AI.LLM.Model) // 初始化认证服务 tokenMgr := auth.NewTokenManager( diff --git a/backend/go.mod b/backend/go.mod index 6a23eff..6f652f0 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,6 +3,8 @@ module github.com/hhs/camtalk go 1.25.0 require ( + github.com/cloudwego/eino v0.9.9 + github.com/cloudwego/eino-ext/components/model/openai v0.1.13 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 @@ -24,8 +26,6 @@ require ( github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect - github.com/cloudwego/eino v0.9.9 // indirect - github.com/cloudwego/eino-ext/components/model/openai v0.1.13 // indirect github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/backend/go.sum b/backend/go.sum index 5c289db..9d58137 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -13,6 +13,8 @@ github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqR github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/mockey v1.3.0 h1:ONLRdvhqmCfr9rTasUB8ZKCfvbdD2tohOg4u+4Q/ed0= +github.com/bytedance/mockey v1.3.0/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= @@ -49,6 +51,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -72,6 +76,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -89,6 +95,8 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -104,10 +112,14 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/meguminnnnnnnnn/go-openai v0.1.2 h1:iXombGGjqjBrmE9WaSidUhhi3YQhf42QTHvHLMkgvCA= github.com/meguminnnnnnnnn/go-openai v0.1.2/go.mod h1:qs96ysDmxhE4BZoU45I43zcyfnaYxU3X+aRzLko/htY= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -137,6 +149,10 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -170,6 +186,8 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= @@ -178,6 +196,8 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= @@ -203,6 +223,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= diff --git a/backend/internal/eino/adapter.go b/backend/internal/eino/adapter.go new file mode 100644 index 0000000..75c1b2c --- /dev/null +++ b/backend/internal/eino/adapter.go @@ -0,0 +1,165 @@ +package eino + +import ( + "context" + "encoding/base64" + "io" + "time" + + "github.com/cloudwego/eino/compose" + + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/orchestrator" + "github.com/hhs/camtalk/internal/session" +) + +// ctxKeySessionID sessionID 的 context key。 +type ctxKeySessionID struct{} + +// WithSessionID 将 sessionID 注入 context。 +func WithSessionID(ctx context.Context, sessionID string) context.Context { + return context.WithValue(ctx, ctxKeySessionID{}, sessionID) +} + +// EinoOrchestrator 实现 orchestrator.Orchestrator 接口。 +// 将 Eino Graph 包装为现有接口，WS Handler 几乎不用改。 +type EinoOrchestrator struct { + graph *PipelineGraph + sessionMgr session.Manager + model string + callbacks compose.Option // 运行时 Callback option +} + +// NewEinoOrchestrator 创建 Eino 编排器适配器。 +func NewEinoOrchestrator(graph *PipelineGraph, sessionMgr session.Manager, model string) *EinoOrchestrator { + return &EinoOrchestrator{ + graph: graph, + sessionMgr: sessionMgr, + model: model, + callbacks: compose.WithCallbacks(BuildCallbackHandler()), + } +} + +// ProcessQuery 实现 orchestrator.Orchestrator 接口。 +func (e *EinoOrchestrator) ProcessQuery( + ctx context.Context, + sessionID string, + req models.WsQuery, + history []models.Message, + sender orchestrator.Sender, +) error { + log := logger.Log + startTime := time.Now() + + // 1. 设置活跃请求 + if err := e.sessionMgr.SetActiveRequest(ctx, sessionID, req.RequestID); err != nil { + log.Errorw("设置活跃请求失败", "error", err) + } + defer e.sessionMgr.ClearActiveRequest(ctx, sessionID) + + // 2. 获取会话配置 + sess, err := e.sessionMgr.Get(ctx, sessionID) + if err != nil { + log.Errorw("获取会话失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "SESSION_NOT_FOUND", + Message: "会话不存在", + }) + return err + } + + // 3. 解码音频和图片 + var audioData []byte + if req.Text == "" && req.Audio != "" { + audioData, err = base64.StdEncoding.DecodeString(req.Audio) + if err != nil { + log.Errorw("音频解码失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "INVALID_MESSAGE", + Message: "音频数据解码失败", + }) + return err + } + } + + var imageData []byte + if req.Image != "" { + imageData, err = base64.StdEncoding.DecodeString(req.Image) + if err != nil { + log.Errorw("图片解码失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "INVALID_MESSAGE", + Message: "图片数据解码失败", + }) + return err + } + } + + // 4. 构建 Graph 输入 + input := buildPipelineInput(req, sessionID, sess, audioData, imageData) + + // 5. 注入 context 值（供 Callback 和 Lambda 节点使用） + ctx = WithSender(ctx, sender) + ctx = WithRequestID(ctx, req.RequestID) + ctx = WithSessionID(ctx, sessionID) + ctx = WithStartTime(ctx, startTime) + ctx = WithPipelineState(ctx, genLocalState(ctx)) + + // 6. 追加用户消息到历史 + if req.Text != "" { + _ = e.sessionMgr.AppendMessage(ctx, sessionID, models.Message{ + Role: "user", + Content: req.Text, + }) + } + + // 7. 调用 Graph（Stream 模式 + 运行时 Callback） + streamReader, err := e.graph.Runnable.Stream(ctx, input, e.callbacks) + if err != nil { + log.Errorw("Graph Stream 启动失败", "error", err) + sender.SendError(models.WsError{ + Type: "error", + RequestID: req.RequestID, + Code: "INTERNAL_ERROR", + Message: "编排器启动失败", + }) + return err + } + + // 8. 消费 StreamReader（触发整条链路执行，side effects 推送消息到客户端） + var output PipelineOutput + for { + o, err := streamReader.Recv() + if err != nil { + if err == io.EOF { + break + } + log.Errorw("Graph Stream 消费错误", "error", err) + break + } + output = o + } + + // 9. 追加助手消息到历史 + if output.FullResponse != "" { + _ = e.sessionMgr.AppendMessage(ctx, sessionID, models.Message{ + Role: "assistant", + Content: output.FullResponse, + }) + } + + latency := time.Since(startTime).Milliseconds() + log.Infow("Eino 编排完成", + "request_id", req.RequestID, + "latency_ms", latency, + "session_id", sessionID) + + return nil +} diff --git a/backend/internal/eino/graph.go b/backend/internal/eino/graph.go new file mode 100644 index 0000000..f8a2449 --- /dev/null +++ b/backend/internal/eino/graph.go @@ -0,0 +1,113 @@ +package eino + +import ( + "context" + "time" + + openaiImpl "github.com/cloudwego/eino-ext/components/model/openai" + "github.com/cloudwego/eino/compose" + + "github.com/hhs/camtalk/internal/ai/stt" + "github.com/hhs/camtalk/internal/ai/tts" + "github.com/hhs/camtalk/internal/config" + "github.com/hhs/camtalk/internal/logger" + "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/session" +) + +const ( + nodeSTT = "stt" + nodeHistory = "history" + nodeLLM = "llm" + nodeSplitter = "splitter" + nodeTTS = "tts" + nodeDone = "done" +) + +// PipelineGraph 封装编译后的 Eino Graph。 +type PipelineGraph struct { + Runnable compose.Runnable[PipelineInput, PipelineOutput] +} + +// NewPipelineGraph 构建 CamTalk AI 编排 Graph。 +// +// 拓扑：START → STT → History → ChatModel → Splitter → TTS → Done → END +// +// Graph 使用 Stream 模式调用，ChatModel 实现真正的 token 级流式输出。 +// LLM token 通过 Callback 的 OnEndWithStreamOutput 实时推送到客户端。 +func NewPipelineGraph( + ctx context.Context, + cfg *config.Config, + sttService stt.Service, + ttsService tts.Service, + sessionMgr session.Manager, +) (*PipelineGraph, error) { + log := logger.Log + + // 1. 创建 eino-ext ChatModel（对接 DashScope OpenAI 兼容接口） + chatModel, err := openaiImpl.NewChatModel(ctx, &openaiImpl.ChatModelConfig{ + APIKey: cfg.AI.LLM.APIKey, + Model: cfg.AI.LLM.Model, + BaseURL: cfg.AI.LLM.Endpoint, + Timeout: time.Duration(cfg.AI.LLM.Timeout) * time.Second, + }) + if err != nil { + return nil, err + } + log.Infow("Eino ChatModel 初始化成功", + "model", cfg.AI.LLM.Model, + "endpoint", cfg.AI.LLM.Endpoint) + + // 2. 构建 Graph（值类型，非指针） + g := compose.NewGraph[PipelineInput, PipelineOutput]( + compose.WithGenLocalState(genLocalState), + ) + + // 3. 添加节点 + maxHistory := cfg.Session.MaxHistory + + _ = g.AddLambdaNode(nodeSTT, NewSTTLambda(sttService)) + _ = g.AddLambdaNode(nodeHistory, NewHistoryLambda(sessionMgr.GetHistory, maxHistory)) + _ = g.AddChatModelNode(nodeLLM, chatModel) + _ = g.AddLambdaNode(nodeSplitter, NewSplitterLambda()) + _ = g.AddLambdaNode(nodeTTS, NewTTSLambda( + ttsService, + cfg.AI.TTS.Voice, + cfg.AI.TTS.Speed, + cfg.AI.TTS.OutputFormat, + cfg.AI.TTS.SampleRate, + )) + _ = g.AddLambdaNode(nodeDone, NewDoneLambda(cfg.AI.LLM.Model)) + + // 4. 连接边 + _ = g.AddEdge(compose.START, nodeSTT) + _ = g.AddEdge(nodeSTT, nodeHistory) + _ = g.AddEdge(nodeHistory, nodeLLM) + _ = g.AddEdge(nodeLLM, nodeSplitter) + _ = g.AddEdge(nodeSplitter, nodeTTS) + _ = g.AddEdge(nodeTTS, nodeDone) + _ = g.AddEdge(nodeDone, compose.END) + + // 5. 编译（回调在运行时通过 Stream option 传入） + runnable, err := g.Compile(ctx) + if err != nil { + return nil, err + } + + log.Infow("Eino Graph 编译成功", "nodes", 6) + return &PipelineGraph{Runnable: runnable}, nil +} + +// buildPipelineInput 从 WebSocket 请求和会话配置构建 Graph 输入。 +func buildPipelineInput(req models.WsQuery, sessionID string, sess *models.Session, audioData, imageData []byte) PipelineInput { + return PipelineInput{ + AudioData: audioData, + ImageData: imageData, + Text: req.Text, + SessionID: sessionID, + RequestID: req.RequestID, + Language: sess.Config.Language, + Scenario: sess.Config.Scenario, + TTSEnabled: sess.Config.TTSEnabled, + } +} diff --git a/backend/internal/eino/nodes_done.go b/backend/internal/eino/nodes_done.go index 90eb362..745c69b 100644 --- a/backend/internal/eino/nodes_done.go +++ b/backend/internal/eino/nodes_done.go @@ -8,49 +8,47 @@ import ( "github.com/hhs/camtalk/internal/logger" "github.com/hhs/camtalk/internal/models" - "github.com/hhs/camtalk/internal/session" ) +// ctxKeyStartTime 请求开始时间的 context key。 +type ctxKeyStartTime struct{} + +// WithStartTime 将请求开始时间注入 context。 +func WithStartTime(ctx context.Context, t time.Time) context.Context { + return context.WithValue(ctx, ctxKeyStartTime{}, t) +} + +// latencyFromCtx 从 context 获取开始时间并计算延迟（毫秒）。 +func latencyFromCtx(ctx context.Context) int64 { + if startTime, ok := ctx.Value(ctxKeyStartTime{}).(time.Time); ok { + return time.Since(startTime).Milliseconds() + } + return 0 +} + // NewDoneLambda 创建 Done Lambda 节点。 -// 输入: struct{}（TTS 完成信号）→ 输出: PipelineOutput +// 输入: struct{}（TTS 完成信号）→ 输出: *PipelineOutput // -// 从 PipelineState 读取完整回复和 token 用量，发送 llm_done 到客户端， -// 追加助手消息到会话历史，返回 PipelineOutput。 -func NewDoneLambda(sessionMgr session.Manager, model string) *compose.Lambda { - return compose.InvokableLambda(func(ctx context.Context, _ struct{}) (*PipelineOutput, error) { +// 从 PipelineState 读取完整回复和 token 用量，发送 llm_done 到客户端。 +// 历史消息追加由适配器负责（避免重复写入）。 +func NewDoneLambda(defaultModel string) *compose.Lambda { + return compose.InvokableLambda(func(ctx context.Context, _ struct{}) (PipelineOutput, error) { log := logger.Log sender := senderFromCtx(ctx) - requestID := requestIDFromCtx(ctx) state := stateFromCtx(ctx) if state == nil { - return &PipelineOutput{}, nil + return PipelineOutput{}, nil } state.mu.Lock() fullResponse := state.FullResponse.String() transcribedText := state.TranscribedText tokenUsage := state.TokenUsage - modelName := model + requestID := state.RequestID + modelName := defaultModel state.mu.Unlock() - // 追加助手消息到会话历史 - sessionID := "" - if state != nil { - // 从 context 获取 sessionID（由适配器注入） - if sid, ok := ctx.Value(ctxKeySessionID{}).(string); ok { - sessionID = sid - } - } - if sessionID != "" && sessionMgr != nil && fullResponse != "" { - if err := sessionMgr.AppendMessage(ctx, sessionID, models.Message{ - Role: "assistant", - Content: fullResponse, - }); err != nil { - log.Errorw("追加助手消息到历史失败", "error", err) - } - } - // 发送 llm_done if sender != nil && requestID != "" { done := models.WsLLMDone{ @@ -58,7 +56,7 @@ func NewDoneLambda(sessionMgr session.Manager, model string) *compose.Lambda { RequestID: requestID, FullText: fullResponse, Model: modelName, - LatencyMs: 0, // 由适配器计算 + LatencyMs: latencyFromCtx(ctx), } if tokenUsage != nil { done.TokensUsed = struct { @@ -78,9 +76,9 @@ func NewDoneLambda(sessionMgr session.Manager, model string) *compose.Lambda { log.Infow("查询处理完成", "request_id", requestID, - "text_length", len(fullResponse)) + "response_length", len(fullResponse)) - return &PipelineOutput{ + return PipelineOutput{ TranscribedText: transcribedText, FullResponse: fullResponse, Model: modelName, @@ -88,27 +86,3 @@ func NewDoneLambda(sessionMgr session.Manager, model string) *compose.Lambda { }, nil }) } - -// ctxKeySessionID sessionID 的 context key。 -type ctxKeySessionID struct{} - -// WithSessionID 将 sessionID 注入 context。 -func WithSessionID(ctx context.Context, sessionID string) context.Context { - return context.WithValue(ctx, ctxKeySessionID{}, sessionID) -} - -// latencyFromCtx 从 context 获取开始时间并计算延迟。 -func latencyFromCtx(ctx context.Context) int64 { - if startTime, ok := ctx.Value(ctxKeyStartTime{}).(time.Time); ok { - return time.Since(startTime).Milliseconds() - } - return 0 -} - -// ctxKeyStartTime 请求开始时间的 context key。 -type ctxKeyStartTime struct{} - -// WithStartTime 将请求开始时间注入 context。 -func WithStartTime(ctx context.Context, t time.Time) context.Context { - return context.WithValue(ctx, ctxKeyStartTime{}, t) -} diff --git a/backend/internal/eino/nodes_history.go b/backend/internal/eino/nodes_history.go index 0858f24..a558a6b 100644 --- a/backend/internal/eino/nodes_history.go +++ b/backend/internal/eino/nodes_history.go @@ -3,6 +3,7 @@ package eino import ( "context" "encoding/base64" + "github.com/cloudwego/eino/compose" "github.com/cloudwego/eino/schema" @@ -11,28 +12,33 @@ import ( "github.com/hhs/camtalk/internal/models" ) -// HistoryInput 历史组装节点的输入，包含 STT 输出和原始请求信息。 -type HistoryInput struct { - STTOutput *STTOutput - SessionID string - RequestID string - ImageData []byte - Scenario string - DetailLevel string -} - // NewHistoryLambda 创建历史组装 Lambda 节点。 -// 输入: HistoryInput → 输出: []*schema.Message +// 输入: *STTOutput → 输出: []*schema.Message // +// 从 PipelineState 读取请求元数据（SessionID、Scenario、ImageData 等）， // 构建系统提示词，组装历史消息和当前用户输入（含多模态图片）。 -func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, maxHistory int) ([]models.Message, error), maxHistory int) *compose.Lambda { - return compose.InvokableLambda(func(ctx context.Context, input *HistoryInput) ([]*schema.Message, error) { +func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, limit int) ([]models.Message, error), maxHistory int) *compose.Lambda { + return compose.InvokableLambda(func(ctx context.Context, sttOut STTOutput) ([]*schema.Message, error) { log := logger.Log - requestID := input.RequestID + + // 从 State 读取请求元数据 + state := stateFromCtx(ctx) + if state == nil { + return []*schema.Message{}, nil + } + + state.mu.Lock() + sessionID := state.SessionID + requestID := state.RequestID + imageData := state.ImageData + scenario := state.Scenario + detailLevel := state.DetailLevel + language := sttOut.Language + state.mu.Unlock() // 构建系统提示词 - scenarioPrompt := llm.GetScenarioPrompt(input.Scenario, input.STTOutput.Language) - systemPrompt := llm.BuildSystemPrompt(input.STTOutput.Language, input.DetailLevel, scenarioPrompt) + scenarioPrompt := llm.GetScenarioPrompt(scenario, language) + systemPrompt := llm.BuildSystemPrompt(language, detailLevel, scenarioPrompt) // 构建 system message（含图片） systemMsg := &schema.Message{ @@ -41,9 +47,9 @@ func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, } // 如果有图片，添加到 system message 的多模态内容中 - if len(input.ImageData) > 0 { - base64Str := base64.StdEncoding.EncodeToString(input.ImageData) - mimeType := detectImageMimeType(input.ImageData) + if len(imageData) > 0 { + base64Str := base64.StdEncoding.EncodeToString(imageData) + mimeType := detectImageMimeType(imageData) systemMsg.UserInputMultiContent = []schema.MessageInputPart{ { Type: schema.ChatMessagePartTypeImageURL, @@ -61,8 +67,8 @@ func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, messages := []*schema.Message{systemMsg} // 获取并追加历史消息 - if historyFetcher != nil && input.SessionID != "" { - history, err := historyFetcher(ctx, input.SessionID, maxHistory) + if historyFetcher != nil && sessionID != "" { + history, err := historyFetcher(ctx, sessionID, maxHistory) if err != nil { log.Warnw("获取历史消息失败，继续处理", "error", err, "request_id", requestID) } else { @@ -78,14 +84,14 @@ func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, // 追加当前用户输入 messages = append(messages, &schema.Message{ Role: schema.User, - Content: input.STTOutput.Text, + Content: sttOut.Text, }) log.Infow("历史组装完成", "request_id", requestID, "message_count", len(messages), - "has_image", len(input.ImageData) > 0, - "scenario", input.Scenario) + "has_image", len(imageData) > 0, + "scenario", scenario) return messages, nil }) @@ -96,22 +102,17 @@ func detectImageMimeType(data []byte) string { if len(data) < 4 { return "image/jpeg" } - // JPEG: FF D8 FF if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { return "image/jpeg" } - // PNG: 89 50 4E 47 if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 { return "image/png" } - // GIF: 47 49 46 38 if data[0] == 0x47 && data[1] == 0x49 && data[2] == 0x46 { return "image/gif" } - // WebP: 52 49 46 46 if data[0] == 0x52 && data[1] == 0x49 && data[2] == 0x46 && data[3] == 0x46 { return "image/webp" } - return "image/jpeg" // 默认 + return "image/jpeg" } - diff --git a/backend/internal/eino/nodes_stt.go b/backend/internal/eino/nodes_stt.go index 6659863..0f63a05 100644 --- a/backend/internal/eino/nodes_stt.go +++ b/backend/internal/eino/nodes_stt.go @@ -19,11 +19,24 @@ import ( // 语音模式：调用 sttService.Recognize() 进行语音识别。 // 识别结果通过 Sender 发送 stt_result 到客户端。 func NewSTTLambda(sttService stt.Service) *compose.Lambda { - return compose.InvokableLambda(func(ctx context.Context, input *PipelineInput) (*STTOutput, error) { + return compose.InvokableLambda(func(ctx context.Context, input PipelineInput) (STTOutput, error) { log := logger.Log sender := senderFromCtx(ctx) requestID := requestIDFromCtx(ctx) + // 将输入元数据写入 State，供下游节点（History、Done）读取 + if state := stateFromCtx(ctx); state != nil { + state.mu.Lock() + state.SessionID = input.SessionID + state.RequestID = input.RequestID + state.ImageData = input.ImageData + state.Scenario = input.Scenario + state.DetailLevel = "low" + state.Language = input.Language + state.TTSEnabled = input.TTSEnabled + state.mu.Unlock() + } + // 文本输入模式：跳过 STT if input.Text != "" { log.Infow("使用文本输入，跳过 STT", @@ -48,7 +61,7 @@ func NewSTTLambda(sttService stt.Service) *compose.Lambda { state.mu.Unlock() } - return &STTOutput{ + return STTOutput{ Text: input.Text, Language: input.Language, IsSkipped: true, @@ -57,7 +70,7 @@ func NewSTTLambda(sttService stt.Service) *compose.Lambda { // 语音模式：解码音频 if len(input.AudioData) == 0 { - return nil, fmt.Errorf("stt: no audio data provided") + return STTOutput{}, fmt.Errorf("stt: no audio data provided") } log.Infow("开始语音识别", @@ -79,7 +92,7 @@ func NewSTTLambda(sttService stt.Service) *compose.Lambda { Message: "语音识别失败: " + err.Error(), }) } - return nil, fmt.Errorf("stt: recognize: %w", err) + return STTOutput{}, fmt.Errorf("stt: recognize: %w", err) } // STT 返回空文本 @@ -109,7 +122,7 @@ func NewSTTLambda(sttService stt.Service) *compose.Lambda { state.mu.Unlock() } - return &STTOutput{ + return STTOutput{ Text: text, Language: input.Language, IsSkipped: false, diff --git a/backend/internal/eino/state.go b/backend/internal/eino/state.go index 613cd9b..aef96af 100644 --- a/backend/internal/eino/state.go +++ b/backend/internal/eino/state.go @@ -7,13 +7,22 @@ import ( ) // PipelineState Graph 全局状态，用于跨节点收集数据。 -// 通过 compose.WithGenLocalState 注册，各节点通过 StatePreHandler/StatePostHandler 读写。 +// 通过 compose.WithGenLocalState 注册，各节点通过 compose.ProcessState 读写。 type PipelineState struct { mu sync.Mutex FullResponse strings.Builder // LLM 完整回复（由 Callback 累积） TranscribedText string // STT 识别文本 Model string // 实际使用的模型名 TokenUsage *TokenUsage // token 用量 + + // 从 PipelineInput 复制的元数据，供下游节点（History、Done）读取 + SessionID string + RequestID string + ImageData []byte + Scenario string + DetailLevel string + Language string + TTSEnabled bool } // genLocalState 创建每请求的 PipelineState 实例。 -- 2.49.1 From 4ffd84510ee12dcde8f1a6a59643603a86c5576f Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Fri, 19 Jun 2026 22:04:39 +0800 Subject: [PATCH 3/7] =?UTF-8?q?refactor:=20=E6=B8=85=E7=90=86=E6=97=A7?= =?UTF-8?q?=E7=BC=96=E6=8E=92=E4=BB=A3=E7=A0=81=EF=BC=8C=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=20eino=20=E5=8C=85=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 删除旧代码： - orchestrator/pipeline.go: 旧 STT→LLM→TTS 手写 goroutine 管道 - orchestrator/splitter.go: 旧句子切分器 - orchestrator/pipeline_test.go: 旧 Pipeline 测试 - ai/llm/openai.go: 旧 LLM OpenAI 实现（被 eino-ext ChatModel 替代） - ai/llm/openai_test.go: 旧 LLM 测试 保留的接口和工具： - orchestrator/orchestrator.go: Orchestrator 接口（ws/handler 依赖） - orchestrator/sender.go: Sender 接口（eino/callback 依赖） - ai/llm/llm.go: Request/Chunk/TokenUsage 类型定义 - ai/llm/prompt.go: BuildSystemPrompt（eino/nodes_history 依赖） - ai/llm/scenarios.go: GetScenarioPrompt（eino/nodes_history 依赖） 新增测试： - eino/graph_test.go: 13 个测试覆盖类型构建、State 并发安全、 Context 注入、延迟计算、接口实现检查等 Co-Authored-By: Claude --- backend/internal/ai/llm/openai.go | 239 ------ backend/internal/ai/llm/openai_test.go | 251 ------ backend/internal/eino/graph_test.go | 235 ++++++ backend/internal/orchestrator/pipeline.go | 403 ---------- .../internal/orchestrator/pipeline_test.go | 713 ------------------ backend/internal/orchestrator/splitter.go | 55 -- 6 files changed, 235 insertions(+), 1661 deletions(-) delete mode 100644 backend/internal/ai/llm/openai.go delete mode 100644 backend/internal/ai/llm/openai_test.go create mode 100644 backend/internal/eino/graph_test.go delete mode 100644 backend/internal/orchestrator/pipeline.go delete mode 100644 backend/internal/orchestrator/pipeline_test.go delete mode 100644 backend/internal/orchestrator/splitter.go diff --git a/backend/internal/ai/llm/openai.go b/backend/internal/ai/llm/openai.go deleted file mode 100644 index 982264d..0000000 --- a/backend/internal/ai/llm/openai.go +++ /dev/null @@ -1,239 +0,0 @@ -package llm - -import ( - "bufio" - "bytes" - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "time" - - "go.uber.org/zap" -) - -// OpenAIService 基于 OpenAI Chat Completions API 的 LLM 实现。 -type OpenAIService struct { - apiKey string - model string - endpoint string - timeout time.Duration - logger *zap.SugaredLogger - client *http.Client -} - -// NewOpenAIService 创建 OpenAI LLM 服务。 -// model、endpoint 由 config 层保证非空。 -func NewOpenAIService(apiKey, model, endpoint string, timeoutSec, httpClientTimeoutSec int, logger *zap.SugaredLogger) *OpenAIService { - timeout := time.Duration(timeoutSec) * time.Second - if timeout <= 0 { - timeout = 10 * time.Second - } - httpClientTimeout := time.Duration(httpClientTimeoutSec) * time.Second - if httpClientTimeout <= 0 { - httpClientTimeout = 60 * time.Second - } - return &OpenAIService{ - apiKey: apiKey, - model: model, - endpoint: endpoint, - timeout: timeout, - logger: logger, - client: &http.Client{Timeout: httpClientTimeout}, - } -} - -// --- OpenAI API 请求/响应结构 --- - -type chatRequest struct { - Model string `json:"model"` - Messages []chatMessage `json:"messages"` - Stream bool `json:"stream"` -} - -type chatMessage struct { - Role string `json:"role"` - Content []contentPart `json:"content"` -} - -type contentPart struct { - Type string `json:"type"` - Text string `json:"text"` - ImageURL *imageURL `json:"image_url,omitempty"` -} - -type imageURL struct { - URL string `json:"url"` -} - -// streamDelta SSE 流式响应的单个 delta。 -type streamDelta struct { - Choices []struct { - Delta struct { - Content string `json:"content"` - } `json:"delta"` - FinishReason *string `json:"finish_reason"` - } `json:"choices"` - Usage *struct { - PromptTokens int `json:"prompt_tokens"` - CompletionTokens int `json:"completion_tokens"` - TotalTokens int `json:"total_tokens"` - } `json:"usage"` - Model string `json:"model"` -} - -// ChatStream 实现 llm.Service。调用 OpenAI Chat Completions API 流式推理。 -func (o *OpenAIService) ChatStream(ctx context.Context, req Request) (<-chan Chunk, error) { - // 构建请求 - messages := o.buildMessages(req) - - body := chatRequest{ - Model: o.model, - Messages: messages, - Stream: true, - } - - payload, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("llm: marshal request: %w", err) - } - if err != nil { - return nil, fmt.Errorf("llm: marshal request: %w", err) - } - - // 创建带超时的 context - ctx, cancel := context.WithTimeout(ctx, o.timeout) - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, o.endpoint+"/chat/completions", bytes.NewReader(payload)) - if err != nil { - cancel() - return nil, fmt.Errorf("llm: create request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") - httpReq.Header.Set("Authorization", "Bearer "+o.apiKey) - - resp, err := o.client.Do(httpReq) - if err != nil { - cancel() - return nil, fmt.Errorf("llm: send request: %w", err) - } - - if resp.StatusCode != http.StatusOK { - cancel() - bodyBytes, _ := io.ReadAll(resp.Body) - resp.Body.Close() - return nil, fmt.Errorf("llm: api error (status %d): %s", resp.StatusCode, string(bodyBytes)) - } - - // 启动 goroutine 解析 SSE 流 - ch := make(chan Chunk, 64) - go func() { - defer close(ch) - defer cancel() - defer resp.Body.Close() - - o.parseSSEStream(resp.Body, ch) - }() - - return ch, nil -} - -// parseSSEStream 解析 SSE 流，将 delta 发送到 channel。 -func (o *OpenAIService) parseSSEStream(body io.Reader, ch chan<- Chunk) { - scanner := bufio.NewScanner(body) - scanner.Buffer(make([]byte, 0, 64*1024), 256*1024) - - var fullText strings.Builder - var lastModel string - - for scanner.Scan() { - line := scanner.Text() - - // SSE 格式：data: {...} - if !strings.HasPrefix(line, "data: ") { - continue - } - data := strings.TrimPrefix(line, "data: ") - if data == "[DONE]" { - // 流结束，发送最终 chunk - ch <- Chunk{Delta: "", Done: true, Model: lastModel} - return - } - - var delta streamDelta - if err := json.Unmarshal([]byte(data), &delta); err != nil { - o.logger.Warnw("llm: unmarshal delta failed", "error", err, "data", data) - continue - } - - if delta.Model != "" { - lastModel = delta.Model - } - - // 提取增量文本 - if len(delta.Choices) > 0 { - content := delta.Choices[0].Delta.Content - if content != "" { - fullText.WriteString(content) - ch <- Chunk{Delta: content, Done: false, Model: lastModel} - } - - // 某些模型在最后一个 choice 中携带 usage - if delta.Choices[0].FinishReason != nil && delta.Usage != nil { - ch <- Chunk{ - Delta: "", - Done: true, - Model: lastModel, - TokensUsed: &TokenUsage{ - Prompt: delta.Usage.PromptTokens, - Completion: delta.Usage.CompletionTokens, - Total: delta.Usage.TotalTokens, - }, - } - return - } - } - } - - // scanner 结束但没收到 [DONE] - if err := scanner.Err(); err != nil { - o.logger.Warnw("llm: scan error", "error", err) - } - ch <- Chunk{Delta: "", Done: true, Model: lastModel} -} - -// buildMessages 构建 OpenAI Chat API 的 messages 数组。 -func (o *OpenAIService) buildMessages(req Request) []chatMessage { - var messages []chatMessage - - // System prompt（情景覆盖优先） - messages = append(messages, chatMessage{ - Role: "system", - Content: []contentPart{{Type: "text", Text: BuildSystemPrompt(req.Language, "", req.SystemPrompt)}}, - }) - - // 历史消息 - for _, msg := range req.History { - messages = append(messages, chatMessage{ - Role: msg.Role, - Content: []contentPart{{Type: "text", Text: msg.Content}}, - }) - } - - // 当前用户消息（图像 + 文本） - var parts []contentPart - if len(req.Image) > 0 { - b64 := base64.StdEncoding.EncodeToString(req.Image) - parts = append(parts, contentPart{ - Type: "image_url", - ImageURL: &imageURL{URL: "data:image/jpeg;base64," + b64}, - }) - } - parts = append(parts, contentPart{Type: "text", Text: req.Text}) - messages = append(messages, chatMessage{Role: "user", Content: parts}) - - return messages -} diff --git a/backend/internal/ai/llm/openai_test.go b/backend/internal/ai/llm/openai_test.go deleted file mode 100644 index f75cbb3..0000000 --- a/backend/internal/ai/llm/openai_test.go +++ /dev/null @@ -1,251 +0,0 @@ -package llm - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "go.uber.org/zap" - - "github.com/hhs/camtalk/internal/models" -) - -// mockLLMServer 创建模拟 OpenAI SSE 流式响应的 HTTP 服务器。 -func mockLLMServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { - t.Helper() - return httptest.NewServer(handler) -} - -func TestOpenAIService_ChatStream_Success(t *testing.T) { - srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { - // 验证请求 - if r.Method != http.MethodPost { - t.Errorf("method = %s, want POST", r.Method) - } - if !strings.Contains(r.URL.Path, "/chat/completions") { - t.Errorf("path = %s, should contain /chat/completions", r.URL.Path) - } - auth := r.Header.Get("Authorization") - if auth != "Bearer test-key" { - t.Errorf("Authorization = %q, want %q", auth, "Bearer test-key") - } - - w.Header().Set("Content-Type", "text/event-stream") - flusher, ok := w.(http.Flusher) - if !ok { - t.Fatal("ResponseWriter does not support Flusher") - } - - // 发送几个 delta - deltas := []string{"你好", "世界", "！"} - for _, d := range deltas { - fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"%s\"}}],\"model\":\"gpt-4o\"}\n\n", d) - flusher.Flush() - } - - // 发送 [DONE] - fmt.Fprintf(w, "data: [DONE]\n\n") - flusher.Flush() - }) - defer srv.Close() - - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) - - ch, err := svc.ChatStream(context.Background(), Request{ - Text: "这是什么？", - Language: "zh-CN", - }) - if err != nil { - t.Fatalf("ChatStream() error: %v", err) - } - - var chunks []Chunk - for c := range ch { - chunks = append(chunks, c) - } - - // 应该有 3 个文本 chunk + 1 个 Done chunk - if len(chunks) != 4 { - t.Fatalf("got %d chunks, want 4", len(chunks)) - } - - // 验证文本内容 - if chunks[0].Delta != "你好" { - t.Errorf("chunk[0].Delta = %q, want %q", chunks[0].Delta, "你好") - } - if chunks[1].Delta != "世界" { - t.Errorf("chunk[1].Delta = %q, want %q", chunks[1].Delta, "世界") - } - - // 验证最后一个 chunk 是 Done - last := chunks[len(chunks)-1] - if !last.Done { - t.Error("last chunk should be Done") - } - if last.Model != "gpt-4o" { - t.Errorf("last chunk Model = %q, want %q", last.Model, "gpt-4o") - } -} - -func TestOpenAIService_ChatStream_WithImage(t *testing.T) { - srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/event-stream") - fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}],\"model\":\"gpt-4o\"}\n\n") - fmt.Fprintf(w, "data: [DONE]\n\n") - }) - defer srv.Close() - - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) - - ch, err := svc.ChatStream(context.Background(), Request{ - Image: []byte("fake-jpeg-data"), - Text: "描述图片", - Language: "zh-CN", - }) - if err != nil { - t.Fatalf("ChatStream() error: %v", err) - } - - // 消费 channel - for range ch { - } -} - -func TestOpenAIService_ChatStream_WithHistory(t *testing.T) { - srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/event-stream") - fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}],\"model\":\"gpt-4o\"}\n\n") - fmt.Fprintf(w, "data: [DONE]\n\n") - }) - defer srv.Close() - - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) - - ch, err := svc.ChatStream(context.Background(), Request{ - Text: "继续", - Language: "zh-CN", - History: []models.Message{ - {Role: "user", Content: "你好"}, - {Role: "assistant", Content: "你好！有什么可以帮助你的吗？"}, - }, - }) - if err != nil { - t.Fatalf("ChatStream() error: %v", err) - } - - for range ch { - } -} - -func TestOpenAIService_ChatStream_APIError(t *testing.T) { - srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusUnauthorized) - fmt.Fprintf(w, `{"error":{"message":"Invalid API key"}}`) - }) - defer srv.Close() - - svc := NewOpenAIService("bad-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) - - _, err := svc.ChatStream(context.Background(), Request{ - Text: "test", - }) - if err == nil { - t.Fatal("ChatStream() should return error for 401") - } - if !strings.Contains(err.Error(), "401") { - t.Errorf("error should mention 401, got: %v", err) - } -} - -func TestOpenAIService_ChatStream_Timeout(t *testing.T) { - srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { - // 模拟慢响应 - time.Sleep(5 * time.Second) - w.Header().Set("Content-Type", "text/event-stream") - fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"late\"}}]}\n\n") - fmt.Fprintf(w, "data: [DONE]\n\n") - }) - defer srv.Close() - - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 1, 60, zap.NewNop().Sugar()) // 1s timeout - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - ch, err := svc.ChatStream(ctx, Request{Text: "test"}) - if err != nil { - // 超时可能在建立连接时或读取时发生 - return - } - - // 如果连接成功，消费 channel 应该超时 - var gotContent bool - for c := range ch { - if c.Delta != "" { - gotContent = true - } - } - if gotContent { - t.Error("should not receive content before timeout") - } -} - -func TestOpenAIService_ChatStream_UsageInResponse(t *testing.T) { - srv := mockLLMServer(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/event-stream") - // 带 usage 的最后一个 chunk - fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"},\"finish_reason\":\"stop\"}],\"model\":\"gpt-4o\",\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":5,\"total_tokens\":15}}\n\n") - fmt.Fprintf(w, "data: [DONE]\n\n") - }) - defer srv.Close() - - svc := NewOpenAIService("test-key", "gpt-4o", srv.URL, 10, 60, zap.NewNop().Sugar()) - - ch, err := svc.ChatStream(context.Background(), Request{Text: "test"}) - if err != nil { - t.Fatalf("ChatStream() error: %v", err) - } - - var last Chunk - for c := range ch { - last = c - } - - if !last.Done { - t.Error("last chunk should be Done") - } - if last.TokensUsed == nil { - t.Fatal("last chunk should have TokensUsed") - } - if last.TokensUsed.Total != 15 { - t.Errorf("TokensUsed.Total = %d, want 15", last.TokensUsed.Total) - } -} - -func TestBuildSystemPrompt(t *testing.T) { - tests := []struct { - name string - language string - detailLevel string - wantContain string - }{ - {"chinese default", "zh-CN", "", "视觉助手"}, - {"chinese high", "zh-CN", "high", "更详细"}, - {"english default", "en", "", "visual assistant"}, - {"english high", "en", "high", "detailed"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := BuildSystemPrompt(tt.language, tt.detailLevel, "") - if !strings.Contains(got, tt.wantContain) { - t.Errorf("BuildSystemPrompt(%q, %q, \"\") should contain %q", tt.language, tt.detailLevel, tt.wantContain) - } - }) - } -} diff --git a/backend/internal/eino/graph_test.go b/backend/internal/eino/graph_test.go new file mode 100644 index 0000000..ae7a692 --- /dev/null +++ b/backend/internal/eino/graph_test.go @@ -0,0 +1,235 @@ +package eino + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/hhs/camtalk/internal/ai/stt" + "github.com/hhs/camtalk/internal/ai/tts" + "github.com/hhs/camtalk/internal/models" + "github.com/hhs/camtalk/internal/orchestrator" +) + +// --- Mock STT Service --- + +type mockSTTService struct { + mock.Mock +} + +func (m *mockSTTService) Recognize(ctx context.Context, audio []byte, opts stt.Options) (string, error) { + args := m.Called(ctx, audio, opts) + return args.String(0), args.Error(1) +} + +// --- Mock TTS Service --- + +type mockTTSService struct { + mock.Mock +} + +func (m *mockTTSService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts tts.Options) (<-chan tts.Chunk, error) { + args := m.Called(ctx, textStream, opts) + return args.Get(0).(<-chan tts.Chunk), args.Error(1) +} + +// --- Mock Sender --- + +type mockSender struct { + mock.Mock + STTResults []models.WsSTTResult + LLMChunks []models.WsLLMChunk + LLMDones []models.WsLLMDone + TTSAudios []models.WsTTSAudio + Errors []models.WsError +} + +func (m *mockSender) SendSTTResult(result models.WsSTTResult) error { + m.STTResults = append(m.STTResults, result) + return m.Called(result).Error(0) +} + +func (m *mockSender) SendLLMChunk(chunk models.WsLLMChunk) error { + m.LLMChunks = append(m.LLMChunks, chunk) + return m.Called(chunk).Error(0) +} + +func (m *mockSender) SendLLMDone(done models.WsLLMDone) error { + m.LLMDones = append(m.LLMDones, done) + return m.Called(done).Error(0) +} + +func (m *mockSender) SendTTSAudio(audio models.WsTTSAudio) error { + m.TTSAudios = append(m.TTSAudios, audio) + return m.Called(audio).Error(0) +} + +func (m *mockSender) SendError(err models.WsError) error { + m.Errors = append(m.Errors, err) + return m.Called(err).Error(0) +} + +// --- Tests --- + +func TestDetectImageMimeType(t *testing.T) { + tests := []struct { + name string + data []byte + expected string + }{ + {"JPEG", []byte{0xFF, 0xD8, 0xFF, 0xE0}, "image/jpeg"}, + {"PNG", []byte{0x89, 0x50, 0x4E, 0x47}, "image/png"}, + {"GIF", []byte{0x47, 0x49, 0x46, 0x38}, "image/gif"}, + {"WebP", []byte{0x52, 0x49, 0x46, 0x46}, "image/webp"}, + {"Unknown", []byte{0x00, 0x00, 0x00}, "image/jpeg"}, + {"Short", []byte{0xFF}, "image/jpeg"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := detectImageMimeType(tt.data) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestBuildPipelineInput(t *testing.T) { + req := models.WsQuery{ + Text: "你好", + RequestID: "req-1", + } + sess := &models.Session{ + Config: models.SessionConfig{ + Language: "zh-CN", + Scenario: "free_chat", + TTSEnabled: true, + }, + } + + input := buildPipelineInput(req, "sess-1", sess, nil, nil) + require.Equal(t, "你好", input.Text) + require.Equal(t, "sess-1", input.SessionID) + require.Equal(t, "req-1", input.RequestID) + require.Equal(t, "zh-CN", input.Language) + require.Equal(t, "free_chat", input.Scenario) + require.True(t, input.TTSEnabled) +} + +func TestBuildPipelineInput_WithAudioData(t *testing.T) { + req := models.WsQuery{ + Audio: "base64audio", + RequestID: "req-2", + } + sess := &models.Session{ + Config: models.SessionConfig{ + Language: "en", + Scenario: "free_chat", + TTSEnabled: false, + }, + } + + audioData := []byte("fake-audio-bytes") + imageData := []byte("fake-image-bytes") + + input := buildPipelineInput(req, "sess-2", sess, audioData, imageData) + require.Equal(t, audioData, input.AudioData) + require.Equal(t, imageData, input.ImageData) + require.False(t, input.TTSEnabled) + require.Equal(t, "en", input.Language) +} + +func TestPipelineState_AppendAndGet(t *testing.T) { + state := genLocalState(context.Background()) + + state.AppendText("Hello ") + state.AppendText("World") + + require.Equal(t, "Hello World", state.GetFullResponse()) +} + +func TestPipelineState_ConcurrentAccess(t *testing.T) { + state := genLocalState(context.Background()) + + done := make(chan struct{}) + go func() { + for i := 0; i < 100; i++ { + state.AppendText("a") + } + close(done) + }() + + for i := 0; i < 100; i++ { + _ = state.GetFullResponse() + } + + <-done + require.Equal(t, 100, len(state.GetFullResponse())) +} + +func TestContextInjection(t *testing.T) { + ctx := context.Background() + + sender := &mockSender{} + ctx = WithSender(ctx, sender) + ctx = WithRequestID(ctx, "req-123") + ctx = WithSessionID(ctx, "sess-456") + ctx = WithStartTime(ctx, time.Now()) + ctx = WithPipelineState(ctx, genLocalState(ctx)) + + require.NotNil(t, senderFromCtx(ctx)) + require.Equal(t, "req-123", requestIDFromCtx(ctx)) + require.NotNil(t, stateFromCtx(ctx)) +} + +func TestLatencyFromCtx(t *testing.T) { + ctx := context.Background() + + // No start time set + require.Equal(t, int64(0), latencyFromCtx(ctx)) + + // With start time + start := time.Now().Add(-100 * time.Millisecond) + ctx = WithStartTime(ctx, start) + latency := latencyFromCtx(ctx) + require.Greater(t, latency, int64(0)) + require.Less(t, latency, int64(1000)) // should be < 1 second +} + +func TestEinoOrchestrator_ImplementsInterface(t *testing.T) { + // Compile-time check that EinoOrchestrator implements orchestrator.Orchestrator + var _ orchestrator.Orchestrator = (*EinoOrchestrator)(nil) +} + +func TestNewSTTLambda_ReturnsNonNil(t *testing.T) { + mockSTT := &mockSTTService{} + lambda := NewSTTLambda(mockSTT) + require.NotNil(t, lambda) +} + +func TestNewHistoryLambda_ReturnsNonNil(t *testing.T) { + fetcher := func(ctx context.Context, sessionID string, limit int) ([]models.Message, error) { + return nil, nil + } + lambda := NewHistoryLambda(fetcher, 10) + require.NotNil(t, lambda) +} + +func TestNewSplitterLambda_ReturnsNonNil(t *testing.T) { + lambda := NewSplitterLambda() + require.NotNil(t, lambda) +} + +func TestNewTTSLambda_ReturnsNonNil(t *testing.T) { + mockTTS := &mockTTSService{} + lambda := NewTTSLambda(mockTTS, "alloy", 1.0, "mp3", 24000) + require.NotNil(t, lambda) +} + +func TestNewDoneLambda_ReturnsNonNil(t *testing.T) { + lambda := NewDoneLambda("test-model") + require.NotNil(t, lambda) +} diff --git a/backend/internal/orchestrator/pipeline.go b/backend/internal/orchestrator/pipeline.go deleted file mode 100644 index 5ed94e6..0000000 --- a/backend/internal/orchestrator/pipeline.go +++ /dev/null @@ -1,403 +0,0 @@ -package orchestrator - -import ( - "context" - "encoding/base64" - "strings" - "sync" - "time" - "unicode/utf8" - - "github.com/hhs/camtalk/internal/ai/llm" - "github.com/hhs/camtalk/internal/ai/stt" - "github.com/hhs/camtalk/internal/ai/tts" - "github.com/hhs/camtalk/internal/config" - "github.com/hhs/camtalk/internal/logger" - "github.com/hhs/camtalk/internal/models" - "github.com/hhs/camtalk/internal/session" -) - -// Pipeline 实现 Orchestrator 接口，管理 STT → LLM → TTS 流式管道。 -type Pipeline struct { - sttService stt.Service - llmService llm.Service - ttsService tts.Service - sessionMgr session.Manager - model string // LLM 模型名，用于 llm_done 上报 - ttsVoice string // TTS 音色 - ttsSpeed float64 // TTS 语速 - ttsOutputFmt string // TTS 输出格式 - ttsSampleRate int // TTS 输出采样率 -} - -// New 创建 Pipeline 实例。 -func New( - sttService stt.Service, - llmService llm.Service, - ttsService tts.Service, - sessionMgr session.Manager, - cfg *config.Config, -) *Pipeline { - return &Pipeline{ - sttService: sttService, - llmService: llmService, - ttsService: ttsService, - sessionMgr: sessionMgr, - model: cfg.AI.LLM.Model, - ttsVoice: cfg.AI.TTS.Voice, - ttsSpeed: cfg.AI.TTS.Speed, - ttsOutputFmt: cfg.AI.TTS.OutputFormat, - ttsSampleRate: cfg.AI.TTS.SampleRate, - } -} - -// ProcessQuery 实现 Orchestrator 接口。 -func (p *Pipeline) ProcessQuery( - ctx context.Context, - sessionID string, - req models.WsQuery, - history []models.Message, - sender Sender, -) error { - log := logger.Log - startTime := time.Now() - - // 解码音频数据（文本输入模式可跳过） - var audio []byte - if req.Text == "" && req.Audio != "" { - var err error - audio, err = base64.StdEncoding.DecodeString(req.Audio) - if err != nil { - log.Errorw("音频解码失败", "error", err) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "INVALID_MESSAGE", - Message: "音频数据解码失败", - }) - return err - } - } - - // 解码图片数据（可选） - var image []byte - if req.Image != "" { - var err error - image, err = base64.StdEncoding.DecodeString(req.Image) - if err != nil { - log.Errorw("图片解码失败", "error", err) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "INVALID_MESSAGE", - Message: "图片数据解码失败", - }) - return err - } - } - - // 设置活跃请求 - if err := p.sessionMgr.SetActiveRequest(ctx, sessionID, req.RequestID); err != nil { - log.Errorw("设置活跃请求失败", "error", err) - } - defer p.sessionMgr.ClearActiveRequest(ctx, sessionID) - - // 获取会话配置 - sess, err := p.sessionMgr.Get(ctx, sessionID) - if err != nil { - log.Errorw("获取会话失败", "error", err) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "SESSION_NOT_FOUND", - Message: "会话不存在", - }) - return err - } - - // Step 1: 获取用户文本（语音识别或直接使用输入文本） - var userText string - if req.Text != "" { - // 文本输入模式：跳过 STT，直接使用用户输入的文本 - log.Infow("使用文本输入", "request_id", req.RequestID, "text", req.Text) - userText = req.Text - - // 发送 stt_result 以保持前端消息流一致性 - if err := sender.SendSTTResult(models.WsSTTResult{ - Type: "stt_result", - RequestID: req.RequestID, - Text: userText, - IsFinal: true, - }); err != nil { - log.Errorw("发送 STT 结果失败", "error", err) - } - } else { - // 语音模式：执行 STT 语音识别 - log.Infow("开始语音识别", "request_id", req.RequestID, "audio_bytes", len(audio)) - sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{ - Encoding: "pcm_s16le", - SampleRate: 16000, - Language: sess.Config.Language, - }) - if err != nil { - log.Errorw("语音识别失败", "error", err, "audio_bytes", len(audio)) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "STT_ERROR", - Message: "语音识别失败: " + err.Error(), - }) - return err - } - userText = sttResult - - // STT 返回空文本：未识别到语音，发送结果后直接返回（不调 LLM） - if strings.TrimSpace(userText) == "" { - log.Infow("语音识别结果为空", "request_id", req.RequestID) - userText = "（未识别到语音）" - if err := sender.SendSTTResult(models.WsSTTResult{ - Type: "stt_result", - RequestID: req.RequestID, - Text: userText, - IsFinal: true, - }); err != nil { - log.Errorw("发送 STT 结果失败", "error", err) - } - // 发送空的 llm_done 以结束本轮处理 - latency := time.Since(startTime).Milliseconds() - _ = sender.SendLLMDone(models.WsLLMDone{ - Type: "llm_done", - RequestID: req.RequestID, - FullText: "", - Model: p.model, - LatencyMs: latency, - }) - return nil - } - - // 发送 STT 结果 - if err := sender.SendSTTResult(models.WsSTTResult{ - Type: "stt_result", - RequestID: req.RequestID, - Text: userText, - IsFinal: true, - }); err != nil { - log.Errorw("发送 STT 结果失败", "error", err) - } - } - - // 追加用户消息到历史 - p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{ - Role: "user", - Content: userText, - }) - - // Step 2+3: LLM 流式推理 + TTS 并行合成 - log.Infow("开始 LLM 推理", "request_id", req.RequestID, "scenario", sess.Config.Scenario) - llmReq := llm.Request{ - Image: image, - Text: userText, - History: history, - Language: sess.Config.Language, - SystemPrompt: llm.GetScenarioPrompt(sess.Config.Scenario, sess.Config.Language), - } - - llmStream, err := p.llmService.ChatStream(ctx, llmReq) - if err != nil { - log.Errorw("LLM 流式推理启动失败", "error", err) - sender.SendError(models.WsError{ - Type: "error", - RequestID: req.RequestID, - Code: "LLM_ERROR", - Message: "LLM 推理失败", - }) - return err - } - - // 创建句子切分器 - sentenceCh := make(chan string, 4) - splitter := NewSplitter(sentenceCh) - - // 并行：LLM 消费 + TTS 合成 - var wg sync.WaitGroup - var fullText string - var ttsErr error - - // goroutine 1: 消费 LLM token + 句子切分 - var tokenUsage *llm.TokenUsage - wg.Add(1) - go func() { - defer wg.Done() - defer close(sentenceCh) - fullText, tokenUsage = p.consumeLLMStream(ctx, llmStream, req.RequestID, sender, splitter) - }() - - // goroutine 2: TTS 合成（如果启用） - if sess.Config.TTSEnabled { - wg.Add(1) - go func() { - defer wg.Done() - log.Infow("开始 TTS 合成", "request_id", req.RequestID) - ttsErr = p.synthesizeTTS(ctx, sentenceCh, req.RequestID, sender) - }() - } else { - // 如果 TTS 未启用，需要消费 sentenceCh 防止阻塞 - go func() { - for range sentenceCh { - } - }() - } - - // 等待所有 goroutine 完成 - wg.Wait() - - // TTS 失败静默跳过 - if ttsErr != nil { - log.Warnw("TTS 合成失败（已跳过）", "error", ttsErr) - } - - // 追加助手消息到历史 - p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{ - Role: "assistant", - Content: fullText, - }) - - // 发送 llm_done - latency := time.Since(startTime).Milliseconds() - done := models.WsLLMDone{ - Type: "llm_done", - RequestID: req.RequestID, - FullText: fullText, - Model: p.model, - LatencyMs: latency, - } - if tokenUsage != nil { - done.TokensUsed = struct { - Prompt int `json:"prompt"` - Completion int `json:"completion"` - Total int `json:"total"` - }{ - Prompt: tokenUsage.Prompt, - Completion: tokenUsage.Completion, - Total: tokenUsage.Total, - } - } - if err := sender.SendLLMDone(done); err != nil { - log.Errorw("发送 llm_done 失败", "error", err) - } - - log.Infow("查询处理完成", - "request_id", req.RequestID, - "latency_ms", latency, - "text_length", utf8.RuneCountInString(fullText), - ) - - return nil -} - -// consumeLLMStream 消费 LLM 流式输出，发送 llm_chunk 并进行句子切分。 -// 返回完整文本和 token 用量。 -func (p *Pipeline) consumeLLMStream( - ctx context.Context, - stream <-chan llm.Chunk, - requestID string, - sender Sender, - splitter *Splitter, -) (string, *llm.TokenUsage) { - log := logger.Log - var fullText strings.Builder - var tokenUsage *llm.TokenUsage - - for chunk := range stream { - // 检查上下文是否已取消 - select { - case <-ctx.Done(): - log.Infow("LLM 流被中断", "request_id", requestID) - return fullText.String(), tokenUsage - default: - } - - if chunk.Done { - // 流结束，记录 token 用量 - if chunk.TokensUsed != nil { - tokenUsage = chunk.TokensUsed - log.Infow("LLM 用量统计", - "request_id", requestID, - "prompt_tokens", tokenUsage.Prompt, - "completion_tokens", tokenUsage.Completion, - "total_tokens", tokenUsage.Total, - ) - } - break - } - - // 累积全文 - fullText.WriteString(chunk.Delta) - - // 发送 llm_chunk - if err := sender.SendLLMChunk(models.WsLLMChunk{ - Type: "llm_chunk", - RequestID: requestID, - Delta: chunk.Delta, - Role: "assistant", - }); err != nil { - log.Errorw("发送 llm_chunk 失败", "error", err) - } - - // 句子切分 - splitter.Feed(chunk.Delta) - } - - // 刷新切分器中的剩余文本 - splitter.Flush() - - return fullText.String(), tokenUsage -} - -// synthesizeTTS 从句子 channel 读取文本，进行 TTS 合成并发送音频。 -func (p *Pipeline) synthesizeTTS( - ctx context.Context, - sentenceCh <-chan string, - requestID string, - sender Sender, -) error { - log := logger.Log - - ttsStream, err := p.ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{ - Voice: p.ttsVoice, - Speed: p.ttsSpeed, - OutputFmt: p.ttsOutputFmt, - SampleRate: p.ttsSampleRate, - }) - if err != nil { - log.Errorw("TTS 合成启动失败", "error", err) - return err - } - - // 消费 TTS 音频流 - for chunk := range ttsStream { - // 检查上下文是否已取消 - select { - case <-ctx.Done(): - log.Infow("TTS 流被中断", "request_id", requestID) - return ctx.Err() - default: - } - - // Base64 编码音频数据 - audioBase64 := base64.StdEncoding.EncodeToString(chunk.Audio) - - if err := sender.SendTTSAudio(models.WsTTSAudio{ - Type: "tts_audio", - RequestID: requestID, - Audio: audioBase64, - MimeType: "audio/mp3", - IsLast: chunk.IsLast, - Final: chunk.Final, - }); err != nil { - log.Errorw("发送 tts_audio 失败", "error", err) - } - } - - return nil -} diff --git a/backend/internal/orchestrator/pipeline_test.go b/backend/internal/orchestrator/pipeline_test.go deleted file mode 100644 index 9000e96..0000000 --- a/backend/internal/orchestrator/pipeline_test.go +++ /dev/null @@ -1,713 +0,0 @@ -package orchestrator - -import ( - "context" - "encoding/base64" - "errors" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - "github.com/hhs/camtalk/internal/ai/llm" - "github.com/hhs/camtalk/internal/ai/stt" - "github.com/hhs/camtalk/internal/ai/tts" - "github.com/hhs/camtalk/internal/config" - "github.com/hhs/camtalk/internal/logger" - "github.com/hhs/camtalk/internal/models" - "github.com/hhs/camtalk/internal/session" -) - -func init() { - logger.Init("debug", "console") -} - -// MockSTTService mock STT 服务 -type MockSTTService struct { - mock.Mock -} - -func (m *MockSTTService) Recognize(ctx context.Context, audio []byte, opts stt.Options) (string, error) { - args := m.Called(ctx, audio, opts) - return args.String(0), args.Error(1) -} - -// MockLLMService mock LLM 服务 -type MockLLMService struct { - mock.Mock -} - -func (m *MockLLMService) ChatStream(ctx context.Context, req llm.Request) (<-chan llm.Chunk, error) { - args := m.Called(ctx, req) - if args.Get(0) == nil { - return nil, args.Error(1) - } - return args.Get(0).(<-chan llm.Chunk), args.Error(1) -} - -// MockTTSService mock TTS 服务 -type MockTTSService struct { - mock.Mock -} - -func (m *MockTTSService) SynthesizeStream(ctx context.Context, textStream <-chan string, opts tts.Options) (<-chan tts.Chunk, error) { - args := m.Called(ctx, textStream, opts) - if args.Get(0) == nil { - return nil, args.Error(1) - } - return args.Get(0).(<-chan tts.Chunk), args.Error(1) -} - -// MockSessionManager mock 会话管理器 -type MockSessionManager struct { - mock.Mock -} - -func (m *MockSessionManager) Create(ctx context.Context, userID string, config models.SessionConfig) (string, error) { - args := m.Called(ctx, userID, config) - return args.String(0), args.Error(1) -} - -func (m *MockSessionManager) UpdateTitle(ctx context.Context, sessionID string, title string) error { - args := m.Called(ctx, sessionID, title) - return args.Error(0) -} - -func (m *MockSessionManager) ListByUser(ctx context.Context, userID string, page, size int) ([]session.ConversationSummary, int, error) { - args := m.Called(ctx, userID, page, size) - return args.Get(0).([]session.ConversationSummary), args.Int(1), args.Error(2) -} - -func (m *MockSessionManager) Get(ctx context.Context, sessionID string) (*models.Session, error) { - args := m.Called(ctx, sessionID) - if args.Get(0) == nil { - return nil, args.Error(1) - } - return args.Get(0).(*models.Session), args.Error(1) -} - -func (m *MockSessionManager) UpdateConfig(ctx context.Context, sessionID string, patch models.SessionConfigPatch) error { - args := m.Called(ctx, sessionID, patch) - return args.Error(0) -} - -func (m *MockSessionManager) GetHistory(ctx context.Context, sessionID string, limit int) ([]models.Message, error) { - args := m.Called(ctx, sessionID, limit) - return args.Get(0).([]models.Message), args.Error(1) -} - -func (m *MockSessionManager) AppendMessage(ctx context.Context, sessionID string, msg models.Message) error { - args := m.Called(ctx, sessionID, msg) - return args.Error(0) -} - -func (m *MockSessionManager) SetActiveRequest(ctx context.Context, sessionID string, requestID string) error { - args := m.Called(ctx, sessionID, requestID) - return args.Error(0) -} - -func (m *MockSessionManager) GetActiveRequestID(ctx context.Context, sessionID string) (string, error) { - args := m.Called(ctx, sessionID) - return args.String(0), args.Error(1) -} - -func (m *MockSessionManager) ClearActiveRequest(ctx context.Context, sessionID string) error { - args := m.Called(ctx, sessionID) - return args.Error(0) -} - -func (m *MockSessionManager) Touch(ctx context.Context, sessionID string) error { - args := m.Called(ctx, sessionID) - return args.Error(0) -} - -func (m *MockSessionManager) Destroy(ctx context.Context, sessionID string) error { - args := m.Called(ctx, sessionID) - return args.Error(0) -} - -func (m *MockSessionManager) ActiveCount() int { - args := m.Called() - return args.Int(0) -} - -// MockSender mock WebSocket 发送器 -type MockSender struct { - mock.Mock - STTResults []models.WsSTTResult - LLMChunks []models.WsLLMChunk - LLMDones []models.WsLLMDone - TTSAudios []models.WsTTSAudio - Errors []models.WsError -} - -func NewMockSender() *MockSender { - return &MockSender{ - STTResults: make([]models.WsSTTResult, 0), - LLMChunks: make([]models.WsLLMChunk, 0), - LLMDones: make([]models.WsLLMDone, 0), - TTSAudios: make([]models.WsTTSAudio, 0), - Errors: make([]models.WsError, 0), - } -} - -func (m *MockSender) SendSTTResult(result models.WsSTTResult) error { - m.STTResults = append(m.STTResults, result) - args := m.Called(result) - return args.Error(0) -} - -func (m *MockSender) SendLLMChunk(chunk models.WsLLMChunk) error { - m.LLMChunks = append(m.LLMChunks, chunk) - args := m.Called(chunk) - return args.Error(0) -} - -func (m *MockSender) SendLLMDone(done models.WsLLMDone) error { - m.LLMDones = append(m.LLMDones, done) - args := m.Called(done) - return args.Error(0) -} - -func (m *MockSender) SendTTSAudio(audio models.WsTTSAudio) error { - m.TTSAudios = append(m.TTSAudios, audio) - args := m.Called(audio) - return args.Error(0) -} - -func (m *MockSender) SendError(err models.WsError) error { - m.Errors = append(m.Errors, err) - args := m.Called(err) - return args.Error(0) -} - -// 辅助函数：创建 LLM 流式响应 -func createLLMStream(chunks []llm.Chunk) <-chan llm.Chunk { - ch := make(chan llm.Chunk, len(chunks)) - for _, chunk := range chunks { - ch <- chunk - } - close(ch) - return ch -} - -// 辅助函数：创建 TTS 流式响应 -func createTTSStream(chunks []tts.Chunk) <-chan tts.Chunk { - ch := make(chan tts.Chunk, len(chunks)) - for _, chunk := range chunks { - ch <- chunk - } - close(ch) - return ch -} - -// TestProcessQuery_Success 测试完整流程 -func TestProcessQuery_Success(t *testing.T) { - // 准备测试数据 - audioData := []byte("test audio") - imageData := []byte("test image") - audioBase64 := base64.StdEncoding.EncodeToString(audioData) - imageBase64 := base64.StdEncoding.EncodeToString(imageData) - - req := models.WsQuery{ - Type: "query", - RequestID: "req-123", - Image: imageBase64, - Audio: audioBase64, - } - - session := &models.Session{ - ID: "session-123", - Config: models.SessionConfig{ - TTSEnabled: true, - DetailLevel: "low", - Language: "zh-CN", - }, - } - - // 创建 mock - mockSTT := new(MockSTTService) - mockLLM := new(MockLLMService) - mockTTS := new(MockTTSService) - mockSession := new(MockSessionManager) - mockSender := NewMockSender() - - // 设置 mock 期望 - mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) - mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) - mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) - mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) - - mockSTT.On("Recognize", mock.Anything, audioData, stt.Options{ - Encoding: "pcm_s16le", - SampleRate: 16000, - Language: "zh-CN", - }).Return("你好，世界", nil) - - mockSender.On("SendSTTResult", mock.Anything).Return(nil) - - llmChunks := []llm.Chunk{ - {Delta: "你好"}, - {Delta: "，世界！"}, - {Done: true, TokensUsed: &llm.TokenUsage{Prompt: 10, Completion: 5, Total: 15}}, - } - mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil) - - mockSender.On("SendLLMChunk", mock.Anything).Return(nil) - mockSender.On("SendLLMDone", mock.Anything).Return(nil) - - ttsChunks := []tts.Chunk{ - {Audio: []byte("audio1"), IsLast: false}, - {Audio: []byte("audio2"), IsLast: true}, - } - mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return(createTTSStream(ttsChunks), nil) - - mockSender.On("SendTTSAudio", mock.Anything).Return(nil) - - // 创建 Pipeline - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ - AI: config.AIConfig{ - LLM: config.LLMConfig{Model: "gpt-4o"}, - TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, - }, - }) - - // 执行 - ctx := context.Background() - err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) - - // 验证 - assert.NoError(t, err) - assert.Len(t, mockSender.STTResults, 1) - assert.Equal(t, "你好，世界", mockSender.STTResults[0].Text) - assert.Len(t, mockSender.LLMChunks, 2) - assert.Len(t, mockSender.LLMDones, 1) - assert.Len(t, mockSender.TTSAudios, 2) - - mockSTT.AssertExpectations(t) - mockLLM.AssertExpectations(t) - mockTTS.AssertExpectations(t) - mockSession.AssertExpectations(t) -} - -// TestProcessQuery_STTError 测试 STT 失败降级 -func TestProcessQuery_STTError(t *testing.T) { - audioData := []byte("test audio") - audioBase64 := base64.StdEncoding.EncodeToString(audioData) - - req := models.WsQuery{ - Type: "query", - RequestID: "req-123", - Audio: audioBase64, - } - - mockSTT := new(MockSTTService) - mockLLM := new(MockLLMService) - mockTTS := new(MockTTSService) - mockSession := new(MockSessionManager) - mockSender := NewMockSender() - - mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) - mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) - mockSession.On("Get", mock.Anything, "session-123").Return(&models.Session{ - ID: "session-123", - Config: models.SessionConfig{ - Language: "zh-CN", - }, - }, nil) - - mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything). - Return("", errors.New("STT service unavailable")) - - mockSender.On("SendError", mock.Anything).Return(nil) - - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ - AI: config.AIConfig{ - LLM: config.LLMConfig{Model: "gpt-4o"}, - TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, - }, - }) - - ctx := context.Background() - err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) - - assert.Error(t, err) - assert.Len(t, mockSender.Errors, 1) - assert.Equal(t, "STT_ERROR", mockSender.Errors[0].Code) - - mockSTT.AssertExpectations(t) - mockLLM.AssertNotCalled(t, "ChatStream") - mockTTS.AssertNotCalled(t, "SynthesizeStream") -} - -// TestProcessQuery_LLMError 测试 LLM 失败降级 -func TestProcessQuery_LLMError(t *testing.T) { - audioData := []byte("test audio") - audioBase64 := base64.StdEncoding.EncodeToString(audioData) - - req := models.WsQuery{ - Type: "query", - RequestID: "req-123", - Audio: audioBase64, - } - - session := &models.Session{ - ID: "session-123", - Config: models.SessionConfig{ - TTSEnabled: true, - Language: "zh-CN", - }, - } - - mockSTT := new(MockSTTService) - mockLLM := new(MockLLMService) - mockTTS := new(MockTTSService) - mockSession := new(MockSessionManager) - mockSender := NewMockSender() - - mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) - mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) - mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) - mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) - - mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil) - mockSender.On("SendSTTResult", mock.Anything).Return(nil) - - mockLLM.On("ChatStream", mock.Anything, mock.Anything). - Return(nil, errors.New("LLM service unavailable")) - - mockSender.On("SendError", mock.Anything).Return(nil) - - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ - AI: config.AIConfig{ - LLM: config.LLMConfig{Model: "gpt-4o"}, - TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, - }, - }) - - ctx := context.Background() - err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) - - assert.Error(t, err) - assert.Len(t, mockSender.Errors, 1) - assert.Equal(t, "LLM_ERROR", mockSender.Errors[0].Code) - - mockSTT.AssertExpectations(t) - mockLLM.AssertExpectations(t) - mockTTS.AssertNotCalled(t, "SynthesizeStream") -} - -// TestProcessQuery_TTSError 测试 TTS 失败静默跳过 -func TestProcessQuery_TTSError(t *testing.T) { - audioData := []byte("test audio") - audioBase64 := base64.StdEncoding.EncodeToString(audioData) - - req := models.WsQuery{ - Type: "query", - RequestID: "req-123", - Audio: audioBase64, - } - - session := &models.Session{ - ID: "session-123", - Config: models.SessionConfig{ - TTSEnabled: true, - Language: "zh-CN", - }, - } - - mockSTT := new(MockSTTService) - mockLLM := new(MockLLMService) - mockTTS := new(MockTTSService) - mockSession := new(MockSessionManager) - mockSender := NewMockSender() - - mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) - mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) - mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) - mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) - - mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil) - mockSender.On("SendSTTResult", mock.Anything).Return(nil) - - llmChunks := []llm.Chunk{ - {Delta: "你好"}, - {Done: true}, - } - mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil) - mockSender.On("SendLLMChunk", mock.Anything).Return(nil) - mockSender.On("SendLLMDone", mock.Anything).Return(nil) - - mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything). - Return(nil, errors.New("TTS service unavailable")) - - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ - AI: config.AIConfig{ - LLM: config.LLMConfig{Model: "gpt-4o"}, - TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, - }, - }) - - ctx := context.Background() - err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) - - // TTS 失败应该静默跳过，不返回错误 - assert.NoError(t, err) - assert.Len(t, mockSender.LLMDones, 1) - assert.Len(t, mockSender.TTSAudios, 0) - - mockSTT.AssertExpectations(t) - mockLLM.AssertExpectations(t) - mockTTS.AssertExpectations(t) -} - -// TestProcessQuery_ContextCancelled 测试上下文取消（Interrupt） -func TestProcessQuery_ContextCancelled(t *testing.T) { - audioData := []byte("test audio") - audioBase64 := base64.StdEncoding.EncodeToString(audioData) - - req := models.WsQuery{ - Type: "query", - RequestID: "req-123", - Audio: audioBase64, - } - - session := &models.Session{ - ID: "session-123", - Config: models.SessionConfig{ - TTSEnabled: true, - Language: "zh-CN", - }, - } - - mockSTT := new(MockSTTService) - mockLLM := new(MockLLMService) - mockTTS := new(MockTTSService) - mockSession := new(MockSessionManager) - mockSender := NewMockSender() - - mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) - mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) - mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) - mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) - - mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil) - mockSender.On("SendSTTResult", mock.Anything).Return(nil) - - // 创建一个会延迟的 LLM 流，以便我们可以取消上下文 - llmCh := make(chan llm.Chunk) - go func() { - time.Sleep(100 * time.Millisecond) - llmCh <- llm.Chunk{Delta: "你"} - time.Sleep(100 * time.Millisecond) - llmCh <- llm.Chunk{Delta: "好"} - close(llmCh) - }() - - mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return((<-chan llm.Chunk)(llmCh), nil) - mockSender.On("SendLLMChunk", mock.Anything).Return(nil) - mockSender.On("SendLLMDone", mock.Anything).Return(nil) - - // 创建一个会延迟的 TTS 流 - ttsCh := make(chan tts.Chunk) - go func() { - time.Sleep(200 * time.Millisecond) - close(ttsCh) - }() - mockTTS.On("SynthesizeStream", mock.Anything, mock.Anything, mock.Anything).Return((<-chan tts.Chunk)(ttsCh), nil) - - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ - AI: config.AIConfig{ - LLM: config.LLMConfig{Model: "gpt-4o"}, - TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, - }, - }) - - // 创建可取消的上下文 - ctx, cancel := context.WithCancel(context.Background()) - - // 在 50ms 后取消 - go func() { - time.Sleep(50 * time.Millisecond) - cancel() - }() - - err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) - - // 上下文取消后，流程应该正常完成（中断流但不返回错误） - assert.NoError(t, err) - - mockSTT.AssertExpectations(t) -} - -// TestProcessQuery_DisabledTTS 测试 TTS 未启用的情况 -func TestProcessQuery_DisabledTTS(t *testing.T) { - audioData := []byte("test audio") - audioBase64 := base64.StdEncoding.EncodeToString(audioData) - - req := models.WsQuery{ - Type: "query", - RequestID: "req-123", - Audio: audioBase64, - } - - session := &models.Session{ - ID: "session-123", - Config: models.SessionConfig{ - TTSEnabled: false, // TTS 未启用 - Language: "zh-CN", - }, - } - - mockSTT := new(MockSTTService) - mockLLM := new(MockLLMService) - mockTTS := new(MockTTSService) - mockSession := new(MockSessionManager) - mockSender := NewMockSender() - - mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) - mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) - mockSession.On("Get", mock.Anything, "session-123").Return(session, nil) - mockSession.On("AppendMessage", mock.Anything, "session-123", mock.Anything).Return(nil) - - mockSTT.On("Recognize", mock.Anything, audioData, mock.Anything).Return("你好", nil) - mockSender.On("SendSTTResult", mock.Anything).Return(nil) - - llmChunks := []llm.Chunk{ - {Delta: "你好"}, - {Done: true}, - } - mockLLM.On("ChatStream", mock.Anything, mock.Anything).Return(createLLMStream(llmChunks), nil) - mockSender.On("SendLLMChunk", mock.Anything).Return(nil) - mockSender.On("SendLLMDone", mock.Anything).Return(nil) - - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ - AI: config.AIConfig{ - LLM: config.LLMConfig{Model: "gpt-4o"}, - TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, - }, - }) - - ctx := context.Background() - err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) - - assert.NoError(t, err) - assert.Len(t, mockSender.LLMDones, 1) - assert.Len(t, mockSender.TTSAudios, 0) - - // TTS 不应该被调用 - mockTTS.AssertNotCalled(t, "SynthesizeStream") -} - -// TestSplitter 测试句子切分器 -func TestSplitter(t *testing.T) { - ch := make(chan string, 10) - splitter := NewSplitter(ch) - - // 输入包含多个句子的文本 - splitter.Feed("你好。") - splitter.Feed("世界！") - splitter.Feed("这是") - splitter.Feed("一个测试。") - splitter.Flush() - - // 应该有 3 个句子 - assert.Equal(t, 3, len(ch)) - assert.Equal(t, "你好。", <-ch) - assert.Equal(t, "世界！", <-ch) - assert.Equal(t, "这是一个测试。", <-ch) -} - -// TestSplitter_NoDelimiter 测试没有分隔符的情况 -func TestSplitter_NoDelimiter(t *testing.T) { - ch := make(chan string, 10) - splitter := NewSplitter(ch) - - splitter.Feed("没有分隔符的文本") - splitter.Flush() - - // 应该有 1 个句子（Flush 会发送剩余内容） - assert.Equal(t, 1, len(ch)) - assert.Equal(t, "没有分隔符的文本", <-ch) -} - -// TestSplitter_Empty 测试空输入 -func TestSplitter_Empty(t *testing.T) { - ch := make(chan string, 10) - splitter := NewSplitter(ch) - - splitter.Flush() - - // 应该没有句子 - assert.Equal(t, 0, len(ch)) -} - -// TestProcessQuery_InvalidAudio 测试无效音频数据 -func TestProcessQuery_InvalidAudio(t *testing.T) { - req := models.WsQuery{ - Type: "query", - RequestID: "req-123", - Audio: "invalid-base64!!!", - } - - mockSTT := new(MockSTTService) - mockLLM := new(MockLLMService) - mockTTS := new(MockTTSService) - mockSession := new(MockSessionManager) - mockSender := NewMockSender() - - mockSender.On("SendError", mock.Anything).Return(nil) - - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ - AI: config.AIConfig{ - LLM: config.LLMConfig{Model: "gpt-4o"}, - TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, - }, - }) - - ctx := context.Background() - err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) - - assert.Error(t, err) - assert.Len(t, mockSender.Errors, 1) - assert.Equal(t, "INVALID_MESSAGE", mockSender.Errors[0].Code) -} - -// TestProcessQuery_SessionNotFound 测试会话不存在 -func TestProcessQuery_SessionNotFound(t *testing.T) { - audioData := []byte("test audio") - audioBase64 := base64.StdEncoding.EncodeToString(audioData) - - req := models.WsQuery{ - Type: "query", - RequestID: "req-123", - Audio: audioBase64, - } - - mockSTT := new(MockSTTService) - mockLLM := new(MockLLMService) - mockTTS := new(MockTTSService) - mockSession := new(MockSessionManager) - mockSender := NewMockSender() - - mockSession.On("SetActiveRequest", mock.Anything, "session-123", "req-123").Return(nil) - mockSession.On("ClearActiveRequest", mock.Anything, "session-123").Return(nil) - mockSession.On("Get", mock.Anything, "session-123").Return(nil, errors.New("session not found")) - - mockSender.On("SendError", mock.Anything).Return(nil) - - pipeline := New(mockSTT, mockLLM, mockTTS, mockSession, &config.Config{ - AI: config.AIConfig{ - LLM: config.LLMConfig{Model: "gpt-4o"}, - TTS: config.TTSConfig{Voice: "alloy", Speed: 1.0, OutputFormat: "mp3", SampleRate: 24000}, - }, - }) - - ctx := context.Background() - err := pipeline.ProcessQuery(ctx, "session-123", req, nil, mockSender) - - assert.Error(t, err) - assert.Len(t, mockSender.Errors, 1) - assert.Equal(t, "SESSION_NOT_FOUND", mockSender.Errors[0].Code) -} diff --git a/backend/internal/orchestrator/splitter.go b/backend/internal/orchestrator/splitter.go deleted file mode 100644 index 57b8b26..0000000 --- a/backend/internal/orchestrator/splitter.go +++ /dev/null @@ -1,55 +0,0 @@ -package orchestrator - -import "strings" - -// sentenceDelimiters 句子分隔符集合。 -var sentenceDelimiters = map[rune]bool{ - '。': true, - '！': true, - '？': true, - '\n': true, - '.': true, - '!': true, - '?': true, -} - -// Splitter 句子切分器。 -// 将流式文本按句子边界切分，发送到 channel 供 TTS 合成。 -type Splitter struct { - ch chan<- string - buffer strings.Builder -} - -// NewSplitter 创建句子切分器。 -// ch 用于接收切分后的句子文本。 -func NewSplitter(ch chan<- string) *Splitter { - return &Splitter{ - ch: ch, - } -} - -// Feed 输入增量文本，遇到句子分隔符时发送完整句子。 -func (s *Splitter) Feed(delta string) { - for _, r := range delta { - s.buffer.WriteRune(r) - if sentenceDelimiters[r] { - s.flushBuffer() - } - } -} - -// Flush 刷新缓冲区中的剩余文本（即使没有句子分隔符）。 -func (s *Splitter) Flush() { - if s.buffer.Len() > 0 { - s.flushBuffer() - } -} - -// flushBuffer 将缓冲区内容发送到 channel 并清空。 -func (s *Splitter) flushBuffer() { - text := strings.TrimSpace(s.buffer.String()) - if text != "" { - s.ch <- text - } - s.buffer.Reset() -} -- 2.49.1 From 9576884619ece8b17e87805a4c4002191d71b1de Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Fri, 19 Jun 2026 22:09:07 +0800 Subject: [PATCH 4/7] =?UTF-8?q?docs:=20=E6=96=B0=E5=A2=9E=20Eino=20?= =?UTF-8?q?=E6=A1=86=E6=9E=B6=E6=8A=80=E6=9C=AF=E6=96=87=E6=A1=A3=E5=92=8C?= =?UTF-8?q?=E9=87=8D=E6=9E=84=E5=AE=9E=E6=96=BD=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - docs/11-Eino框架技术文档.md: 框架简介、技术选型对比、核心概念（Lambda/Graph/ChatModel/StreamReader/Callback/State）、CamTalk Graph 设计、目录结构、注意事项 - docs/12-Eino重构实施记录.md: 重构背景、架构变更、四阶段实施详情、代码统计、遗留事项 Co-Authored-By: Claude --- docs/11-Eino框架技术文档.md | 246 ++++++++++++++++++++++++++++++++++++ docs/12-Eino重构实施记录.md | 204 ++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 docs/11-Eino框架技术文档.md create mode 100644 docs/12-Eino重构实施记录.md diff --git a/docs/11-Eino框架技术文档.md b/docs/11-Eino框架技术文档.md new file mode 100644 index 0000000..0995b97 --- /dev/null +++ b/docs/11-Eino框架技术文档.md @@ -0,0 +1,246 @@ +# CamTalk Eino 框架技术文档 + +> 创建日期：2026-06-19 +> 状态：已实施 + +## 1. 框架简介 + +[CloudWeGo Eino](https://github.com/cloudwego/eino) 是字节跳动 CloudWeGo 团队开源的 AI 应用开发框架，提供基于图（Graph）的编排能力、组件抽象和流式处理支持。 + +CamTalk 使用 Eino 替代原有的手写 goroutine 管道，实现 STT → LLM → TTS 的声明式编排。 + +## 2. 技术选型 + +### 2.1 为什么选 Eino + +| 维度 | 手写 goroutine（旧方案） | Eino Graph（新方案） | +|------|------------------------|---------------------| +| 编排方式 | 手动 `go func()` + `sync.WaitGroup` | 声明式 DAG，类型安全 | +| 流式处理 | 自定义 `chan` 传递 | `StreamReader` + `Pipe`，自动转换 | +| 错误处理 | 各节点独立处理，不一致 | Graph 级别统一错误传播 | +| 回调/AOP | 日志散落各处 | `callbacks.Handler` 统一注入 | +| 配置灵活性 | Pipeline 创建时固定 | 每请求 `Option` 动态注入 | +| 可测试性 | 需启动 goroutine | `Graph.Invoke()` 直接测试 | +| 扩展性 | 修改 Pipeline 代码 | 添加节点 + 边，无侵入 | +| 并发安全 | 手动 `sync` | State 自动加锁 | + +### 2.2 Eino vs 其他编排框架 + +| 框架 | 特点 | CamTalk 适用性 | +|------|------|---------------| +| **Eino** | Go 原生、类型安全、流式原生 | ✅ 完美匹配 | +| LangChain Go | 生态丰富但较重 | ❌ 过度抽象 | +| 自研编排 | 完全可控 | ❌ 维护成本高 | + +**选择 Eino 的核心理由**： +1. Go 原生，泛型支持，编译时类型检查 +2. 原生流式处理（`StreamReader`），适合 LLM token 级推送 +3. Graph 支持分支、并行、循环，满足当前和未来需求 +4. Callback 机制实现 AOP（日志、指标、消息推送） +5. eino-ext 提供 OpenAI ChatModel 实现，直接对接 DashScope + +### 2.3 核心依赖版本 + +``` +github.com/cloudwego/eino v0.9.9 +github.com/cloudwego/eino-ext/components/model/openai v0.1.13 +``` + +## 3. Eino 核心概念 + +### 3.1 Lambda + +Lambda 是 Graph 中的可组合函数单元，支持四种模式： + +| 模式 | 函数签名 | 构造方法 | 说明 | +|------|---------|---------|------| +| Invoke | `I → O` | `compose.InvokableLambda()` | 同步调用 | +| Stream | `I → StreamReader[O]` | `compose.StreamableLambda()` | 流式输出 | +| Collect | `StreamReader[I] → O` | `compose.CollectableLambda()` | 流式输入 | +| Transform | `StreamReader[I] → StreamReader[O]` | `compose.TransformableLambda()` | 双向流式 | + +**返回类型**：所有 Lambda 构造函数返回 `*compose.Lambda`。 + +### 3.2 Graph + +Graph 是有向无环图（DAG）编排器，支持： +- **节点**：Lambda、ChatModel、ToolsNode 等 +- **边**：`g.AddEdge(from, to)` 定义数据流向 +- **分支**：`g.AddBranch()` 条件路由 +- **State**：`compose.WithGenLocalState()` 跨节点共享状态 + +```go +g := compose.NewGraph[PipelineInput, PipelineOutput]() +g.AddLambdaNode("stt", sttLambda) +g.AddChatModelNode("llm", chatModel) +g.AddEdge(compose.START, "stt") +g.AddEdge("stt", "llm") +g.AddEdge("llm", compose.END) + +runnable, err := g.Compile(ctx) +output, err := runnable.Invoke(ctx, input) // 同步调用 +stream, err := runnable.Stream(ctx, input) // 流式调用 +``` + +### 3.3 ChatModel + +ChatModel 是 LLM 组件抽象，接口定义： + +```go +type BaseChatModel interface { + Generate(ctx, []*schema.Message, ...Option) (*schema.Message, error) + Stream(ctx, []*schema.Message, ...Option) (*schema.StreamReader[*schema.Message], error) +} +``` + +CamTalk 使用 `eino-ext/components/model/openai` 实现，通过 `BaseURL` 对接 DashScope： + +```go +chatModel, _ := openai.NewChatModel(ctx, &openai.ChatModelConfig{ + APIKey: cfg.AI.LLM.APIKey, + Model: cfg.AI.LLM.Model, + BaseURL: cfg.AI.LLM.Endpoint, // "https://dashscope.aliyuncs.com/compatible-mode/v1" +}) +``` + +### 3.4 StreamReader + +`schema.StreamReader[T]` 是 Eino 的流式数据抽象： +- `sr.Recv()` 读取一帧，`io.EOF` 表示流结束 +- `schema.Pipe[T](bufSize)` 创建 `StreamReader` + `StreamWriter` 对 +- 框架自动处理 `T ↔ StreamReader[T]` 的转换（装箱/concat） + +### 3.5 Callback + +Callback 是 Eino 的 AOP 机制，支持节点生命周期钩子： + +```go +type Handler interface { + OnStart(ctx, *RunInfo, CallbackInput) context.Context + OnEnd(ctx, *RunInfo, CallbackOutput) context.Context + OnError(ctx, *RunInfo, error) context.Context + OnStartWithStreamInput(ctx, *RunInfo, *StreamReader[CallbackInput]) context.Context + OnEndWithStreamOutput(ctx, *RunInfo, *StreamReader[CallbackOutput]) context.Context +} +``` + +CamTalk 使用 `utils/callbacks.NewHandlerHelper()` 构建 typed handler： +- `ModelCallbackHandler.OnEndWithStreamOutput`：逐 token 推送 `llm_chunk` + +### 3.6 State + +Graph 全局状态，通过 `WithGenLocalState` 注册： + +```go +type PipelineState struct { + FullResponse strings.Builder + TranscribedText string + TokenUsage *TokenUsage +} + +g := compose.NewGraph[I, O](compose.WithGenLocalState(func(ctx context.Context) *PipelineState { + return &PipelineState{} +})) +``` + +节点通过 `compose.ProcessState` 读写 State。 + +## 4. CamTalk Graph 设计 + +### 4.1 拓扑 + +``` +START → STT → History → ChatModel → Splitter → TTS → Done → END +``` + +| 节点 | 类型 | 输入 → 输出 | 职责 | +|------|------|------------|------| +| STT | InvokableLambda | `PipelineInput → STTOutput` | 语音识别，写入 State | +| History | InvokableLambda | `STTOutput → []*schema.Message` | 组装提示词和历史 | +| ChatModel | ChatModel（原生） | `[]*schema.Message → StreamReader[*Message]` | LLM 流式推理 | +| Splitter | TransformableLambda | `StreamReader[string] → StreamReader[[]string]` | 句子切分 | +| TTS | InvokableLambda | `[]string → struct{}` | 语音合成，推送音频 | +| Done | InvokableLambda | `struct{} → PipelineOutput` | 发送 llm_done | + +### 4.2 流式模式 + +Graph 使用 **Stream 模式**调用： +- 内部所有节点以 Transform 模式运行 +- ChatModel 的 `Stream()` 方法实现真正的 token 级流式 +- 适配器消费 `StreamReader[PipelineOutput]` 触发整条链路 + +### 4.3 消息推送机制 + +| 消息 | 推送方式 | 时机 | +|------|---------|------| +| `stt_result` | Lambda 内部直接调用 Sender | STT 完成后 | +| `llm_chunk` | Callback `OnEndWithStreamOutput` | ChatModel 逐 token | +| `tts_audio` | Lambda 内部直接调用 Sender | TTS 逐句合成 | +| `llm_done` | Lambda 内部直接调用 Sender | Done 节点执行时 | + +**Context 注入**：Sender、RequestID、SessionID、PipelineState 通过 `context.WithValue` 传递。 + +### 4.4 多模态支持 + +History 节点将图片构建为 `schema.Message.UserInputMultiContent`： + +```go +systemMsg.UserInputMultiContent = []schema.MessageInputPart{ + { + Type: schema.ChatMessagePartTypeImageURL, + Image: &schema.MessageInputImage{ + MessagePartCommon: schema.MessagePartCommon{ + Base64Data: &base64Str, + MIMEType: "image/jpeg", + }, + Detail: schema.ImageURLDetailAuto, + }, + }, +} +``` + +## 5. 目录结构 + +``` +backend/internal/eino/ +├── types.go # PipelineInput/Output、STTOutput、TokenUsage +├── state.go # PipelineState（跨节点状态） +├── callback.go # Callback handler（LLM token 推送） +├── graph.go # Graph 构建与编译 +├── adapter.go # EinoOrchestrator（Orchestrator 接口适配器） +├── nodes_stt.go # STT Lambda +├── nodes_history.go # 历史组装 Lambda +├── nodes_splitter.go # 句子分割 Transform Lambda +├── nodes_tts.go # TTS Lambda +├── nodes_done.go # Done Lambda +└── graph_test.go # 单元测试 +``` + +## 6. 注意事项 + +### 6.1 值类型 vs 指针类型 + +Graph 泛型参数必须使用值类型（`PipelineInput`/`PipelineOutput`），所有 Lambda 的输入输出也使用值类型。框架在 Transform 模式下会自动处理 `T` 和 `StreamReader[T]` 的转换。 + +### 6.2 Callback 运行时传入 + +Callback 通过 `Stream()` 的 option 传入，不在 `Compile()` 时注册： + +```go +streamReader, err := runnable.Stream(ctx, input, compose.WithCallbacks(handler)) +``` + +### 6.3 eino-ext 与 DashScope 兼容性 + +eino-ext OpenAI ChatModel 通过 `BaseURL` 对接 DashScope 兼容接口。需注意： +- 多模态图片使用 `Base64Data` + `MIMEType` 格式 +- `Timeout` 控制单次请求超时 +- 流式输出通过 `Stream()` 方法获取 `StreamReader[*schema.Message]` + +### 6.4 框架自动类型转换 + +Eino 框架在编排场景中自动处理以下转换： +- **T → StreamReader[T]**：将完整值装箱为单帧流（非流式 → 假流式） +- **StreamReader[T] → T**：将流 concat 为完整值（流式 → 非流式） + +这使得不同流式模式的节点可以无缝连接。 diff --git a/docs/12-Eino重构实施记录.md b/docs/12-Eino重构实施记录.md new file mode 100644 index 0000000..cafcdca --- /dev/null +++ b/docs/12-Eino重构实施记录.md @@ -0,0 +1,204 @@ +# CamTalk Eino 重构实施记录 + +> 创建日期：2026-06-19 +> 状态：已完成 + +## 1. 重构背景 + +CamTalk 原 AI 编排层（`internal/orchestrator/pipeline.go`）使用手写 goroutine + WaitGroup + channel 实现 STT → LLM → TTS 流式管道，存在以下问题： + +1. **编排逻辑硬编码**：流程写死在 `ProcessQuery()` 中，扩展需重写 goroutine 调度 +2. **并发控制粗糙**：手动 `go func()` + `sync.WaitGroup`，缺乏结构化流式传递 +3. **无回调/AOP 机制**：日志、指标、追踪散落各处 +4. **配置耦合**：模型名、TTS 参数硬编码在 Pipeline 结构体 +5. **错误处理不一致**：TTS 错误被静默吞掉，缺乏统一模式 + +**重构目标**：使用 Eino Graph 替换手写 Pipeline，实现声明式编排、统一回调、按请求动态配置，保持 WebSocket 协议和 REST API 不变。 + +## 2. 整体架构变更 + +### 2.1 重构前 + +``` +WS Handler → Orchestrator.Pipeline.ProcessQuery() + ├→ goroutine: STT.Recognize() + ├→ goroutine: LLM.ChatStream() ──→ chan chunk ──→ Sender + └→ goroutine: Splitter → TTS.SynthesizeStream() ──→ chan audio ──→ Sender + WaitGroup.Wait() + Sender.SendLLMDone() +``` + +### 2.2 重构后 + +``` +WS Handler → EinoOrchestrator.ProcessQuery() + ├→ Graph.Stream(ctx, input) + │ ├→ STT Lambda ─→ History Lambda ─→ ChatModel ─→ Splitter ─→ TTS ─→ Done + │ │ (State 写入) (Callback (Transform) (Invoke) (Invoke) + │ │ 流式推送) + │ └→ 消费 StreamReader（触发整条链路惰性执行） + └→ 追加助手消息到历史 +``` + +### 2.3 关键设计决策 + +| 决策 | 选择 | 理由 | +|------|------|------| +| Graph 调用模式 | Stream | ChatModel 需要真正的 token 级流式输出 | +| LLM 组件 | eino-ext ChatModel | 原生 Eino 组件，直接对接 DashScope | +| 消息推送 | Callback（LLM）+ Sender（其他） | LLM token 流式推送需要 Callback | +| 值类型 vs 指针 | 值类型统一 | 避免框架类型转换不匹配 | +| 历史追加 | 适配器负责 | Done 节点只负责发送 llm_done | + +## 3. 分阶段实施 + +### Phase 1：基础设施（提交 `fd5c771`） + +**目标**：引入 Eino 依赖，创建基础类型和 Callback。 + +**任务清单**： + +| 任务 | 文件 | 说明 | +|------|------|------| +| 引入 Eino 依赖 | `go.mod` | `eino v0.9.9` + `eino-ext/components/model/openai v0.1.13` | +| 数据类型定义 | `eino/types.go` | `PipelineInput`、`PipelineOutput`、`STTOutput`、`TokenUsage` | +| State 定义 | `eino/state.go` | `PipelineState` 含 `sync.Mutex` 并发保护 | +| 消息推送 Callback | `eino/callback.go` | `BuildCallbackHandler()` 使用 `callbacks.NewHandlerHelper()` | + +**关键实现**： +- `PipelineState` 使用 `strings.Builder` + `sync.Mutex` 累积 LLM 完整回复 +- Callback 通过 `ModelCallbackHandler.OnEndWithStreamOutput` 逐 token 推送 `llm_chunk` +- Sender/RequestID/PipelineState 通过 `context.WithValue` 注入 + +**验证**：`go build ./cmd/server` ✓ + +--- + +### Phase 2：节点实现（提交 `fd5c771`） + +**目标**：实现 Graph 中的 5 个 Lambda 节点。 + +**任务清单**： + +| 任务 | 文件 | Lambda 类型 | 输入 → 输出 | +|------|------|------------|------------| +| STT Lambda | `eino/nodes_stt.go` | InvokableLambda | `PipelineInput → STTOutput` | +| 历史组装 Lambda | `eino/nodes_history.go` | InvokableLambda | `STTOutput → []*schema.Message` | +| 句子分割 Lambda | `eino/nodes_splitter.go` | TransformableLambda | `StreamReader[string] → StreamReader[[]string]` | +| TTS Lambda | `eino/nodes_tts.go` | InvokableLambda | `[]string → struct{}` | +| Done Lambda | `eino/nodes_done.go` | InvokableLambda | `struct{} → PipelineOutput` | + +**关键实现**： +- STT 节点将输入元数据写入 State，供下游节点读取 +- History 节点从 State 读取 SessionID/Scenario/ImageData，构建系统提示词 + 多模态消息 +- Splitter 使用 `TransformableLambda` 按句子分隔符切分，逐句输出给 TTS +- TTS 节点调用 `ttsService.SynthesizeStream()`，逐 chunk 推送 `tts_audio` +- Done 节点从 State 读取完整回复，发送 `llm_done` +- 所有 Lambda 使用值类型（非指针），返回 `*compose.Lambda` + +**验证**：`go build ./internal/eino/...` ✓ + +--- + +### Phase 3：Graph 构建与适配器（提交 `4b731b5`） + +**目标**：构建 Graph、实现适配器、切换 main.go。 + +**任务清单**： + +| 任务 | 文件 | 说明 | +|------|------|------| +| Graph 构建 | `eino/graph.go` | `NewPipelineGraph()` 组装 6 个节点 + 边 + 编译 | +| 适配器 | `eino/adapter.go` | `EinoOrchestrator` 实现 `orchestrator.Orchestrator` 接口 | +| main.go 切换 | `cmd/server/main.go` | 移除旧 LLM + orchestrator，替换为 Eino | + +**Graph 拓扑**： +``` +START → STT → History → ChatModel → Splitter → TTS → Done → END +``` + +**适配器职责**： +1. 解码 base64 音频/图片 +2. 获取会话配置 +3. 注入 Sender/RequestID/SessionID/StartTime/State 到 context +4. 追加用户消息到历史 +5. 调用 `graph.Stream(ctx, input, callbacks)` 触发惰性执行 +6. 消费 `StreamReader[PipelineOutput]` +7. 追加助手消息到历史 + +**关键实现**： +- eino-ext ChatModel 配置：`BaseURL` 对接 DashScope，`Timeout` 控制请求超时 +- Callback 在运行时通过 `compose.WithCallbacks()` 传入，不在编译时注册 +- 元数据（SessionID/Scenario 等）通过 State 跨节点传递，不通过 Graph 边传递 + +**变更文件**： +- 修改 `state.go`：新增 SessionID/RequestID/ImageData 等字段 +- 修改 `nodes_stt.go`：写入元数据到 State +- 修改 `nodes_history.go`：从 State 读取元数据（移除 HistoryInput 依赖） +- 修改 `nodes_done.go`：移除历史追加（由适配器负责） + +**验证**：`go build ./cmd/server` ✓，`go vet ./...` ✓ + +--- + +### Phase 4：清理与测试（提交 `4ffd845`） + +**目标**：删除旧代码，编写单元测试。 + +**删除的文件**： + +| 文件 | 说明 | +|------|------| +| `orchestrator/pipeline.go` | 旧 STT→LLM→TTS 手写 goroutine 管道（-547 行） | +| `orchestrator/splitter.go` | 旧句子切分器（-114 行） | +| `orchestrator/pipeline_test.go` | 旧 Pipeline 测试（-309 行） | +| `ai/llm/openai.go` | 旧 LLM OpenAI 实现（-548 行） | +| `ai/llm/openai_test.go` | 旧 LLM 测试（-143 行） | + +**保留的文件**： + +| 文件 | 保留原因 | +|------|---------| +| `orchestrator/orchestrator.go` | Orchestrator 接口（ws/handler 依赖） | +| `orchestrator/sender.go` | Sender 接口（eino/callback 依赖） | +| `ai/llm/llm.go` | Request/Chunk/TokenUsage 类型定义 | +| `ai/llm/prompt.go` | BuildSystemPrompt（eino/nodes_history 依赖） | +| `ai/llm/scenarios.go` | GetScenarioPrompt（eino/nodes_history 依赖） | + +**新增测试**：`eino/graph_test.go`（13 个测试） + +| 测试 | 覆盖内容 | +|------|---------| +| `TestDetectImageMimeType` | JPEG/PNG/GIF/WebP/未知格式检测 | +| `TestBuildPipelineInput` | 文本输入构建 | +| `TestBuildPipelineInput_WithAudioData` | 音频+图片输入构建 | +| `TestPipelineState_AppendAndGet` | State 文本追加和读取 | +| `TestPipelineState_ConcurrentAccess` | State 并发安全（100 goroutine） | +| `TestContextInjection` | Sender/RequestID/State 注入和提取 | +| `TestLatencyFromCtx` | 延迟计算 | +| `TestEinoOrchestrator_ImplementsInterface` | 接口实现检查 | +| `TestNew*Lambda_ReturnsNonNil` | 5 个 Lambda 构造函数非空检查 | + +**验证**：`go build ./...` ✓，`go vet ./...` ✓，`go test ./...` ✓ + +## 4. 代码变更统计 + +| 阶段 | 提交 | 新增 | 删除 | 净变化 | +|------|------|------|------|--------| +| Phase 1 + 2 | `fd5c771` | +946 | -24 | +922 | +| Phase 3 | `4b731b5` | +395 | -98 | +297 | +| Phase 4 | `4ffd845` | +235 | -1661 | -1426 | +| **合计** | | **+1576** | **-1783** | **-207** | + +重构后代码量净减少 207 行，同时获得了更好的可维护性、可测试性和可扩展性。 + +## 5. 遗留事项 + +| 事项 | 优先级 | 说明 | +|------|--------|------| +| eino-ext ChatModel DashScope 兼容性端到端验证 | 高 | 需要真实 API Key 验证流式输出和多模态 | +| LLM 超时控制 | 中 | eino-ext ChatModel 的 `Timeout` 配置需验证 | +| TTS 流式优化 | 中 | 当前 TTS 是 InvokableLambda，可改为 StreamableLambda | +| ReAct Agent 扩展 | 低 | 基于 Graph Branch 实现工具调用循环 | +| Model Router | 低 | 按场景/成本路由不同 LLM | +| 指标监控 | 低 | 通过 Callback 接入 Prometheus | -- 2.49.1 From eb1b90445fb124f3db1e209ca254d8a13e124f65 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Fri, 19 Jun 2026 23:24:01 +0800 Subject: [PATCH 5/7] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20Eino=20Graph=20?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E4=B8=8D=E5=8C=B9=E9=85=8D=E5=92=8C=E5=A4=9A?= =?UTF-8?q?=E6=A8=A1=E6=80=81=E6=B6=88=E6=81=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 添加 msg2str 转换节点解决 ChatModel 输出 *schema.Message 与 Splitter 期望 string 的类型不匹配 - 将多模态图片内容从 system 消息移到 user 消息（DashScope API 仅支持 user/tool 角色的多模态内容） - 修复 Content 和 UserInputMultiContent 不能同时设置的问题 - Splitter 输出改为 StreamReader[string]（单句），TTS 改为 TransformableLambda 流式消费 - 修复 .env 中 PostgreSQL DSN 和 Redis ADDR 的 http:// 前缀问题 --- backend/internal/eino/graph.go | 9 +- backend/internal/eino/nodes_history.go | 56 +++++---- backend/internal/eino/nodes_splitter.go | 54 ++++++-- backend/internal/eino/nodes_tts.go | 158 ++++++++++++++---------- 4 files changed, 171 insertions(+), 106 deletions(-) diff --git a/backend/internal/eino/graph.go b/backend/internal/eino/graph.go index f8a2449..8bf3a13 100644 --- a/backend/internal/eino/graph.go +++ b/backend/internal/eino/graph.go @@ -18,8 +18,9 @@ import ( const ( nodeSTT = "stt" nodeHistory = "history" - nodeLLM = "llm" - nodeSplitter = "splitter" + nodeLLM = "llm" + nodeMessageToString = "msg2str" + nodeSplitter = "splitter" nodeTTS = "tts" nodeDone = "done" ) @@ -69,6 +70,7 @@ func NewPipelineGraph( _ = g.AddLambdaNode(nodeSTT, NewSTTLambda(sttService)) _ = g.AddLambdaNode(nodeHistory, NewHistoryLambda(sessionMgr.GetHistory, maxHistory)) _ = g.AddChatModelNode(nodeLLM, chatModel) + _ = g.AddLambdaNode(nodeMessageToString, NewMessageToStringLambda()) _ = g.AddLambdaNode(nodeSplitter, NewSplitterLambda()) _ = g.AddLambdaNode(nodeTTS, NewTTSLambda( ttsService, @@ -83,7 +85,8 @@ func NewPipelineGraph( _ = g.AddEdge(compose.START, nodeSTT) _ = g.AddEdge(nodeSTT, nodeHistory) _ = g.AddEdge(nodeHistory, nodeLLM) - _ = g.AddEdge(nodeLLM, nodeSplitter) + _ = g.AddEdge(nodeLLM, nodeMessageToString) + _ = g.AddEdge(nodeMessageToString, nodeSplitter) _ = g.AddEdge(nodeSplitter, nodeTTS) _ = g.AddEdge(nodeTTS, nodeDone) _ = g.AddEdge(nodeDone, compose.END) diff --git a/backend/internal/eino/nodes_history.go b/backend/internal/eino/nodes_history.go index a558a6b..c48b488 100644 --- a/backend/internal/eino/nodes_history.go +++ b/backend/internal/eino/nodes_history.go @@ -40,30 +40,12 @@ func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, scenarioPrompt := llm.GetScenarioPrompt(scenario, language) systemPrompt := llm.BuildSystemPrompt(language, detailLevel, scenarioPrompt) - // 构建 system message（含图片） + // 构建 system message（仅文本，多模态内容只能放在 user 角色） systemMsg := &schema.Message{ Role: schema.System, Content: systemPrompt, } - // 如果有图片，添加到 system message 的多模态内容中 - if len(imageData) > 0 { - base64Str := base64.StdEncoding.EncodeToString(imageData) - mimeType := detectImageMimeType(imageData) - systemMsg.UserInputMultiContent = []schema.MessageInputPart{ - { - Type: schema.ChatMessagePartTypeImageURL, - Image: &schema.MessageInputImage{ - MessagePartCommon: schema.MessagePartCommon{ - Base64Data: &base64Str, - MIMEType: mimeType, - }, - Detail: schema.ImageURLDetailAuto, - }, - }, - } - } - messages := []*schema.Message{systemMsg} // 获取并追加历史消息 @@ -81,11 +63,37 @@ func NewHistoryLambda(historyFetcher func(ctx context.Context, sessionID string, } } - // 追加当前用户输入 - messages = append(messages, &schema.Message{ - Role: schema.User, - Content: sttOut.Text, - }) + // 追加当前用户输入（含图片，多模态内容只能放在 user 角色） + // 注意：不能同时设置 Content 和 UserInputMultiContent，需要统一放到 MultiContent 中 + if len(imageData) > 0 { + base64Str := base64.StdEncoding.EncodeToString(imageData) + mimeType := detectImageMimeType(imageData) + parts := []schema.MessageInputPart{ + { + Type: schema.ChatMessagePartTypeText, + Text: sttOut.Text, + }, + { + Type: schema.ChatMessagePartTypeImageURL, + Image: &schema.MessageInputImage{ + MessagePartCommon: schema.MessagePartCommon{ + Base64Data: &base64Str, + MIMEType: mimeType, + }, + Detail: schema.ImageURLDetailAuto, + }, + }, + } + messages = append(messages, &schema.Message{ + Role: schema.User, + UserInputMultiContent: parts, + }) + } else { + messages = append(messages, &schema.Message{ + Role: schema.User, + Content: sttOut.Text, + }) + } log.Infow("历史组装完成", "request_id", requestID, diff --git a/backend/internal/eino/nodes_splitter.go b/backend/internal/eino/nodes_splitter.go index 6f3de8b..e3cfafb 100644 --- a/backend/internal/eino/nodes_splitter.go +++ b/backend/internal/eino/nodes_splitter.go @@ -20,18 +20,49 @@ var sentenceDelimiters = map[rune]bool{ '?': true, } -// NewSplitterLambda 创建句子分割 Transform Lambda 节点。 -// 输入: StreamReader[string]（LLM 完整文本的单帧流）→ 输出: StreamReader[[]string]（句子数组流） +// NewMessageToStringLambda 创建 Message → String 转换 Lambda 节点。 +// 输入: *schema.Message → 输出: string // -// 在 Stream 模式下，框架自动将 ChatModel 的 StreamReader[*schema.Message] -// concat 为 string 后传入此节点。此节点将文本按句子边界切分， -// 每切出一个句子就输出一次，供 TTS 节点实时合成。 -func NewSplitterLambda() *compose.Lambda { - return compose.TransformableLambda(func(ctx context.Context, input *schema.StreamReader[string]) (*schema.StreamReader[[]string], error) { - sr, sw := schema.Pipe[[]string](8) +// 提取 Message.Content 文本，供 Splitter 节点消费。 +func NewMessageToStringLambda() *compose.Lambda { + return compose.TransformableLambda(func(ctx context.Context, input *schema.StreamReader[*schema.Message]) (*schema.StreamReader[string], error) { + sr, sw := schema.Pipe[string](8) go func() { defer sw.Close() + defer input.Close() + + for { + msg, err := input.Recv() + if err != nil { + if err == io.EOF { + return + } + sw.Send("", err) + return + } + if msg != nil && msg.Content != "" { + sw.Send(msg.Content, nil) + } + } + }() + + return sr, nil + }) +} + +// NewSplitterLambda 创建句子分割 Transform Lambda 节点。 +// 输入: StreamReader[string]（LLM token 流）→ 输出: StreamReader[string]（完整句子流） +// +// 逐字符累积，按句子分隔符切分。每切出一个完整句子就输出一次， +// 供下游 TTS 节点实时合成。 +func NewSplitterLambda() *compose.Lambda { + return compose.TransformableLambda(func(ctx context.Context, input *schema.StreamReader[string]) (*schema.StreamReader[string], error) { + sr, sw := schema.Pipe[string](8) + + go func() { + defer sw.Close() + defer input.Close() var buffer strings.Builder @@ -43,23 +74,22 @@ func NewSplitterLambda() *compose.Lambda { if buffer.Len() > 0 { text := strings.TrimSpace(buffer.String()) if text != "" { - sw.Send([]string{text}, nil) + sw.Send(text, nil) } } return } - sw.Send(nil, err) + sw.Send("", err) return } - // chunk 是 concat 后的完整文本（单帧流） // 逐字符累积，按句子分隔符切分 for _, r := range chunk { buffer.WriteRune(r) if sentenceDelimiters[r] { text := strings.TrimSpace(buffer.String()) if text != "" { - sw.Send([]string{text}, nil) + sw.Send(text, nil) } buffer.Reset() } diff --git a/backend/internal/eino/nodes_tts.go b/backend/internal/eino/nodes_tts.go index 5d4e7b7..09c86d1 100644 --- a/backend/internal/eino/nodes_tts.go +++ b/backend/internal/eino/nodes_tts.go @@ -3,90 +3,114 @@ package eino import ( "context" "encoding/base64" + "io" "github.com/cloudwego/eino/compose" + "github.com/cloudwego/eino/schema" "github.com/hhs/camtalk/internal/ai/tts" "github.com/hhs/camtalk/internal/logger" "github.com/hhs/camtalk/internal/models" ) -// NewTTSLambda 创建 TTS Lambda 节点。 -// 输入: []string（句子数组，框架自动从 StreamReader concat）→ 输出: struct{} +// NewTTSLambda 创建 TTS Transform Lambda 节点。 +// 输入: StreamReader[string]（句子流）→ 输出: StreamReader[struct{}]（结果流） // -// 将句子数组转为 channel，调用 ttsService.SynthesizeStream() 流式合成， +// 流式消费每个句子，调用 ttsService.SynthesizeStream() 合成， // 逐 chunk 推送 tts_audio 到客户端。TTS 失败静默跳过。 func NewTTSLambda(ttsService tts.Service, ttsVoice string, ttsSpeed float64, ttsOutputFmt string, ttsSampleRate int) *compose.Lambda { - return compose.InvokableLambda(func(ctx context.Context, sentences []string) (struct{}, error) { - log := logger.Log - sender := senderFromCtx(ctx) - requestID := requestIDFromCtx(ctx) - state := stateFromCtx(ctx) + return compose.TransformableLambda(func(ctx context.Context, input *schema.StreamReader[string]) (*schema.StreamReader[struct{}], error) { + sr, sw := schema.Pipe[struct{}](8) - // 检查 TTS 是否启用（从 State 或 context 获取） - // TTSEnabled 信息在 PipelineInput 中，通过 State 传递 - if state != nil { - state.mu.Lock() - ttsEnabled := true // 默认启用，由适配器通过 State 设置 - state.mu.Unlock() - if !ttsEnabled { - return struct{}{}, nil - } - } + go func() { + defer sw.Close() + defer input.Close() - if len(sentences) == 0 { - return struct{}{}, nil - } + log := logger.Log + sender := senderFromCtx(ctx) + requestID := requestIDFromCtx(ctx) - if sender == nil || requestID == "" { - return struct{}{}, nil - } - - log.Infow("开始 TTS 合成", "request_id", requestID, "sentence_count", len(sentences)) - - // 将句子数组转为 channel（ttsService.SynthesizeStream 需要 <-chan string） - sentenceCh := make(chan string, len(sentences)) - for _, s := range sentences { - sentenceCh <- s - } - close(sentenceCh) - - // 调用 TTS 服务 - ttsStream, err := ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{ - Voice: ttsVoice, - Speed: ttsSpeed, - OutputFmt: ttsOutputFmt, - SampleRate: ttsSampleRate, - }) - if err != nil { - log.Errorw("TTS 合成启动失败（已跳过）", "error", err, "request_id", requestID) - return struct{}{}, nil // TTS 失败不中断流程 - } - - // 消费 TTS 音频流，推送到客户端 - for chunk := range ttsStream { - select { - case <-ctx.Done(): - log.Infow("TTS 流被中断", "request_id", requestID) - return struct{}{}, ctx.Err() - default: + if sender == nil || requestID == "" { + // 消费并丢弃流 + for { + _, err := input.Recv() + if err != nil { + return + } + } } - audioBase64 := base64.StdEncoding.EncodeToString(chunk.Audio) - - if err := sender.SendTTSAudio(models.WsTTSAudio{ - Type: "tts_audio", - RequestID: requestID, - Audio: audioBase64, - MimeType: "audio/mp3", - IsLast: chunk.IsLast, - Final: chunk.Final, - }); err != nil { - log.Errorw("发送 tts_audio 失败", "error", err) + // 收集句子，按批次合成 TTS + var sentences []string + for { + sentence, err := input.Recv() + if err != nil { + if err == io.EOF { + break + } + log.Errorw("TTS: stream recv error", "error", err, "request_id", requestID) + break + } + if sentence != "" { + sentences = append(sentences, sentence) + } } - } - log.Infow("TTS 合成完成", "request_id", requestID) - return struct{}{}, nil + if len(sentences) == 0 { + sw.Send(struct{}{}, nil) + return + } + + log.Infow("开始 TTS 合成", "request_id", requestID, "sentence_count", len(sentences)) + + // 将句子数组转为 channel + sentenceCh := make(chan string, len(sentences)) + for _, s := range sentences { + sentenceCh <- s + } + close(sentenceCh) + + // 调用 TTS 服务 + ttsStream, err := ttsService.SynthesizeStream(ctx, sentenceCh, tts.Options{ + Voice: ttsVoice, + Speed: ttsSpeed, + OutputFmt: ttsOutputFmt, + SampleRate: ttsSampleRate, + }) + if err != nil { + log.Errorw("TTS 合成启动失败（已跳过）", "error", err, "request_id", requestID) + sw.Send(struct{}{}, nil) + return + } + + // 消费 TTS 音频流，推送到客户端 + for chunk := range ttsStream { + select { + case <-ctx.Done(): + log.Infow("TTS 流被中断", "request_id", requestID) + sw.Send(struct{}{}, ctx.Err()) + return + default: + } + + audioBase64 := base64.StdEncoding.EncodeToString(chunk.Audio) + + if err := sender.SendTTSAudio(models.WsTTSAudio{ + Type: "tts_audio", + RequestID: requestID, + Audio: audioBase64, + MimeType: "audio/mp3", + IsLast: chunk.IsLast, + Final: chunk.Final, + }); err != nil { + log.Errorw("发送 tts_audio 失败", "error", err) + } + } + + log.Infow("TTS 合成完成", "request_id", requestID) + sw.Send(struct{}{}, nil) + }() + + return sr, nil }) } -- 2.49.1 From b8fd6a933078028189dbe8699366c33dbc2ce6f0 Mon Sep 17 00:00:00 2001 From: hhs <386998068@qq.com> Date: Sat, 20 Jun 2026 11:54:21 +0800 Subject: [PATCH 6/7] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20TieredManager?= =?UTF-8?q?=20=E5=88=9B=E5=BB=BA=20session=20=E6=97=B6=20L1/L2=20ID=20?= =?UTF-8?q?=E4=B8=8D=E4=B8=80=E8=87=B4=E5=AF=BC=E8=87=B4=20Redis=20?= =?UTF-8?q?=E5=86=99=E5=85=A5=E5=A4=B1=E8=B4=A5=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/session/redis.go | 7 ++++++- backend/internal/session/tiered.go | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/internal/session/redis.go b/backend/internal/session/redis.go index 2210331..c497dd5 100644 --- a/backend/internal/session/redis.go +++ b/backend/internal/session/redis.go @@ -47,7 +47,12 @@ func userSessKey(id string) string { return fmt.Sprintf("user:%s:sessions", id) // Create 创建新会话。userID 为空表示匿名会话。 func (m *RedisManager) Create(ctx context.Context, userID string, config models.SessionConfig) (string, error) { - id := uuidNew() + return m.CreateWithID(ctx, uuidNew(), userID, config) +} + +// CreateWithID 使用指定 ID 创建新会话。 +// 供 TieredManager 调用，确保 L1/L2 使用相同的 session ID。 +func (m *RedisManager) CreateWithID(ctx context.Context, id string, userID string, config models.SessionConfig) (string, error) { now := time.Now().UTC() pipe := m.rdb.Pipeline() diff --git a/backend/internal/session/tiered.go b/backend/internal/session/tiered.go index 5fc008b..9f9d3b7 100644 --- a/backend/internal/session/tiered.go +++ b/backend/internal/session/tiered.go @@ -127,9 +127,9 @@ func (m *TieredManager) Create(ctx context.Context, userID string, config models return "", err } - // L2: Redis（同步） + // L2: Redis（同步），使用 L1 生成的 ID 保证一致性 if m.isRedisOK() { - if _, err := m.l2.Create(ctx, userID, config); err != nil { + if _, err := m.l2.CreateWithID(ctx, id, userID, config); err != nil { logger.Log.Warnw("Redis Create failed, continuing without L2", "session", id, "error", err) } -- 2.49.1 From 5910e02a6652a079e7cd4d93e2b4c541756578d9 Mon Sep 17 00:00:00 2001 From: hhs <386998068@qq.com> Date: Sat, 20 Jun 2026 11:59:07 +0800 Subject: [PATCH 7/7] =?UTF-8?q?ci:=20deploy=20=E5=B7=A5=E4=BD=9C=E6=B5=81?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20main=20=E5=88=86=E6=94=AF=E8=A7=A6?= =?UTF-8?q?=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 2e29362..44abebb 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -2,7 +2,7 @@ name: Deploy on: push: - branches: [v2] + branches: [main, v2] jobs: deploy: -- 2.49.1