feat: 添加文字输入对话功能

- 后端 WsQuery 添加 text 字段,支持文本输入模式
- Pipeline 支持文本查询时跳过 STT 直接使用输入文本
- 前端 ChatPanel 添加文字输入框,连接状态下可用
- useVisionSession 添加 sendTextMessage 方法
- 更新接口文档,添加文本输入模式说明
This commit is contained in:
2026-06-14 12:52:39 +08:00
parent 563ca12790
commit 51ed6aa563
8 changed files with 252 additions and 73 deletions

View File

@@ -62,22 +62,27 @@ func (p *Pipeline) ProcessQuery(
log := logger.Log
startTime := time.Now()
// 解码音频数据
audio, err := base64.StdEncoding.DecodeString(req.Audio)
if err != nil {
log.Errorw("音频解码失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "INVALID_MESSAGE",
Message: "音频数据解码失败",
})
return err
// 解码音频数据(文本输入模式可跳过)
var audio []byte
if req.Text == "" && req.Audio != "" {
var err error
audio, err = base64.StdEncoding.DecodeString(req.Audio)
if err != nil {
log.Errorw("音频解码失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "INVALID_MESSAGE",
Message: "音频数据解码失败",
})
return err
}
}
// 解码图片数据(可选)
var image []byte
if req.Image != "" {
var err error
image, err = base64.StdEncoding.DecodeString(req.Image)
if err != nil {
log.Errorw("图片解码失败", "error", err)
@@ -110,45 +115,64 @@ func (p *Pipeline) ProcessQuery(
return err
}
// Step 1: STT 语音识别
log.Infow("开始语音识别", "request_id", req.RequestID)
sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{
Encoding: "pcm_s16le",
SampleRate: 16000,
Language: sess.Config.Language,
})
if err != nil {
log.Errorw("语音识别失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "STT_ERROR",
Message: "语音识别失败",
})
return err
}
// Step 1: 获取用户文本(语音识别或直接使用输入文本)
var userText string
if req.Text != "" {
// 文本输入模式:跳过 STT直接使用用户输入的文本
log.Infow("使用文本输入", "request_id", req.RequestID, "text", req.Text)
userText = req.Text
// 发送 STT 结果
if err := sender.SendSTTResult(models.WsSTTResult{
Type: "stt_result",
RequestID: req.RequestID,
Text: sttResult,
IsFinal: true,
}); err != nil {
log.Errorw("发送 STT 结果失败", "error", err)
// 发送 stt_result 以保持前端消息流一致性
if err := sender.SendSTTResult(models.WsSTTResult{
Type: "stt_result",
RequestID: req.RequestID,
Text: userText,
IsFinal: true,
}); err != nil {
log.Errorw("发送 STT 结果失败", "error", err)
}
} else {
// 语音模式:执行 STT 语音识别
log.Infow("开始语音识别", "request_id", req.RequestID)
sttResult, err := p.sttService.Recognize(ctx, audio, stt.Options{
Encoding: "pcm_s16le",
SampleRate: 16000,
Language: sess.Config.Language,
})
if err != nil {
log.Errorw("语音识别失败", "error", err)
sender.SendError(models.WsError{
Type: "error",
RequestID: req.RequestID,
Code: "STT_ERROR",
Message: "语音识别失败",
})
return err
}
userText = sttResult
// 发送 STT 结果
if err := sender.SendSTTResult(models.WsSTTResult{
Type: "stt_result",
RequestID: req.RequestID,
Text: userText,
IsFinal: true,
}); err != nil {
log.Errorw("发送 STT 结果失败", "error", err)
}
}
// 追加用户消息到历史
p.sessionMgr.AppendMessage(ctx, sessionID, models.Message{
Role: "user",
Content: sttResult,
Content: userText,
})
// Step 2+3: LLM 流式推理 + TTS 并行合成
log.Infow("开始 LLM 推理", "request_id", req.RequestID)
llmReq := llm.Request{
Image: image,
Text: sttResult,
Text: userText,
History: history,
Language: sess.Config.Language,
}