Merge pull request 'feat: 添加 Xiaomi MiMo ASR 语音识别提供者' (#50) from fix/redis-config into develop
Reviewed-on: http://8.161.227.145:3000/XEngineers/CamTalk/pulls/50
This commit was merged in pull request #50.
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -46,7 +47,13 @@ func main() {
|
||||
defer sessionMgr.(*session.MemoryManager).Stop()
|
||||
|
||||
// 初始化 AI 服务
|
||||
sttService := stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, logger.Log)
|
||||
var sttService stt.Service
|
||||
switch strings.ToLower(cfg.AI.STT.Provider) {
|
||||
case "mimo", "xiaomi":
|
||||
sttService = stt.NewMiMoService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, logger.Log)
|
||||
default:
|
||||
sttService = stt.NewDeepgramService(cfg.AI.STT.APIKey, cfg.AI.STT.Model, cfg.AI.STT.Endpoint, logger.Log)
|
||||
}
|
||||
llmService := llm.NewOpenAIService(cfg.AI.LLM.APIKey, cfg.AI.LLM.Model, cfg.AI.LLM.Endpoint, cfg.AI.LLM.Timeout, logger.Log)
|
||||
ttsService := tts.NewOpenAIService(cfg.AI.TTS.APIKey, cfg.AI.TTS.Model, cfg.AI.TTS.Voice, cfg.AI.TTS.Endpoint, cfg.AI.TTS.Speed, cfg.AI.TTS.Timeout, logger.Log)
|
||||
|
||||
|
||||
@@ -15,8 +15,8 @@ redis:
|
||||
|
||||
ai:
|
||||
stt:
|
||||
provider: Xiaomi MiMo
|
||||
model: mimo-v2.5
|
||||
provider: mimo
|
||||
model: mimo-v2.5-asr
|
||||
endpoint: "https://token-plan-cn.xiaomimimo.com/v1"
|
||||
llm:
|
||||
provider: dashscope
|
||||
|
||||
232
backend/internal/ai/stt/mimo.go
Normal file
232
backend/internal/ai/stt/mimo.go
Normal file
@@ -0,0 +1,232 @@
|
||||
package stt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// MiMoService 基于 Xiaomi MiMo ASR HTTP API 的语音识别实现。
|
||||
// 接口兼容 OpenAI chat/completions 格式，音频仅支持 mp3/wav。
|
||||
type MiMoService struct {
|
||||
apiKey string
|
||||
model string
|
||||
endpoint string
|
||||
logger *zap.SugaredLogger
|
||||
}
|
||||
|
||||
// NewMiMoService 创建 MiMo STT 服务。
|
||||
func NewMiMoService(apiKey, model, endpoint string, logger *zap.SugaredLogger) *MiMoService {
|
||||
if model == "" {
|
||||
model = "mimo-v2.5-asr"
|
||||
}
|
||||
if endpoint == "" {
|
||||
endpoint = "https://api.xiaomimimo.com/v1"
|
||||
}
|
||||
return &MiMoService{
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
endpoint: endpoint,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// mimoRequest MiMo ASR 请求体。
|
||||
type mimoRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []mimoMessage `json:"messages"`
|
||||
ASROptions *mimoASROptions `json:"asr_options,omitempty"`
|
||||
}
|
||||
|
||||
type mimoMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content []mimoContent `json:"content"`
|
||||
}
|
||||
|
||||
type mimoContent struct {
|
||||
Type string `json:"type"`
|
||||
InputAudio *mimoAudioIn `json:"input_audio,omitempty"`
|
||||
}
|
||||
|
||||
type mimoAudioIn struct {
|
||||
Data string `json:"data"` // data URL: data:{mime};base64,{data}
|
||||
}
|
||||
|
||||
type mimoASROptions struct {
|
||||
Language string `json:"language"`
|
||||
}
|
||||
|
||||
// mimoResponse MiMo ASR 非流式响应。
|
||||
type mimoResponse struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
|
||||
// Recognize 实现 stt.Service。将音频发送到 MiMo ASR API，返回识别文本。
|
||||
func (m *MiMoService) Recognize(ctx context.Context, audio []byte, opts Options) (string, error) {
|
||||
if len(audio) == 0 {
|
||||
return "", fmt.Errorf("stt: empty audio")
|
||||
}
|
||||
|
||||
// MiMo 仅支持 mp3/wav，若输入为原始 PCM 则封装为 WAV
|
||||
audioData := audio
|
||||
mimeType := "audio/wav"
|
||||
if !isWAV(audio) && !isMP3(audio) {
|
||||
wav, err := pcmToWAV(audio, opts.SampleRate, 1)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stt: pcm to wav: %w", err)
|
||||
}
|
||||
audioData = wav
|
||||
} else if isMP3(audio) {
|
||||
mimeType = "audio/mpeg"
|
||||
}
|
||||
|
||||
b64 := base64.StdEncoding.EncodeToString(audioData)
|
||||
dataURL := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
|
||||
|
||||
// 映射语言代码
|
||||
language := mapLanguage(opts.Language)
|
||||
|
||||
reqBody := mimoRequest{
|
||||
Model: m.model,
|
||||
Messages: []mimoMessage{
|
||||
{
|
||||
Role: "user",
|
||||
Content: []mimoContent{
|
||||
{
|
||||
Type: "input_audio",
|
||||
InputAudio: &mimoAudioIn{
|
||||
Data: dataURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if language != "" {
|
||||
reqBody.ASROptions = &mimoASROptions{Language: language}
|
||||
}
|
||||
|
||||
body, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stt: marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := strings.TrimRight(m.endpoint, "/") + "/chat/completions"
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stt: create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("api-key", m.apiKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stt: request mimo: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stt: read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("stt: mimo returned %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
var mResp mimoResponse
|
||||
if err := json.Unmarshal(respBody, &mResp); err != nil {
|
||||
return "", fmt.Errorf("stt: unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
if len(mResp.Choices) == 0 {
|
||||
return "", fmt.Errorf("stt: mimo returned empty choices")
|
||||
}
|
||||
|
||||
text := strings.TrimSpace(mResp.Choices[0].Message.Content)
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// mapLanguage 将标准语言代码映射为 MiMo 支持的值（auto/zh/en）。
|
||||
func mapLanguage(lang string) string {
|
||||
switch {
|
||||
case lang == "":
|
||||
return "auto"
|
||||
case strings.HasPrefix(lang, "zh"):
|
||||
return "zh"
|
||||
case strings.HasPrefix(lang, "en"):
|
||||
return "en"
|
||||
default:
|
||||
return "auto"
|
||||
}
|
||||
}
|
||||
|
||||
// isWAV 检查数据是否为 WAV 格式（RIFF 头）。
|
||||
func isWAV(data []byte) bool {
|
||||
return len(data) > 4 && string(data[:4]) == "RIFF"
|
||||
}
|
||||
|
||||
// isMP3 检查数据是否为 MP3 格式（ID3 标签或帧同步字）。
|
||||
func isMP3(data []byte) bool {
|
||||
if len(data) > 3 && string(data[:3]) == "ID3" {
|
||||
return true
|
||||
}
|
||||
// 帧同步字：0xFF 0xFB/0xF3/0xF2
|
||||
return len(data) > 2 && data[0] == 0xFF && (data[1]&0xE0) == 0xE0
|
||||
}
|
||||
|
||||
// pcmToWAV 将原始 PCM 数据封装为 WAV 文件。
|
||||
func pcmToWAV(pcm []byte, sampleRate, channels int) ([]byte, error) {
|
||||
if sampleRate == 0 {
|
||||
sampleRate = 16000
|
||||
}
|
||||
if channels == 0 {
|
||||
channels = 1
|
||||
}
|
||||
|
||||
bitsPerSample := 16
|
||||
byteRate := sampleRate * channels * bitsPerSample / 8
|
||||
blockAlign := channels * bitsPerSample / 8
|
||||
dataSize := len(pcm)
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
// RIFF header
|
||||
buf.WriteString("RIFF")
|
||||
binary.Write(&buf, binary.LittleEndian, uint32(36+dataSize))
|
||||
buf.WriteString("WAVE")
|
||||
|
||||
// fmt 子块
|
||||
buf.WriteString("fmt ")
|
||||
binary.Write(&buf, binary.LittleEndian, uint32(16)) // 子块大小
|
||||
binary.Write(&buf, binary.LittleEndian, uint16(1)) // PCM 格式
|
||||
binary.Write(&buf, binary.LittleEndian, uint16(channels)) // 通道数
|
||||
binary.Write(&buf, binary.LittleEndian, uint32(sampleRate)) // 采样率
|
||||
binary.Write(&buf, binary.LittleEndian, uint32(byteRate)) // 字节率
|
||||
binary.Write(&buf, binary.LittleEndian, uint16(blockAlign)) // 块对齐
|
||||
binary.Write(&buf, binary.LittleEndian, uint16(bitsPerSample)) // 每样本位数
|
||||
|
||||
// data 子块
|
||||
buf.WriteString("data")
|
||||
binary.Write(&buf, binary.LittleEndian, uint32(dataSize))
|
||||
buf.Write(pcm)
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
255
backend/internal/ai/stt/mimo_test.go
Normal file
255
backend/internal/ai/stt/mimo_test.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package stt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func newTestMiMoService(handler http.HandlerFunc) (*MiMoService, *httptest.Server) {
|
||||
srv := httptest.NewServer(handler)
|
||||
s := NewMiMoService("test-key", "mimo-v2.5-asr", srv.URL, zap.NewNop().Sugar())
|
||||
return s, srv
|
||||
}
|
||||
|
||||
func TestMiMoService_Recognize_Success(t *testing.T) {
|
||||
s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) {
|
||||
// 验证请求
|
||||
if r.Header.Get("api-key") != "test-key" {
|
||||
t.Errorf("expected api-key test-key, got %s", r.Header.Get("api-key"))
|
||||
}
|
||||
if r.URL.Path != "/chat/completions" {
|
||||
t.Errorf("expected path /chat/completions, got %s", r.URL.Path)
|
||||
}
|
||||
|
||||
var req mimoRequest
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
t.Fatalf("unmarshal request: %v", err)
|
||||
}
|
||||
if req.Model != "mimo-v2.5-asr" {
|
||||
t.Errorf("expected model mimo-v2.5-asr, got %s", req.Model)
|
||||
}
|
||||
if len(req.Messages) == 0 || req.Messages[0].Role != "user" {
|
||||
t.Error("expected user message")
|
||||
}
|
||||
if req.ASROptions == nil || req.ASROptions.Language != "zh" {
|
||||
t.Errorf("expected language zh, got %v", req.ASROptions)
|
||||
}
|
||||
|
||||
resp := mimoResponse{
|
||||
Choices: []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
}{
|
||||
{Message: struct {
|
||||
Content string `json:"content"`
|
||||
}{Content: "你好世界"}},
|
||||
},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
// 发送一个简单的有效 WAV（44 字节头 + 少量 PCM）
|
||||
wav := makeValidWAV([]byte{0x00, 0x00, 0x00, 0x00})
|
||||
text, err := s.Recognize(context.Background(), wav, Options{Language: "zh-CN"})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if text != "你好世界" {
|
||||
t.Errorf("expected '你好世界', got '%s'", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiMoService_Recognize_EmptyAudio(t *testing.T) {
|
||||
s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) {})
|
||||
defer srv.Close()
|
||||
|
||||
_, err := s.Recognize(context.Background(), nil, Options{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty audio")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiMoService_Recognize_ServerError(t *testing.T) {
|
||||
s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte("internal error"))
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
wav := makeValidWAV([]byte{0x00, 0x00})
|
||||
_, err := s.Recognize(context.Background(), wav, Options{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for 500 response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiMoService_Recognize_EmptyChoices(t *testing.T) {
|
||||
s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) {
|
||||
resp := mimoResponse{Choices: nil}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
wav := makeValidWAV([]byte{0x00, 0x00})
|
||||
_, err := s.Recognize(context.Background(), wav, Options{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty choices")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiMoService_Recognize_PCMAutoWrap(t *testing.T) {
|
||||
// 测试原始 PCM 数据自动封装为 WAV
|
||||
s, srv := newTestMiMoService(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req mimoRequest
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
t.Fatalf("unmarshal request: %v", err)
|
||||
}
|
||||
|
||||
// 验证 data URL 格式
|
||||
if len(req.Messages) == 0 || len(req.Messages[0].Content) == 0 {
|
||||
t.Fatal("empty message content")
|
||||
}
|
||||
dataURL := req.Messages[0].Content[0].InputAudio.Data
|
||||
if len(dataURL) < 22 || dataURL[:14] != "data:audio/wav" {
|
||||
t.Errorf("expected wav data URL, got prefix: %s", dataURL[:min(len(dataURL), 30)])
|
||||
}
|
||||
|
||||
// 验证 base64 可解码
|
||||
b64Part := dataURL[22:] // skip "data:audio/wav;base64,"
|
||||
decoded, err := base64.StdEncoding.DecodeString(b64Part)
|
||||
if err != nil {
|
||||
t.Fatalf("base64 decode failed: %v", err)
|
||||
}
|
||||
// 应该是有效 WAV（RIFF 头）
|
||||
if len(decoded) < 44 || string(decoded[:4]) != "RIFF" {
|
||||
t.Error("decoded data is not a valid WAV")
|
||||
}
|
||||
|
||||
resp := mimoResponse{
|
||||
Choices: []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
}{
|
||||
{Message: struct {
|
||||
Content string `json:"content"`
|
||||
}{Content: "test"}},
|
||||
},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
})
|
||||
defer srv.Close()
|
||||
|
||||
// 发送原始 PCM（非 WAV/MP3）
|
||||
pcm := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}
|
||||
text, err := s.Recognize(context.Background(), pcm, Options{SampleRate: 16000})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if text != "test" {
|
||||
t.Errorf("expected 'test', got '%s'", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapLanguage(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"", "auto"},
|
||||
{"zh-CN", "zh"},
|
||||
{"zh", "zh"},
|
||||
{"en-US", "en"},
|
||||
{"en", "en"},
|
||||
{"ja", "auto"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := mapLanguage(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("mapLanguage(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsWAV(t *testing.T) {
|
||||
if !isWAV([]byte("RIFF....")) {
|
||||
t.Error("expected true for RIFF header")
|
||||
}
|
||||
if isWAV([]byte("ID3...")) {
|
||||
t.Error("expected false for ID3 header")
|
||||
}
|
||||
if isWAV([]byte{0x00}) {
|
||||
t.Error("expected false for short data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMP3(t *testing.T) {
|
||||
if !isMP3([]byte("ID3\x03")) {
|
||||
t.Error("expected true for ID3 header")
|
||||
}
|
||||
if !isMP3([]byte{0xFF, 0xFB, 0x00}) {
|
||||
t.Error("expected true for MP3 sync word")
|
||||
}
|
||||
if isMP3([]byte("RIFF")) {
|
||||
t.Error("expected false for RIFF header")
|
||||
}
|
||||
}
|
||||
|
||||
func makeValidWAV(pcm []byte) []byte {
|
||||
// 构造一个最小有效 WAV
|
||||
wav := make([]byte, 44+len(pcm))
|
||||
copy(wav[:4], "RIFF")
|
||||
// little-endian size = 36 + len(pcm)
|
||||
size := uint32(36 + len(pcm))
|
||||
wav[4] = byte(size)
|
||||
wav[5] = byte(size >> 8)
|
||||
wav[6] = byte(size >> 16)
|
||||
wav[7] = byte(size >> 24)
|
||||
copy(wav[8:12], "WAVE")
|
||||
copy(wav[12:16], "fmt ")
|
||||
// fmt chunk size = 16
|
||||
wav[16] = 16
|
||||
// PCM format = 1
|
||||
wav[20] = 1
|
||||
// channels = 1
|
||||
wav[22] = 1
|
||||
// sample rate = 16000
|
||||
wav[24] = 0x80
|
||||
wav[25] = 0x3E
|
||||
// byte rate = 32000
|
||||
wav[28] = 0x00
|
||||
wav[29] = 0x7D
|
||||
// block align = 2
|
||||
wav[32] = 2
|
||||
// bits per sample = 16
|
||||
wav[34] = 16
|
||||
copy(wav[36:40], "data")
|
||||
dSize := uint32(len(pcm))
|
||||
wav[40] = byte(dSize)
|
||||
wav[41] = byte(dSize >> 8)
|
||||
wav[42] = byte(dSize >> 16)
|
||||
wav[43] = byte(dSize >> 24)
|
||||
copy(wav[44:], pcm)
|
||||
return wav
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
Reference in New Issue
Block a user