fix: 修复语音与文字不同步的问题,改为句子级流式播放
之前前端 TTSPlayer 攒齐所有音频片段后才播放,导致文字全部显示后才开始语音。 改为后端每句 TTS 发送 is_last: true,前端收到每句即加入播放队列,第一句到达即开始播放。 - 后端 Chunk 结构体新增 Final 字段,区分句子结束和整轮结束 - 前端 TTSPlayer 重写为队列式播放,onended 回调自动衔接下一句 - 同步更新接口文档和测试用例
This commit is contained in:
@@ -266,9 +266,7 @@ export function useVisionSession() {
|
||||
|
||||
case "tts_audio":
|
||||
getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
|
||||
if (!msg.is_last) {
|
||||
setIsAudioPlaying(true);
|
||||
}
|
||||
setIsAudioPlaying(true);
|
||||
break;
|
||||
|
||||
case "error":
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
// ============================================================
|
||||
// TTS Player — 语音播放器
|
||||
// 职责:收集后端流式 tts_audio 片段,拼接后播放
|
||||
// 格式:MVP 仅支持 audio/mp3,pcm 为 TODO
|
||||
// 职责:接收后端流式 tts_audio 片段,按句子排队播放
|
||||
// 设计:第一句到达即开始播放,后续句子在 onended 回调中自动衔接
|
||||
// ============================================================
|
||||
|
||||
type OnEndCallback = () => void;
|
||||
|
||||
export class TTSPlayer {
|
||||
private chunks: string[] = [];
|
||||
/** 当前句子的音频片段缓冲(每句可能含多个 chunk) */
|
||||
private sentenceChunks: string[] = [];
|
||||
/** 已就绪的句子 Blob URL 播放队列 */
|
||||
private queue: string[] = [];
|
||||
private audio: HTMLAudioElement | null = null;
|
||||
private _isPlaying = false;
|
||||
private onEndCallback: OnEndCallback | null = null;
|
||||
|
||||
/** 注册播放完成回调 */
|
||||
/** 注册播放完成回调(所有句子播完后触发) */
|
||||
onEnd(cb: OnEndCallback): void {
|
||||
this.onEndCallback = cb;
|
||||
}
|
||||
@@ -25,38 +28,40 @@ export class TTSPlayer {
|
||||
/**
|
||||
* 入队一个 TTS 音频片段
|
||||
* @param base64 Base64 编码的音频数据
|
||||
* @param mimeType 音频格式("audio/mp3" 或 "audio/pcm")
|
||||
* @param isLast 是否为最后一个片段
|
||||
* @param mimeType 音频格式("audio/mp3")
|
||||
* @param isLast 当前句子是否合成完毕(每句结束时为 true)
|
||||
*/
|
||||
enqueue(base64: string, mimeType: string, isLast: boolean): void {
|
||||
this.chunks.push(base64);
|
||||
this.sentenceChunks.push(base64);
|
||||
|
||||
if (isLast) {
|
||||
this.play(mimeType);
|
||||
// 当前句子的音频已完整,拼接并加入播放队列
|
||||
const combined = this.sentenceChunks.join("");
|
||||
this.sentenceChunks = [];
|
||||
|
||||
const url = this.base64ToBlobUrl(combined, mimeType);
|
||||
this.queue.push(url);
|
||||
|
||||
// 如果当前没有在播,立即开始播放
|
||||
if (!this._isPlaying) {
|
||||
this.playNext();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 停止播放并清空缓冲区 */
|
||||
/** 停止播放并清空所有队列 */
|
||||
stop(): void {
|
||||
if (this.audio) {
|
||||
this.audio.pause();
|
||||
this.audio.removeAttribute("src");
|
||||
this.audio = null;
|
||||
}
|
||||
this.chunks = [];
|
||||
this._isPlaying = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 停止当前正在播放的音频(不清空 chunks)
|
||||
* 用于 play() 开始前,防止新旧音频重叠
|
||||
*/
|
||||
private stopCurrentAudio(): void {
|
||||
if (this.audio) {
|
||||
this.audio.pause();
|
||||
this.audio.removeAttribute("src");
|
||||
this.audio = null;
|
||||
// 释放队列中的 Blob URL
|
||||
for (const url of this.queue) {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
this.queue = [];
|
||||
this.sentenceChunks = [];
|
||||
this._isPlaying = false;
|
||||
}
|
||||
|
||||
@@ -70,51 +75,48 @@ export class TTSPlayer {
|
||||
this.audio?.play();
|
||||
}
|
||||
|
||||
/** 拼接所有片段并播放 */
|
||||
private play(mimeType: string): void {
|
||||
if (this.chunks.length === 0) return;
|
||||
|
||||
// 停止当前正在播放的音频,防止重叠
|
||||
this.stopCurrentAudio();
|
||||
|
||||
// 拼接所有 Base64 片段
|
||||
const combined = this.chunks.join("");
|
||||
this.chunks = [];
|
||||
|
||||
// Base64 → Uint8Array → Blob
|
||||
const binary = atob(combined);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
/** 播放队列中的下一个句子 */
|
||||
private playNext(): void {
|
||||
if (this.queue.length === 0) {
|
||||
this._isPlaying = false;
|
||||
this.onEndCallback?.();
|
||||
return;
|
||||
}
|
||||
const blob = new Blob([bytes], { type: mimeType });
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
// 播放
|
||||
const url = this.queue.shift()!;
|
||||
const audio = new Audio(url);
|
||||
this.audio = audio;
|
||||
this._isPlaying = true;
|
||||
|
||||
audio.onended = () => {
|
||||
URL.revokeObjectURL(url);
|
||||
this._isPlaying = false;
|
||||
this.audio = null;
|
||||
this.onEndCallback?.();
|
||||
this.playNext();
|
||||
};
|
||||
|
||||
audio.onerror = () => {
|
||||
console.error("[TTS] 播放失败");
|
||||
console.error("[TTS] 播放失败,跳过");
|
||||
URL.revokeObjectURL(url);
|
||||
this._isPlaying = false;
|
||||
this.audio = null;
|
||||
this.onEndCallback?.();
|
||||
this.playNext();
|
||||
};
|
||||
|
||||
audio.play().catch((err) => {
|
||||
console.error("[TTS] play() 被拒绝:", err);
|
||||
this._isPlaying = false;
|
||||
URL.revokeObjectURL(url);
|
||||
this.audio = null;
|
||||
this.onEndCallback?.();
|
||||
this.playNext();
|
||||
});
|
||||
}
|
||||
|
||||
/** Base64 字符串转 Blob URL */
|
||||
private base64ToBlobUrl(base64: string, mimeType: string): string {
|
||||
const binary = atob(base64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
const blob = new Blob([bytes], { type: mimeType });
|
||||
return URL.createObjectURL(blob);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +115,8 @@ export interface TTSAudioMessage {
|
||||
request_id: string;
|
||||
audio: string; // Base64 音频片段
|
||||
mime_type: string; // "audio/mp3" 或 "audio/pcm"
|
||||
is_last: boolean;
|
||||
is_last: boolean; // 当前句子的音频是否完整(每句结束时为 true)
|
||||
final: boolean; // 整轮 TTS 是否结束
|
||||
}
|
||||
|
||||
export interface ErrorMessage {
|
||||
|
||||
Reference in New Issue
Block a user