feat: 实现 TTS 语音播放,完成 MVP P0 全部用户故事

- 新增 TTSPlayer:收集流式 tts_audio 片段,is_last 时拼接解码播放
- useVisionSession 接入 TTS 播放器,interrupt/stopSession 时停止播放
- App 显示 '🔊 正在播放...' 指示器
- MVP P0 四个用户故事前端全部实现(US-01 ~ US-04)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-06-13 13:57:36 +08:00
parent 0ef7aa657d
commit 6a47c4dfbb
4 changed files with 137 additions and 3 deletions

View File

@@ -112,6 +112,12 @@ body {
animation: none;
}
.vad-indicator--audio {
left: auto;
right: 12px;
color: var(--color-primary);
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.6; }

View File

@@ -13,6 +13,7 @@ function App() {
messages,
currentReply,
isProcessing,
isAudioPlaying,
isSpeaking,
isVADReady,
vadError,
@@ -43,6 +44,7 @@ function App() {
<div className="video-section">
<VideoPreview ref={videoRef} isStreaming={!!stream} />
{isSpeaking && <div className="vad-indicator">🎤 ...</div>}
{isAudioPlaying && <div className="vad-indicator vad-indicator--audio">🔊 ...</div>}
{isConnected && !isVADReady && !vadError && (
<div className="vad-indicator vad-indicator--loading">
...

View File

@@ -9,6 +9,7 @@ import { v4 as uuidv4 } from "uuid";
import { wsClient } from "../lib/websocket";
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
import { getErrorMessage } from "../lib/errors";
import { TTSPlayer } from "../lib/ttsPlayer";
import { showToast } from "../lib/toast";
import { useCamera } from "../components/CameraManager";
import { useMicrophone } from "../components/MicManager";
@@ -20,6 +21,18 @@ export function useVisionSession() {
const [messages, setMessages] = useState<ChatMessage[]>([]);
const [currentReply, setCurrentReply] = useState<string>("");
const [isProcessing, setIsProcessing] = useState(false);
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
// TTS 播放器
const ttsPlayerRef = useRef<TTSPlayer | null>(null);
const getTTSPlayer = useCallback(() => {
if (!ttsPlayerRef.current) {
const player = new TTSPlayer();
player.onEnd(() => setIsAudioPlaying(false));
ttsPlayerRef.current = player;
}
return ttsPlayerRef.current;
}, []);
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
const { startMic, stopMic } = useMicrophone();
@@ -115,7 +128,10 @@ export function useVisionSession() {
}
case "tts_audio":
// TODO: 阶段 5 音频流播放
getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
if (!msg.is_last) {
setIsAudioPlaying(true);
}
break;
case "error":
@@ -127,7 +143,7 @@ export function useVisionSession() {
});
return unsub;
}, []);
}, [getTTSPlayer]);
// 连接断开时显示提示
useEffect(() => {
@@ -160,7 +176,9 @@ export function useVisionSession() {
stopMic();
stopCamera();
disconnect();
// 清理所有对话状态
// 停止 TTS 并清理状态
ttsPlayerRef.current?.stop();
setIsAudioPlaying(false);
setMessages([]);
setCurrentReply("");
setIsProcessing(false);
@@ -169,6 +187,9 @@ export function useVisionSession() {
/** 打断当前回复 */
const interrupt = useCallback(() => {
send({ type: "interrupt" });
// 停止 TTS 播放
ttsPlayerRef.current?.stop();
setIsAudioPlaying(false);
// 将未完成的流式内容保存为最终消息
if (currentReply) {
setMessages((prev) => [
@@ -184,6 +205,7 @@ export function useVisionSession() {
messages,
currentReply,
isProcessing,
isAudioPlaying,
isSpeaking,
isVADReady,
vadError,

View File

@@ -0,0 +1,104 @@
// ============================================================
// TTS Player — 语音播放器
// 职责:收集后端流式 tts_audio 片段,拼接后播放
// 格式MVP 仅支持 audio/mp3pcm 为 TODO
// ============================================================
type OnEndCallback = () => void;
export class TTSPlayer {
private chunks: string[] = [];
private audio: HTMLAudioElement | null = null;
private _isPlaying = false;
private onEndCallback: OnEndCallback | null = null;
/** 注册播放完成回调 */
onEnd(cb: OnEndCallback): void {
this.onEndCallback = cb;
}
/** 当前是否正在播放 */
get isPlaying(): boolean {
return this._isPlaying;
}
/**
* 入队一个 TTS 音频片段
* @param base64 Base64 编码的音频数据
* @param mimeType 音频格式("audio/mp3" 或 "audio/pcm"
* @param isLast 是否为最后一个片段
*/
enqueue(base64: string, mimeType: string, isLast: boolean): void {
this.chunks.push(base64);
if (isLast) {
this.play(mimeType);
}
}
/** 停止播放并清空缓冲区 */
stop(): void {
if (this.audio) {
this.audio.pause();
this.audio.removeAttribute("src");
this.audio = null;
}
this.chunks = [];
this._isPlaying = false;
}
/** 暂停播放 */
pause(): void {
this.audio?.pause();
}
/** 恢复播放 */
resume(): void {
this.audio?.play();
}
/** 拼接所有片段并播放 */
private play(mimeType: string): void {
if (this.chunks.length === 0) return;
// 拼接所有 Base64 片段
const combined = this.chunks.join("");
this.chunks = [];
// Base64 → Uint8Array → Blob
const binary = atob(combined);
const bytes = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i++) {
bytes[i] = binary.charCodeAt(i);
}
const blob = new Blob([bytes], { type: mimeType });
const url = URL.createObjectURL(blob);
// 播放
const audio = new Audio(url);
this.audio = audio;
this._isPlaying = true;
audio.onended = () => {
URL.revokeObjectURL(url);
this._isPlaying = false;
this.audio = null;
this.onEndCallback?.();
};
audio.onerror = () => {
console.error("[TTS] 播放失败");
URL.revokeObjectURL(url);
this._isPlaying = false;
this.audio = null;
this.onEndCallback?.();
};
audio.play().catch((err) => {
console.error("[TTS] play() 被拒绝:", err);
this._isPlaying = false;
this.audio = null;
this.onEndCallback?.();
});
}
}