From 6a47c4dfbb11ff9491e229e78c0f85dcd0904c95 Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Sat, 13 Jun 2026 13:57:36 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=20TTS=20=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E6=92=AD=E6=94=BE=EF=BC=8C=E5=AE=8C=E6=88=90=20MVP=20?= =?UTF-8?q?P0=20=E5=85=A8=E9=83=A8=E7=94=A8=E6=88=B7=E6=95=85=E4=BA=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 TTSPlayer：收集流式 tts_audio 片段，is_last 时拼接解码播放 - useVisionSession 接入 TTS 播放器，interrupt/stopSession 时停止播放 - App 显示 '🔊 正在播放...' 指示器 - MVP P0 四个用户故事前端全部实现（US-01 ~ US-04） Co-Authored-By: Claude --- frontend/src/App.css | 6 ++ frontend/src/App.tsx | 2 + frontend/src/hooks/useVisionSession.ts | 28 ++++++- frontend/src/lib/ttsPlayer.ts | 104 +++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 frontend/src/lib/ttsPlayer.ts diff --git a/frontend/src/App.css b/frontend/src/App.css index 4cf2215..cc45dce 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -112,6 +112,12 @@ body { animation: none; } +.vad-indicator--audio { + left: auto; + right: 12px; + color: var(--color-primary); +} + @keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.6; } diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 9bdb7b4..dd55531 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -13,6 +13,7 @@ function App() { messages, currentReply, isProcessing, + isAudioPlaying, isSpeaking, isVADReady, vadError, @@ -43,6 +44,7 @@ function App() {
{isSpeaking &&
🎤 正在聆听...
} + {isAudioPlaying &&
🔊 正在播放...
} {isConnected && !isVADReady && !vadError && (
正在初始化语音检测... diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts index a9112f6..dfbb554 100644 --- a/frontend/src/hooks/useVisionSession.ts +++ b/frontend/src/hooks/useVisionSession.ts @@ -9,6 +9,7 @@ import { v4 as uuidv4 } from "uuid"; import { wsClient } from "../lib/websocket"; import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio"; import { getErrorMessage } from "../lib/errors"; +import { TTSPlayer } from "../lib/ttsPlayer"; import { showToast } from "../lib/toast"; import { useCamera } from "../components/CameraManager"; import { useMicrophone } from "../components/MicManager"; @@ -20,6 +21,18 @@ export function useVisionSession() { const [messages, setMessages] = useState([]); const [currentReply, setCurrentReply] = useState(""); const [isProcessing, setIsProcessing] = useState(false); + const [isAudioPlaying, setIsAudioPlaying] = useState(false); + + // TTS 播放器 + const ttsPlayerRef = useRef(null); + const getTTSPlayer = useCallback(() => { + if (!ttsPlayerRef.current) { + const player = new TTSPlayer(); + player.onEnd(() => setIsAudioPlaying(false)); + ttsPlayerRef.current = player; + } + return ttsPlayerRef.current; + }, []); const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); const { startMic, stopMic } = useMicrophone(); @@ -115,7 +128,10 @@ export function useVisionSession() { } case "tts_audio": - // TODO: 阶段 5 音频流播放 + getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last); + if (!msg.is_last) { + setIsAudioPlaying(true); + } break; case "error": @@ -127,7 +143,7 @@ export function useVisionSession() { }); return unsub; - }, []); + }, [getTTSPlayer]); // 连接断开时显示提示 useEffect(() => { @@ -160,7 +176,9 @@ export function useVisionSession() { stopMic(); stopCamera(); disconnect(); - // 清理所有对话状态 + // 停止 TTS 并清理状态 + ttsPlayerRef.current?.stop(); + setIsAudioPlaying(false); setMessages([]); setCurrentReply(""); setIsProcessing(false); @@ -169,6 +187,9 @@ export function useVisionSession() { /** 打断当前回复 */ const interrupt = useCallback(() => { send({ type: "interrupt" }); + // 停止 TTS 播放 + ttsPlayerRef.current?.stop(); + setIsAudioPlaying(false); // 将未完成的流式内容保存为最终消息 if (currentReply) { setMessages((prev) => [ @@ -184,6 +205,7 @@ export function useVisionSession() { messages, currentReply, isProcessing, + isAudioPlaying, isSpeaking, isVADReady, vadError, diff --git a/frontend/src/lib/ttsPlayer.ts b/frontend/src/lib/ttsPlayer.ts new file mode 100644 index 0000000..a7f283f --- /dev/null +++ b/frontend/src/lib/ttsPlayer.ts @@ -0,0 +1,104 @@ +// ============================================================ +// TTS Player — 语音播放器 +// 职责：收集后端流式 tts_audio 片段，拼接后播放 +// 格式：MVP 仅支持 audio/mp3，pcm 为 TODO +// ============================================================ + +type OnEndCallback = () => void; + +export class TTSPlayer { + private chunks: string[] = []; + private audio: HTMLAudioElement | null = null; + private _isPlaying = false; + private onEndCallback: OnEndCallback | null = null; + + /** 注册播放完成回调 */ + onEnd(cb: OnEndCallback): void { + this.onEndCallback = cb; + } + + /** 当前是否正在播放 */ + get isPlaying(): boolean { + return this._isPlaying; + } + + /** + * 入队一个 TTS 音频片段 + * @param base64 Base64 编码的音频数据 + * @param mimeType 音频格式（"audio/mp3" 或 "audio/pcm"） + * @param isLast 是否为最后一个片段 + */ + enqueue(base64: string, mimeType: string, isLast: boolean): void { + this.chunks.push(base64); + + if (isLast) { + this.play(mimeType); + } + } + + /** 停止播放并清空缓冲区 */ + stop(): void { + if (this.audio) { + this.audio.pause(); + this.audio.removeAttribute("src"); + this.audio = null; + } + this.chunks = []; + this._isPlaying = false; + } + + /** 暂停播放 */ + pause(): void { + this.audio?.pause(); + } + + /** 恢复播放 */ + resume(): void { + this.audio?.play(); + } + + /** 拼接所有片段并播放 */ + private play(mimeType: string): void { + if (this.chunks.length === 0) return; + + // 拼接所有 Base64 片段 + const combined = this.chunks.join(""); + this.chunks = []; + + // Base64 → Uint8Array → Blob + const binary = atob(combined); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + const blob = new Blob([bytes], { type: mimeType }); + const url = URL.createObjectURL(blob); + + // 播放 + const audio = new Audio(url); + this.audio = audio; + this._isPlaying = true; + + audio.onended = () => { + URL.revokeObjectURL(url); + this._isPlaying = false; + this.audio = null; + this.onEndCallback?.(); + }; + + audio.onerror = () => { + console.error("[TTS] 播放失败"); + URL.revokeObjectURL(url); + this._isPlaying = false; + this.audio = null; + this.onEndCallback?.(); + }; + + audio.play().catch((err) => { + console.error("[TTS] play() 被拒绝:", err); + this._isPlaying = false; + this.audio = null; + this.onEndCallback?.(); + }); + } +}