feat: 实现 TTS 语音播放,完成 MVP P0 全部用户故事
- 新增 TTSPlayer:收集流式 tts_audio 片段,is_last 时拼接解码播放 - useVisionSession 接入 TTS 播放器,interrupt/stopSession 时停止播放 - App 显示 '🔊 正在播放...' 指示器 - MVP P0 四个用户故事前端全部实现(US-01 ~ US-04) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import { v4 as uuidv4 } from "uuid";
|
||||
import { wsClient } from "../lib/websocket";
|
||||
import { encodeAudioToBase64, dataUrlToBase64 } from "../lib/audio";
|
||||
import { getErrorMessage } from "../lib/errors";
|
||||
import { TTSPlayer } from "../lib/ttsPlayer";
|
||||
import { showToast } from "../lib/toast";
|
||||
import { useCamera } from "../components/CameraManager";
|
||||
import { useMicrophone } from "../components/MicManager";
|
||||
@@ -20,6 +21,18 @@ export function useVisionSession() {
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const [currentReply, setCurrentReply] = useState<string>("");
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [isAudioPlaying, setIsAudioPlaying] = useState(false);
|
||||
|
||||
// TTS 播放器
|
||||
const ttsPlayerRef = useRef<TTSPlayer | null>(null);
|
||||
const getTTSPlayer = useCallback(() => {
|
||||
if (!ttsPlayerRef.current) {
|
||||
const player = new TTSPlayer();
|
||||
player.onEnd(() => setIsAudioPlaying(false));
|
||||
ttsPlayerRef.current = player;
|
||||
}
|
||||
return ttsPlayerRef.current;
|
||||
}, []);
|
||||
|
||||
const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
|
||||
const { startMic, stopMic } = useMicrophone();
|
||||
@@ -115,7 +128,10 @@ export function useVisionSession() {
|
||||
}
|
||||
|
||||
case "tts_audio":
|
||||
// TODO: 阶段 5 音频流播放
|
||||
getTTSPlayer().enqueue(msg.audio, msg.mime_type, msg.is_last);
|
||||
if (!msg.is_last) {
|
||||
setIsAudioPlaying(true);
|
||||
}
|
||||
break;
|
||||
|
||||
case "error":
|
||||
@@ -127,7 +143,7 @@ export function useVisionSession() {
|
||||
});
|
||||
|
||||
return unsub;
|
||||
}, []);
|
||||
}, [getTTSPlayer]);
|
||||
|
||||
// 连接断开时显示提示
|
||||
useEffect(() => {
|
||||
@@ -160,7 +176,9 @@ export function useVisionSession() {
|
||||
stopMic();
|
||||
stopCamera();
|
||||
disconnect();
|
||||
// 清理所有对话状态
|
||||
// 停止 TTS 并清理状态
|
||||
ttsPlayerRef.current?.stop();
|
||||
setIsAudioPlaying(false);
|
||||
setMessages([]);
|
||||
setCurrentReply("");
|
||||
setIsProcessing(false);
|
||||
@@ -169,6 +187,9 @@ export function useVisionSession() {
|
||||
/** 打断当前回复 */
|
||||
const interrupt = useCallback(() => {
|
||||
send({ type: "interrupt" });
|
||||
// 停止 TTS 播放
|
||||
ttsPlayerRef.current?.stop();
|
||||
setIsAudioPlaying(false);
|
||||
// 将未完成的流式内容保存为最终消息
|
||||
if (currentReply) {
|
||||
setMessages((prev) => [
|
||||
@@ -184,6 +205,7 @@ export function useVisionSession() {
|
||||
messages,
|
||||
currentReply,
|
||||
isProcessing,
|
||||
isAudioPlaying,
|
||||
isSpeaking,
|
||||
isVADReady,
|
||||
vadError,
|
||||
|
||||
Reference in New Issue
Block a user