From 6967dd7b2e31c5587c7363968df0d9026839da0e Mon Sep 17 00:00:00 2001 From: cfy666 <3087823110@qq.com> Date: Sun, 21 Jun 2026 16:49:48 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=E6=91=84=E5=83=8F?= =?UTF-8?q?=E5=A4=B4=E5=92=8C=E9=BA=A6=E5=85=8B=E9=A3=8E=E8=AE=BE=E5=A4=87?= =?UTF-8?q?=E9=80=89=E6=8B=A9=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 useDeviceList hook，枚举音视频输入设备并监听热插拔 - CameraManager/MicManager 的 startCamera/startMic 支持可选 deviceId 参数 - useVisionSession 集成设备选择：授权后自动枚举、切换设备时热重启 - 连接后显示设备下拉选择器，未连接时隐藏（避免未授权时空列表） - SessionConfig 新增 cameraDeviceId/micDeviceId 持久化到 localStorage --- frontend/src/App.tsx | 47 +++++++++++++------ .../src/components/CameraManager/index.tsx | 7 ++- frontend/src/components/MicManager/index.tsx | 12 ++--- frontend/src/hooks/useDeviceList.ts | 44 +++++++++++++++++ frontend/src/hooks/useVisionSession.ts | 47 ++++++++++++++++--- frontend/src/lib/i18n/en-US.ts | 2 - frontend/src/lib/i18n/ja-JP.ts | 2 - frontend/src/lib/i18n/zh-CN.ts | 2 - frontend/src/types/index.ts | 2 + 9 files changed, 128 insertions(+), 37 deletions(-) create mode 100644 frontend/src/hooks/useDeviceList.ts diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index ef349a3..89e84d6 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -96,6 +96,9 @@ function AppContent() { toggleCamera, toggleMic, sendTextMessage, + cameras, + mics, + switchDevice, } = useVisionSession(accessToken, activeSessionId); const isConnected = connectionStatus === "connected"; @@ -406,21 +409,6 @@ function AppContent() { - {/* 设备选择器 */} -
-
- - -
-
- - -
-
) : isCameraOn ? ( <> @@ -457,6 +445,35 @@ function AppContent() { {tr("controls.stopVideo")} + {/* 设备选择器 */} +
+
+ + +
+
+ + +
+
) : ( <> diff --git a/frontend/src/components/CameraManager/index.tsx b/frontend/src/components/CameraManager/index.tsx index 7b950a4..6c89f69 100644 --- a/frontend/src/components/CameraManager/index.tsx +++ b/frontend/src/components/CameraManager/index.tsx @@ -19,10 +19,13 @@ export function useCamera() { const [stream, setStream] = useState(null); const [error, setError] = useState(null); - const startCamera = useCallback(async () => { + const startCamera = useCallback(async (deviceId?: string) => { try { + const videoConstraints: MediaTrackConstraints = deviceId + ? { deviceId: { exact: deviceId }, width: 640, height: 480 } + : { facingMode: "environment", width: 640, height: 480 }; const mediaStream = await navigator.mediaDevices.getUserMedia({ - video: { facingMode: "environment", width: 640, height: 480 }, + video: videoConstraints, audio: false, }); setStream(mediaStream); diff --git a/frontend/src/components/MicManager/index.tsx b/frontend/src/components/MicManager/index.tsx index 4537c7f..383ba38 100644 --- a/frontend/src/components/MicManager/index.tsx +++ b/frontend/src/components/MicManager/index.tsx @@ -12,15 +12,13 @@ export function useMicrophone() { const [error, setError] = useState(null); const audioContextRef = useRef(null); - const startMic = useCallback(async () => { + const startMic = useCallback(async (deviceId?: string) => { try { + const audioConstraints: MediaTrackConstraints = deviceId + ? { deviceId: { exact: deviceId }, sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true } + : { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true }; const mediaStream = await navigator.mediaDevices.getUserMedia({ - audio: { - sampleRate: 16000, - channelCount: 1, - echoCancellation: true, - noiseSuppression: true, - }, + audio: audioConstraints, video: false, }); setStream(mediaStream); diff --git a/frontend/src/hooks/useDeviceList.ts b/frontend/src/hooks/useDeviceList.ts new file mode 100644 index 0000000..54fde10 --- /dev/null +++ b/frontend/src/hooks/useDeviceList.ts @@ -0,0 +1,44 @@ +// ============================================================ +// useDeviceList — 设备枚举 Hook +// 职责：枚举摄像头/麦克风设备列表，监听设备热插拔 +// 注意：首次枚举需要先有一次成功的 getUserMedia 授权 +// ============================================================ + +import { useCallback, useEffect, useState } from "react"; + +export interface DeviceInfo { + deviceId: string; + label: string; +} + +export function useDeviceList() { + const [cameras, setCameras] = useState([]); + const [mics, setMics] = useState([]); + + /** 枚举当前可用的音视频输入设备 */ + const refreshDevices = useCallback(async () => { + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + setCameras( + devices + .filter((d) => d.kind === "videoinput") + .map((d) => ({ deviceId: d.deviceId, label: d.label || `摄像头 ${d.deviceId.slice(0, 4)}` })) + ); + setMics( + devices + .filter((d) => d.kind === "audioinput") + .map((d) => ({ deviceId: d.deviceId, label: d.label || `麦克风 ${d.deviceId.slice(0, 4)}` })) + ); + } catch (err) { + console.warn("[DeviceList] 枚举设备失败:", err); + } + }, []); + + // 监听设备热插拔 + useEffect(() => { + navigator.mediaDevices.addEventListener("devicechange", refreshDevices); + return () => navigator.mediaDevices.removeEventListener("devicechange", refreshDevices); + }, [refreshDevices]); + + return { cameras, mics, refreshDevices }; +} diff --git a/frontend/src/hooks/useVisionSession.ts b/frontend/src/hooks/useVisionSession.ts index a226b53..dc7ed9c 100644 --- a/frontend/src/hooks/useVisionSession.ts +++ b/frontend/src/hooks/useVisionSession.ts @@ -17,6 +17,7 @@ import { useCamera } from "../components/CameraManager"; import { useMicrophone } from "../components/MicManager"; import { useVAD, sampleFrame } from "../components/EdgeProcessor"; import { useWebSocketManager } from "../components/WebSocketManager"; +import { useDeviceList } from "./useDeviceList"; import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types"; export interface SessionStats { @@ -55,6 +56,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera(); const { startMic, stopMic } = useMicrophone(); const { status, connect, disconnect, send } = useWebSocketManager(); + const { cameras, mics, refreshDevices } = useDeviceList(); // 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题 const isProcessingRef = useRef(false); @@ -277,14 +279,14 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s // 2. 尝试获取摄像头（可选） try { - await startCamera(); + await startCamera(config.cameraDeviceId || undefined); setIsCameraOn(true); } catch { console.warn("[Session] 无法获取摄像头权限，将以纯文本模式运行"); } // 3. 尝试获取麦克风（可选） - const micStream = await startMic(); + const micStream = await startMic(config.micDeviceId || undefined); if (micStream) { setIsMicOn(true); // 4. 启动 VAD（仅在麦克风可用时） @@ -292,7 +294,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s } else { console.warn("[Session] 无法获取麦克风权限，将以文本输入模式运行"); } - }, [startCamera, startMic, connect, startVAD, accessToken]); + + // 5. 授权后刷新设备列表 + refreshDevices(); + }, [startCamera, startMic, connect, startVAD, accessToken, config.cameraDeviceId, config.micDeviceId, refreshDevices]); /** 结束会话 */ const stopSession = useCallback(async () => { @@ -332,10 +337,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s stopCamera(); setIsCameraOn(false); } else { - await startCamera(); + await startCamera(config.cameraDeviceId || undefined); setIsCameraOn(true); } - }, [isCameraOn, startCamera, stopCamera]); + }, [isCameraOn, startCamera, stopCamera, config.cameraDeviceId]); /** 麦克风开关 */ const toggleMic = useCallback(async () => { @@ -344,13 +349,13 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s stopMic(); setIsMicOn(false); } else { - const micStream = await startMic(); + const micStream = await startMic(config.micDeviceId || undefined); if (micStream) { await startVAD(micStream); setIsMicOn(true); } } - }, [isMicOn, startMic, stopMic, startVAD, stopVAD]); + }, [isMicOn, startMic, stopMic, startVAD, stopVAD, config.micDeviceId]); /** 打断当前回复 */ const interrupt = useCallback(() => { @@ -370,6 +375,31 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s setIsProcessing(false); }, [send, currentReply]); + /** 切换设备（摄像头或麦克风） */ + const switchDevice = useCallback(async (kind: "camera" | "mic", deviceId: string) => { + if (kind === "camera") { + updateConfig({ cameraDeviceId: deviceId || undefined }); + if (isCameraOn) { + stopCamera(); + try { + await startCamera(deviceId || undefined); + } catch { + console.warn("[Session] 切换摄像头失败"); + } + } + } else { + updateConfig({ micDeviceId: deviceId || undefined }); + if (isMicOn) { + await stopVAD(); + stopMic(); + const micStream = await startMic(deviceId || undefined); + if (micStream) { + await startVAD(micStream); + } + } + } + }, [isCameraOn, isMicOn, stopCamera, startCamera, stopMic, startMic, stopVAD, startVAD, updateConfig]); + /** 发送文本消息（手动输入） */ const sendTextMessage = useCallback( (text: string) => { @@ -439,5 +469,8 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s toggleMic, sendTextMessage, captureFrame, + cameras, + mics, + switchDevice, }; } diff --git a/frontend/src/lib/i18n/en-US.ts b/frontend/src/lib/i18n/en-US.ts index 528f7e4..83bab31 100644 --- a/frontend/src/lib/i18n/en-US.ts +++ b/frontend/src/lib/i18n/en-US.ts @@ -133,8 +133,6 @@ export const enUS: TranslationMap = { // Video controls (enhanced) "controls.recognize": "Analyze Scene", - "controls.device.camera": "Camera", - "controls.device.mic": "Microphone", "controls.device.default": "Default", // Status bar diff --git a/frontend/src/lib/i18n/ja-JP.ts b/frontend/src/lib/i18n/ja-JP.ts index 784b480..770efec 100644 --- a/frontend/src/lib/i18n/ja-JP.ts +++ b/frontend/src/lib/i18n/ja-JP.ts @@ -133,8 +133,6 @@ export const jaJP: TranslationMap = { // Video controls (enhanced) "controls.recognize": "シーンを分析", - "controls.device.camera": "カメラ", - "controls.device.mic": "マイク", "controls.device.default": "デフォルト", // Status bar diff --git a/frontend/src/lib/i18n/zh-CN.ts b/frontend/src/lib/i18n/zh-CN.ts index 4b7b109..f934f0d 100644 --- a/frontend/src/lib/i18n/zh-CN.ts +++ b/frontend/src/lib/i18n/zh-CN.ts @@ -133,8 +133,6 @@ export const zhCN: TranslationMap = { // Video controls (enhanced) "controls.recognize": "识别画面", - "controls.device.camera": "摄像头", - "controls.device.mic": "麦克风", "controls.device.default": "默认", // Status bar diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 9f6b3b1..aec4780 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -10,6 +10,8 @@ export interface SessionConfig { detailLevel: "low" | "high"; language: string; scenario: string; // 情景 ID，如 "free_chat"、"interviewer" + cameraDeviceId?: string; // 摄像头设备 ID，空 = 系统默认 + micDeviceId?: string; // 麦克风设备 ID，空 = 系统默认 } export type Theme = "dark" | "light";