feat: 实现摄像头和麦克风设备选择功能

- 新增 useDeviceList hook，枚举音视频输入设备并监听热插拔 - CameraManager/MicManager 的 startCamera/startMic 支持可选 deviceId 参数 - useVisionSession 集成设备选择：授权后自动枚举、切换设备时热重启 - 连接后显示设备下拉选择器，未连接时隐藏（避免未授权时空列表） - SessionConfig 新增 cameraDeviceId/micDeviceId 持久化到 localStorage
2026-06-21 16:49:48 +08:00
parent eea5c07eaa
commit 6967dd7b2e
9 changed files with 128 additions and 37 deletions
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -96,6 +96,9 @@ function AppContent() {
    toggleCamera,
    toggleMic,
    sendTextMessage,
+    cameras,
+    mics,
+    switchDevice,
  } = useVisionSession(accessToken, activeSessionId);

  const isConnected = connectionStatus === "connected";
@@ -406,21 +409,6 @@ function AppContent() {
                <button className="btn btn--primary btn--lg" onClick={startSession}>
                  {connectionStatus === "connecting" ? tr("controls.connecting") : tr("controls.startVideo")}
                </button>
-                {/* 设备选择器 */}
-                <div className="video-controls__devices">
-                  <div className="device-select-wrapper">
-                    <label className="device-select-label">📷 {tr("controls.device.camera")}</label>
-                    <select className="device-select" defaultValue="default">
-                      <option value="default">{tr("controls.device.default")}</option>
-                    </select>
-                  </div>
-                  <div className="device-select-wrapper">
-                    <label className="device-select-label">🎤 {tr("controls.device.mic")}</label>
-                    <select className="device-select" defaultValue="default">
-                      <option value="default">{tr("controls.device.default")}</option>
-                    </select>
-                  </div>
-                </div>
              </>
            ) : isCameraOn ? (
              <>
@@ -457,6 +445,35 @@ function AppContent() {
                    {tr("controls.stopVideo")}
                  </button>
                </div>
+                {/* 设备选择器 */}
+                <div className="video-controls__devices">
+                  <div className="device-select-wrapper">
+                    <label className="device-select-label">📷</label>
+                    <select
+                      className="device-select"
+                      value={config.cameraDeviceId || "default"}
+                      onChange={(e) => switchDevice("camera", e.target.value === "default" ? "" : e.target.value)}
+                    >
+                      <option value="default">{tr("controls.device.default")}</option>
+                      {cameras.map((d) => (
+                        <option key={d.deviceId} value={d.deviceId}>{d.label}</option>
+                      ))}
+                    </select>
+                  </div>
+                  <div className="device-select-wrapper">
+                    <label className="device-select-label">🎤</label>
+                    <select
+                      className="device-select"
+                      value={config.micDeviceId || "default"}
+                      onChange={(e) => switchDevice("mic", e.target.value === "default" ? "" : e.target.value)}
+                    >
+                      <option value="default">{tr("controls.device.default")}</option>
+                      {mics.map((d) => (
+                        <option key={d.deviceId} value={d.deviceId}>{d.label}</option>
+                      ))}
+                    </select>
+                  </div>
+                </div>
              </>
            ) : (
              <>
--- a/frontend/src/components/CameraManager/index.tsx
+++ b/frontend/src/components/CameraManager/index.tsx
@@ -19,10 +19,13 @@ export function useCamera() {
  const [stream, setStream] = useState<MediaStream | null>(null);
  const [error, setError] = useState<string | null>(null);

-  const startCamera = useCallback(async () => {
+  const startCamera = useCallback(async (deviceId?: string) => {
    try {
+      const videoConstraints: MediaTrackConstraints = deviceId
+        ? { deviceId: { exact: deviceId }, width: 640, height: 480 }
+        : { facingMode: "environment", width: 640, height: 480 };
      const mediaStream = await navigator.mediaDevices.getUserMedia({
-        video: { facingMode: "environment", width: 640, height: 480 },
+        video: videoConstraints,
        audio: false,
      });
      setStream(mediaStream);
--- a/frontend/src/components/MicManager/index.tsx
+++ b/frontend/src/components/MicManager/index.tsx
@@ -12,15 +12,13 @@ export function useMicrophone() {
  const [error, setError] = useState<string | null>(null);
  const audioContextRef = useRef<AudioContext | null>(null);

-  const startMic = useCallback(async () => {
+  const startMic = useCallback(async (deviceId?: string) => {
    try {
+      const audioConstraints: MediaTrackConstraints = deviceId
+        ? { deviceId: { exact: deviceId }, sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true }
+        : { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true };
      const mediaStream = await navigator.mediaDevices.getUserMedia({
-        audio: {
-          sampleRate: 16000,
-          channelCount: 1,
-          echoCancellation: true,
-          noiseSuppression: true,
-        },
+        audio: audioConstraints,
        video: false,
      });
      setStream(mediaStream);
--- a/frontend/src/hooks/useDeviceList.ts
+++ b/frontend/src/hooks/useDeviceList.ts
@@ -0,0 +1,44 @@
+// ============================================================
+// useDeviceList — 设备枚举 Hook
+// 职责：枚举摄像头/麦克风设备列表，监听设备热插拔
+// 注意：首次枚举需要先有一次成功的 getUserMedia 授权
+// ============================================================
+
+import { useCallback, useEffect, useState } from "react";
+
+export interface DeviceInfo {
+  deviceId: string;
+  label: string;
+}
+
+export function useDeviceList() {
+  const [cameras, setCameras] = useState<DeviceInfo[]>([]);
+  const [mics, setMics] = useState<DeviceInfo[]>([]);
+
+  /** 枚举当前可用的音视频输入设备 */
+  const refreshDevices = useCallback(async () => {
+    try {
+      const devices = await navigator.mediaDevices.enumerateDevices();
+      setCameras(
+        devices
+          .filter((d) => d.kind === "videoinput")
+          .map((d) => ({ deviceId: d.deviceId, label: d.label || `摄像头 ${d.deviceId.slice(0, 4)}` }))
+      );
+      setMics(
+        devices
+          .filter((d) => d.kind === "audioinput")
+          .map((d) => ({ deviceId: d.deviceId, label: d.label || `麦克风 ${d.deviceId.slice(0, 4)}` }))
+      );
+    } catch (err) {
+      console.warn("[DeviceList] 枚举设备失败:", err);
+    }
+  }, []);
+
+  // 监听设备热插拔
+  useEffect(() => {
+    navigator.mediaDevices.addEventListener("devicechange", refreshDevices);
+    return () => navigator.mediaDevices.removeEventListener("devicechange", refreshDevices);
+  }, [refreshDevices]);
+
+  return { cameras, mics, refreshDevices };
+}
--- a/frontend/src/hooks/useVisionSession.ts
+++ b/frontend/src/hooks/useVisionSession.ts
@@ -17,6 +17,7 @@ import { useCamera } from "../components/CameraManager";
 import { useMicrophone } from "../components/MicManager";
 import { useVAD, sampleFrame } from "../components/EdgeProcessor";
 import { useWebSocketManager } from "../components/WebSocketManager";
+import { useDeviceList } from "./useDeviceList";
 import type { ChatMessage, SessionConfig, ServerMessage, LLMDoneMessage } from "../types";

 export interface SessionStats {
@@ -55,6 +56,7 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
  const { videoRef, captureFrame, startCamera, stopCamera, stream } = useCamera();
  const { startMic, stopMic } = useMicrophone();
  const { status, connect, disconnect, send } = useWebSocketManager();
+  const { cameras, mics, refreshDevices } = useDeviceList();

  // 用 ref 跟踪 isProcessing，避免 VAD 回调闭包问题
  const isProcessingRef = useRef(false);
@@ -277,14 +279,14 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s

    // 2. 尝试获取摄像头（可选）
    try {
-      await startCamera();
+      await startCamera(config.cameraDeviceId || undefined);
      setIsCameraOn(true);
    } catch {
      console.warn("[Session] 无法获取摄像头权限，将以纯文本模式运行");
    }

    // 3. 尝试获取麦克风（可选）
-    const micStream = await startMic();
+    const micStream = await startMic(config.micDeviceId || undefined);
    if (micStream) {
      setIsMicOn(true);
      // 4. 启动 VAD（仅在麦克风可用时）
@@ -292,7 +294,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
    } else {
      console.warn("[Session] 无法获取麦克风权限，将以文本输入模式运行");
    }
-  }, [startCamera, startMic, connect, startVAD, accessToken]);
+
+    // 5. 授权后刷新设备列表
+    refreshDevices();
+  }, [startCamera, startMic, connect, startVAD, accessToken, config.cameraDeviceId, config.micDeviceId, refreshDevices]);

  /** 结束会话 */
  const stopSession = useCallback(async () => {
@@ -332,10 +337,10 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
      stopCamera();
      setIsCameraOn(false);
    } else {
-      await startCamera();
+      await startCamera(config.cameraDeviceId || undefined);
      setIsCameraOn(true);
    }
-  }, [isCameraOn, startCamera, stopCamera]);
+  }, [isCameraOn, startCamera, stopCamera, config.cameraDeviceId]);

  /** 麦克风开关 */
  const toggleMic = useCallback(async () => {
@@ -344,13 +349,13 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
      stopMic();
      setIsMicOn(false);
    } else {
-      const micStream = await startMic();
+      const micStream = await startMic(config.micDeviceId || undefined);
      if (micStream) {
        await startVAD(micStream);
        setIsMicOn(true);
      }
    }
-  }, [isMicOn, startMic, stopMic, startVAD, stopVAD]);
+  }, [isMicOn, startMic, stopMic, startVAD, stopVAD, config.micDeviceId]);

  /** 打断当前回复 */
  const interrupt = useCallback(() => {
@@ -370,6 +375,31 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
    setIsProcessing(false);
  }, [send, currentReply]);

+  /** 切换设备（摄像头或麦克风） */
+  const switchDevice = useCallback(async (kind: "camera" | "mic", deviceId: string) => {
+    if (kind === "camera") {
+      updateConfig({ cameraDeviceId: deviceId || undefined });
+      if (isCameraOn) {
+        stopCamera();
+        try {
+          await startCamera(deviceId || undefined);
+        } catch {
+          console.warn("[Session] 切换摄像头失败");
+        }
+      }
+    } else {
+      updateConfig({ micDeviceId: deviceId || undefined });
+      if (isMicOn) {
+        await stopVAD();
+        stopMic();
+        const micStream = await startMic(deviceId || undefined);
+        if (micStream) {
+          await startVAD(micStream);
+        }
+      }
+    }
+  }, [isCameraOn, isMicOn, stopCamera, startCamera, stopMic, startMic, stopVAD, startVAD, updateConfig]);
+
  /** 发送文本消息（手动输入） */
  const sendTextMessage = useCallback(
    (text: string) => {
@@ -439,5 +469,8 @@ export function useVisionSession(accessToken?: string | null, conversationId?: s
    toggleMic,
    sendTextMessage,
    captureFrame,
+    cameras,
+    mics,
+    switchDevice,
  };
 }
--- a/frontend/src/lib/i18n/en-US.ts
+++ b/frontend/src/lib/i18n/en-US.ts
@@ -133,8 +133,6 @@ export const enUS: TranslationMap = {

  // Video controls (enhanced)
  "controls.recognize": "Analyze Scene",
-  "controls.device.camera": "Camera",
-  "controls.device.mic": "Microphone",
  "controls.device.default": "Default",

  // Status bar
--- a/frontend/src/lib/i18n/ja-JP.ts
+++ b/frontend/src/lib/i18n/ja-JP.ts
@@ -133,8 +133,6 @@ export const jaJP: TranslationMap = {

  // Video controls (enhanced)
  "controls.recognize": "シーンを分析",
-  "controls.device.camera": "カメラ",
-  "controls.device.mic": "マイク",
  "controls.device.default": "デフォルト",

  // Status bar
--- a/frontend/src/lib/i18n/zh-CN.ts
+++ b/frontend/src/lib/i18n/zh-CN.ts
@@ -133,8 +133,6 @@ export const zhCN: TranslationMap = {

  // Video controls (enhanced)
  "controls.recognize": "识别画面",
-  "controls.device.camera": "摄像头",
-  "controls.device.mic": "麦克风",
  "controls.device.default": "默认",

  // Status bar
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -10,6 +10,8 @@ export interface SessionConfig {
  detailLevel: "low" | "high";
  language: string;
  scenario: string;    // 情景 ID，如 "free_chat"、"interviewer"
+  cameraDeviceId?: string;  // 摄像头设备 ID，空 = 系统默认
+  micDeviceId?: string;     // 麦克风设备 ID，空 = 系统默认
 }

 export type Theme = "dark" | "light";