feat: hi-fi audio (#2305)

### 💡 Overview

**Hi-fi audio**
* Public api for stereo output enabling is simplified.
`callManager.start({ audioRole: 'listener' })` enables stereo output for
both platforms. Should be invoked on pre-join stage.
* Wired stereo input enable/disable flag for Android. For now it is
omitted.

**Presented media engine**
* Media engine is a layer responsible for creating/disposing WebRTC peer
connection factory in runtime during call join/leave stages. Peer
connection factory and audio device module instances are defined on
per-call basis. Only one factory instance can be created at a time. Web
resolves to a no-op engine.
* Guarded join/leave race. There are several implicit factory creators
during join flow: `getGenericSdp` and `initPublisherAndSubscriber`
(RTCPeerConnection constructor). The guard prevents creating new
instance if the join flow was interrupted by leave invocation.

**Made RN lobby independent from webrtc**
* Presented new video preview component, which uses video capturer
directly without creating local tracks. That video capturer instance is
passed later to a local track as a source of media
* Made device managers mute/unmute on pre-join stage update
corresponding state in an optimistic manner. That state is applied
during tracks publishing.

**Hardened call manager and audio wiring pipeline**
* Public call manager now won't invoke audio session configuration
explicitly. Instead it stores configuration params, which are applied
during join flow. Those params may override default call settings (e.g.
deviceEndpointType). Params are disposed during leave stage.
* Audio engine interruptions subscription for callingx is now managed
during join/leave, instead of single global subscription.

🎫 Ticket: https://linear.app/stream/issue/RN-402/hi-fi-audio

📑 Docs: GetStream/docs-content#1444

Corresponding WebRTC PR:
GetStream/react-native-webrtc#50


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added per-call WebRTC media engine support with configurable
providers.
* Added React Native lobby camera previews with on-demand permissions
and optimistic camera state.
* Added iOS controls for microphone mute mode and recording preparation.
* Added audio-engine subscription management for CallingX and in-call
behavior.

* **Bug Fixes**
* Prevented join/leave race conditions from continuing setup after
leaving.
* Improved React Native media capture, camera controls, and device-state
handling.
* Improved iOS audio stability when no active audio device is available.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: greenfrvr

packages/client/index.ts

@@ -11,6 +11,7 @@ export * from './src/stats/types';
 
 export * from './src/Call';
 export * from './src/CallType';
+export * from './src/rtc/mediaEngine';
 export * from './src/StreamVideoClient';
 export * from './src/StreamSfuClient';
 export * from './src/devices';

packages/client/src/Call.ts

@@ -2,7 +2,9 @@ import { StreamSfuClient } from './StreamSfuClient';
 import { SfuJoinError } from './errors';
 import {
   BasePeerConnectionOpts,
+  type CallMediaEngine,
   Dispatcher,
+  getCallMediaEngineProvider,
   getGenericSdp,
   isAudioTrackType,
   isSfuEvent,
@@ -332,6 +334,8 @@ export class Call {
   private allowOwnTracksLoopback = false;
   private hasJoinedOnce = false;
   private deviceSettingsAppliedOnce = false;
+  private callManagerStarted = false;
+  private leaveGeneration = 0;
   private credentials?: Credentials;
 
   private initialized = false;
@@ -355,6 +359,14 @@ export class Call {
     ClientCapability.SUBSCRIBER_VIDEO_PAUSE,
   ]);
 
+  /**
+   * The in-flight per-call media engine. On web/React this resolves to a thin
+   * globals-backed engine (no provider registered); React Native registers a
+   * provider that owns a per-call native factory.
+   * @internal
+   */
+  private mediaEnginePromise?: Promise<CallMediaEngine>;
+
   /**
    * Constructs a new `Call` instance.
    *
@@ -703,6 +715,8 @@ export class Call {
         return;
       }
 
+      this.leaveGeneration += 1;
+
       if (callingState === CallingState.JOINING) {
         const waitUntilCallJoined = () => {
           return new Promise<void>((resolve) => {
@@ -803,13 +817,15 @@ export class Call {
 
       globalThis.streamRNVideoSDK?.callManager.stop({
         isRingingTypeCall: this.ringing,
+        shouldStopCallManager: this.callManagerStarted,
       });
 
       this.camera.dispose();
       this.microphone.dispose();
       this.screenShare.dispose();
       this.speaker.dispose();
       this.deviceSettingsAppliedOnce = false;
+      this.callManagerStarted = false;
 
       const stopOnLeavePromises: Promise<void>[] = [];
       if (this.camera.stopOnLeave) {
@@ -822,6 +838,23 @@ export class Call {
         stopOnLeavePromises.push(this.screenShare.disable(true));
       }
       await Promise.all(stopOnLeavePromises);
+
+      // Dispose the per-call media engine last — after peer connections and
+      // local tracks are gone — so the backing factory tears down with no
+      // owned PCs/tracks. A fresh `join()` builds a new engine.
+      if (this.mediaEnginePromise) {
+        const enginePromise = this.mediaEnginePromise;
+        this.mediaEnginePromise = undefined;
+        this.logger.debug('Disposing per-call media factory');
+        await enginePromise
+          .then((engine) => {
+            globalThis.streamRNVideoSDK?.callingX?.unwireAudioEngineSubscription();
+            return engine.dispose();
+          })
+          .catch((err) => {
+            this.logger.warn('Failed to dispose media engine', err);
+          });
+      }
     });
   };
 
@@ -941,11 +974,7 @@ export class Call {
       this.clientStore.registerOrUpdateCall(this);
     }
     // Skip speaker setup on RN if ringing was requested or the call is already ringing
-    const skipSpeakerApply = isReactNative()
-      ? params?.ring === true
-        ? true
-        : this.ringing
-      : false;
+    const skipSpeakerApply = isReactNative();
     await this.applyDeviceConfig(
       response.call.settings,
       false,
@@ -982,11 +1011,7 @@ export class Call {
     }
 
     // Skip speaker setup on RN if ringing was requested or the call is already ringing
-    const skipSpeakerApply = isReactNative()
-      ? data?.ring === true
-        ? true
-        : this.ringing
-      : false;
+    const skipSpeakerApply = isReactNative();
     await this.applyDeviceConfig(
       response.call.settings,
       false,
@@ -1194,12 +1219,26 @@ export class Call {
   private doJoin = async (data?: JoinCallData): Promise<void> => {
     const connectStartTime = Date.now();
     const callingState = this.state.callingState;
+    const joinLeaveGeneration = this.leaveGeneration;
+    const supersededByLeave = () =>
+      this.leaveGeneration !== joinLeaveGeneration;
 
     this.joinCallData = data;
 
     this.logger.debug('Starting join flow');
     this.state.setCallingState(CallingState.JOINING);
 
+    // Ensure the per-call media engine exists before any peer connection
+    // (codec probe, subscriber, publisher) or capture happens, so the WebRTC
+    // globals resolve to the call's factory. Idempotent across
+    // reconnect/migration attempts.
+    await this.ensureMediaFactory();
+
+    const callingX = globalThis.streamRNVideoSDK?.callingX;
+    if (callingX) {
+      callingX.wireAudioEngineSubscription();
+    }
+
     const performingMigration =
       this.reconnectStrategy === WebsocketReconnectStrategy.MIGRATE;
     const performingRejoin =
@@ -1270,6 +1309,11 @@ export class Call {
       // the capabilities of the client (codec support, etc.)
       const { dangerouslyForceCodec, fmtpLine, subscriberFmtpLine } =
         this.clientPublishOptions || {};
+      // skip if a leave superseded this join so codec detection doesn't resolve to a default factory.
+      if (supersededByLeave()) {
+        this.logger.debug('Join superseded by leave; skipping codec detection');
+        return;
+      }
       const [subscriberSdp, publisherSdp] = await Promise.all([
         getGenericSdp('recvonly', dangerouslyForceCodec, subscriberFmtpLine),
         getGenericSdp('sendonly', dangerouslyForceCodec, fmtpLine),
@@ -1329,6 +1373,13 @@ export class Call {
       }
     }
 
+    // If the user left while this join was in flight, bail before re-setting JOINED and before
+    // peer-connection setup below (both run synchronously after this, so one check covers them).
+    if (supersededByLeave()) {
+      this.logger.debug('Join superseded by leave; aborting join flow');
+      return;
+    }
+
     if (!performingMigration) {
       // in MIGRATION, `JOINED` state is set in `this.reconnectMigrate()`
       this.state.setCallingState(CallingState.JOINED);
@@ -1375,12 +1426,21 @@ export class Call {
 
     // device settings should be applied only once, we don't have to
     // re-apply them on later reconnections or server-side data fetches
-    if (!this.deviceSettingsAppliedOnce && this.state.settings) {
+    if (
+      !this.deviceSettingsAppliedOnce &&
+      this.state.settings &&
+      !supersededByLeave()
+    ) {
       await this.applyDeviceConfig(this.state.settings, true, false);
+      this.deviceSettingsAppliedOnce = true;
+    }
+
+    if (!this.callManagerStarted && !supersededByLeave()) {
       globalThis.streamRNVideoSDK?.callManager.start({
         isRingingTypeCall: this.ringing,
+        cid: this.cid,
       });
-      this.deviceSettingsAppliedOnce = true;
+      this.callManagerStarted = true;
     }
 
     // We shouldn't persist the `ring` and `notify` state after joining the call
@@ -1669,6 +1729,39 @@ export class Call {
     return joinResponse;
   };
 
+  /**
+   * Whether the per-call media engine currently exists. True from join until leave.
+   *
+   * @internal an internal getter and should not be used outside the SDK.
+   */
+  get hasMediaEngine(): boolean {
+    return !!this.mediaEnginePromise;
+  }
+
+  /**
+   * Ensures a {@link CallMediaEngine} exists for this call's media session and
+   * returns it. Idempotent: the engine is created once via the registered
+   * provider (see `setCallMediaEngineProvider`) and cached until `leave()`
+   * disposes it. Concurrent callers (e.g. camera + microphone enabling in
+   * parallel) share the same engine because the in-flight creation promise is
+   * cached, never the unresolved result.
+   *
+   * @internal
+   */
+  ensureMediaFactory = async (): Promise<CallMediaEngine> => {
+    if (!this.mediaEnginePromise) {
+      const provider = getCallMediaEngineProvider();
+
+      this.logger.debug(`Requesting per-call media factory creation`);
+      this.mediaEnginePromise = Promise.resolve(provider()).catch((err) => {
+        // Drop the cached rejection so a retried join() can rebuild the engine
+        this.mediaEnginePromise = undefined;
+        throw err;
+      });
+    }
+    return this.mediaEnginePromise;
+  };
+
   /**
    * Handles the closing of the SFU signal connection.
    *

packages/client/src/devices/AudioDeviceManager.ts

@@ -2,6 +2,7 @@ import { DeviceManager } from './DeviceManager';
 import { AudioDeviceManagerState } from './AudioDeviceManagerState';
 import { AudioBitrateProfile } from '../gen/video/sfu/models/models';
 import { TrackPublishOptions } from '../rtc';
+import { isReactNative } from '../helpers/platforms';
 
 /**
  * Base class for High Fidelity enabled Device Managers.
@@ -17,6 +18,11 @@ export abstract class AudioDeviceManager<
     if (!this.call.state.settings?.audio.hifi_audio_enabled) {
       throw new Error('High Fidelity audio is not enabled for this call');
     }
+    if (isReactNative() && this.call.hasMediaEngine) {
+      throw new Error(
+        'setAudioBitrateProfile must be called before joining the call.',
+      );
+    }
     this.doSetAudioBitrateProfile(profile);
     this.state.setAudioBitrateProfile(profile);
     if (this.enabled) {

packages/client/src/devices/CameraManager.ts

@@ -7,6 +7,7 @@ import { VideoSettingsResponse } from '../gen/coordinator';
 import { TrackType } from '../gen/video/sfu/models/models';
 import { isMobile } from '../helpers/compatibility';
 import { isReactNative } from '../helpers/platforms';
+import { CallingState } from '../store';
 import { DevicePersistenceOptions } from './devicePersistence';
 
 export class CameraManager extends DeviceManager<CameraManagerState> {
@@ -124,6 +125,53 @@ export class CameraManager extends DeviceManager<CameraManagerState> {
     }
   }
 
+  override enable(): Promise<void> {
+    if (
+      isReactNative() &&
+      this.call.state.callingState !== CallingState.JOINED
+    ) {
+      this.state.setPendingStatus('enabled');
+      return Promise.resolve();
+    }
+
+    return super.enable();
+  }
+
+  override disable(options: { forceStop?: boolean }): Promise<void>;
+  override disable(forceStop?: boolean): Promise<void>;
+  override async disable(
+    forceStopOrOptions?: boolean | { forceStop?: boolean },
+  ): Promise<void> {
+    if (
+      isReactNative() &&
+      this.call.state.callingState !== CallingState.JOINED
+    ) {
+      this.state.setPendingStatus('disabled');
+      return;
+    }
+
+    // forward verbatim to the base, narrowing so the right overload is selected
+    if (forceStopOrOptions === undefined) return super.disable();
+    if (typeof forceStopOrOptions === 'boolean') {
+      return super.disable(forceStopOrOptions);
+    }
+    return super.disable(forceStopOrOptions);
+  }
+
+  override toggle(): Promise<void> {
+    if (
+      isReactNative() &&
+      this.call.state.callingState !== CallingState.JOINED
+    ) {
+      this.state.setPendingStatus(
+        this.state.optimisticStatus === 'enabled' ? 'disabled' : 'enabled',
+      );
+      return Promise.resolve();
+    }
+
+    return super.toggle();
+  }
+
   /**
    * Applies the video settings to the camera.
    *
@@ -166,9 +214,15 @@ export class CameraManager extends DeviceManager<CameraManagerState> {
       }
     }
 
-    const { mediaStream } = this.state;
-    if (canPublish && publish && this.enabled && mediaStream) {
-      await this.publishStream(mediaStream);
+    if (isReactNative() && publish && canPublish) {
+      // On RN the camera is enabled/disabled optimistically before JOINED. Reconcile now
+      // acquires the track and publishes it, so it fully owns the publish.
+      await this.reconcileOptimisticStatus();
+    } else {
+      const { mediaStream } = this.state;
+      if (canPublish && publish && this.enabled && mediaStream) {
+        await this.publishStream(mediaStream);
+      }
     }
   }
 
@@ -196,9 +250,12 @@ export class CameraManager extends DeviceManager<CameraManagerState> {
     return constraints;
   }
 
-  protected override getStream(
+  protected override async getStream(
     constraints: MediaTrackConstraints,
   ): Promise<MediaStream> {
+    // Ensure the call's media factory exists before capture so the resulting
+    // track is owned by it (the WebRTC globals resolve to the live factory).
+    await this.call.ensureMediaFactory();
     return getVideoStream(constraints, this.call.tracer);
   }
 }

packages/client/src/devices/DeviceManager.ts

@@ -509,6 +509,23 @@ export abstract class DeviceManager<
     }
   }
 
+  protected reconcileOptimisticStatus = async (): Promise<void> => {
+    const target = this.state.optimisticStatus;
+    await withCancellation(this.statusChangeConcurrencyTag, async (signal) => {
+      try {
+        if (target === 'enabled' && this.state.status !== 'enabled') {
+          await this.unmuteStream();
+          if (!signal.aborted) this.state.setStatus('enabled');
+        } else if (target === 'disabled' && this.state.status === 'enabled') {
+          // mirror whatever disable() does to stop/pause the track per disableMode
+          if (!signal.aborted) this.state.setStatus('disabled');
+        }
+      } finally {
+        if (!signal.aborted) this.state.setPendingStatus(this.state.status);
+      }
+    });
+  };
+
   private disableTracks() {
     this.getTracks().forEach((track) => {
       if (track.enabled) track.enabled = false;