Stabilize WebRTC Safari setup

DjDeveloperr · DjDeveloperr · commit f3059032837c · 2026-04-30T13:47:13.000-04:00
diff --git a/cli/DFPrivateSimulatorDisplayBridge.h b/cli/DFPrivateSimulatorDisplayBridge.h
@@ -61,6 +61,7 @@ NS_SWIFT_NAME(PrivateSimulatorDisplayBridge)
                error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(sendKey(keyCode:down:));
 
 - (BOOL)pressHomeButton:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(pressHomeButton());
+- (BOOL)openAppSwitcher:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(openAppSwitcher());
 - (BOOL)pressHardwareButtonNamed:(NSString *)buttonName
                        durationMs:(NSUInteger)durationMs
                             error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(pressHardwareButton(named:durationMs:));
diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m
@@ -2015,6 +2015,67 @@ static BOOL DFPressHomeViaHIDClient(id hidClient, NSError **error) {
     return NO;
 }
 
+static BOOL DFOpenAppSwitcherViaHIDClient(id hidClient, NSError **error) {
+    if (hidClient == nil) {
+        if (error != NULL) {
+            *error = DFMakeError(
+                DFPrivateSimulatorErrorCodeTouchDispatchFailed,
+                @"SimulatorKit did not provide a headless HID client for App Switcher."
+            );
+        }
+        return NO;
+    }
+
+    static const DFHomeButtonHIDStrategy strategies[] = {
+        { "IndigoHIDMessageForHIDArbitrary page=0x0c usage=0x40 (Menu) target=0x32", NO, 0, DFConsumerControlUsagePage, 0x40, DFIndigoTouchTarget },
+        { "IndigoHIDMessageForHIDArbitrary page=0x0c usage=0x65 (Home) target=0x32", NO, 0, DFConsumerControlUsagePage, DFHomeConsumerUsage, DFIndigoTouchTarget },
+        { "IndigoHIDMessageForButton code=0x191 target=0x2",  YES, DFHomeButtonCode, 0, 0, 0x2 },
+        { "IndigoHIDMessageForButton code=0x191 target=0x32", YES, DFHomeButtonCode, 0, 0, DFIndigoTouchTarget },
+    };
+
+    NSError *lastError = nil;
+    for (size_t index = 0; index < sizeof(strategies) / sizeof(strategies[0]); index++) {
+        const DFHomeButtonHIDStrategy *strategy = &strategies[index];
+        BOOL dispatched = YES;
+        for (NSUInteger pressIndex = 0; pressIndex < 2; pressIndex += 1) {
+            NSError *downError = nil;
+            if (!DFSendHomeStrategyEdge(hidClient, strategy, DFButtonDirectionDown, &downError)) {
+                DFLog(@"App Switcher strategy rejected (down): %s — %@", strategy->label, downError.localizedDescription ?: @"no error");
+                lastError = downError;
+                dispatched = NO;
+                break;
+            }
+
+            [NSThread sleepForTimeInterval:0.06];
+
+            NSError *upError = nil;
+            if (!DFSendHomeStrategyEdge(hidClient, strategy, DFButtonDirectionUp, &upError)) {
+                DFLog(@"App Switcher strategy rejected (up): %s — %@", strategy->label, upError.localizedDescription ?: @"no error");
+                lastError = upError;
+                dispatched = NO;
+                break;
+            }
+
+            if (pressIndex == 0) {
+                [NSThread sleepForTimeInterval:0.12];
+            }
+        }
+
+        if (dispatched) {
+            DFLog(@"App Switcher dispatched via %s", strategy->label);
+            return YES;
+        }
+    }
+
+    if (error != NULL) {
+        *error = lastError ?: DFMakeError(
+            DFPrivateSimulatorErrorCodeTouchDispatchFailed,
+            @"SimulatorKit rejected every App Switcher HID strategy."
+        );
+    }
+    return NO;
+}
+
 @interface DFPrivateSimulatorDisplayBridge ()
 
 @property (nonatomic, strong) NSView *displayView;
@@ -2912,6 +2973,37 @@ - (BOOL)pressHomeButton:(NSError * _Nullable __autoreleasing *)error {
     return success;
 }
 
+- (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error {
+    __block BOOL success = NO;
+    __block NSError *dispatchError = nil;
+
+    dispatch_block_t work = ^{
+        NSError *hidError = nil;
+        if (DFOpenAppSwitcherViaHIDClient(self->_hidClient, &hidError)) {
+            success = YES;
+            return;
+        }
+
+        DFLog(@"HID App Switcher path failed: %@", hidError.localizedDescription ?: @"unknown error");
+        dispatchError = hidError ?: DFMakeError(
+            DFPrivateSimulatorErrorCodeTouchDispatchFailed,
+            @"No App Switcher path succeeded."
+        );
+    };
+
+    if (dispatch_get_specific(DFPrivateSimulatorCallbackQueueKey) != NULL) {
+        work();
+    } else {
+        dispatch_sync(_callbackQueue, work);
+    }
+
+    if (!success && error != NULL) {
+        *error = dispatchError;
+    }
+
+    return success;
+}
+
 - (BOOL)rotateByDegrees:(double)deltaDegrees error:(NSError * _Nullable __autoreleasing *)error {
     __block BOOL success = NO;
     __block NSError *dispatchError = nil;
diff --git a/cli/XCWPrivateSimulatorSession.h b/cli/XCWPrivateSimulatorSession.h
@@ -49,6 +49,7 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData,
               error:(NSError * _Nullable * _Nullable)error;
 
 - (BOOL)pressHomeButton:(NSError * _Nullable * _Nullable)error;
+- (BOOL)openAppSwitcher:(NSError * _Nullable * _Nullable)error;
 - (BOOL)rotateRight:(NSError * _Nullable * _Nullable)error;
 - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error;
 - (void)disconnect;
diff --git a/cli/XCWPrivateSimulatorSession.m b/cli/XCWPrivateSimulatorSession.m
@@ -302,6 +302,10 @@ - (BOOL)pressHomeButton:(NSError * _Nullable __autoreleasing *)error {
     return [_displayBridge pressHomeButton:error];
 }
 
+- (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error {
+    return [_displayBridge openAppSwitcher:error];
+}
+
 - (BOOL)rotateRight:(NSError * _Nullable __autoreleasing *)error {
     return [_displayBridge rotateRight:error];
 }
diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h
@@ -49,6 +49,7 @@ bool xcw_native_send_touch(const char * _Nonnull udid, double x, double y, const
 bool xcw_native_send_key(const char * _Nonnull udid, uint16_t key_code, uint32_t modifiers, char * _Nullable * _Nullable error_message);
 bool xcw_native_send_key_event(const char * _Nonnull udid, uint16_t key_code, bool down, char * _Nullable * _Nullable error_message);
 bool xcw_native_press_home(const char * _Nonnull udid, char * _Nullable * _Nullable error_message);
+bool xcw_native_open_app_switcher(const char * _Nonnull udid, char * _Nullable * _Nullable error_message);
 bool xcw_native_press_button(const char * _Nonnull udid, const char * _Nonnull button_name, uint32_t duration_ms, char * _Nullable * _Nullable error_message);
 bool xcw_native_rotate_right(const char * _Nonnull udid, char * _Nullable * _Nullable error_message);
 bool xcw_native_rotate_left(const char * _Nonnull udid, char * _Nullable * _Nullable error_message);
@@ -75,6 +76,7 @@ bool xcw_native_session_send_touch(void * _Nonnull handle, double x, double y, c
 bool xcw_native_session_send_multitouch(void * _Nonnull handle, double x1, double y1, double x2, double y2, const char * _Nonnull phase, char * _Nullable * _Nullable error_message);
 bool xcw_native_session_send_key(void * _Nonnull handle, uint16_t key_code, uint32_t modifiers, char * _Nullable * _Nullable error_message);
 bool xcw_native_session_press_home(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
+bool xcw_native_session_open_app_switcher(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
 bool xcw_native_session_rotate_right(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
 bool xcw_native_session_rotate_left(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
 void xcw_native_session_set_frame_callback(void * _Nonnull handle, xcw_native_frame_callback _Nullable callback, void * _Nullable user_data);
diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m
@@ -502,6 +502,22 @@ bool xcw_native_press_home(const char *udid, char **error_message) {
     }
 }
 
+bool xcw_native_open_app_switcher(const char *udid, char **error_message) {
+    @autoreleasepool {
+        DFPrivateSimulatorDisplayBridge *bridge = XCWInputBridgeForUDID(udid, error_message);
+        if (bridge == nil) {
+            return false;
+        }
+        NSError *error = nil;
+        BOOL ok = [bridge openAppSwitcher:&error];
+        [bridge disconnect];
+        if (!ok) {
+            XCWSetErrorMessage(error_message, error);
+        }
+        return ok;
+    }
+}
+
 bool xcw_native_press_button(const char *udid, const char *button_name, uint32_t duration_ms, char **error_message) {
     @autoreleasepool {
         DFPrivateSimulatorDisplayBridge *bridge = XCWInputBridgeForUDID(udid, error_message);
@@ -728,6 +744,17 @@ bool xcw_native_session_press_home(void *handle, char **error_message) {
     }
 }
 
+bool xcw_native_session_open_app_switcher(void *handle, char **error_message) {
+    @autoreleasepool {
+        NSError *error = nil;
+        BOOL ok = [XCWNativeSessionFromHandle(handle) openAppSwitcher:&error];
+        if (!ok) {
+            XCWSetErrorMessage(error_message, error);
+        }
+        return ok;
+    }
+}
+
 bool xcw_native_session_rotate_right(void *handle, char **error_message) {
     @autoreleasepool {
         NSError *error = nil;
diff --git a/cli/native/XCWNativeSession.h b/cli/native/XCWNativeSession.h
@@ -28,6 +28,7 @@ NS_ASSUME_NONNULL_BEGIN
           modifiers:(uint32_t)modifiers
               error:(NSError * _Nullable * _Nullable)error;
 - (BOOL)pressHome:(NSError * _Nullable * _Nullable)error;
+- (BOOL)openAppSwitcher:(NSError * _Nullable * _Nullable)error;
 - (BOOL)rotateRight:(NSError * _Nullable * _Nullable)error;
 - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error;
 - (void)setFrameCallback:(xcw_native_frame_callback _Nullable)callback
diff --git a/cli/native/XCWNativeSession.m b/cli/native/XCWNativeSession.m
@@ -129,6 +129,10 @@ - (BOOL)pressHome:(NSError * _Nullable __autoreleasing *)error {
     return [self.session pressHomeButton:error];
 }
 
+- (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error {
+    return [self.session openAppSwitcher:error];
+}
+
 - (BOOL)rotateRight:(NSError * _Nullable __autoreleasing *)error {
     return [self.session rotateRight:error];
 }
diff --git a/client/src/api/types.ts b/client/src/api/types.ts
@@ -24,6 +24,10 @@ export interface SimulatorsResponse {
 export interface HealthResponse {
   ok: boolean;
   videoCodec?: string;
+  webRtc?: {
+    iceServers?: RTCIceServer[];
+    iceTransportPolicy?: RTCIceTransportPolicy;
+  };
 }
 
 export interface SimulatorResponse {
diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts
@@ -1,4 +1,5 @@
 import { apiHeaders, fetchHealth } from "../../api/client";
+import type { HealthResponse } from "../../api/types";
 import { createEmptyStreamStats } from "./stats";
 import type {
   StreamConnectTarget,
@@ -87,9 +88,13 @@ class WebRtcStreamClient implements StreamClientBackend {
     });
 
     try {
+      const health = await fetchHealth().catch(() => null);
+      if (generation !== this.connectGeneration) {
+        return;
+      }
       const peerConnection = new RTCPeerConnection({
-        iceServers: iceServers(),
-        iceTransportPolicy: iceTransportPolicy(),
+        iceServers: iceServers(health),
+        iceTransportPolicy: iceTransportPolicy(health),
       });
       this.peerConnection = peerConnection;
       this.attachDiagnostics(peerConnection, target, generation);
@@ -619,27 +624,47 @@ function configureReceiverCodecPreferences(transceiver: RTCRtpTransceiver) {
   ]);
 }
 
-function iceServers(): RTCIceServer[] {
+function iceServers(health?: HealthResponse | null): RTCIceServer[] {
   const params = new URLSearchParams(window.location.search);
-  const raw = params.get("iceServers") ?? "stun:stun.l.google.com:19302";
+  const queryValue = params.get("iceServers");
+  const raw = queryValue ?? "";
   if (raw === "none") {
     return [];
   }
-  return [
-    {
+  if (raw.trim()) {
+    const server: RTCIceServer = {
       urls: raw
         .split(",")
         .map((value) => value.trim())
         .filter(Boolean),
-    },
-  ];
+    };
+    const username = params.get("iceUsername");
+    const credential = params.get("iceCredential");
+    if (username) {
+      server.username = username;
+    }
+    if (credential) {
+      server.credential = credential;
+    }
+    return [server];
+  }
+  if (health?.webRtc?.iceServers?.length) {
+    return health.webRtc.iceServers;
+  }
+  return [{ urls: ["stun:stun.l.google.com:19302"] }];
 }
 
-function iceTransportPolicy(): RTCIceTransportPolicy {
+function iceTransportPolicy(
+  health?: HealthResponse | null,
+): RTCIceTransportPolicy {
   const value = new URLSearchParams(window.location.search).get(
     "iceTransportPolicy",
   );
-  return value === "relay" || value === "all" ? value : "all";
+  if (value === "relay" || value === "all") {
+    return value;
+  }
+  const healthValue = health?.webRtc?.iceTransportPolicy;
+  return healthValue === "relay" || healthValue === "all" ? healthValue : "all";
 }
 
 interface WebRtcDiagnostics {
diff --git a/docs/api/health.md b/docs/api/health.md
@@ -12,17 +12,23 @@ Returns the static bootstrap information the browser client needs, plus a freshn
   "httpPort": 4310,
   "timestamp": 1714094761.234,
   "videoCodec": "h264-software",
-  "lowLatency": false
+  "lowLatency": false,
+  "webRtc": {
+    "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }],
+    "iceTransportPolicy": "all"
+  }
 }
 ```
 
-| Field        | Notes                                                                                   |
-| ------------ | --------------------------------------------------------------------------------------- |
-| `ok`         | Always `true` if the route is reachable. Network failures are signalled by HTTP errors. |
-| `httpPort`   | HTTP port for the REST API, browser UI, and WebRTC offer endpoint.                      |
-| `timestamp`  | Server-side `time.now()` as a fractional Unix epoch in seconds.                         |
-| `videoCodec` | Active encoder. One of `h264` or `h264-software`. See [Video Pipeline](/guide/video).   |
-| `lowLatency` | `true` when software H.264 low-latency mode was enabled at daemon startup.              |
+| Field                       | Notes                                                                                   |
+| --------------------------- | --------------------------------------------------------------------------------------- |
+| `ok`                        | Always `true` if the route is reachable. Network failures are signalled by HTTP errors. |
+| `httpPort`                  | HTTP port for the REST API, browser UI, and WebRTC offer endpoint.                      |
+| `timestamp`                 | Server-side `time.now()` as a fractional Unix epoch in seconds.                         |
+| `videoCodec`                | Active encoder. One of `h264` or `h264-software`. See [Video Pipeline](/guide/video).   |
+| `lowLatency`                | `true` when software H.264 low-latency mode was enabled at daemon startup.              |
+| `webRtc.iceServers`         | ICE servers the browser should use when creating the WebRTC peer connection.            |
+| `webRtc.iceTransportPolicy` | Browser ICE transport policy. One of `all` or `relay`.                                  |
 
 The default access token is regenerated every time the server restarts. A client should refetch `/api/health` after any disconnection.
 
diff --git a/docs/api/rest.md b/docs/api/rest.md
@@ -23,11 +23,16 @@ Returns server health and the active video encoder mode.
   "httpPort": 4310,
   "timestamp": 1714094761.234,
   "videoCodec": "h264-software",
-  "lowLatency": false
+  "lowLatency": false,
+  "webRtc": {
+    "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }],
+    "iceTransportPolicy": "all"
+  }
 }
 ```
 
-The browser client polls this endpoint at startup and again after a long disconnect to detect server restarts.
+The browser client polls this endpoint at startup to detect server restarts and
+to mirror the daemon's WebRTC ICE configuration.
 
 ### `GET /api/metrics`
 
@@ -259,7 +264,7 @@ Presses the home button:
 
 ### `POST /api/simulators/{udid}/app-switcher`
 
-Double-presses the home button to invoke the app switcher.
+Invokes the app switcher as one server-side native action.
 
 ### `POST /api/simulators/{udid}/rotate-left`
 
diff --git a/docs/guide/lan-access.md b/docs/guide/lan-access.md
@@ -49,7 +49,11 @@ Whatever you advertise must be resolvable from the remote client.
   "ok": true,
   "httpPort": 4310,
   "videoCodec": "h264-software",
-  "lowLatency": false
+  "lowLatency": false,
+  "webRtc": {
+    "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }],
+    "iceTransportPolicy": "all"
+  }
 }
 ```
 
diff --git a/docs/guide/video.md b/docs/guide/video.md
@@ -32,6 +32,25 @@ matters more than maximum smoothness.
 
 The chosen codec is reported to clients in the JSON `videoCodec` field on `GET /api/health`.
 
+## Remote WebRTC ICE
+
+By default SimDeck advertises Google's public STUN server to both the Rust
+WebRTC peer and the browser. For remote browsers, especially Safari connecting
+to a GitHub Actions runner, provide a TURN server and force relay candidates:
+
+```sh
+SIMDECK_WEBRTC_ICE_SERVERS=turns:turn.example.com:5349?transport=tcp \
+SIMDECK_WEBRTC_ICE_USERNAME=simdeck \
+SIMDECK_WEBRTC_ICE_CREDENTIAL=secret \
+SIMDECK_WEBRTC_ICE_TRANSPORT_POLICY=relay \
+simdeck daemon start --video-codec h264-software --low-latency
+```
+
+The browser reads these settings from `GET /api/health` before creating its
+peer connection, so the local and remote peers use the same ICE configuration.
+Use `SIMDECK_WEBRTC_ICE_TRANSPORT_POLICY=all` or leave it unset for local LAN
+and localhost sessions.
+
 ## Keyframe handshake
 
 When a browser connects through `/api/simulators/{udid}/webrtc/offer`:
diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs
diff --git a/server/src/main.rs b/server/src/main.rs
diff --git a/server/src/native/bridge.rs b/server/src/native/bridge.rs
diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs
diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs
diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md

Original file line number	Diff line number	Diff line change
`@@ -302,6 +302,10 @@ - (BOOL)pressHomeButton:(NSError * _Nullable __autoreleasing *)error {`
`302`	`302`	`return [_displayBridge pressHomeButton:error];`
`303`	`303`	`}`
`304`	`304`
	`305`	`+- (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error {`
	`306`	`+ return [_displayBridge openAppSwitcher:error];`
	`307`	`+}`
	`308`	`+`
`305`	`309`	`- (BOOL)rotateRight:(NSError * _Nullable __autoreleasing *)error {`
`306`	`310`	`return [_displayBridge rotateRight:error];`
`307`	`311`	`}`
Original file line number	Diff line number	Diff line change
`@@ -129,6 +129,10 @@ - (BOOL)pressHome:(NSError * _Nullable __autoreleasing *)error {`
`129`	`129`	`return [self.session pressHomeButton:error];`
`130`	`130`	`}`
`131`	`131`
	`132`	`+- (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error {`
	`133`	`+ return [self.session openAppSwitcher:error];`
	`134`	`+}`
	`135`	`+`
`132`	`136`	`- (BOOL)rotateRight:(NSError * _Nullable __autoreleasing *)error {`
`133`	`137`	`return [self.session rotateRight:error];`
`134`	`138`	`}`