diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f29ddfe3..65e98ed5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,3 +232,51 @@ jobs: - name: JS API simulator integration tests run: npm run test:integration:js-api + + integration-android: + name: Android emulator integration + runs-on: ubuntu-latest + timeout-minutes: 35 + needs: + - client + - packages + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: package-lock.json + + - uses: dtolnay/rust-toolchain@stable + + - name: Enable KVM access + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Install root dependencies + run: npm ci --ignore-scripts --force + + - name: Build Linux Android integration artifacts + run: | + npm run build:cli + npm run build:simdeck-test + + - name: Android emulator integration tests + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 35 + target: google_apis + arch: x86_64 + profile: pixel_6 + avd-name: SimDeck_Pixel_CI + disable-animations: true + script: npm run test:integration:android + env: + SIMDECK_INTEGRATION_ANDROID_AVD: SimDeck_Pixel_CI + SIMDECK_INTEGRATION_REQUIRE_RUNNING_ANDROID: "1" + SIMDECK_INTEGRATION_VERBOSE: "1" diff --git a/README.md b/README.md index faa38e4c..2093f80f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

SimDeck is a developer tool built for streamlining mobile app development for coding agents. - Drive Simulator from the CLI using agents, browser, and automated tests on macOS. + Drive iOS Simulators and Android emulators from the CLI using agents, browser, and automated tests on macOS.

@@ -35,8 +35,9 @@ view inside the editor. ## Features -- Local simulator video stream over browser-native WebRTC H.264 with H.264 WebSocket fallback -- Full simulator control & inspection using private accessibility APIs - available using `simdeck` CLI +- Local iOS Simulator video over browser-native WebRTC H.264 with H.264 WebSocket fallback +- Android emulator frames are sourced from emulator gRPC; loopback browsers use raw RGBA over WebRTC, and non-loopback browsers use VideoToolbox-encoded H.264 +- Full simulator control & inspection using private iOS accessibility APIs and Android UIAutomator - available using `simdeck` CLI - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - CoreSimulator chrome asset rendering for device bezels - NativeScript, React Native, Flutter, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live @@ -120,6 +121,7 @@ simdeck boot simdeck shutdown simdeck erase simdeck install /path/to/App.app +simdeck install android: /path/to/app.apk simdeck uninstall com.example.App simdeck open-url https://example.com simdeck launch com.apple.Preferences @@ -161,6 +163,14 @@ simdeck logs --seconds 30 --limit 200 without launching Simulator.app, then falls back to `xcrun simctl` when private booting is unavailable. +Android emulators appear in `simdeck list` with IDs like +`android:SimDeck_Pixel_8_API_36`. For Android IDs, lifecycle, install, launch, +URL, screenshot, logs, UIAutomator `describe`, tap, swipe, text, key, home, app +switcher, rotation, pasteboard, and browser live view route through the Android +SDK tools (`emulator` and `adb`) plus the emulator gRPC screenshot stream for +live video. `simdeck stream` remains iOS-only because it writes the iOS H.264 +transport stream. + `stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or external tools such as `ffplay`. diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index a7d81d61..ff4ac7f1 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -89,6 +89,11 @@ bool xcw_native_session_rotate_right(void * _Nonnull handle, char * _Nullable * bool xcw_native_session_rotate_left(void * _Nonnull handle, char * _Nullable * _Nullable error_message); void xcw_native_session_set_frame_callback(void * _Nonnull handle, xcw_native_frame_callback _Nullable callback, void * _Nullable user_data); +void * _Nullable xcw_native_h264_encoder_create(xcw_native_frame_callback _Nullable callback, void * _Nullable user_data, char * _Nullable * _Nullable error_message); +void xcw_native_h264_encoder_destroy(void * _Nullable handle); +bool xcw_native_h264_encoder_encode_rgba(void * _Nonnull handle, const uint8_t * _Nonnull rgba, size_t length, uint32_t width, uint32_t height, uint64_t timestamp_us, char * _Nullable * _Nullable error_message); +void xcw_native_h264_encoder_request_keyframe(void * _Nonnull handle); + void xcw_native_free_string(char * _Nullable value); void xcw_native_free_bytes(xcw_native_owned_bytes bytes); void xcw_native_release_shared_bytes(xcw_native_shared_bytes bytes); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index 3fc2376a..e93241e7 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -3,11 +3,13 @@ #import "DFPrivateSimulatorDisplayBridge.h" #import "XCWAccessibilityBridge.h" #import "XCWChromeRenderer.h" +#import "XCWH264Encoder.h" #import "XCWNativeSession.h" #import "XCWSimctl.h" #import #import +#import #include #include @@ -63,10 +65,190 @@ static xcw_native_owned_bytes XCWOwnedBytesFromData(NSData *data) { return bytes; } +static xcw_native_shared_bytes XCWSharedBytesFromData(NSData *data) { + if (data.length == 0) { + return (xcw_native_shared_bytes){0}; + } + + CFTypeRef owner = CFRetain((__bridge CFTypeRef)data); + return (xcw_native_shared_bytes){ + .data = data.bytes, + .length = data.length, + .owner = (const void *)owner, + }; +} + static XCWNativeSession *XCWNativeSessionFromHandle(void *handle) { return (__bridge XCWNativeSession *)handle; } +@interface XCWNativeH264Encoder : NSObject + +- (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback + userData:(void *)userData; +- (BOOL)encodeRGBA:(const uint8_t *)rgba + length:(size_t)length + width:(uint32_t)width + height:(uint32_t)height + error:(NSError * _Nullable __autoreleasing *)error; +- (void)requestKeyFrame; +- (void)invalidate; + +@end + +@implementation XCWNativeH264Encoder { + XCWH264Encoder *_encoder; + xcw_native_frame_callback _callback; + void *_callbackUserData; + uint64_t _frameSequence; +} + +- (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback + userData:(void *)userData { + self = [super init]; + if (self == nil) { + return nil; + } + + _callback = callback; + _callbackUserData = userData; + __weak typeof(self) weakSelf = self; + @synchronized (XCWNativeH264Encoder.class) { + const char *previousCodec = getenv("SIMDECK_VIDEO_CODEC"); + char *previousCodecCopy = previousCodec != NULL ? strdup(previousCodec) : NULL; + const char *androidCodec = getenv("SIMDECK_ANDROID_VIDEO_CODEC"); + if (androidCodec == NULL || strlen(androidCodec) == 0) { + androidCodec = "software"; + } + setenv("SIMDECK_VIDEO_CODEC", androidCodec, 1); + _encoder = [[XCWH264Encoder alloc] initWithOutputHandler:^(NSData *sampleData, + uint64_t timestampUs, + BOOL isKeyFrame, + NSString * _Nullable codec, + NSData * _Nullable decoderConfig, + CGSize dimensions) { + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf == nil || strongSelf->_callback == NULL || sampleData.length == 0) { + return; + } + strongSelf->_frameSequence += 1; + xcw_native_frame frame = { + .frame_sequence = strongSelf->_frameSequence, + .timestamp_us = timestampUs, + .is_keyframe = isKeyFrame, + .width = (uint32_t)llround(dimensions.width), + .height = (uint32_t)llround(dimensions.height), + .codec = codec.UTF8String, + .description = XCWSharedBytesFromData(decoderConfig), + .data = XCWSharedBytesFromData(sampleData), + }; + strongSelf->_callback(&frame, strongSelf->_callbackUserData); + }]; + if (previousCodecCopy != NULL) { + setenv("SIMDECK_VIDEO_CODEC", previousCodecCopy, 1); + free(previousCodecCopy); + } else { + unsetenv("SIMDECK_VIDEO_CODEC"); + } + } + return self; +} + +- (void)dealloc { + [self invalidate]; +} + +- (BOOL)encodeRGBA:(const uint8_t *)rgba + length:(size_t)length + width:(uint32_t)width + height:(uint32_t)height + error:(NSError * _Nullable __autoreleasing *)error { + if (rgba == NULL || width == 0 || height == 0) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:1 + userInfo:@{ NSLocalizedDescriptionKey: @"RGBA frame input was empty." }]; + } + return NO; + } + size_t expectedLength = (size_t)width * (size_t)height * 4; + if (length < expectedLength) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:2 + userInfo:@{ NSLocalizedDescriptionKey: @"RGBA frame input was truncated." }]; + } + return NO; + } + + NSDictionary *attributes = @{ + (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (__bridge NSString *)kCVPixelBufferWidthKey: @(width), + (__bridge NSString *)kCVPixelBufferHeightKey: @(height), + (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + CVPixelBufferRef pixelBuffer = NULL; + CVReturn createStatus = CVPixelBufferCreate(kCFAllocatorDefault, + (size_t)width, + (size_t)height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)attributes, + &pixelBuffer); + if (createStatus != kCVReturnSuccess || pixelBuffer == NULL) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:createStatus + userInfo:@{ NSLocalizedDescriptionKey: @"Unable to allocate a VideoToolbox pixel buffer." }]; + } + return NO; + } + + CVReturn lockStatus = CVPixelBufferLockBaseAddress(pixelBuffer, 0); + if (lockStatus != kCVReturnSuccess) { + CVPixelBufferRelease(pixelBuffer); + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:lockStatus + userInfo:@{ NSLocalizedDescriptionKey: @"Unable to lock a VideoToolbox pixel buffer." }]; + } + return NO; + } + + uint8_t *dst = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t dstRowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer); + size_t srcRowBytes = (size_t)width * 4; + for (uint32_t y = 0; y < height; y += 1) { + const uint8_t *srcRow = rgba + ((size_t)y * srcRowBytes); + uint8_t *dstRow = dst + ((size_t)y * dstRowBytes); + for (uint32_t x = 0; x < width; x += 1) { + const uint8_t *src = srcRow + ((size_t)x * 4); + uint8_t *pixel = dstRow + ((size_t)x * 4); + pixel[0] = src[2]; + pixel[1] = src[1]; + pixel[2] = src[0]; + pixel[3] = src[3]; + } + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + [_encoder encodePixelBuffer:pixelBuffer]; + CVPixelBufferRelease(pixelBuffer); + return YES; +} + +- (void)requestKeyFrame { + [_encoder requestKeyFrame]; +} + +- (void)invalidate { + [_encoder invalidate]; +} + +@end + +static XCWNativeH264Encoder *XCWNativeH264EncoderFromHandle(void *handle) { + return (__bridge XCWNativeH264Encoder *)handle; +} + static BOOL XCWPerformSimctlAction(char **errorMessage, BOOL (^action)(XCWSimctl *simctl, NSError **error)) { XCWSimctl *simctl = [[XCWSimctl alloc] init]; NSError *error = nil; @@ -889,6 +1071,58 @@ void xcw_native_session_set_frame_callback(void *handle, xcw_native_frame_callba } } +void *xcw_native_h264_encoder_create(xcw_native_frame_callback callback, void *user_data, char **error_message) { + @autoreleasepool { + XCWNativeH264Encoder *encoder = [[XCWNativeH264Encoder alloc] initWithFrameCallback:callback + userData:user_data]; + if (encoder == nil) { + if (error_message != NULL) { + *error_message = XCWCopyCString(@"Unable to create the native H.264 encoder."); + } + return NULL; + } + return (__bridge_retained void *)encoder; + } +} + +void xcw_native_h264_encoder_destroy(void *handle) { + if (handle == NULL) { + return; + } + @autoreleasepool { + XCWNativeH264Encoder *encoder = CFBridgingRelease(handle); + [encoder invalidate]; + } +} + +bool xcw_native_h264_encoder_encode_rgba(void *handle, + const uint8_t *rgba, + size_t length, + uint32_t width, + uint32_t height, + uint64_t timestamp_us, + char **error_message) { + (void)timestamp_us; + @autoreleasepool { + NSError *error = nil; + BOOL ok = [XCWNativeH264EncoderFromHandle(handle) encodeRGBA:rgba + length:length + width:width + height:height + error:&error]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + +void xcw_native_h264_encoder_request_keyframe(void *handle) { + @autoreleasepool { + [XCWNativeH264EncoderFromHandle(handle) requestKeyFrame]; + } +} + void xcw_native_free_string(char *value) { if (value != NULL) { free(value); diff --git a/client/src/api/types.ts b/client/src/api/types.ts index 0f4b3c9d..fe0533ce 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -28,11 +28,17 @@ export interface PrivateDisplayInfo { export interface SimulatorMetadata { udid: string; name: string; + platform?: "ios-simulator" | "android-emulator" | string; runtimeName?: string; runtimeIdentifier?: string; deviceTypeName?: string; deviceTypeIdentifier?: string; isBooted: boolean; + android?: { + avdName?: string; + grpcPort?: number; + serial?: string; + }; privateDisplay?: PrivateDisplayInfo; } @@ -112,6 +118,13 @@ export interface ChromeProfile { screenWidth: number; screenHeight: number; cornerRadius: number; + cornerRadii?: { + topLeft?: number; + topRight?: number; + bottomRight?: number; + bottomLeft?: number; + }; + chromeStyle?: "asset" | string; hasScreenMask?: boolean; buttons?: ChromeButtonProfile[]; } @@ -158,15 +171,23 @@ export interface AccessibilityNode { AXUniqueId?: string | null; AXValue?: string | null; alpha?: number | null; + androidClass?: string | null; + androidPackage?: string | null; + androidResourceId?: string | null; backgroundColor?: Record | null; bounds?: AccessibilityFrame | null; + checkable?: boolean | null; + checked?: boolean | null; className?: string | null; + clickable?: boolean | null; children?: AccessibilityNode[]; control?: Record | null; content_required?: boolean | null; custom_actions?: string[] | null; debugDescription?: string | null; enabled?: boolean | null; + focusable?: boolean | null; + focused?: boolean | null; frame?: AccessibilityFrame | null; frameInScreen?: AccessibilityFrame | null; flutter?: Record | null; @@ -176,8 +197,10 @@ export interface AccessibilityNode { isHidden?: boolean | null; isOpaque?: boolean | null; isUserInteractionEnabled?: boolean | null; + longClickable?: boolean | null; moduleName?: string | null; nativeScript?: Record | null; + password?: boolean | null; pid?: number | null; placeholder?: string | null; reactNative?: Record | null; @@ -185,6 +208,8 @@ export interface AccessibilityNode { role_description?: string | null; scroll?: Record | null; semantics?: Record | null; + scrollable?: boolean | null; + selected?: boolean | null; source?: | "native-ax" | "in-app-inspector" @@ -211,6 +236,7 @@ export interface AccessibilityNode { } export type AccessibilitySource = + | "android-uiautomator" | "native-ax" | "in-app-inspector" | "nativescript" diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index f984108f..78b29066 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -105,6 +105,7 @@ import { const ACCESSIBILITY_REFRESH_MS = 1500; const REACT_NATIVE_ACCESSIBILITY_REFRESH_MS = 500; const FLUTTER_ACCESSIBILITY_REFRESH_MS = 1000; +const ANDROID_METADATA_REFRESH_MS = 1000; const DEFAULT_ACCESSIBILITY_MAX_DEPTH = 10; const LOGICAL_INSPECTOR_MAX_DEPTH = 80; const FLUTTER_INSPECTOR_MAX_DEPTH = 48; @@ -457,8 +458,12 @@ export function AppShell({ udid: string; } | null>(null); const touchMoveFrameRef = useRef(0); + const refreshRef = useRef(refresh); + const previousAndroidDisplayKeyRef = useRef(""); + const previousAndroidViewportSizeKeyRef = useRef(""); const canvasSize = useElementSize(outerCanvasElement); const zoomDockSize = useElementSize(zoomDockElement); + refreshRef.current = refresh; const handleOuterCanvasRef = useCallback((node: HTMLDivElement | null) => { outerCanvasRef.current = node; @@ -688,17 +693,36 @@ export function AppShell({ const shouldRenderChrome = selectedSimulator != null && shouldRenderNativeChrome(selectedSimulator); const viewportChromeProfile = shouldRenderChrome ? chromeProfile : null; - const effectiveDeviceNaturalSize = useMemo( - () => + const isAndroidViewport = isAndroidSimulator(selectedSimulator); + const androidDisplayKey = + isAndroidViewport && selectedSimulator + ? androidDisplayKeyForSimulator(selectedSimulator) + : ""; + const effectiveDeviceNaturalSize = useMemo(() => { + const displaySize = simulatorDisplaySize(selectedSimulator); + if (isAndroidViewport) { + return deviceNaturalSize ?? displaySize; + } + return ( deviceNaturalSize ?? (!shouldRenderChrome && chromeProfile ? { width: chromeProfile.screenWidth, height: chromeProfile.screenHeight, } - : simulatorDisplaySize(selectedSimulator)), - [chromeProfile, deviceNaturalSize, selectedSimulator, shouldRenderChrome], - ); + : displaySize) + ); + }, [ + chromeProfile, + deviceNaturalSize, + isAndroidViewport, + selectedSimulator, + shouldRenderChrome, + ]); + const androidViewportSizeKey = + isAndroidViewport && effectiveDeviceNaturalSize + ? `${Math.round(effectiveDeviceNaturalSize.width)}x${Math.round(effectiveDeviceNaturalSize.height)}` + : ""; const zoomDockReservedHeight = zoomDockElement && typeof window !== "undefined" @@ -745,9 +769,9 @@ export function AppShell({ : "", [selectedSimulator?.udid, streamStamp], ); + const chromeUsesAsset = Boolean(viewportChromeProfile && chromeUrl); const chromeRequired = Boolean( - (shouldRenderChrome && !chromeProfileReady) || - (viewportChromeProfile && chromeUrl), + (shouldRenderChrome && !chromeProfileReady) || chromeUsesAsset, ); const simulatorRotationQuarterTurns = normalizeSimulatorRotationQuarterTurns(selectedSimulator); @@ -1035,6 +1059,12 @@ export function AppShell({ }, [isBooted]); useEffect(() => { + if (isAndroidViewport) { + setRotationQuarterTurns((current) => + normalizeQuarterTurns(current) === 0 ? current : 0, + ); + return; + } if (simulatorRotationQuarterTurns == null) { return; } @@ -1046,7 +1076,56 @@ export function AppShell({ beginZoomAnimation(); return simulatorRotationQuarterTurns; }); - }, [simulatorRotationQuarterTurns]); + }, [isAndroidViewport, simulatorRotationQuarterTurns]); + + useEffect(() => { + if (!isAndroidViewport || !selectedSimulator?.isBooted) { + return; + } + + let cancelled = false; + const refreshAndroidMetadata = () => { + if (cancelled || document.visibilityState !== "visible") { + return; + } + void refreshRef.current(); + }; + + const intervalId = window.setInterval( + refreshAndroidMetadata, + ANDROID_METADATA_REFRESH_MS, + ); + return () => { + cancelled = true; + window.clearInterval(intervalId); + }; + }, [isAndroidViewport, selectedSimulator?.isBooted, selectedSimulator?.udid]); + + useEffect(() => { + if (!isAndroidViewport || !androidDisplayKey) { + previousAndroidDisplayKeyRef.current = ""; + return; + } + + const previousKey = previousAndroidDisplayKeyRef.current; + previousAndroidDisplayKeyRef.current = androidDisplayKey; + if (previousKey && previousKey !== androidDisplayKey) { + beginZoomAnimation(); + } + }, [androidDisplayKey, isAndroidViewport]); + + useEffect(() => { + if (!isAndroidViewport || !androidViewportSizeKey) { + previousAndroidViewportSizeKeyRef.current = ""; + return; + } + + const previousKey = previousAndroidViewportSizeKeyRef.current; + previousAndroidViewportSizeKeyRef.current = androidViewportSizeKey; + if (previousKey && previousKey !== androidViewportSizeKey) { + beginZoomAnimation(); + } + }, [androidViewportSizeKey, isAndroidViewport]); useEffect(() => { setChromeLoaded(!chromeRequired); @@ -1079,7 +1158,12 @@ export function AppShell({ return () => { cancelled = true; }; - }, [selectedSimulator?.udid]); + }, [ + selectedSimulator?.privateDisplay?.displayHeight, + selectedSimulator?.privateDisplay?.displayWidth, + selectedSimulator?.privateDisplay?.rotationQuarterTurns, + selectedSimulator?.udid, + ]); useEffect(() => { if (!menuOpen) { @@ -1311,13 +1395,15 @@ export function AppShell({ : null; const screenOnlyStyle = !viewportChromeProfile && chromeProfile && chromeProfile.screenWidth > 0 - ? ({ - borderRadius: `${Math.min( - chromeProfile.cornerRadius * - (DEVICE_SCREEN_WIDTH / chromeProfile.screenWidth), - DEVICE_SCREEN_WIDTH / 2, - )}px`, - } satisfies CSSProperties) + ? isAndroidViewport + ? androidScreenRadiusStyle(chromeProfile, effectiveDeviceNaturalSize) + : ({ + borderRadius: `${Math.min( + chromeProfile.cornerRadius * + (DEVICE_SCREEN_WIDTH / chromeProfile.screenWidth), + DEVICE_SCREEN_WIDTH / 2, + )}px`, + } satisfies CSSProperties) : null; const viewportScreenStyle = chromeScreenStyle ?? screenOnlyStyle; const shellStyle = viewportChromeProfile @@ -1881,7 +1967,17 @@ export function AppShell({ if (!selectedSimulator) { return; } + const androidViewport = isAndroidSimulator(selectedSimulator); beginZoomAnimation(); + if (androidViewport) { + void runAction(async () => { + await rotateRight(selectedSimulator.udid); + setRotationQuarterTurns(0); + beginZoomAnimation(); + await refresh(); + }, false); + return; + } if (sendControl(selectedSimulator.udid, { type: "rotateRight" })) { setRotationQuarterTurns((current) => (current + 1) % 4); return; @@ -2046,6 +2142,7 @@ export function AppShell({ outerCanvasRef={handleOuterCanvasRef} rotationQuarterTurns={rotationQuarterTurns} screenAspect={screenAspect} + screenClassName={isAndroidViewport ? "android-screen" : undefined} selectedSimulator={selectedSimulator} shellStyle={shellStyle} streamCanvasRef={handleStreamCanvasRef} @@ -2070,6 +2167,77 @@ export function AppShell({ ); } +function androidScreenRadiusStyle( + chromeProfile: ChromeProfile, + displaySize: Size | null, +): CSSProperties | null { + const screenWidth = + displaySize && displaySize.width > 0 + ? displaySize.width + : chromeProfile.screenWidth; + if (screenWidth <= 0) { + return null; + } + + const scale = DEVICE_SCREEN_WIDTH / screenWidth; + const maxRadius = DEVICE_SCREEN_WIDTH / 2; + const radii = chromeProfile.cornerRadii; + const topLeft = scaledScreenRadius( + radii?.topLeft ?? chromeProfile.cornerRadius, + scale, + maxRadius, + ); + const topRight = scaledScreenRadius( + radii?.topRight ?? chromeProfile.cornerRadius, + scale, + maxRadius, + ); + const bottomRight = scaledScreenRadius( + radii?.bottomRight ?? chromeProfile.cornerRadius, + scale, + maxRadius, + ); + const bottomLeft = scaledScreenRadius( + radii?.bottomLeft ?? chromeProfile.cornerRadius, + scale, + maxRadius, + ); + + if (topLeft <= 0 && topRight <= 0 && bottomRight <= 0 && bottomLeft <= 0) { + return null; + } + + const borderRadius = `${topLeft}px ${topRight}px ${bottomRight}px ${bottomLeft}px`; + return { + borderRadius, + borderTopLeftRadius: `${topLeft}px`, + borderTopRightRadius: `${topRight}px`, + borderBottomRightRadius: `${bottomRight}px`, + borderBottomLeftRadius: `${bottomLeft}px`, + clipPath: `inset(0 round ${borderRadius})`, + }; +} + +function scaledScreenRadius(radius: number, scale: number, maxRadius: number) { + if (!Number.isFinite(radius) || radius <= 0) { + return 0; + } + return Math.min(radius * scale, maxRadius); +} + +function androidDisplayKeyForSimulator(simulator: SimulatorMetadata): string { + const display = simulator.privateDisplay; + if (!display) { + return simulator.udid; + } + return [ + simulator.udid, + Math.round(display.displayWidth), + Math.round(display.displayHeight), + display.rotationQuarterTurns ?? 0, + ].join("|"); +} + function readDeviceQueryParam(): string | undefined { if (typeof window === "undefined") { return undefined; @@ -2217,6 +2385,14 @@ function normalizeMaxEdge( : fallback; } +function isAndroidSimulator(simulator: SimulatorMetadata | null): boolean { + return Boolean( + simulator?.platform === "android-emulator" || + simulator?.deviceTypeIdentifier === "android-emulator" || + simulator?.udid.startsWith("android:"), + ); +} + function streamConfigsEqual(left: StreamConfig, right: StreamConfig): boolean { return ( left.encoder === right.encoder && diff --git a/client/src/app/uiState.ts b/client/src/app/uiState.ts index 98871d0c..0a00e027 100644 --- a/client/src/app/uiState.ts +++ b/client/src/app/uiState.ts @@ -37,6 +37,7 @@ const ACCESSIBILITY_SOURCE_ORDER: AccessibilitySource[] = [ "flutter", "swiftui", "in-app-inspector", + "android-uiautomator", "native-ax", ]; @@ -162,6 +163,7 @@ export function isAccessibilitySource( value === "flutter" || value === "swiftui" || value === "in-app-inspector" || + value === "android-uiautomator" || value === "native-ax" ); } diff --git a/client/src/features/accessibility/AccessibilityInspector.tsx b/client/src/features/accessibility/AccessibilityInspector.tsx index bf2c8d44..42a866c3 100644 --- a/client/src/features/accessibility/AccessibilityInspector.tsx +++ b/client/src/features/accessibility/AccessibilityInspector.tsx @@ -464,17 +464,24 @@ function NodeDetails({ node: AccessibilityNode; selectedSimulator: SimulatorMetadata | null; }) { + const isAndroid = isAndroidSimulator(selectedSimulator); const details = [ ["Type", accessibilityKind(node)], ["Label", primaryAccessibilityText(node)], ["Source", sourceLocationText(node)], - ["Identifier", accessibilityIdentifier(node)], + [ + isAndroid ? "Resource ID" : "Identifier", + isAndroid + ? (node.androidResourceId ?? "") + : accessibilityIdentifier(node), + ], ["Inspector ID", node.inspectorId ?? ""], ["Module", node.moduleName ?? ""], ["NativeScript", nativeScriptDescription(node.nativeScript)], ["React Native", reactNativeDescription(node.reactNative)], ["Flutter", flutterDescription(node.flutter)], - ["UIKit Class", node.className ?? ""], + [isAndroid ? "Android Class" : "UIKit Class", node.className ?? ""], + ["Package", isAndroid ? (node.androidPackage ?? "") : ""], ["Last JS", lastUIKitScriptText(node)], ["Value", node.AXValue ?? ""], ["Role", node.role ?? ""], @@ -483,6 +490,15 @@ function NodeDetails({ ["SwiftUI", swiftUIDescription(node.swiftUI)], ["Enabled", node.enabled == null ? "" : node.enabled ? "true" : "false"], ["Hidden", node.isHidden == null ? "" : node.isHidden ? "true" : "false"], + ["Clickable", boolDetail(isAndroid, node.clickable)], + ["Long Clickable", boolDetail(isAndroid, node.longClickable)], + ["Focusable", boolDetail(isAndroid, node.focusable)], + ["Focused", boolDetail(isAndroid, node.focused)], + ["Scrollable", boolDetail(isAndroid, node.scrollable)], + ["Checkable", boolDetail(isAndroid, node.checkable)], + ["Checked", boolDetail(isAndroid, node.checked)], + ["Selected", boolDetail(isAndroid, node.selected)], + ["Password", boolDetail(isAndroid, node.password)], ["Alpha", node.alpha == null ? "" : String(round(node.alpha))], ["Frame", validFrame(node.frame) ? frameText(node.frame) : ""], ["PID", node.pid == null ? "" : String(node.pid)], @@ -504,6 +520,18 @@ function NodeDetails({ ); } +function isAndroidSimulator(simulator: SimulatorMetadata | null): boolean { + return Boolean( + simulator?.platform === "android-emulator" || + simulator?.deviceTypeIdentifier === "android-emulator" || + simulator?.udid.startsWith("android:"), + ); +} + +function boolDetail(include: boolean, value: boolean | null | undefined) { + return include && value != null ? (value ? "true" : "false") : ""; +} + function UIKitScriptEditor({ node, selectedSimulator, @@ -737,6 +765,7 @@ const HIERARCHY_SOURCE_ORDER: AccessibilitySource[] = [ "flutter", "swiftui", "in-app-inspector", + "android-uiautomator", "native-ax", ]; @@ -770,6 +799,9 @@ function sourceLabel(source: AccessibilitySource): string { if (source === "swiftui") { return "SwiftUI"; } + if (source === "android-uiautomator") { + return "Android"; + } return source === "in-app-inspector" ? "UIKit" : "Native AX"; } diff --git a/client/src/features/simulators/simulatorDisplay.test.ts b/client/src/features/simulators/simulatorDisplay.test.ts index 58c0b2f9..f39a5507 100644 --- a/client/src/features/simulators/simulatorDisplay.test.ts +++ b/client/src/features/simulators/simulatorDisplay.test.ts @@ -51,4 +51,16 @@ describe("simulatorDisplay", () => { ), ).toBe(false); }); + + it("keeps native chrome off for Android emulators", () => { + expect( + shouldRenderNativeChrome( + simulator({ + deviceTypeIdentifier: "android-emulator", + name: "SimDeck Pixel", + platform: "android-emulator", + }), + ), + ).toBe(false); + }); }); diff --git a/client/src/features/stream/streamTypes.ts b/client/src/features/stream/streamTypes.ts index f55eb58c..4e9b6099 100644 --- a/client/src/features/stream/streamTypes.ts +++ b/client/src/features/stream/streamTypes.ts @@ -2,6 +2,7 @@ import type { Size } from "../viewport/types"; export interface StreamConnectTarget { clientId?: string; + platform?: string; remote?: boolean; streamConfig?: StreamConfig; transport?: StreamTransport; diff --git a/client/src/features/stream/streamWorkerClient.test.ts b/client/src/features/stream/streamWorkerClient.test.ts new file mode 100644 index 00000000..eddc2054 --- /dev/null +++ b/client/src/features/stream/streamWorkerClient.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; + +import { + buildStreamTarget, + initialStreamBackend, + preferredStreamBackend, + shouldUseLocalAndroidRgbaWebRtc, +} from "./streamWorkerClient"; + +describe("streamWorkerClient", () => { + it("uses the common H264 WebSocket preference for Android emulator streams", () => { + const target = buildStreamTarget("android:emulator-5554", { + platform: "android-emulator", + transport: "h264", + }); + + expect(preferredStreamBackend(target)).toBe("h264-ws"); + }); + + it("uses the common WebRTC preference for Android emulator streams", () => { + const target = buildStreamTarget("android:Pixel_8", { + transport: "webrtc", + }); + + expect(preferredStreamBackend(target)).toBe("webrtc"); + }); + + it("treats explicit RGBA transport as a WebRTC backend", () => { + const previousWindow = globalThis.window; + Object.defineProperty(globalThis, "window", { + configurable: true, + value: { location: { search: "?stream=rgba" } }, + }); + + try { + expect(preferredStreamBackend(buildStreamTarget("android:Pixel_8"))).toBe( + "webrtc", + ); + } finally { + Object.defineProperty(globalThis, "window", { + configurable: true, + value: previousWindow, + }); + } + }); + + it("defaults Android auto streams to WebRTC when the browser supports it", () => { + const previousPeerConnection = globalThis.RTCPeerConnection; + ( + globalThis as unknown as { RTCPeerConnection: unknown } + ).RTCPeerConnection = function RTCPeerConnection() {}; + const target = buildStreamTarget("android:Pixel_8", { + transport: "auto", + }); + + try { + expect(preferredStreamBackend(target)).toBe("auto"); + expect(initialStreamBackend(target)).toBe("webrtc"); + } finally { + ( + globalThis as unknown as { RTCPeerConnection: unknown } + ).RTCPeerConnection = previousPeerConnection; + } + }); + + it("uses RGBA WebRTC transport for local loopback Android streams", () => { + const previousWindow = globalThis.window; + Object.defineProperty(globalThis, "window", { + configurable: true, + value: { location: { hostname: "127.0.0.1", search: "" } }, + }); + + try { + expect( + shouldUseLocalAndroidRgbaWebRtc( + buildStreamTarget("android:Pixel_8", { + platform: "android-emulator", + transport: "auto", + }), + ), + ).toBe(true); + } finally { + Object.defineProperty(globalThis, "window", { + configurable: true, + value: previousWindow, + }); + } + }); + + it("keeps Android RGBA disabled for h264 or remote streams", () => { + const previousWindow = globalThis.window; + const location = { hostname: "127.0.0.1", search: "?stream=h264" }; + Object.defineProperty(globalThis, "window", { + configurable: true, + value: { location }, + }); + + try { + expect( + shouldUseLocalAndroidRgbaWebRtc( + buildStreamTarget("android:Pixel_8", { + platform: "android-emulator", + transport: "auto", + }), + ), + ).toBe(false); + location.search = ""; + expect( + shouldUseLocalAndroidRgbaWebRtc( + buildStreamTarget("android:Pixel_8", { + platform: "android-emulator", + remote: true, + transport: "auto", + }), + ), + ).toBe(false); + } finally { + Object.defineProperty(globalThis, "window", { + configurable: true, + value: previousWindow, + }); + } + }); +}); diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index e5397753..00ea6d36 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -18,6 +18,7 @@ import type { const HAVE_CURRENT_DATA = 2; const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control"; const WEBRTC_TELEMETRY_CHANNEL_LABEL = "simdeck-telemetry"; +const WEBRTC_RGBA_CHANNEL_LABEL = "simdeck-rgba"; const WEBRTC_FIRST_FRAME_TIMEOUT_MS = 10000; const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 3000; const WEBRTC_LOCAL_RECEIVER_BUFFER_SECONDS = 0.001; @@ -33,6 +34,10 @@ const H264_WS_HEADER_BYTES = 40; const H264_WS_MAGIC = 0x53444831; const H264_WS_FLAG_KEYFRAME = 1 << 0; const H264_WS_FLAG_CONFIG = 1 << 1; +const WEBRTC_RGBA_CHUNK_HEADER_BYTES = 48; +const WEBRTC_RGBA_CHUNK_MAGIC = 0x53445243; +const WEBRTC_RGBA_VERSION = 1; +const WEBRTC_RGBA_FORMAT_RGBA8888 = 1; const H264_WS_LOCAL_AUTO_PROFILES: StreamQualityPreset[] = [ "low", "economy", @@ -224,6 +229,7 @@ export function buildStreamTarget( udid: string, options: { clientId?: string; + platform?: string; remote?: boolean; streamConfig?: StreamConfig; transport?: StreamTransport; @@ -231,6 +237,7 @@ export function buildStreamTarget( ): StreamConnectTarget { return { clientId: options.clientId, + platform: options.platform, remote: options.remote, streamConfig: options.streamConfig, transport: options.transport, @@ -300,6 +307,34 @@ interface H264WebSocketFrame { width: number; } +interface WebRtcRgbaFrame { + height: number; + payload: Uint8Array; + sequence: number; + timestampUs: number; + width: number; +} + +interface WebRtcRgbaChunk { + chunkOffset: number; + height: number; + payload: Uint8Array; + payloadBytes: number; + sequence: number; + timestampUs: number; + width: number; +} + +interface WebRtcRgbaAssembly { + buffer: Uint8Array; + height: number; + receivedBytes: number; + receivedRanges: Array<[number, number]>; + sequence: number; + timestampUs: number; + width: number; +} + interface WebCodecsVideoFrame { close(): void; codedHeight?: number; @@ -351,6 +386,13 @@ interface WebCodecsVideoDecoderConstructor { }>; } +interface WebRtcAnswerPayload extends RTCSessionDescriptionInit { + video?: { + height?: number; + width?: number; + }; +} + interface PendingVideoFrame { frame: WebCodecsVideoFrame; sequence: number | null; @@ -1141,6 +1183,53 @@ function parseH264WebSocketFrame(data: unknown): H264WebSocketFrame | null { }; } +function parseWebRtcRgbaChunk(data: unknown): WebRtcRgbaChunk | null { + const bytes = bytesFromBinaryMessage(data); + if (!bytes || bytes.byteLength < WEBRTC_RGBA_CHUNK_HEADER_BYTES) { + return null; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if ( + view.getUint32(0, false) !== WEBRTC_RGBA_CHUNK_MAGIC || + view.getUint8(4) !== WEBRTC_RGBA_VERSION || + view.getUint8(5) !== WEBRTC_RGBA_FORMAT_RGBA8888 + ) { + return null; + } + const headerBytes = view.getUint16(6, false); + if ( + headerBytes < WEBRTC_RGBA_CHUNK_HEADER_BYTES || + headerBytes > bytes.byteLength + ) { + return null; + } + const width = view.getUint32(24, false); + const height = view.getUint32(28, false); + const payloadBytes = view.getUint32(32, false); + const chunkOffset = view.getUint32(36, false); + const chunkBytes = view.getUint32(40, false); + const payloadEnd = headerBytes + chunkBytes; + if ( + width <= 0 || + height <= 0 || + payloadBytes !== width * height * 4 || + chunkBytes <= 0 || + chunkOffset + chunkBytes > payloadBytes || + payloadEnd > bytes.byteLength + ) { + return null; + } + return { + chunkOffset, + height, + payload: bytes.subarray(headerBytes, payloadEnd), + payloadBytes, + sequence: Number(view.getBigUint64(8, false)), + timestampUs: Number(view.getBigUint64(16, false)), + width, + }; +} + function bytesFromBinaryMessage(data: unknown): Uint8Array | null { if (ArrayBuffer.isView(data)) { return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); @@ -1168,6 +1257,16 @@ function bytesFromBinaryMessage(data: unknown): Uint8Array | null { return null; } +function rangeAlreadyReceived( + ranges: Array<[number, number]>, + start: number, + end: number, +): boolean { + return ranges.some( + ([rangeStart, rangeEnd]) => start >= rangeStart && end <= rangeEnd, + ); +} + function h264CodecStringFromAvcC(config: Uint8Array): string | null { if (config.byteLength < 4 || config[0] !== 1) { return null; @@ -1196,6 +1295,7 @@ function hexByte(byte: number): string { class WebRtcStreamClient implements StreamClientBackend { private animationFrame = 0; private canvas: HTMLCanvasElement | null = null; + private canvasContext: CanvasRenderingContext2D | null = null; private connectGeneration = 0; private controlChannel: RTCDataChannel | null = null; private diagnostics = createWebRtcDiagnostics(); @@ -1215,6 +1315,10 @@ class WebRtcStreamClient implements StreamClientBackend { private receiverStatsInterval = 0; private receiverStatsSeen = false; private shouldReconnect = false; + private latestRgbaSequence = -1; + private rgbaAssemblies = new Map(); + private rgbaChannel: RTCDataChannel | null = null; + private rgbaMode = false; private streamConfigGeneration = 0; private streamTarget: StreamConnectTarget | null = null; private telemetryChannel: RTCDataChannel | null = null; @@ -1228,6 +1332,10 @@ class WebRtcStreamClient implements StreamClientBackend { attachCanvas(canvasElement: HTMLCanvasElement) { this.canvas = canvasElement; + this.canvasContext = canvasElement.getContext("2d", { + alpha: false, + desynchronized: true, + }); if ( this.video && this.video.parentElement !== canvasElement.parentElement @@ -1240,9 +1348,12 @@ class WebRtcStreamClient implements StreamClientBackend { } clear() { - this.canvas - ?.getContext("2d") - ?.clearRect(0, 0, this.canvas.width, this.canvas.height); + this.ensureCanvasContext()?.clearRect( + 0, + 0, + this.canvas?.width ?? 0, + this.canvas?.height ?? 0, + ); } async collectVisualArtifactSample( @@ -1317,16 +1428,20 @@ class WebRtcStreamClient implements StreamClientBackend { iceTransportPolicy: iceTransportPolicy(health), }); this.peerConnection = peerConnection; + const useRgbaTransport = shouldUseLocalAndroidRgbaWebRtc(target); + this.rgbaMode = useRgbaTransport; this.attachDiagnostics(peerConnection, target, generation); - this.startReceiverStatsPolling(peerConnection, target, generation); - const transceiver = peerConnection.addTransceiver("video", { - direction: "recvonly", - }); - configureReceiverCodecPreferences(transceiver); - configureLowLatencyReceiver( - transceiver.receiver, - receiverBufferSeconds(target), - ); + if (!useRgbaTransport) { + this.startReceiverStatsPolling(peerConnection, target, generation); + const transceiver = peerConnection.addTransceiver("video", { + direction: "recvonly", + }); + configureReceiverCodecPreferences(transceiver); + configureLowLatencyReceiver( + transceiver.receiver, + receiverBufferSeconds(target), + ); + } const controlChannel = peerConnection.createDataChannel( WEBRTC_CONTROL_CHANNEL_LABEL, { @@ -1355,7 +1470,20 @@ class WebRtcStreamClient implements StreamClientBackend { } }); + peerConnection.ondatachannel = (event) => { + if ( + generation !== this.connectGeneration || + event.channel.label !== WEBRTC_RGBA_CHANNEL_LABEL + ) { + return; + } + this.attachRgbaDataChannel(event.channel, generation); + }; + peerConnection.ontrack = (event) => { + if (useRgbaTransport) { + return; + } if (generation !== this.connectGeneration) { return; } @@ -1419,7 +1547,12 @@ class WebRtcStreamClient implements StreamClientBackend { if (this.reportedVideoWidth > 0 && this.reportedVideoHeight > 0) { this.onMessage({ type: "status", - status: { detail: "WebRTC media connected", state: "streaming" }, + status: { + detail: useRgbaTransport + ? "WebRTC RGBA stream connected" + : "WebRTC media connected", + state: "streaming", + }, }); } return; @@ -1454,6 +1587,7 @@ class WebRtcStreamClient implements StreamClientBackend { await this.negotiatePeerConnection(peerConnection, target, generation, { detailPrefix: "local", + transport: useRgbaTransport ? "rgba" : undefined, }); this.scheduleFrameWatchdog(target, generation); } catch (error) { @@ -1496,7 +1630,11 @@ class WebRtcStreamClient implements StreamClientBackend { peerConnection: RTCPeerConnection, target: StreamConnectTarget, generation: number, - options: { detailPrefix: string; iceRestart?: boolean }, + options: { + detailPrefix: string; + iceRestart?: boolean; + transport?: "rgba"; + }, ) { const offer = safariBaselineH264Offer( await peerConnection.createOffer({ iceRestart: options.iceRestart }), @@ -1529,8 +1667,9 @@ class WebRtcStreamClient implements StreamClientBackend { const response = await postWebRtcOfferWithAuthRetry( target, localDescription, + options.transport, ); - const answer = (await response.json()) as RTCSessionDescriptionInit; + const answer = (await response.json()) as WebRtcAnswerPayload; if (generation !== this.connectGeneration) { return; } @@ -1539,6 +1678,15 @@ class WebRtcStreamClient implements StreamClientBackend { ); this.postDiagnostics(target, `${options.detailPrefix}-answer`); await peerConnection.setRemoteDescription(answer); + if ( + typeof answer.video?.width === "number" && + typeof answer.video?.height === "number" && + answer.video.width > 0 && + answer.video.height > 0 + ) { + this.syncCanvasSize(answer.video.width, answer.video.height); + this.reportVideoConfig(answer.video.width, answer.video.height); + } } destroy() { @@ -1573,6 +1721,11 @@ class WebRtcStreamClient implements StreamClientBackend { activeWebRtcTelemetryChannel = null; } this.telemetryChannel = null; + this.rgbaChannel?.close(); + this.rgbaChannel = null; + this.rgbaAssemblies.clear(); + this.latestRgbaSequence = -1; + this.rgbaMode = false; this.peerConnection?.close(); this.peerConnection = null; } @@ -1685,10 +1838,14 @@ class WebRtcStreamClient implements StreamClientBackend { return; } const now = performance.now(); - const hasRenderedFrame = this.stats.renderedFrames > 0; + const hasMediaProgress = + this.hasRenderedFrame || + this.stats.renderedFrames > 0 || + this.stats.decodedFrames > 0 || + this.stats.receivedPackets > 0; const frameAgeMs = this.lastVideoFrameAt > 0 ? now - this.lastVideoFrameAt : Infinity; - if (!hasRenderedFrame) { + if (!hasMediaProgress) { this.handleConnectionError( target, generation, @@ -1697,6 +1854,10 @@ class WebRtcStreamClient implements StreamClientBackend { ); return; } + if (!this.hasRenderedFrame) { + this.scheduleFrameWatchdog(target, generation); + return; + } if (frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) { this.sendControl({ snapshot: true, type: "streamControl" }); this.scheduleFrameWatchdog(target, generation); @@ -1966,6 +2127,171 @@ class WebRtcStreamClient implements StreamClientBackend { } } + private attachRgbaDataChannel(channel: RTCDataChannel, generation: number) { + this.rgbaChannel?.close(); + this.rgbaChannel = channel; + channel.binaryType = "arraybuffer"; + channel.addEventListener("message", (event) => { + if ( + generation !== this.connectGeneration || + channel !== this.rgbaChannel + ) { + return; + } + if (hasArrayBufferMethod(event.data)) { + void event.data.arrayBuffer().then((buffer) => { + if ( + generation === this.connectGeneration && + channel === this.rgbaChannel + ) { + this.handleRgbaMessage(buffer); + } + }); + return; + } + this.handleRgbaMessage(event.data); + }); + channel.addEventListener("close", () => { + if (this.rgbaChannel === channel) { + this.rgbaChannel = null; + } + }); + } + + private handleRgbaMessage(data: unknown) { + const chunk = parseWebRtcRgbaChunk(data); + if (!chunk) { + this.stats.h264ParseFailures += 1; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + return; + } + this.stats.receivedPackets += 1; + if (chunk.sequence < this.latestRgbaSequence) { + this.stats.droppedFrames += 1; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + return; + } + const frame = this.appendRgbaChunk(chunk); + if (!frame) { + return; + } + this.stats.decodedFrames += 1; + this.stats.width = frame.width; + this.stats.height = frame.height; + this.stats.frameSequence = frame.sequence; + this.stats.codec = "webrtc-rgba"; + this.latestRgbaSequence = frame.sequence; + this.paintRgbaFrame(frame); + } + + private appendRgbaChunk(chunk: WebRtcRgbaChunk): WebRtcRgbaFrame | null { + let assembly = this.rgbaAssemblies.get(chunk.sequence); + if ( + assembly && + (assembly.width !== chunk.width || + assembly.height !== chunk.height || + assembly.buffer.byteLength !== chunk.payloadBytes) + ) { + this.rgbaAssemblies.delete(chunk.sequence); + assembly = undefined; + } + if (!assembly) { + assembly = { + buffer: new Uint8Array(chunk.payloadBytes), + height: chunk.height, + receivedBytes: 0, + receivedRanges: [], + sequence: chunk.sequence, + timestampUs: chunk.timestampUs, + width: chunk.width, + }; + this.rgbaAssemblies.set(chunk.sequence, assembly); + this.trimRgbaAssemblies(chunk.sequence); + } + assembly.buffer.set(chunk.payload, chunk.chunkOffset); + const chunkEnd = chunk.chunkOffset + chunk.payload.byteLength; + if ( + !rangeAlreadyReceived( + assembly.receivedRanges, + chunk.chunkOffset, + chunkEnd, + ) + ) { + assembly.receivedRanges.push([chunk.chunkOffset, chunkEnd]); + assembly.receivedBytes += chunk.payload.byteLength; + } + if (assembly.receivedBytes < assembly.buffer.byteLength) { + return null; + } + this.rgbaAssemblies.delete(chunk.sequence); + return { + height: assembly.height, + payload: assembly.buffer, + sequence: assembly.sequence, + timestampUs: assembly.timestampUs, + width: assembly.width, + }; + } + + private trimRgbaAssemblies(latestSequence: number) { + while (this.rgbaAssemblies.size > 3) { + const firstKey = this.rgbaAssemblies.keys().next().value; + if (typeof firstKey !== "number") { + break; + } + this.rgbaAssemblies.delete(firstKey); + } + for (const sequence of this.rgbaAssemblies.keys()) { + if (latestSequence - sequence > 2) { + this.rgbaAssemblies.delete(sequence); + } + } + } + + private paintRgbaFrame(frame: WebRtcRgbaFrame) { + const canvas = this.canvas; + const context = this.ensureCanvasContext(); + if (!canvas || !context) { + return; + } + this.syncCanvasSize(frame.width, frame.height); + const startedAt = performance.now(); + const rgba = new Uint8ClampedArray( + frame.payload.buffer as ArrayBuffer, + frame.payload.byteOffset, + frame.payload.byteLength, + ); + try { + context.putImageData( + new ImageData(rgba, frame.width, frame.height), + 0, + 0, + ); + } catch { + this.stats.droppedFrames += 1; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + return; + } + const finishedAt = performance.now(); + const previousFrameAt = this.lastVideoFrameAt; + this.lastVideoFrameAt = finishedAt; + this.hasRenderedFrame = true; + this.stats.renderedFrames += 1; + this.stats.latestRenderMs = finishedAt - startedAt; + this.stats.maxRenderMs = Math.max( + this.stats.maxRenderMs, + this.stats.latestRenderMs, + ); + this.stats.averageRenderMs = + this.stats.averageRenderMs <= 0 + ? this.stats.latestRenderMs + : this.stats.averageRenderMs * 0.9 + this.stats.latestRenderMs * 0.1; + this.stats.latestFrameGapMs = + previousFrameAt > 0 ? finishedAt - previousFrameAt : 0; + this.reportVideoConfig(frame.width, frame.height); + this.onMessage({ type: "stats", stats: { ...this.stats } }); + } + private drawVideoFrame = () => { this.videoFrameCallback = 0; if (!this.canvas || !this.video) { @@ -2031,7 +2357,12 @@ class WebRtcStreamClient implements StreamClientBackend { }); this.onMessage({ type: "status", - status: { detail: "WebRTC media connected", state: "streaming" }, + status: { + detail: this.rgbaMode + ? "WebRTC RGBA stream connected" + : "WebRTC media connected", + state: "streaming", + }, }); } @@ -2145,6 +2476,22 @@ class WebRtcStreamClient implements StreamClientBackend { this.videoFrameCallback = 0; } + private ensureCanvasContext(): CanvasRenderingContext2D | null { + const canvas = this.canvas; + if (!canvas) { + this.canvasContext = null; + return null; + } + if (this.canvasContext?.canvas === canvas) { + return this.canvasContext; + } + this.canvasContext = canvas.getContext("2d", { + alpha: false, + desynchronized: true, + }); + return this.canvasContext; + } + private syncCanvasSize(width: number, height: number) { if (!this.canvas) { return; @@ -2195,8 +2542,9 @@ function streamErrorIsServerUnreachable(message: string): boolean { async function postWebRtcOfferWithAuthRetry( target: StreamConnectTarget, localDescription: RTCSessionDescription, + transport?: "rgba", ): Promise { - const response = await postWebRtcOffer(target, localDescription); + const response = await postWebRtcOffer(target, localDescription, transport); if (response.status !== 401) { if (!response.ok) { throw new Error(await response.text()); @@ -2207,7 +2555,7 @@ async function postWebRtcOfferWithAuthRetry( throw new Error(await response.text()); } await fetchHealth(); - const retry = await postWebRtcOffer(target, localDescription); + const retry = await postWebRtcOffer(target, localDescription, transport); if (!retry.ok) { throw new Error(await retry.text()); } @@ -2273,6 +2621,7 @@ function streamQualityQuery(config: StreamConfig | undefined): string { function postWebRtcOffer( target: StreamConnectTarget, localDescription: RTCSessionDescription, + transport?: "rgba", ): Promise { return fetch( apiUrl(`/api/simulators/${encodeURIComponent(target.udid)}/webrtc/offer`), @@ -2283,6 +2632,7 @@ function postWebRtcOffer( streamConfig: target.streamConfig ? streamQualityPayload(target.streamConfig) : undefined, + transport, type: localDescription.type, }), headers: apiHeaders(), @@ -2334,6 +2684,25 @@ function shouldUseRemoteH264AutoProfile( return Boolean(target?.remote) || !isLoopbackHost(window.location.hostname); } +export function shouldUseLocalAndroidRgbaWebRtc( + target: StreamConnectTarget, +): boolean { + const stream = new URLSearchParams(window.location.search).get("stream"); + if ( + target.transport === "h264" || + stream === "h264" || + stream === "h264-ws" + ) { + return false; + } + return ( + !target.remote && + isLoopbackHost(window.location.hostname) && + (target.platform === "android-emulator" || + target.udid.startsWith("android:")) + ); +} + function configureLowLatencyReceiver( receiver: RTCRtpReceiver, bufferSeconds: number | null, @@ -2687,10 +3056,11 @@ export class StreamWorkerClient { return; } this.backend?.destroy(); - this.backend = - kind === "h264-ws" - ? new H264WebSocketStreamClient(this.handleBackendMessage) - : new WebRtcStreamClient(this.handleBackendMessage); + if (kind === "h264-ws") { + this.backend = new H264WebSocketStreamClient(this.handleBackendMessage); + } else { + this.backend = new WebRtcStreamClient(this.handleBackendMessage); + } this.backendKind = kind; if (this.canvasElement) { this.backend.attachCanvas(this.canvasElement); @@ -2725,7 +3095,7 @@ export class StreamWorkerClient { }; } -function preferredStreamBackend( +export function preferredStreamBackend( target?: StreamConnectTarget | null, ): "auto" | StreamBackend { const value = @@ -2734,10 +3104,14 @@ function preferredStreamBackend( if (value === "h264" || value === "h264-ws") { return "h264-ws"; } - return value === "webrtc" ? "webrtc" : "auto"; + return value === "webrtc" || value === "rgba" || value === "webrtc-rgba" + ? "webrtc" + : "auto"; } -function initialStreamBackend(target: StreamConnectTarget): StreamBackend { +export function initialStreamBackend( + target: StreamConnectTarget, +): StreamBackend { const preferredBackend = preferredStreamBackend(target); if (preferredBackend === "h264-ws") { return canUseH264WebSocket() ? "h264-ws" : "webrtc"; diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index 55f40118..5cf02fbe 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -299,10 +299,21 @@ export function useLiveStream({ return; } + const display = simulator.privateDisplay; + const displayKey = + simulator.platform === "android-emulator" && display + ? [ + Math.round(display.displayWidth), + Math.round(display.displayHeight), + display.rotationQuarterTurns ?? 0, + ].join("x") + : ""; const targetKey = [ simulator.udid, + simulator.platform ?? "", remote ? "remote" : "local", streamTransport, + displayKey, ].join("|"); if (connectedStreamTargetKeyRef.current === targetKey) { return; @@ -316,6 +327,7 @@ export function useLiveStream({ workerClient.connect( buildStreamTarget(simulator.udid, { clientId: clientTelemetryIdRef.current, + platform: simulator.platform, remote, streamConfig, transport: streamTransport, @@ -324,6 +336,10 @@ export function useLiveStream({ }, [ canvasElement, simulator?.isBooted, + simulator?.platform, + simulator?.privateDisplay?.displayHeight, + simulator?.privateDisplay?.displayWidth, + simulator?.privateDisplay?.rotationQuarterTurns, simulator?.udid, paused, remote, diff --git a/client/src/features/viewport/DeviceChrome.tsx b/client/src/features/viewport/DeviceChrome.tsx index dfa0bb8c..41f96325 100644 --- a/client/src/features/viewport/DeviceChrome.tsx +++ b/client/src/features/viewport/DeviceChrome.tsx @@ -43,6 +43,7 @@ interface DeviceChromeProps { onStartPanning: (event: React.PointerEvent) => void; rotationQuarterTurns: number; screenAspect: string; + screenClassName?: string; shellStyle: CSSProperties | null; simulatorName: string; streamBackend: string; @@ -82,6 +83,7 @@ export function DeviceChrome({ onStartPanning, rotationQuarterTurns, screenAspect, + screenClassName, shellStyle, simulatorName, streamBackend, @@ -141,6 +143,7 @@ export function DeviceChrome({ onSimulatorInteraction={onSimulatorInteraction} rotationQuarterTurns={rotationQuarterTurns} simulatorName={simulatorName} + screenClassName={screenClassName} streamBackend={streamBackend} streamCanvasRef={streamCanvasRef} streamCanvasKey={streamCanvasKey} @@ -183,6 +186,7 @@ export function DeviceChrome({ onSimulatorInteraction={onSimulatorInteraction} rotationQuarterTurns={rotationQuarterTurns} simulatorName={simulatorName} + screenClassName={screenClassName} streamBackend={streamBackend} streamCanvasRef={streamCanvasRef} streamCanvasKey={streamCanvasKey} @@ -423,6 +427,7 @@ interface ScreenLayerProps { onPickerSelect: (id: string) => void; onSimulatorInteraction: () => void; rotationQuarterTurns: number; + screenClassName?: string; simulatorName: string; streamBackend: string; streamCanvasRef: Ref; @@ -451,6 +456,7 @@ function ScreenLayer({ onPickerSelect, onSimulatorInteraction, rotationQuarterTurns, + screenClassName, simulatorName, streamBackend, streamCanvasRef, @@ -462,7 +468,13 @@ function ScreenLayer({ }: ScreenLayerProps) { return (
{ onSimulatorInteraction(); diff --git a/client/src/features/viewport/SimulatorViewport.tsx b/client/src/features/viewport/SimulatorViewport.tsx index f4ebe360..000f75ff 100644 --- a/client/src/features/viewport/SimulatorViewport.tsx +++ b/client/src/features/viewport/SimulatorViewport.tsx @@ -57,6 +57,7 @@ interface SimulatorViewportProps { outerCanvasRef: Ref; rotationQuarterTurns: number; screenAspect: string; + screenClassName?: string; selectedSimulator: SimulatorMetadata | null; shellStyle: CSSProperties | null; streamBackend: string; @@ -114,6 +115,7 @@ export function SimulatorViewport({ outerCanvasRef, rotationQuarterTurns, screenAspect, + screenClassName, selectedSimulator, shellStyle, streamBackend, @@ -200,6 +202,7 @@ export function SimulatorViewport({ onStartPanning={onStartPanning} rotationQuarterTurns={rotationQuarterTurns} screenAspect={screenAspect} + screenClassName={screenClassName} shellStyle={shellStyle} simulatorName={selectedSimulator.name} streamBackend={streamBackend} diff --git a/client/src/styles/components.css b/client/src/styles/components.css index df0b0127..8f073478 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -667,6 +667,12 @@ color: color-mix(in srgb, #d7ba7d 82%, var(--text)); } +.hierarchy-source-pill.source-android-uiautomator { + border-color: color-mix(in srgb, #7fd97f 55%, var(--border)); + background: color-mix(in srgb, #7fd97f 13%, transparent); + color: color-mix(in srgb, #7fd97f 82%, var(--text)); +} + .hierarchy-source-pill.active { gap: 5px; padding-inline: 7px 8px; @@ -1601,6 +1607,10 @@ box-shadow: 0 0 0 1px var(--screen-bg); } +.device-screen.android-screen { + background: transparent; +} + .stream-canvas { position: absolute; inset: 0; diff --git a/docs/api/rest.md b/docs/api/rest.md index 2d9a01ec..30dd39b8 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -102,7 +102,8 @@ When ### `GET /api/simulators` -Returns every simulator known to the native bridge, enriched with any session state SimDeck has attached: +Returns every iOS Simulator known to the native bridge plus every Android AVD +found in the Android SDK, enriched with any session state SimDeck has attached: ```json { @@ -113,6 +114,7 @@ Returns every simulator known to the native bridge, enriched with any session st "runtimeName": "iOS 18.0", "deviceTypeIdentifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-15-Pro", "isBooted": true, + "platform": "ios-simulator", "privateDisplay": { "displayReady": true, "displayStatus": "running", @@ -126,13 +128,32 @@ Returns every simulator known to the native bridge, enriched with any session st } ``` -`privateDisplay` is `null` until a stream attaches. +Android emulators use IDs prefixed with `android:` and include Android metadata: + +```json +{ + "udid": "android:SimDeck_Pixel_8_API_36", + "name": "SimDeck_Pixel_8_API_36", + "platform": "android-emulator", + "runtimeName": "Android", + "deviceTypeName": "Android Emulator", + "isBooted": true, + "android": { + "avdName": "SimDeck_Pixel_8_API_36", + "serial": "emulator-5554", + "grpcPort": 8554 + } +} +``` + +For iOS, `privateDisplay` is `null` until a stream attaches. For Android, +SimDeck fills display size from `adb shell wm size` when the emulator is booted. ## Simulator lifecycle ### `POST /api/simulators/{udid}/boot` -Boots the simulator and returns the refreshed simulator metadata: +Boots the simulator or Android emulator and returns the refreshed device metadata: ```json { "simulator": { ... } } @@ -140,11 +161,12 @@ Boots the simulator and returns the refreshed simulator metadata: ### `POST /api/simulators/{udid}/shutdown` -Tears down the live session (if any) and shuts the simulator down. +Tears down the live session (if any) and shuts the simulator or emulator down. ### `POST /api/simulators/{udid}/toggle-appearance` -Toggles between light and dark appearance via `simctl ui appearance`. +Toggles between light and dark appearance via `simctl ui appearance` on iOS or +`cmd uimode night` on Android. ```json { "ok": true } @@ -152,7 +174,10 @@ Toggles between light and dark appearance via `simctl ui appearance`. ### `POST /api/simulators/{udid}/refresh` -Forces the encoder to emit a fresh keyframe. Useful after a discontinuity or when the client decoder drifts. +Forces the iOS encoder to emit a fresh frame. For Android IDs, this route is a +no-op that returns `{ "ok": true, "stream": "screenshot" }`; Android WebRTC +keyframe requests are handled through the WebRTC control channel and RTCP +feedback. ```json { "ok": true } @@ -182,8 +207,10 @@ and the server responds with an SDP answer for a receive-only H.264 video track: } ``` -The endpoint requires the active simulator stream to produce H.264-compatible -samples. The bundled browser client always uses this endpoint. +For iOS, samples come from the native simulator display session and are sent as +an H.264 media track. Android loopback clients use the same endpoint and control +channels, but receive raw RGBA frames over the `simdeck-rgba` data channel. +Non-loopback Android clients receive VideoToolbox-encoded H.264. The browser also opens `simdeck-control` and `simdeck-telemetry` data channels. In addition to input messages, clients can request a keyframe or tune the @@ -274,6 +301,13 @@ Content-Type: application/json { "ok": true } ``` +### `GET /api/simulators/{udid}/screenshot.png` + +Returns a PNG screenshot for the selected device. iOS screenshots come from the +native simulator bridge; Android screenshots come from `adb exec-out screencap +-p`. The browser client uses this endpoint for still-image diagnostics and +fallbacks. + ## Input ### `POST /api/simulators/{udid}/touch` @@ -446,6 +480,7 @@ Returns the current accessibility tree. The server merges framework inspectors, | `swiftui` / `swift-ui` | Force the published SwiftUI logical tree if the Swift agent root publisher is installed in the app. | | `uikit` / `in-app-inspector` | Force the raw UIKit hierarchy from the in-app inspector agent (NativeScript or Swift). | | `native-ax` / `ax` | Always use the native accessibility snapshot. | +| `android-uiautomator` | Force the Android emulator UIAutomator hierarchy. | | Parameter | Default | Description | | --------------- | ------- | ----------------------------------------------------------------------------------------------- | diff --git a/docs/cli/commands.md b/docs/cli/commands.md index c67386e9..101aac04 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -177,12 +177,17 @@ simdeck shutdown simdeck erase ``` -`list` returns the same simulator inventory the browser UI renders. Lifecycle commands return JSON and use the native bridge, preferring private CoreSimulator paths when available and falling back to `xcrun simctl`. +`list` returns the same simulator inventory the browser UI renders, including +Android AVDs as IDs like `android:Pixel_8_API_36`. iOS lifecycle commands use +the native bridge, preferring private CoreSimulator paths when available and +falling back to `xcrun simctl`. Android lifecycle commands use the Android SDK +`emulator` and `adb` tools. ## Apps And URLs ```sh simdeck install /path/to/App.app +simdeck install android: /path/to/app.apk simdeck uninstall com.example.App simdeck launch com.example.App simdeck open-url https://example.com @@ -202,6 +207,7 @@ simdeck describe --source react-native simdeck describe --source flutter simdeck describe --source uikit simdeck describe --source native-ax +simdeck describe --source android-uiautomator simdeck describe --point 120,240 simdeck describe --direct ``` @@ -269,9 +275,13 @@ simdeck chrome-profile `stream` writes Annex B H.264 samples to stdout and runs until interrupted, or until `--frames` samples have been written. It is intended for diagnostics and -external tools. +external tools, and is iOS-only. Android live viewing in the browser uses the +WebRTC H.264 endpoint; raw frames come from emulator gRPC and are encoded +through VideoToolbox. -`logs` fetches recent simulator logs. `chrome-profile` returns the CoreSimulator chrome layout used by the browser viewport. +`logs` fetches recent simulator logs or Android `logcat` output. `chrome-profile` +returns the CoreSimulator chrome layout for iOS and a screen-sized profile for +Android. ## HTTP Fast Path diff --git a/docs/cli/flags.md b/docs/cli/flags.md index 6afeb1e7..784e7562 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -47,14 +47,14 @@ The public commands generate an access token automatically. Use `simdeck daemon ## `describe` -| Flag | Default | Description | -| ------------------ | ------------------------------ | --------------------------------------------------------------------------------------------- | -| `--format` | `json` | Output format: `json`, `compact-json`, or `agent`. | -| `--source` | `auto` | Hierarchy source: `auto`, `nativescript`, `react-native`, `flutter`, `uikit`, or `native-ax`. | -| `--max-depth` | unlimited native / `80` daemon | Trim descendants after the requested depth. | -| `--include-hidden` | `false` | Include hidden in-app inspector views when supported. | -| `--direct` | `false` | Skip the daemon and use the private native accessibility bridge directly. | -| `--point ,` | unset | Return the native element at a screen point. | +| Flag | Default | Description | +| ------------------ | ------------------------------ | -------------------------------------------------------------------------------------------------------------------- | +| `--format` | `json` | Output format: `json`, `compact-json`, or `agent`. | +| `--source` | `auto` | Hierarchy source: `auto`, `nativescript`, `react-native`, `flutter`, `uikit`, `native-ax`, or `android-uiautomator`. | +| `--max-depth` | unlimited native / `80` daemon | Trim descendants after the requested depth. | +| `--include-hidden` | `false` | Include hidden in-app inspector views when supported. | +| `--direct` | `false` | Skip the daemon and use the private native accessibility bridge directly. | +| `--point ,` | unset | Return the native element at a screen point. | ## Input Flags diff --git a/docs/contributing.md b/docs/contributing.md index ad3e7df0..b6ed66b0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -145,9 +145,15 @@ Set `SIMDECK_INTEGRATION_TRACE_HTTP=1` if you also need raw HTTP request logs. Set `SIMDECK_INTEGRATION_KEEP_SIMULATOR=1` with the verbose command if you want the temporary simulator left around for inspection after the suite exits. -GitHub Actions runs this suite on macOS after the normal build/test pipeline. -The integration suite does not require the live video display bridge; REST input -routes use the non-display native input path, and the video stream is covered by +GitHub Actions runs the iOS suite on macOS after the normal build/test pipeline. +It also runs the Android integration suite on an Ubuntu runner with a real +Android emulator. Linux builds use a native iOS stub so the Android bridge, +daemon, CLI, and `simdeck/test` API can be exercised without macOS frameworks. +The Android CI job preboots the AVD with `android-emulator-runner` before +starting SimDeck; local Android integration runs skip cleanly unless an emulator +is already booted or `SIMDECK_INTEGRATION_BOOT_ANDROID=1` is set. +The integration suites do not require the live video display bridge; REST input +routes use the non-display input path, and the video stream is covered by lower-level protocol tests. ## Full CI pipeline @@ -164,8 +170,10 @@ This is the normal local CI script: 4. `npm run package:vscode-extension` — VS Code `.vsix`. GitHub Actions runs `npm run ci`, then `npm run test:integration:cli` for the -temp-simulator CLI and REST control sweep. A clean `npm run ci` and integration -run are required for any PR that changes simulator control behavior. +temp-simulator CLI and REST control sweep, plus +`npm run test:integration:android` on Ubuntu for Android emulator coverage. A +clean `npm run ci` and integration run are required for any PR that changes +simulator control behavior. ## Documentation diff --git a/docs/extensions/browser-client.md b/docs/extensions/browser-client.md index b9124c30..04229894 100644 --- a/docs/extensions/browser-client.md +++ b/docs/extensions/browser-client.md @@ -39,15 +39,15 @@ client/ └── styles/ ``` -| Folder | Responsibility | -| ------------------------- | ----------------------------------------------------------------------- | -| `api/` | Typed wrappers around the SimDeck REST API and shared TypeScript types. | -| `features/simulators/` | Sidebar list of simulators plus boot/shutdown affordances. | -| `features/viewport/` | Frame canvas, chrome compositing, hit testing. | -| `features/stream/` | WebRTC client, receiver stats, and video frame plumbing. | -| `features/input/` | Touch / keyboard / hardware-button affordances. | -| `features/accessibility/` | Accessibility tree pane and source switcher. | -| `features/toolbar/` | Top toolbar (rotate, home, app switcher, dark mode toggle, refresh). | +| Folder | Responsibility | +| ------------------------- | ------------------------------------------------------------------------------------ | +| `api/` | Typed wrappers around the SimDeck REST API and shared TypeScript types. | +| `features/simulators/` | Sidebar list of simulators plus boot/shutdown affordances. | +| `features/viewport/` | Frame canvas, chrome compositing, hit testing. | +| `features/stream/` | WebRTC media/RGBA stream client, H.264 fallback, receiver stats, and frame plumbing. | +| `features/input/` | Touch / keyboard / hardware-button affordances. | +| `features/accessibility/` | Accessibility tree pane and source switcher. | +| `features/toolbar/` | Top toolbar (rotate, home, app switcher, dark mode toggle, refresh). | ## Bootstrap flow @@ -55,8 +55,8 @@ client/ 2. `main.tsx` mounts the React tree at `#root`. 3. `AppShell` calls `GET /api/health` to learn the active encoder mode. 4. The simulator sidebar fetches `GET /api/simulators` and renders the list. -5. Selecting a simulator posts an SDP offer to `/api/simulators//webrtc/offer`. -6. The browser renders the H.264 video track through native WebRTC playback. +5. Selecting a device posts an SDP offer to `/api/simulators//webrtc/offer`. +6. The browser renders the WebRTC stream. iOS uses an H.264 media track; Android loopback clients use a raw RGBA data channel, while non-loopback Android clients use VideoToolbox-encoded H.264. 7. Touch and key events round-trip through `POST /api/simulators//touch` and `/key`. ## Dev workflow diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 4468b188..6ba4a190 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -4,13 +4,13 @@ SimDeck is intentionally split into a small number of clearly-scoped layers. Eve ## High-level layout -SimDeck has three layers stacked between the browser and the iOS Simulator: +SimDeck has three layers stacked between the browser and the target device: -1. **Browser / VS Code** runs the React client from `client/`. It speaks HTTP for control and WebRTC for live video, served by the Rust server. +1. **Browser / VS Code** runs the React client from `client/`. It speaks HTTP for control and WebRTC H.264 for live video, served by the Rust server. 2. **The Rust server** (`server/`, built on `axum` + `tokio`) owns the CLI entrypoint, project daemon lifecycle, REST routes (`api/`), the stream transports (`transport/`), the inspector WebSocket hub (`inspector.rs`), the per-UDID session registry (`simulators/`), metrics, and log streaming. -3. **The Objective-C bridge** (`cli/`) is reached through a narrow C ABI in `cli/native/XCWNativeBridge.*`. It wraps `xcrun simctl`, the private `CoreSimulator` direct-boot path, the per-session hardware/software H.264 encoder, the headless display bridge that produces frames and accepts HID input, and the device-chrome renderer. +3. **Native device bridges** own platform-specific work. The Objective-C bridge (`cli/`) is reached through a narrow C ABI in `cli/native/XCWNativeBridge.*` for iOS. The Rust Android bridge (`server/src/android.rs`) shells out to the Android SDK for AVD discovery, emulator lifecycle, ADB input, screenshots, UIAutomator, and logcat. -Underneath all of that is the iOS Simulator itself — `CoreSimulator` for lifecycle, `SimulatorKit` for chrome assets. +Underneath all of that are the iOS Simulator (`CoreSimulator` and `SimulatorKit`) and the Android emulator (`emulator` and `adb`). ## Layer responsibilities @@ -20,17 +20,18 @@ Owns the public CLI shape (`simdeck`, `simdeck ui`, `daemon`, `boot`, `shutdown` Key modules: -| Module | Responsibility | -| ----------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| `server/src/main.rs` | CLI entrypoint, project daemon management, AppKit main-thread shim, tokio runtime bootstrap. | -| `server/src/api/routes.rs` | Every `/api/*` route, including simulator control, accessibility, and inspector proxy. | -| `server/src/transport/webrtc.rs` | WebRTC offer/answer endpoint for H.264 browser video. | -| `server/src/transport/packet.rs` | Shared encoded frame type used between simulator sessions and transports. | -| `server/src/inspector.rs` | WebSocket hub for the NativeScript runtime inspector. | -| `server/src/simulators/registry.rs` | Per-UDID session registry with lazy attachment to the native bridge. | -| `server/src/simulators/session.rs` | Frame broadcast channel, keyframe gating, refresh requests. | -| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted from stream transports or `/api/client-stream-stats`. | -| `server/src/logs.rs` | `os_log` log streaming and filtering. | +| Module | Responsibility | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `server/src/main.rs` | CLI entrypoint, project daemon management, AppKit main-thread shim, tokio runtime bootstrap. | +| `server/src/api/routes.rs` | Every `/api/*` route, including simulator control, accessibility, and inspector proxy. | +| `server/src/android.rs` | Android AVD discovery, emulator lifecycle, ADB input, emulator gRPC video, screenshots, UIAutomator, and logcat. | +| `server/src/transport/webrtc.rs` | WebRTC offer/answer endpoint for H.264 browser video. | +| `server/src/transport/packet.rs` | Shared encoded frame type used between simulator sessions and transports. | +| `server/src/inspector.rs` | WebSocket hub for the NativeScript runtime inspector. | +| `server/src/simulators/registry.rs` | Per-UDID session registry with lazy attachment to the native bridge. | +| `server/src/simulators/session.rs` | Frame broadcast channel, keyframe gating, refresh requests. | +| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted from stream transports or `/api/client-stream-stats`. | +| `server/src/logs.rs` | `os_log` log streaming and filtering. | The Rust server runs the tokio runtime on a worker thread while the AppKit main loop spins on the main thread. The native bridge needs the main loop to deliver display callbacks and HID events. @@ -53,13 +54,16 @@ Inside the bridge: ### `client/` — React browser UI -The React app served at `/` is a thin shell that calls the REST API and consumes live video over WebRTC H.264. +The React app served at `/` is a thin shell that calls the REST API. It consumes +live device video over WebRTC H.264. iOS frames come from the native simulator +display bridge; Android frames come from emulator gRPC `streamScreenshot` and +are encoded through VideoToolbox on the server. Layout under `client/src/`: - `app/AppShell.tsx` — top-level shell. - `api/` — typed wrappers around `/api/*` (`client.ts`, `controls.ts`, `simulators.ts`, `types.ts`). -- `features/stream/` — WebRTC client, receiver stats, and video frame plumbing. +- `features/stream/` — WebRTC client, receiver stats, and frame plumbing. - `features/viewport/` — frame canvas, hit testing, chrome compositing. - `features/input/` — touch/keyboard/hardware button affordances. - `features/accessibility/` — accessibility tree pane and source switcher. @@ -85,7 +89,7 @@ Most control endpoints follow the same path: a typed Rust handler in `server/src ### Live video -The browser posts an SDP offer to `/api/simulators/{udid}/webrtc/offer`. The handler in `transport::webrtc` ensures the per-UDID `SimulatorSession` is started, waits up to ~3 s for the first H.264 keyframe, returns an SDP answer, and writes the simulator frame source to a WebRTC video track. +The browser posts an SDP offer to `/api/simulators/{udid}/webrtc/offer`. The handler in `transport::webrtc` starts the selected frame source, waits for the first H.264 keyframe, returns an SDP answer, and writes H.264 samples to a WebRTC video track. For Android, that source is emulator gRPC raw pixels passed through the shared VideoToolbox encoder path. ### Input diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 43e33926..0fabaa30 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -10,6 +10,7 @@ SimDeck only runs on macOS. The native bridge links private `CoreSimulator` and | ---------------------------------- | ------------------------------------------------------------------------------------ | | **macOS 13+** | Required for current `CoreSimulator` and Apple's VideoToolbox H.264 encoder. | | **Xcode + iOS Simulator runtimes** | The native bridge invokes `xcrun simctl` and the Simulator app. | +| **Android SDK tools** | Optional. Required for Android emulator support (`emulator`, `adb`, and AVD images). | | **Node.js ≥ 18** | The launcher (`bin/simdeck.mjs`) and the bundled client tooling. | | **Rust (stable)** | Required only when building from source. Installed via [rustup](https://rustup.rs/). | diff --git a/docs/guide/testing.md b/docs/guide/testing.md index acbe6d56..0fe8c2a2 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -28,27 +28,39 @@ try { The current session object exposes: -| Method | Purpose | -| -------------- | ----------------------------------------------------------------- | -| `list()` | Fetch simulator inventory from `GET /api/simulators`. | -| `launch()` | Launch an installed bundle ID. | -| `openUrl()` | Open a URL or deep link. | -| `tap()` | Tap normalized screen coordinates. | -| `key()` | Send one HID key code. | -| `button()` | Press a hardware button. | -| `tree()` | Fetch an accessibility hierarchy. | -| `query()` | Return compact matches for a selector. | -| `waitFor()` | Poll until a selector appears. | -| `assert()` | Assert a selector is present. | -| `batch()` | Run multiple REST actions through `/api/simulators/{udid}/batch`. | -| `screenshot()` | Return a PNG buffer. | -| `close()` | Stop the daemon if this session started it. | +| Method | Purpose | +| -------------------------------------- | ----------------------------------------------------------------- | +| `list()` | Fetch simulator inventory from `GET /api/simulators`. | +| `boot()`, `shutdown()`, `erase()` | Manage simulator or Android emulator lifecycle. | +| `install()`, `uninstall()` | Install or remove an app. | +| `launch()` | Launch an installed bundle ID or Android package. | +| `openUrl()` | Open a URL or deep link. | +| `tap()`, `tapElement()` | Tap normalized coordinates or a matching accessibility element. | +| `touch()`, `swipe()`, `gesture()` | Send normalized pointer gestures. | +| `typeText()`, `key()`, `keySequence()` | Send text or HID keyboard input. | +| `button()` | Press a hardware button. | +| `home()`, `dismissKeyboard()` | Trigger common system controls. | +| `appSwitcher()` | Open the app switcher. | +| `rotateLeft()`, `rotateRight()` | Rotate the simulator display. | +| `toggleAppearance()` | Toggle light/dark appearance. | +| `pasteboardSet()`, `pasteboardGet()` | Read or write pasteboard text. | +| `chromeProfile()` | Fetch screen/chrome geometry. | +| `logs()` | Fetch recent simulator or Android log entries. | +| `tree()` | Fetch an accessibility hierarchy. | +| `query()` | Return compact matches for a selector. | +| `waitFor()` | Poll until a selector appears. | +| `assert()` | Assert a selector is present. | +| `batch()` | Run multiple REST actions through `/api/simulators/{udid}/batch`. | +| `screenshot()` | Return a PNG buffer. | +| `close()` | Stop the daemon if this session started it. | Selectors can match `id`, `label`, `value`, or `type`. Query options accept `source`, `maxDepth`, and `includeHidden`. ## Repository Integration Suite -The repo includes a macOS-only integration runner that creates a temporary simulator, builds and installs a small UIKit fixture app, then sweeps the CLI and REST control surface. +The repo includes simulator-backed integration runners. The iOS runner is +macOS-only; it creates a temporary simulator, builds and installs a small UIKit +fixture app, then sweeps the CLI and REST control surface. ```sh npm run build:cli @@ -85,6 +97,31 @@ does not exceed the active `iphonesimulator` SDK version, falling back to the same major version when needed. This keeps CI off newer installed runtimes that do not match the selected Xcode toolchain. +Android coverage is opt-in because it requires a locally installed Android SDK +and at least one existing AVD. It runs on macOS or Linux. On Linux, SimDeck +builds the daemon with a native iOS stub and leaves the Android bridge active. +The runner starts an isolated SimDeck daemon and sweeps the Android CLI and +`simdeck/test` surface for lifecycle, tree, screenshot, pasteboard behavior, +app launch, URL opening, touch/swipe/gesture, keyboard, system buttons, +rotation, appearance, logs, and batch controls: + +```sh +npm run build:cli +npm run build:simdeck-test +npm run test:integration:android +``` + +Set `SIMDECK_INTEGRATION_ANDROID_AVD=` to pick a specific AVD. The +runner expects that emulator to already be booted, which is how the Linux CI job +uses `reactivecircus/android-emulator-runner`. If no AVD is configured, or a +local AVD exists but is not running, the Android runner prints a skip message +and exits successfully. Set `SIMDECK_INTEGRATION_REQUIRE_RUNNING_ANDROID=1` to +turn that skip into a failure. Set `SIMDECK_INTEGRATION_BOOT_ANDROID=1` to let +SimDeck cold-boot a local AVD before the suite. Set +`SIMDECK_INTEGRATION_KEEP_ANDROID=1` to leave an emulator booted when the runner +started it. Set `SIMDECK_INTEGRATION_STEP_TIMEOUT_MS` to override the per-step +timeout. + ## Stress and Leak Checks Use the stress runner against an already-running daemon when you want to shake out diff --git a/package.json b/package.json index d147d6ef..b8de4fc2 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,8 @@ "test:integration:fixture": "node scripts/integration/prebuild-fixture.mjs", "test:integration:js-api": "node scripts/integration/js-api.mjs", "test:integration:js-api:verbose": "SIMDECK_INTEGRATION_SHOW_SIMULATOR=1 node scripts/integration/js-api.mjs", + "test:integration:android": "node scripts/integration/android.mjs", + "test:integration:android:verbose": "SIMDECK_INTEGRATION_VERBOSE=1 node scripts/integration/android.mjs", "test:stream:reliability": "node scripts/check-stream-reliability.mjs", "test:e2e:webrtc": "node scripts/e2e-webrtc-reliability.mjs", "test:e2e:webrtc:headed": "SIMDECK_E2E_HEADFUL=1 node scripts/e2e-webrtc-reliability.mjs", diff --git a/packages/simdeck-test/dist/index.d.ts b/packages/simdeck-test/dist/index.d.ts index 78875ccf..596e4f82 100644 --- a/packages/simdeck-test/dist/index.d.ts +++ b/packages/simdeck-test/dist/index.d.ts @@ -13,7 +13,8 @@ export type QueryOptions = { | "react-native" | "flutter" | "uikit" - | "native-ax"; + | "native-ax" + | "android-uiautomator"; maxDepth?: number; includeHidden?: boolean; }; @@ -28,11 +29,35 @@ export type TapOptions = QueryOptions & { waitTimeoutMs?: number; pollMs?: number; }; +export type SwipeOptions = { + durationMs?: number; + steps?: number; +}; +export type GestureOptions = SwipeOptions & { + delta?: number; +}; +export type TypeTextOptions = { + delayMs?: number; +}; +export type KeySequenceOptions = { + delayMs?: number; +}; +export type LogsOptions = { + backfill?: boolean; + seconds?: number; + limit?: number; + levels?: string[]; + processes?: string[]; + q?: string; +}; export type SimDeckSession = { endpoint: string; pid: number; projectRoot: string; list(): Promise; + boot(udid: string): Promise; + shutdown(udid: string): Promise; + erase(udid: string): Promise; install(udid: string, appPath: string): Promise; uninstall(udid: string, bundleId: string): Promise; launch(udid: string, bundleId: string): Promise; @@ -44,11 +69,41 @@ export type SimDeckSession = { options?: TapOptions, ): Promise; touch(udid: string, x: number, y: number, phase: string): Promise; + swipe( + udid: string, + startX: number, + startY: number, + endX: number, + endY: number, + options?: SwipeOptions, + ): Promise; + gesture( + udid: string, + preset: string, + options?: GestureOptions, + ): Promise; + typeText( + udid: string, + text: string, + options?: TypeTextOptions, + ): Promise; key(udid: string, keyCode: number, modifiers?: number): Promise; + keySequence( + udid: string, + keyCodes: number[], + options?: KeySequenceOptions, + ): Promise; button(udid: string, button: string, durationMs?: number): Promise; + home(udid: string): Promise; + dismissKeyboard(udid: string): Promise; + appSwitcher(udid: string): Promise; + rotateLeft(udid: string): Promise; + rotateRight(udid: string): Promise; + toggleAppearance(udid: string): Promise; pasteboardSet(udid: string, text: string): Promise; pasteboardGet(udid: string): Promise; chromeProfile(udid: string): Promise; + logs(udid: string, options?: LogsOptions): Promise; tree(udid: string, options?: QueryOptions): Promise; query( udid: string, diff --git a/packages/simdeck-test/dist/index.js b/packages/simdeck-test/dist/index.js index 0259c11d..a067eac5 100644 --- a/packages/simdeck-test/dist/index.js +++ b/packages/simdeck-test/dist/index.js @@ -18,6 +18,27 @@ export async function connect(options = {}) { pid: result.pid, projectRoot: result.projectRoot, list: () => requestJson(endpoint, "GET", "/api/simulators"), + boot: (udid) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/boot`, + null, + ), + shutdown: (udid) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/shutdown`, + null, + ), + erase: (udid) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/erase`, + null, + ), install: (udid, appPath) => requestOk( endpoint, @@ -67,11 +88,68 @@ export async function connect(options = {}) { y, phase, }), + swipe: (udid, startX, startY, endX, endY, swipeOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "swipe", + startX, + startY, + endX, + endY, + ...swipeOptions, + }, + ], + }, + ), + gesture: (udid, preset, gestureOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "gesture", + preset, + ...gestureOptions, + }, + ], + }, + ), + typeText: (udid, text, typeOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "type", + text, + ...typeOptions, + }, + ], + }, + ), key: (udid, keyCode, modifiers = 0) => requestOk(endpoint, `/api/simulators/${encodeURIComponent(udid)}/key`, { keyCode, modifiers, }), + keySequence: (udid, keyCodes, keySequenceOptions = {}) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/key-sequence`, + { + keyCodes, + ...keySequenceOptions, + }, + ), button: (udid, button, durationMs = 0) => requestOk( endpoint, @@ -81,6 +159,42 @@ export async function connect(options = {}) { durationMs, }, ), + home: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/home`, + null, + ), + dismissKeyboard: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/dismiss-keyboard`, + null, + ), + appSwitcher: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/app-switcher`, + null, + ), + rotateLeft: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/rotate-left`, + null, + ), + rotateRight: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/rotate-right`, + null, + ), + toggleAppearance: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/toggle-appearance`, + null, + ), pasteboardSet: (udid, text) => requestOk( endpoint, @@ -103,6 +217,14 @@ export async function connect(options = {}) { "GET", `/api/simulators/${encodeURIComponent(udid)}/chrome-profile`, ), + logs: async (udid, logsOptions) => { + const result = await requestJson( + endpoint, + "GET", + `/api/simulators/${encodeURIComponent(udid)}/logs?${logsQuery(logsOptions)}`, + ); + return result.entries ?? []; + }, tree: (udid, treeOptions) => requestJson( endpoint, @@ -366,6 +488,19 @@ function treeQuery(options = {}) { if (options.includeHidden) params.set("includeHidden", "true"); return params.toString(); } +function logsQuery(options = {}) { + const params = new URLSearchParams(); + if (options.backfill !== undefined) + params.set("backfill", String(options.backfill)); + if (options.seconds !== undefined) + params.set("seconds", String(options.seconds)); + if (options.limit !== undefined) params.set("limit", String(options.limit)); + if (options.levels?.length) params.set("levels", options.levels.join(",")); + if (options.processes?.length) + params.set("processes", options.processes.join(",")); + if (options.q) params.set("q", options.q); + return params.toString(); +} function selectorPayload(selector) { return { id: selector.id, diff --git a/packages/simdeck-test/src/index.ts b/packages/simdeck-test/src/index.ts index db2ffd9f..306b4cbf 100644 --- a/packages/simdeck-test/src/index.ts +++ b/packages/simdeck-test/src/index.ts @@ -22,7 +22,8 @@ export type QueryOptions = { | "react-native" | "flutter" | "uikit" - | "native-ax"; + | "native-ax" + | "android-uiautomator"; maxDepth?: number; includeHidden?: boolean; }; @@ -40,11 +41,40 @@ export type TapOptions = QueryOptions & { pollMs?: number; }; +export type SwipeOptions = { + durationMs?: number; + steps?: number; +}; + +export type GestureOptions = SwipeOptions & { + delta?: number; +}; + +export type TypeTextOptions = { + delayMs?: number; +}; + +export type KeySequenceOptions = { + delayMs?: number; +}; + +export type LogsOptions = { + backfill?: boolean; + seconds?: number; + limit?: number; + levels?: string[]; + processes?: string[]; + q?: string; +}; + export type SimDeckSession = { endpoint: string; pid: number; projectRoot: string; list(): Promise; + boot(udid: string): Promise; + shutdown(udid: string): Promise; + erase(udid: string): Promise; install(udid: string, appPath: string): Promise; uninstall(udid: string, bundleId: string): Promise; launch(udid: string, bundleId: string): Promise; @@ -56,11 +86,41 @@ export type SimDeckSession = { options?: TapOptions, ): Promise; touch(udid: string, x: number, y: number, phase: string): Promise; + swipe( + udid: string, + startX: number, + startY: number, + endX: number, + endY: number, + options?: SwipeOptions, + ): Promise; + gesture( + udid: string, + preset: string, + options?: GestureOptions, + ): Promise; + typeText( + udid: string, + text: string, + options?: TypeTextOptions, + ): Promise; key(udid: string, keyCode: number, modifiers?: number): Promise; + keySequence( + udid: string, + keyCodes: number[], + options?: KeySequenceOptions, + ): Promise; button(udid: string, button: string, durationMs?: number): Promise; + home(udid: string): Promise; + dismissKeyboard(udid: string): Promise; + appSwitcher(udid: string): Promise; + rotateLeft(udid: string): Promise; + rotateRight(udid: string): Promise; + toggleAppearance(udid: string): Promise; pasteboardSet(udid: string, text: string): Promise; pasteboardGet(udid: string): Promise; chromeProfile(udid: string): Promise; + logs(udid: string, options?: LogsOptions): Promise; tree(udid: string, options?: QueryOptions): Promise; query( udid: string, @@ -114,6 +174,27 @@ export async function connect( pid: result.pid, projectRoot: result.projectRoot, list: () => requestJson(endpoint, "GET", "/api/simulators"), + boot: (udid) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/boot`, + null, + ), + shutdown: (udid) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/shutdown`, + null, + ), + erase: (udid) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/erase`, + null, + ), install: (udid, appPath) => requestOk( endpoint, @@ -163,11 +244,68 @@ export async function connect( y, phase, }), + swipe: (udid, startX, startY, endX, endY, swipeOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "swipe", + startX, + startY, + endX, + endY, + ...swipeOptions, + }, + ], + }, + ), + gesture: (udid, preset, gestureOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "gesture", + preset, + ...gestureOptions, + }, + ], + }, + ), + typeText: (udid, text, typeOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "type", + text, + ...typeOptions, + }, + ], + }, + ), key: (udid, keyCode, modifiers = 0) => requestOk(endpoint, `/api/simulators/${encodeURIComponent(udid)}/key`, { keyCode, modifiers, }), + keySequence: (udid, keyCodes, keySequenceOptions = {}) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/key-sequence`, + { + keyCodes, + ...keySequenceOptions, + }, + ), button: (udid, button, durationMs = 0) => requestOk( endpoint, @@ -177,6 +315,42 @@ export async function connect( durationMs, }, ), + home: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/home`, + null, + ), + dismissKeyboard: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/dismiss-keyboard`, + null, + ), + appSwitcher: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/app-switcher`, + null, + ), + rotateLeft: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/rotate-left`, + null, + ), + rotateRight: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/rotate-right`, + null, + ), + toggleAppearance: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/toggle-appearance`, + null, + ), pasteboardSet: (udid, text) => requestOk( endpoint, @@ -199,6 +373,14 @@ export async function connect( "GET", `/api/simulators/${encodeURIComponent(udid)}/chrome-profile`, ), + logs: async (udid, logsOptions) => { + const result = await requestJson<{ entries?: unknown[] }>( + endpoint, + "GET", + `/api/simulators/${encodeURIComponent(udid)}/logs?${logsQuery(logsOptions)}`, + ); + return result.entries ?? []; + }, tree: (udid, treeOptions) => requestJson( endpoint, @@ -503,6 +685,20 @@ function treeQuery(options: QueryOptions = {}): string { return params.toString(); } +function logsQuery(options: LogsOptions = {}): string { + const params = new URLSearchParams(); + if (options.backfill !== undefined) + params.set("backfill", String(options.backfill)); + if (options.seconds !== undefined) + params.set("seconds", String(options.seconds)); + if (options.limit !== undefined) params.set("limit", String(options.limit)); + if (options.levels?.length) params.set("levels", options.levels.join(",")); + if (options.processes?.length) + params.set("processes", options.processes.join(",")); + if (options.q) params.set("q", options.q); + return params.toString(); +} + function selectorPayload( selector: ElementSelector, ): Record { diff --git a/scripts/build-cli.sh b/scripts/build-cli.sh index 56526237..b1c51baa 100755 --- a/scripts/build-cli.sh +++ b/scripts/build-cli.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env zsh +#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" @@ -8,11 +8,11 @@ OUTPUT_BIN="$BUILD_DIR/simdeck-bin" MANIFEST_PATH="$ROOT_DIR/server/Cargo.toml" SERVER_TARGET_DIR="$ROOT_DIR/server/target" -# SimDeck depends on AArch64 inline asm in cli/*.m and Apple Silicon-only -# private CoreSimulator behavior, so the binary is arm64-only by design. +# SimDeck's full iOS bridge is macOS-only. Non-macOS builds compile a native +# stub so Android-only integration tests can run on Linux CI. # Optionally pin the build to an explicit Rust target triple via # SIMDECK_BUILD_TARGET (the release workflow uses aarch64-apple-darwin); when -# unset we use the host triple so local dev on Apple Silicon stays fast. +# unset we use the host triple so local dev stays fast. mkdir -p "$BUILD_DIR" @@ -43,7 +43,7 @@ echo "Built $OUTPUT_BIN" file "$OUTPUT_BIN" cat > "$OUTPUT" < { + cleanupSync(); + process.exit(130); +}); +process.on("SIGTERM", () => { + cleanupSync(); + process.exit(143); +}); + +main() + .then(() => process.exit(0)) + .catch((error) => { + console.error(error?.stack ?? error); + process.exit(1); + }); + +async function main() { + try { + if (!fs.existsSync(simdeck)) { + throw new Error(`Missing ${simdeck}. Run npm run build:cli first.`); + } + if ( + !fs.existsSync( + path.join(root, "packages", "simdeck-test", "dist", "index.js"), + ) + ) { + throw new Error( + "Missing simdeck/test dist. Run npm run build:simdeck-test first.", + ); + } + + session = await measuredStep("simdeck/test isolated connect", () => + connect({ + cliPath: simdeck, + projectRoot: root, + isolated: true, + videoCodec: "software", + }), + ); + console.log(`daemon ${session.endpoint}`); + + const target = await measuredStep("resolve Android AVD", () => + resolveAndroidDevice(), + ); + if (!target) { + console.log( + "No Android AVDs discovered; skipping Android integration suite.", + ); + return; + } + + androidUDID = target.udid; + console.log( + `android target ${androidUDID} (${target.name})${ + target.isBooted ? " already booted" : "" + }`, + ); + if (!target.isBooted && !bootAndroid) { + const message = `${androidUDID} is not running. Start the emulator first or set SIMDECK_INTEGRATION_BOOT_ANDROID=1 to let SimDeck boot it.`; + if (requireRunningAndroid) { + throw new Error(message); + } + console.log(`${message} Skipping Android integration suite.`); + return; + } + shutdownAndroidAfterRun = !target.isBooted && bootAndroid && !keepAndroid; + + await measuredStep( + target.isBooted + ? "JS boot Android idempotent" + : "JS boot Android emulator", + async () => { + await session.boot(androidUDID); + await waitForBooted(androidUDID); + }, + { timeoutMs: target.isBooted ? 120_000 : 300_000 }, + ); + + await runCliSurface(); + await runJsSurface(); + console.log("SimDeck Android integration suite passed"); + } finally { + await cleanupAndroid(); + cleanupSync(); + printTimingSummary(); + } +} + +async function runCliSurface() { + await measuredStep("CLI list includes Android", () => { + assertAndroidListed(simdeckJson(["list"]), androidUDID); + }); + await measuredStep( + "CLI boot Android idempotent", + () => assertJson(simdeckJson(["boot", androidUDID]), "boot"), + { timeoutMs: 180_000 }, + ); + await measuredStep("CLI chrome-profile", () => { + const profile = simdeckJson(["chrome-profile", androidUDID]); + assertJson(profile, "chrome-profile"); + assert.ok(Number(profile.screenWidth) > 0, "missing Android screenWidth"); + assert.ok(Number(profile.screenHeight) > 0, "missing Android screenHeight"); + }); + const tree = await measuredStep("CLI describe Android tree", () => { + const payload = simdeckJson([ + "describe", + androidUDID, + "--source", + "android-uiautomator", + "--format", + "compact-json", + "--max-depth", + "2", + ]); + assertRoots(payload, "CLI describe"); + return payload; + }); + await measuredStep("CLI describe Android point", () => { + const point = centerOfFirstRoot(tree); + if (!point) { + throw new Error("Unable to derive point from Android tree root."); + } + assertRoots( + simdeckJson([ + "describe", + androidUDID, + "--source", + "android-uiautomator", + "--format", + "compact-json", + "--point", + `${point.x},${point.y}`, + "--max-depth", + "1", + ]), + "CLI point describe", + ); + }); + await measuredStep("CLI screenshot stdout", () => { + assertPngBuffer( + runBuffer( + simdeck, + [ + "--server-url", + session.endpoint, + "screenshot", + androidUDID, + "--stdout", + ], + { timeoutMs: 120_000, maxBuffer: 64 * 1024 * 1024 }, + ), + ); + }); + await measuredStep("CLI pasteboard behavior", () => { + const setResult = runCommand( + simdeck, + [ + "--server-url", + session.endpoint, + "pasteboard", + "set", + androidUDID, + "simdeck android cli", + ], + { timeoutMs: 60_000 }, + ); + if (setResult.status !== 0) { + assertClipboardUnsupported(setResult); + return; + } + try { + const payload = simdeckJson(["pasteboard", "get", androidUDID]); + assert.equal(payload.text, "simdeck android cli"); + } catch (error) { + assertClipboardUnsupported(error); + } + }); + await measuredStep("CLI app launch and URL", () => { + simdeckJson(["launch", androidUDID, "com.android.settings"], { + timeoutMs: 60_000, + }); + simdeckJson(["open-url", androidUDID, "https://example.com"], { + timeoutMs: 60_000, + }); + simdeckJson(["home", androidUDID]); + }); + await measuredStep("CLI pointer gestures", () => { + simdeckJson([ + "tap", + androidUDID, + "0.5", + "0.5", + "--normalized", + "--duration-ms", + "20", + ]); + simdeckJson([ + "touch", + androidUDID, + "0.5", + "0.5", + "--phase", + "began", + "--normalized", + ]); + simdeckJson([ + "touch", + androidUDID, + "0.5", + "0.5", + "--phase", + "ended", + "--normalized", + ]); + simdeckJson([ + "swipe", + androidUDID, + "0.5", + "0.75", + "0.5", + "0.25", + "--normalized", + "--duration-ms", + "120", + "--steps", + "4", + ]); + simdeckJson([ + "gesture", + androidUDID, + "scroll-down", + "--duration-ms", + "120", + "--delta", + "0.2", + ]); + }); + await measuredStep("CLI keyboard and system controls", () => { + simdeckJson(["key", androidUDID, "enter"]); + simdeckJson([ + "key-sequence", + androidUDID, + "--keycodes", + "h,e,l,l,o", + "--delay-ms", + "5", + ]); + simdeckJson([ + "key-combo", + androidUDID, + "--modifiers", + "shift", + "--key", + "h", + ]); + simdeckJson(["type", androidUDID, "simdeck"]); + simdeckJson(["dismiss-keyboard", androidUDID]); + simdeckJson(["button", androidUDID, "back"]); + simdeckJson(["app-switcher", androidUDID]); + simdeckJson(["home", androidUDID]); + }); + await measuredStep("CLI rotation and appearance restore", () => { + simdeckJson(["rotate-left", androidUDID], { timeoutMs: 60_000 }); + simdeckJson(["rotate-right", androidUDID], { timeoutMs: 60_000 }); + simdeckJson(["toggle-appearance", androidUDID], { timeoutMs: 60_000 }); + simdeckJson(["toggle-appearance", androidUDID], { timeoutMs: 60_000 }); + }); + await measuredStep("CLI logs", () => { + const payload = simdeckJson([ + "logs", + androidUDID, + "--seconds", + "1", + "--limit", + "5", + ]); + assert.ok( + Array.isArray(payload.entries), + "CLI logs did not return entries", + ); + }); +} + +async function runJsSurface() { + await measuredStep("JS list includes Android", async () => { + assertAndroidListed(await session.list(), androidUDID); + }); + await measuredStep("JS chromeProfile", async () => { + const profile = await session.chromeProfile(androidUDID); + assertJson(profile, "chromeProfile"); + }); + await measuredStep("JS Android tree", async () => { + assertRoots( + await session.tree(androidUDID, { + source: "android-uiautomator", + maxDepth: 2, + }), + "JS tree", + ); + }); + await measuredStep("JS screenshot", async () => { + assertPngBuffer(await session.screenshot(androidUDID)); + }); + await measuredStep("JS pasteboard behavior", async () => { + try { + await session.pasteboardSet(androidUDID, "simdeck android js"); + assert.equal( + await session.pasteboardGet(androidUDID), + "simdeck android js", + ); + } catch (error) { + assertClipboardUnsupported(error); + } + }); + await measuredStep("JS app launch and URL", async () => { + await session.launch(androidUDID, "com.android.settings"); + await session.openUrl(androidUDID, "https://example.com"); + await session.home(androidUDID); + }); + await measuredStep("JS pointer gestures", async () => { + await session.tap(androidUDID, 0.5, 0.5); + await session.touch(androidUDID, 0.5, 0.5, "began"); + await session.touch(androidUDID, 0.5, 0.5, "ended"); + await session.swipe(androidUDID, 0.5, 0.75, 0.5, 0.25, { + durationMs: 120, + steps: 4, + }); + await session.gesture(androidUDID, "scroll-down", { + durationMs: 120, + delta: 0.2, + }); + }); + await measuredStep("JS keyboard and system controls", async () => { + await session.key(androidUDID, 40); + await session.keySequence(androidUDID, [11, 8, 15, 15, 18], { + delayMs: 5, + }); + await session.typeText(androidUDID, "simdeck"); + await session.dismissKeyboard(androidUDID); + await session.button(androidUDID, "back"); + await session.appSwitcher(androidUDID); + await session.home(androidUDID); + }); + await measuredStep("JS rotation and appearance restore", async () => { + await session.rotateLeft(androidUDID); + await session.rotateRight(androidUDID); + await session.toggleAppearance(androidUDID); + await session.toggleAppearance(androidUDID); + }); + await measuredStep("JS logs", async () => { + assert.ok( + Array.isArray(await session.logs(androidUDID, { limit: 5, seconds: 1 })), + "JS logs did not return entries", + ); + }); + await measuredStep("JS batch Android controls", async () => { + const payload = await session.batch(androidUDID, [ + { + action: "touchSequence", + events: [ + { x: 0.5, y: 0.5, phase: "began", delayMsAfter: 20 }, + { x: 0.5, y: 0.5, phase: "ended" }, + ], + }, + { + action: "swipe", + startX: 0.5, + startY: 0.75, + endX: 0.5, + endY: 0.25, + durationMs: 100, + steps: 4, + }, + { action: "key", keyCode: 40 }, + { action: "home" }, + { action: "describe", source: "android-uiautomator", maxDepth: 1 }, + ]); + assertJson(payload, "JS batch"); + }); +} + +function resolveAndroidDevice() { + const emulator = androidSdkTool("emulator/emulator"); + if (!emulator) { + if (requestedAvd) { + throw new Error("Android SDK emulator binary was not found."); + } + return null; + } + const avds = runText(emulator, ["-list-avds"], { timeoutMs: 30_000 }) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + const running = runningAndroidAvds(); + if (requestedAvd) { + const avdName = requestedAvd.replace(/^android:/, ""); + if (!avds.includes(avdName)) { + throw new Error( + `SIMDECK_INTEGRATION_ANDROID_AVD=${requestedAvd} was not found. Available Android AVDs: ${avds.join(", ")}`, + ); + } + return { + udid: `android:${avdName}`, + name: avdName, + isBooted: running.has(avdName), + }; + } + const avdName = avds[0]; + return avdName + ? { + udid: `android:${avdName}`, + name: avdName, + isBooted: running.has(avdName), + } + : null; +} + +function runningAndroidAvds() { + const adb = androidSdkTool("platform-tools/adb"); + if (!adb) { + return new Set(); + } + const devices = runText(adb, ["devices"], { timeoutMs: 30_000 }) + .split(/\r?\n/) + .map((line) => line.trim().split(/\s+/)) + .filter( + ([serial, state]) => + serial?.startsWith("emulator-") && state === "device", + ) + .map(([serial]) => serial); + const avds = new Set(); + for (const serial of devices) { + const name = androidAvdNameForSerial(adb, serial); + if (name) { + avds.add(name); + } + } + return avds; +} + +function androidAvdNameForSerial(adb, serial) { + for (const property of ["ro.boot.qemu.avd_name", "ro.kernel.qemu.avd_name"]) { + try { + const name = runText(adb, ["-s", serial, "shell", "getprop", property], { + timeoutMs: 10_000, + }).trim(); + if (name) { + return name; + } + } catch { + // Fall through to the next lookup. ADB can briefly expose devices + // before every shell service is ready. + } + } + try { + return runText(adb, ["-s", serial, "emu", "avd", "name"], { + timeoutMs: 10_000, + }) + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line && line !== "OK"); + } catch { + return null; + } +} + +function androidSdkTool(relativePath) { + const roots = [ + process.env.ANDROID_HOME, + process.env.ANDROID_SDK_ROOT, + path.join(os.homedir(), "Library", "Android", "sdk"), + path.join(os.homedir(), "Android", "Sdk"), + ].filter(Boolean); + for (const root of roots) { + const candidate = path.join(root, relativePath); + if (fs.existsSync(candidate)) { + return candidate; + } + } + return null; +} + +function simulatorList(payload) { + return Array.isArray(payload?.simulators) ? payload.simulators : []; +} + +async function waitForBooted(udid) { + const avdName = udid.replace(/^android:/, ""); + const deadline = Date.now() + 180_000; + while (Date.now() < deadline) { + if (runningAndroidAvds().has(avdName)) { + return; + } + await sleep(1_000); + } + throw new Error(`Timed out waiting for ${udid} to boot.`); +} + +async function cleanupAndroid() { + if (!session || !androidUDID || !shutdownAndroidAfterRun) { + return; + } + try { + await measuredStep("shutdown Android emulator", () => + session.shutdown(androidUDID), + ); + } catch (error) { + console.error(`Failed to shutdown ${androidUDID}: ${error.message}`); + } +} + +function cleanupSync() { + if (session) { + session.close(); + session = null; + } +} + +async function measuredStep(label, run, options = {}) { + const started = process.hrtime.bigint(); + let timeoutId = null; + if (verbose) { + console.log(`> ${label}`); + } + try { + const timeoutMs = options.timeoutMs ?? defaultStepTimeoutMs; + const result = await Promise.race([ + Promise.resolve().then(run), + new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + }), + ]); + const elapsedMs = Number(process.hrtime.bigint() - started) / 1_000_000; + stepTimings.push({ label, elapsedMs }); + if (verbose) { + console.log(`ok ${label} ${elapsedMs.toFixed(1)}ms`); + } + if (options.timeoutMs && elapsedMs > options.timeoutMs) { + throw new Error( + `${label} exceeded ${options.timeoutMs}ms budget (${elapsedMs.toFixed( + 1, + )}ms)`, + ); + } + return result; + } catch (error) { + const elapsedMs = Number(process.hrtime.bigint() - started) / 1_000_000; + console.error(`fail ${label} ${elapsedMs.toFixed(1)}ms`); + throw error; + } finally { + if (timeoutId) { + clearTimeout(timeoutId); + } + } +} + +function printTimingSummary() { + if (!verbose || stepTimings.length === 0) { + return; + } + console.log("\nTiming summary:"); + for (const entry of stepTimings) { + console.log(` ${entry.label.padEnd(36)} ${entry.elapsedMs.toFixed(1)}ms`); + } +} + +function simdeckJson(args, options = {}) { + return JSON.parse( + runText(simdeck, ["--server-url", session.endpoint, ...args], options), + ); +} + +function runText(command, args, options = {}) { + return runBuffer(command, args, options).toString("utf8").trim(); +} + +function runBuffer(command, args, options = {}) { + const result = runCommand(command, args, options); + if (result.status !== 0) { + throw new Error( + `${[command, ...args].join(" ")} failed with ${result.status}\n${result.stderr}`, + ); + } + return result.stdoutBuffer ?? Buffer.alloc(0); +} + +function runCommand(command, args, options = {}) { + if (verbose) { + console.log(`$ ${[command, ...args].join(" ")}`); + } + const result = spawnSync(command, args, { + cwd: root, + encoding: null, + timeout: options.timeoutMs ?? 120_000, + maxBuffer: options.maxBuffer ?? 4 * 1024 * 1024, + }); + return { + status: result.status ?? 1, + stdout: (result.stdout ?? Buffer.alloc(0)).toString("utf8"), + stderr: (result.stderr ?? Buffer.alloc(0)).toString("utf8"), + stdoutBuffer: result.stdout ?? Buffer.alloc(0), + }; +} + +function assertClipboardUnsupported(errorOrResult) { + const message = + errorOrResult instanceof Error + ? errorOrResult.message + : `${errorOrResult.stdout}\n${errorOrResult.stderr}`; + assert.match( + message, + /clipboard shell service is not implemented|No shell command implementation/i, + ); +} + +function assertAndroidListed(payload, udid) { + assert.ok( + simulatorList(payload).some( + (simulator) => simulator.udid === udid || simulator.id === udid, + ), + `${udid} was not present in simulator list`, + ); +} + +function assertJson(payload, label) { + assert.ok(payload && typeof payload === "object", `${label} was not JSON`); +} + +function assertRoots(payload, label) { + assertJson(payload, label); + assert.ok(Array.isArray(payload.roots), `${label} missing roots array`); + assert.ok(payload.roots.length > 0, `${label} returned no roots`); +} + +function centerOfFirstRoot(payload) { + const frame = payload?.roots?.[0]?.frame; + if (!frame || typeof frame !== "object") { + return null; + } + const x = Number( + Array.isArray(frame) ? frame[0] : (frame.x ?? frame.minX ?? 0), + ); + const y = Number( + Array.isArray(frame) ? frame[1] : (frame.y ?? frame.minY ?? 0), + ); + const width = Number(Array.isArray(frame) ? frame[2] : (frame.width ?? 0)); + const height = Number(Array.isArray(frame) ? frame[3] : (frame.height ?? 0)); + if ( + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { + return null; + } + return { + x: Math.round(x + width / 2), + y: Math.round(y + height / 2), + }; +} + +function assertPngBuffer(buffer) { + const signature = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + ]); + assert.ok( + buffer.length > signature.length && + buffer.subarray(0, signature.length).equals(signature), + "screenshot did not return a PNG buffer", + ); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/scripts/integration/cli.mjs b/scripts/integration/cli.mjs index 906167eb..de5ad119 100644 --- a/scripts/integration/cli.mjs +++ b/scripts/integration/cli.mjs @@ -761,6 +761,28 @@ async function ensureFixtureForeground(label, options = {}) { if (launchError === null) { throw verifyError; } + logStep(`${label}: opening fixture URL after launch timeout`); + } + + try { + await retrySimdeckJson( + cliArgs(["open-url", simulatorUDID, fixtureUrl]), + `${label} fixture URL fallback`, + { + attempts: 2, + delayMs: 2_000, + timeoutMs: 180_000, + }, + ); + return await verifyUi(label, { + expectFixture: true, + attempts: options.fallbackVerifyAttempts ?? 12, + delayMs: options.fallbackVerifyDelayMs ?? 1_500, + }); + } catch (urlError) { + logStep( + `${label}: fixture URL fallback failed: ${summarizeError(urlError)}`, + ); logStep(`${label}: tapping fixture icon after launch timeout`); } diff --git a/server/Cargo.lock b/server/Cargo.lock index a0120b67..9c7c7db3 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -150,6 +150,28 @@ dependencies = [ "syn", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -173,13 +195,40 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "axum-core", + "axum-core 0.5.6", "base64", "bytes", "form_urlencoded", @@ -190,7 +239,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -203,12 +252,32 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-tungstenite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.5.6" @@ -314,9 +383,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.60" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "shlex", @@ -486,9 +555,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der" @@ -561,6 +630,12 @@ dependencies = [ "spki", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "elliptic-curve" version = "0.13.8" @@ -620,6 +695,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -791,6 +872,31 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.5" @@ -802,9 +908,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -897,6 +1003,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -905,6 +1012,20 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -914,12 +1035,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "libc", "pin-project-lite", + "socket2 0.6.3", "tokio", "tower-service", + "tracing", ] [[package]] @@ -1023,14 +1149,24 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1038,7 +1174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -1085,6 +1221,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1093,10 +1238,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1115,9 +1262,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "litemap" @@ -1149,6 +1296,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matchit" version = "0.8.4" @@ -1378,6 +1531,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1407,7 +1580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ "base64", - "indexmap", + "indexmap 2.14.0", "quick-xml", "serde", "time", @@ -1483,6 +1656,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quick-xml" version = "0.39.4" @@ -1648,6 +1844,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + [[package]] name = "rtcp" version = "0.12.0" @@ -1694,9 +1896,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "once_cell", "ring", @@ -1708,9 +1910,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] @@ -1904,7 +2106,7 @@ name = "simdeck-server" version = "0.1.0" dependencies = [ "anyhow", - "axum", + "axum 0.8.9", "base64", "bytes", "cc", @@ -1914,6 +2116,8 @@ dependencies = [ "http", "libc", "plist", + "prost", + "roxmltree", "serde", "serde_json", "sha2", @@ -1921,6 +2125,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-tungstenite", + "tonic", "tower-http", "tracing", "tracing-subscriber", @@ -2144,9 +2349,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -2206,6 +2411,56 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.7.9", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.6", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.3" @@ -2224,9 +2479,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" dependencies = [ "bitflags 2.11.1", "bytes", @@ -2322,6 +2577,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "tungstenite" version = "0.29.0" @@ -2455,6 +2716,15 @@ dependencies = [ "atomic-waker", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2481,9 +2751,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -2494,9 +2764,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2504,9 +2774,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -2517,9 +2787,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -2541,7 +2811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -2554,7 +2824,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", "semver", ] @@ -2911,7 +3181,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap", + "indexmap 2.14.0", "prettyplease", "syn", "wasm-metadata", @@ -2942,7 +3212,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags 2.11.1", - "indexmap", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -2961,7 +3231,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.14.0", "log", "semver", "serde", diff --git a/server/Cargo.toml b/server/Cargo.toml index 90270a9b..0a376bc6 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -15,6 +15,8 @@ hex = "0.4" http = "1.1" libc = "0.2" plist = "1.7" +prost = "0.13" +roxmltree = "0.20" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10" @@ -22,6 +24,7 @@ thiserror = "2.0" tokio = { version = "1.42", features = ["fs", "io-util", "macros", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = "0.1" tokio-tungstenite = "0.29" +tonic = { version = "0.12", features = ["transport"] } tower-http = { version = "0.6", features = ["cors", "fs", "trace"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } diff --git a/server/build.rs b/server/build.rs index 1e69e127..7f445c1f 100644 --- a/server/build.rs +++ b/server/build.rs @@ -5,6 +5,18 @@ fn main() { .parent() .unwrap() .to_path_buf(); + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + if target_os != "macos" { + let stub = root.join("server/native_stubs.c"); + println!("cargo:rerun-if-changed={}", stub.display()); + cc::Build::new() + .file(&stub) + .flag("-Wall") + .flag("-Wextra") + .compile("xcw_native_bridge"); + return; + } + let cli = root.join("cli"); let native = cli.join("native"); diff --git a/server/native_stubs.c b/server/native_stubs.c new file mode 100644 index 00000000..b838229a --- /dev/null +++ b/server/native_stubs.c @@ -0,0 +1,474 @@ +#include +#include +#include +#include +#include + +typedef struct { + uint8_t *data; + uintptr_t length; +} xcw_native_owned_bytes; + +typedef struct { + const uint8_t *data; + uintptr_t length; + const void *owner; +} xcw_native_shared_bytes; + +typedef struct { + uint64_t frame_sequence; + uint64_t timestamp_us; + bool is_keyframe; + uint32_t width; + uint32_t height; + const char *codec; + xcw_native_shared_bytes description; + xcw_native_shared_bytes data; +} xcw_native_frame; + +typedef void (*xcw_native_frame_callback)(const xcw_native_frame *frame, + void *user_data); + +static char *xcw_strdup(const char *value) { + if (value == NULL) { + value = ""; + } + size_t length = strlen(value); + char *copy = (char *)malloc(length + 1); + if (copy != NULL) { + memcpy(copy, value, length + 1); + } + return copy; +} + +static void xcw_set_error(char **error_message, const char *message) { + if (error_message != NULL) { + *error_message = xcw_strdup(message); + } +} + +static bool xcw_unsupported(char **error_message) { + xcw_set_error(error_message, + "iOS simulator native bridge is only available on macOS."); + return false; +} + +static xcw_native_owned_bytes xcw_empty_bytes(char **error_message) { + xcw_unsupported(error_message); + xcw_native_owned_bytes bytes = {0}; + return bytes; +} + +void xcw_native_initialize_app(void) {} + +void xcw_native_run_main_loop_slice(double duration_seconds) { + if (duration_seconds <= 0.0) { + return; + } + time_t seconds = (time_t)duration_seconds; + long nanos = (long)((duration_seconds - (double)seconds) * 1000000000.0); + if (nanos < 0) { + nanos = 0; + } + struct timespec delay = {.tv_sec = seconds, .tv_nsec = nanos}; + nanosleep(&delay, NULL); +} + +char *xcw_native_list_simulators(char **error_message) { + (void)error_message; + return xcw_strdup("{\"simulators\":[]}"); +} + +bool xcw_native_boot_simulator(const char *udid, char **error_message) { + (void)udid; + return xcw_unsupported(error_message); +} + +bool xcw_native_shutdown_simulator(const char *udid, char **error_message) { + (void)udid; + return xcw_unsupported(error_message); +} + +bool xcw_native_toggle_appearance(const char *udid, char **error_message) { + (void)udid; + return xcw_unsupported(error_message); +} + +bool xcw_native_open_url(const char *udid, const char *url, + char **error_message) { + (void)udid; + (void)url; + return xcw_unsupported(error_message); +} + +bool xcw_native_launch_bundle(const char *udid, const char *bundle_id, + char **error_message) { + (void)udid; + (void)bundle_id; + return xcw_unsupported(error_message); +} + +char *xcw_native_get_chrome_profile(const char *udid, char **error_message) { + (void)udid; + xcw_unsupported(error_message); + return NULL; +} + +xcw_native_owned_bytes xcw_native_render_chrome_png(const char *udid, + bool include_buttons, + char **error_message) { + (void)udid; + (void)include_buttons; + return xcw_empty_bytes(error_message); +} + +xcw_native_owned_bytes xcw_native_render_chrome_button_png( + const char *udid, const char *button_name, bool pressed, + char **error_message) { + (void)udid; + (void)button_name; + (void)pressed; + return xcw_empty_bytes(error_message); +} + +xcw_native_owned_bytes xcw_native_render_screen_mask_png( + const char *udid, char **error_message) { + (void)udid; + return xcw_empty_bytes(error_message); +} + +xcw_native_owned_bytes xcw_native_screenshot_png(const char *udid, + char **error_message) { + (void)udid; + return xcw_empty_bytes(error_message); +} + +char *xcw_native_recent_logs(const char *udid, double seconds, uintptr_t limit, + char **error_message) { + (void)udid; + (void)seconds; + (void)limit; + xcw_unsupported(error_message); + return NULL; +} + +char *xcw_native_accessibility_snapshot(const char *udid, bool has_point, + double x, double y, + uintptr_t max_depth, + char **error_message) { + (void)udid; + (void)has_point; + (void)x; + (void)y; + (void)max_depth; + xcw_unsupported(error_message); + return NULL; +} + +bool xcw_native_send_touch(const char *udid, double x, double y, + const char *phase, char **error_message) { + (void)udid; + (void)x; + (void)y; + (void)phase; + return xcw_unsupported(error_message); +} + +bool xcw_native_send_key(const char *udid, uint16_t key_code, + uint32_t modifiers, char **error_message) { + (void)udid; + (void)key_code; + (void)modifiers; + return xcw_unsupported(error_message); +} + +bool xcw_native_press_home(const char *udid, char **error_message) { + (void)udid; + return xcw_unsupported(error_message); +} + +bool xcw_native_open_app_switcher(const char *udid, char **error_message) { + (void)udid; + return xcw_unsupported(error_message); +} + +bool xcw_native_press_button(const char *udid, const char *button_name, + uint32_t duration_ms, char **error_message) { + (void)udid; + (void)button_name; + (void)duration_ms; + return xcw_unsupported(error_message); +} + +bool xcw_native_send_button(const char *udid, const char *button_name, + bool pressed, bool has_usage, uint32_t usage_page, + uint32_t usage, char **error_message) { + (void)udid; + (void)button_name; + (void)pressed; + (void)has_usage; + (void)usage_page; + (void)usage; + return xcw_unsupported(error_message); +} + +bool xcw_native_rotate_right(const char *udid, char **error_message) { + (void)udid; + return xcw_unsupported(error_message); +} + +bool xcw_native_rotate_left(const char *udid, char **error_message) { + (void)udid; + return xcw_unsupported(error_message); +} + +bool xcw_native_erase_simulator(const char *udid, char **error_message) { + (void)udid; + return xcw_unsupported(error_message); +} + +bool xcw_native_install_app(const char *udid, const char *app_path, + char **error_message) { + (void)udid; + (void)app_path; + return xcw_unsupported(error_message); +} + +bool xcw_native_uninstall_app(const char *udid, const char *bundle_id, + char **error_message) { + (void)udid; + (void)bundle_id; + return xcw_unsupported(error_message); +} + +bool xcw_native_set_pasteboard_text(const char *udid, const char *text, + char **error_message) { + (void)udid; + (void)text; + return xcw_unsupported(error_message); +} + +char *xcw_native_get_pasteboard_text(const char *udid, char **error_message) { + (void)udid; + xcw_unsupported(error_message); + return NULL; +} + +void *xcw_native_input_create(const char *udid, char **error_message) { + (void)udid; + xcw_unsupported(error_message); + return NULL; +} + +void xcw_native_input_destroy(void *handle) { (void)handle; } + +bool xcw_native_input_display_size(void *handle, double *width, + double *height) { + (void)handle; + if (width != NULL) { + *width = 0.0; + } + if (height != NULL) { + *height = 0.0; + } + return false; +} + +bool xcw_native_input_send_touch(void *handle, double x, double y, + const char *phase, char **error_message) { + (void)handle; + (void)x; + (void)y; + (void)phase; + return xcw_unsupported(error_message); +} + +bool xcw_native_input_send_multitouch(void *handle, double x1, double y1, + double x2, double y2, + const char *phase, + char **error_message) { + (void)handle; + (void)x1; + (void)y1; + (void)x2; + (void)y2; + (void)phase; + return xcw_unsupported(error_message); +} + +bool xcw_native_input_send_key(void *handle, uint16_t key_code, + uint32_t modifiers, char **error_message) { + (void)handle; + (void)key_code; + (void)modifiers; + return xcw_unsupported(error_message); +} + +bool xcw_native_input_send_key_event(void *handle, uint16_t key_code, + bool down, char **error_message) { + (void)handle; + (void)key_code; + (void)down; + return xcw_unsupported(error_message); +} + +void *xcw_native_session_create(const char *udid, char **error_message) { + (void)udid; + xcw_unsupported(error_message); + return NULL; +} + +void xcw_native_session_destroy(void *handle) { (void)handle; } + +bool xcw_native_session_start(void *handle, char **error_message) { + (void)handle; + return xcw_unsupported(error_message); +} + +void xcw_native_session_request_refresh(void *handle) { (void)handle; } + +void xcw_native_session_request_keyframe(void *handle) { (void)handle; } + +void xcw_native_session_reconfigure_video_encoder(void *handle) { + (void)handle; +} + +char *xcw_native_session_video_encoder_stats(void *handle, + char **error_message) { + (void)handle; + (void)error_message; + return xcw_strdup("{}"); +} + +int32_t xcw_native_session_rotation_quarter_turns(void *handle) { + (void)handle; + return 0; +} + +void xcw_native_session_set_frame_callback( + void *handle, xcw_native_frame_callback callback, void *user_data) { + (void)handle; + (void)callback; + (void)user_data; +} + +bool xcw_native_session_send_touch(void *handle, double x, double y, + const char *phase, + char **error_message) { + (void)handle; + (void)x; + (void)y; + (void)phase; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_send_edge_touch(void *handle, double x, double y, + const char *phase, uint32_t edge, + char **error_message) { + (void)handle; + (void)x; + (void)y; + (void)phase; + (void)edge; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_send_multitouch(void *handle, double x1, double y1, + double x2, double y2, + const char *phase, + char **error_message) { + (void)handle; + (void)x1; + (void)y1; + (void)x2; + (void)y2; + (void)phase; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_send_key(void *handle, uint16_t key_code, + uint32_t modifiers, char **error_message) { + (void)handle; + (void)key_code; + (void)modifiers; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_press_home(void *handle, char **error_message) { + (void)handle; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_press_button(void *handle, const char *button_name, + uint32_t duration_ms, + char **error_message) { + (void)handle; + (void)button_name; + (void)duration_ms; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_send_button(void *handle, const char *button_name, + bool pressed, bool has_usage, + uint32_t usage_page, uint32_t usage, + char **error_message) { + (void)handle; + (void)button_name; + (void)pressed; + (void)has_usage; + (void)usage_page; + (void)usage; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_open_app_switcher(void *handle, char **error_message) { + (void)handle; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_rotate_right(void *handle, char **error_message) { + (void)handle; + return xcw_unsupported(error_message); +} + +bool xcw_native_session_rotate_left(void *handle, char **error_message) { + (void)handle; + return xcw_unsupported(error_message); +} + +void *xcw_native_h264_encoder_create(xcw_native_frame_callback callback, + void *user_data, char **error_message) { + (void)callback; + (void)user_data; + xcw_set_error(error_message, + "H.264 encoding is only available in the macOS native bridge."); + return NULL; +} + +void xcw_native_h264_encoder_destroy(void *handle) { (void)handle; } + +bool xcw_native_h264_encoder_encode_rgba(void *handle, const uint8_t *rgba, + uintptr_t length, uint32_t width, + uint32_t height, + uint64_t timestamp_us, + char **error_message) { + (void)handle; + (void)rgba; + (void)length; + (void)width; + (void)height; + (void)timestamp_us; + xcw_set_error(error_message, + "H.264 encoding is only available in the macOS native bridge."); + return false; +} + +void xcw_native_h264_encoder_request_keyframe(void *handle) { (void)handle; } + +void xcw_native_free_string(char *value) { free(value); } + +void xcw_native_free_bytes(xcw_native_owned_bytes bytes) { free(bytes.data); } + +void xcw_native_release_shared_bytes(xcw_native_shared_bytes bytes) { + (void)bytes; +} diff --git a/server/src/android.rs b/server/src/android.rs new file mode 100644 index 00000000..168865ff --- /dev/null +++ b/server/src/android.rs @@ -0,0 +1,2228 @@ +use crate::error::AppError; +use bytes::BytesMut; +use http::uri::PathAndQuery; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::env; +use std::ffi::OsString; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Mutex, OnceLock}; +use std::thread; +use std::time::{Duration, Instant}; +use tonic::metadata::MetadataValue; +use tonic::transport::Endpoint; + +const ANDROID_ID_PREFIX: &str = "android:"; +const DEFAULT_GRPC_PORT_BASE: u16 = 8554; +const ANDROID_GRPC_FRAME_MESSAGE_LIMIT: usize = 64 * 1024 * 1024; +const ANDROID_TOUCH_SWIPE_THRESHOLD: f64 = 0.025; +const ANDROID_TOUCH_MIN_DURATION_MS: u128 = 80; +const ANDROID_TOUCH_MAX_DURATION_MS: u128 = 1500; +const ANDROID_COMMAND_TIMEOUT: Duration = Duration::from_secs(30); +const RUNNING_EMULATOR_CACHE_TTL: Duration = Duration::from_secs(2); +const AVD_GRPC_PORT_CACHE_TTL: Duration = Duration::from_secs(60); +const SCREEN_SIZE_CACHE_TTL: Duration = Duration::from_secs(1); +const MODIFIER_SHIFT: u32 = 1 << 0; +const MODIFIER_CONTROL: u32 = 1 << 1; +const MODIFIER_OPTION: u32 = 1 << 2; +const MODIFIER_COMMAND: u32 = 1 << 3; +const MODIFIER_CAPS_LOCK: u32 = 1 << 4; + +type TimedMap = Option<(Instant, HashMap)>; +type DisplayMetricsCache = HashMap; + +#[derive(Clone, Copy, Debug, PartialEq)] +struct AndroidDisplayMetrics { + width: f64, + height: f64, + rotation_quarter_turns: u16, + corner_radii: AndroidCornerRadii, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +struct AndroidCornerRadii { + top_left: f64, + top_right: f64, + bottom_right: f64, + bottom_left: f64, +} + +impl AndroidCornerRadii { + const ZERO: Self = Self { + top_left: 0.0, + top_right: 0.0, + bottom_right: 0.0, + bottom_left: 0.0, + }; + + fn max(self) -> f64 { + self.top_left + .max(self.top_right) + .max(self.bottom_right) + .max(self.bottom_left) + } +} + +#[derive(Clone, Default)] +pub struct AndroidBridge; + +#[derive(Clone, Debug)] +pub struct AndroidDevice { + pub avd_name: String, + pub serial: Option, + pub is_booted: bool, + pub grpc_port: u16, +} + +#[derive(Debug)] +pub struct AndroidFrame { + pub width: u32, + pub height: u32, + pub timestamp_us: u64, + pub rgba: Vec, +} + +pub struct AndroidGrpcFrameStream { + inner: tonic::Streaming, + target: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct AndroidFrameTarget { + width: u32, + height: u32, + rotation_quarter_turns: u16, +} + +#[derive(Debug)] +pub struct AndroidTouchGesture { + started_at: Instant, + start_x: f64, + start_y: f64, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum AndroidTouchAction { + None, + Tap { + x: f64, + y: f64, + }, + Swipe { + start_x: f64, + start_y: f64, + end_x: f64, + end_y: f64, + duration_ms: u64, + }, +} + +impl AndroidTouchAction { + pub fn perform(self, bridge: &AndroidBridge, id: &str) -> Result<(), AppError> { + match self { + AndroidTouchAction::None => Ok(()), + AndroidTouchAction::Tap { x, y } => bridge.send_tap_adb(id, x, y), + AndroidTouchAction::Swipe { + start_x, + start_y, + end_x, + end_y, + duration_ms, + } => bridge.send_swipe_adb(id, start_x, start_y, end_x, end_y, duration_ms), + } + } +} + +pub fn is_android_id(id: &str) -> bool { + id.starts_with(ANDROID_ID_PREFIX) +} + +pub fn avd_from_id(id: &str) -> Result { + id.strip_prefix(ANDROID_ID_PREFIX) + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| AppError::bad_request(format!("Invalid Android emulator id `{id}`."))) +} + +pub fn id_for_avd(avd_name: &str) -> String { + format!("{ANDROID_ID_PREFIX}{avd_name}") +} + +pub fn update_touch_gesture( + active_touch: &mut Option, + x: f64, + y: f64, + phase: &str, +) -> Result { + if !x.is_finite() || !y.is_finite() { + return Err(AppError::bad_request( + "`x` and `y` must be finite normalized numbers.", + )); + } + let x = x.clamp(0.0, 1.0); + let y = y.clamp(0.0, 1.0); + + match phase { + "began" => { + *active_touch = Some(AndroidTouchGesture { + started_at: Instant::now(), + start_x: x, + start_y: y, + }); + Ok(AndroidTouchAction::None) + } + "moved" => Ok(AndroidTouchAction::None), + "ended" => { + let touch = active_touch.take().unwrap_or(AndroidTouchGesture { + started_at: Instant::now(), + start_x: x, + start_y: y, + }); + let distance = ((x - touch.start_x).powi(2) + (y - touch.start_y).powi(2)).sqrt(); + if distance < ANDROID_TOUCH_SWIPE_THRESHOLD { + return Ok(AndroidTouchAction::Tap { x, y }); + } + Ok(AndroidTouchAction::Swipe { + start_x: touch.start_x, + start_y: touch.start_y, + end_x: x, + end_y: y, + duration_ms: touch + .started_at + .elapsed() + .as_millis() + .clamp(ANDROID_TOUCH_MIN_DURATION_MS, ANDROID_TOUCH_MAX_DURATION_MS) + as u64, + }) + } + "cancelled" => { + *active_touch = None; + Ok(AndroidTouchAction::None) + } + _ => Ok(AndroidTouchAction::None), + } +} + +impl AndroidBridge { + pub fn list_devices(&self) -> Result, AppError> { + if !self.emulator_path().exists() { + return Ok(Vec::new()); + } + + let avds = self + .run_emulator(["-list-avds"])? + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToOwned::to_owned) + .collect::>(); + if avds.is_empty() { + return Ok(Vec::new()); + } + + let running = self.running_emulators().unwrap_or_default(); + Ok(avds + .into_iter() + .enumerate() + .map(|(index, avd_name)| AndroidDevice { + serial: running.get(&avd_name).cloned(), + is_booted: running.contains_key(&avd_name), + grpc_port: DEFAULT_GRPC_PORT_BASE + index as u16, + avd_name, + }) + .collect()) + } + + pub fn enrich_devices(&self, devices: Vec) -> Vec { + devices + .into_iter() + .map(|device| self.device_value(device)) + .collect() + } + + pub fn boot(&self, id: &str) -> Result { + let avd_name = avd_from_id(id)?; + if self.resolve_serial(&avd_name).is_ok() { + return Ok(false); + } + let grpc_port = self.grpc_port_for_avd(&avd_name)?; + Command::new(self.emulator_path()) + .args([ + "-avd", + &avd_name, + "-no-window", + "-no-audio", + "-gpu", + "swiftshader_indirect", + "-grpc", + &grpc_port.to_string(), + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| { + AppError::native(format!( + "Unable to start Android emulator `{avd_name}`: {error}" + )) + })?; + Ok(true) + } + + pub fn shutdown(&self, id: &str) -> Result<(), AppError> { + let avd_name = avd_from_id(id)?; + let serial = self.resolve_serial(&avd_name)?; + let _ = self.run_adb(["-s", &serial, "emu", "kill"])?; + Ok(()) + } + + pub fn erase(&self, id: &str) -> Result<(), AppError> { + let avd_name = avd_from_id(id)?; + if self.resolve_serial(&avd_name).is_ok() { + return Err(AppError::bad_request( + "Shutdown the Android emulator before erasing it.", + )); + } + let avd_dir = self.avd_dir(&avd_name); + for file_name in [ + "userdata-qemu.img", + "cache.img", + "data.img", + "sdcard.img", + "snapshots.img", + ] { + let path = avd_dir.join(file_name); + if path.exists() { + std::fs::remove_file(&path).map_err(|error| { + AppError::native(format!("Unable to remove {}: {error}", path.display())) + })?; + } + } + Ok(()) + } + + pub fn wait_until_booted(&self, id: &str, timeout_duration: Duration) -> Result<(), AppError> { + let avd_name = avd_from_id(id)?; + let deadline = Instant::now() + timeout_duration; + loop { + if let Ok(serial) = self.resolve_serial(&avd_name) { + if self + .run_adb(["-s", &serial, "shell", "getprop", "sys.boot_completed"]) + .unwrap_or_default() + .trim() + == "1" + { + return Ok(()); + } + } + if Instant::now() >= deadline { + return Err(AppError::native(format!( + "Android emulator `{avd_name}` did not finish booting in time." + ))); + } + thread::sleep(Duration::from_millis(500)); + } + } + + pub fn screenshot_png(&self, id: &str) -> Result, AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb_bytes(["-s", &serial, "exec-out", "screencap", "-p"]) + } + + pub fn install_app(&self, id: &str, app_path: &str) -> Result<(), AppError> { + if !app_path.ends_with(".apk") { + return Err(AppError::bad_request( + "Android install expects an `.apk` path.", + )); + } + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "install", "-r", app_path])?; + Ok(()) + } + + pub fn uninstall_app(&self, id: &str, package_name: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "uninstall", package_name])?; + Ok(()) + } + + pub fn open_url(&self, id: &str, url: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb([ + "-s", + &serial, + "shell", + "am", + "start", + "-a", + "android.intent.action.VIEW", + "-d", + url, + ])?; + Ok(()) + } + + pub fn launch_package(&self, id: &str, package: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb([ + "-s", + &serial, + "shell", + "am", + "start", + "-a", + "android.intent.action.MAIN", + "-c", + "android.intent.category.LAUNCHER", + "-p", + package, + ])?; + Ok(()) + } + + pub fn set_pasteboard_text(&self, id: &str, text: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + let output = + self.run_adb_shell(&serial, &format!("cmd clipboard set {}", shell_quote(text)))?; + ensure_android_clipboard_available(&output)?; + Ok(()) + } + + pub fn pasteboard_text(&self, id: &str) -> Result { + let serial = self.serial_for_id(id)?; + let output = self.run_adb_shell(&serial, "cmd clipboard get")?; + ensure_android_clipboard_available(&output)?; + Ok(output.trim_end_matches(['\r', '\n']).to_owned()) + } + + pub fn send_touch(&self, id: &str, x: f64, y: f64, phase: &str) -> Result<(), AppError> { + match phase { + "ended" => self.send_tap_adb(id, x, y), + _ => Ok(()), + } + } + + fn send_tap_adb(&self, id: &str, x: f64, y: f64) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + let (width, height) = self.screen_size_for_serial(&serial)?; + let px = (x.clamp(0.0, 1.0) * (width - 1.0)).round().max(0.0); + let py = (y.clamp(0.0, 1.0) * (height - 1.0)).round().max(0.0); + self.run_adb([ + "-s", + &serial, + "shell", + "input", + "tap", + &px.to_string(), + &py.to_string(), + ])?; + Ok(()) + } + + pub fn send_swipe( + &self, + id: &str, + start_x: f64, + start_y: f64, + end_x: f64, + end_y: f64, + duration_ms: u64, + ) -> Result<(), AppError> { + self.send_swipe_adb(id, start_x, start_y, end_x, end_y, duration_ms) + } + + fn send_swipe_adb( + &self, + id: &str, + start_x: f64, + start_y: f64, + end_x: f64, + end_y: f64, + duration_ms: u64, + ) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + let (width, height) = self.screen_size_for_serial(&serial)?; + let coords = [start_x, start_y, end_x, end_y] + .into_iter() + .enumerate() + .map(|(index, value)| { + let max = if index % 2 == 0 { + width - 1.0 + } else { + height - 1.0 + }; + (value.clamp(0.0, 1.0) * max).round().max(0.0).to_string() + }) + .collect::>(); + self.run_adb([ + "-s", + &serial, + "shell", + "input", + "swipe", + &coords[0], + &coords[1], + &coords[2], + &coords[3], + &duration_ms.to_string(), + ])?; + Ok(()) + } + + pub fn send_key(&self, id: &str, key_code: u16, modifiers: u32) -> Result<(), AppError> { + if let Some(text) = hid_text_for_key(key_code, modifiers) { + return self.type_text_adb(id, &text); + } + + let serial = self.serial_for_id(id)?; + let android_key = android_key_code(key_code); + if has_android_key_modifiers(modifiers) { + return self.press_android_key_combination(&serial, android_key, modifiers); + } + self.press_android_key(&serial, android_key) + } + + pub fn type_text(&self, id: &str, text: &str) -> Result<(), AppError> { + self.type_text_adb(id, text) + } + + pub fn dismiss_keyboard(&self, id: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.press_android_key(&serial, 4) + } + + fn type_text_adb(&self, id: &str, text: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + let escaped = android_input_text_arg(text); + self.run_adb(["-s", &serial, "shell", "input", "text", &escaped])?; + Ok(()) + } + + fn press_android_key(&self, serial: &str, key_code: u16) -> Result<(), AppError> { + let key_code = key_code.to_string(); + self.run_adb(["-s", serial, "shell", "input", "keyevent", &key_code])?; + Ok(()) + } + + fn press_android_key_combination( + &self, + serial: &str, + key_code: u16, + modifiers: u32, + ) -> Result<(), AppError> { + let mut parts = vec!["input".to_owned(), "keycombination".to_owned()]; + parts.extend( + android_modifier_key_codes(modifiers) + .into_iter() + .map(|key| key.to_string()), + ); + parts.push(key_code.to_string()); + match self.run_adb_shell(serial, &parts.join(" ")) { + Ok(_) => Ok(()), + Err(_) => self.press_android_key(serial, key_code), + } + } + + pub fn press_home(&self, id: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "3"])?; + Ok(()) + } + + pub fn open_app_switcher(&self, id: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "187"])?; + Ok(()) + } + + pub fn press_button(&self, id: &str, button: &str, duration_ms: u32) -> Result<(), AppError> { + match button { + "home" => self.press_home(id), + "lock" | "side-button" => { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "26"])?; + if duration_ms > 500 { + thread::sleep(Duration::from_millis(u64::from(duration_ms))); + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "26"])?; + } + Ok(()) + } + "back" => { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "4"])?; + Ok(()) + } + _ => Err(AppError::bad_request(format!( + "Unsupported Android hardware button `{button}`." + ))), + } + } + + pub fn rotate_right(&self, id: &str) -> Result<(), AppError> { + self.rotate_by_quarter_turns(id, 1) + } + + pub fn rotate_left(&self, id: &str) -> Result<(), AppError> { + self.rotate_by_quarter_turns(id, -1) + } + + fn rotate_by_quarter_turns(&self, id: &str, delta: i16) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.invalidate_display_metrics_for_serial(&serial); + let current = self + .display_metrics_for_serial(&serial) + .map(|metrics| metrics.rotation_quarter_turns) + .unwrap_or(0); + let next = (i16::try_from(current).unwrap_or(0) + delta).rem_euclid(4) as u16; + let rotation = next.to_string(); + let _ = self.run_adb([ + "-s", + &serial, + "shell", + "cmd", + "window", + "set-ignore-orientation-request", + "-d", + "0", + "true", + ]); + let _ = self.run_adb([ + "-s", + &serial, + "shell", + "cmd", + "window", + "fixed-to-user-rotation", + "-d", + "0", + "enabled", + ]); + self.run_adb([ + "-s", + &serial, + "shell", + "cmd", + "window", + "user-rotation", + "-d", + "0", + "lock", + &rotation, + ])?; + self.invalidate_display_metrics_for_serial(&serial); + self.wait_for_display_rotation(&serial, next); + Ok(()) + } + + fn wait_for_display_rotation(&self, serial: &str, rotation: u16) { + let deadline = Instant::now() + Duration::from_secs(4); + while Instant::now() < deadline { + self.invalidate_display_metrics_for_serial(serial); + if self + .display_metrics_for_serial(serial) + .map(|metrics| metrics.rotation_quarter_turns == rotation) + .unwrap_or(false) + { + return; + } + thread::sleep(Duration::from_millis(100)); + } + self.invalidate_display_metrics_for_serial(serial); + } + + pub fn toggle_appearance(&self, id: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + let current = self.run_adb_shell(&serial, "cmd uimode night")?; + let mode = if current.to_lowercase().contains("yes") { + "no" + } else { + "yes" + }; + self.run_adb(["-s", &serial, "shell", "cmd", "uimode", "night", mode])?; + Ok(()) + } + + pub fn logs(&self, id: &str, limit: usize) -> Result, AppError> { + let serial = self.serial_for_id(id)?; + let raw = self.run_adb([ + "-s", + &serial, + "logcat", + "-d", + "-v", + "threadtime", + "-t", + &limit.max(1).to_string(), + ])?; + Ok(raw + .lines() + .map(|line| { + json!({ + "timestamp": "", + "level": android_log_level(line), + "process": "", + "pid": Value::Null, + "subsystem": "android", + "category": "logcat", + "message": line, + }) + }) + .collect()) + } + + pub fn chrome_profile(&self, id: &str) -> Result { + let serial = self.serial_for_id(id)?; + let metrics = self.display_metrics_for_serial(&serial)?; + let width = metrics.width; + let height = metrics.height; + let radii = metrics.corner_radii; + Ok(json!({ + "totalWidth": width, + "totalHeight": height, + "screenX": 0, + "screenY": 0, + "screenWidth": width, + "screenHeight": height, + "cornerRadius": radii.max(), + "cornerRadii": { + "topLeft": radii.top_left, + "topRight": radii.top_right, + "bottomRight": radii.bottom_right, + "bottomLeft": radii.bottom_left, + }, + "hasScreenMask": false, + })) + } + + pub async fn grpc_frame_stream( + &self, + id: &str, + max_edge: Option, + ) -> Result { + let avd_name = avd_from_id(id)?; + let port = self.grpc_port_for_avd(&avd_name)?; + let mut format = grpc::ImageFormat { + format: grpc::image_format::ImgFormat::Rgba8888 as i32, + width: 0, + height: 0, + display: 0, + transport: None, + }; + let target = self + .resolve_serial(&avd_name) + .ok() + .and_then(|serial| self.display_metrics_for_serial(&serial).ok()) + .map(|metrics| AndroidFrameTarget { + width: metrics.width.round().max(1.0) as u32, + height: metrics.height.round().max(1.0) as u32, + rotation_quarter_turns: metrics.rotation_quarter_turns, + }); + if let (Some(max_edge), Some(target)) = (max_edge, target) { + let max_edge = max_edge.clamp(240, 2400); + let largest = target.width.max(target.height); + if largest > max_edge { + if target.width >= target.height { + format.width = max_edge; + } else { + format.height = max_edge; + } + } + } + + let endpoint = Endpoint::from_shared(format!("http://127.0.0.1:{port}")) + .map_err(|error| AppError::native(format!("Invalid Android gRPC endpoint: {error}")))? + .connect() + .await + .map_err(|error| { + AppError::native(format!( + "Unable to connect to Android emulator gRPC: {error}" + )) + })?; + let mut grpc = tonic::client::Grpc::new(endpoint) + .max_decoding_message_size(ANDROID_GRPC_FRAME_MESSAGE_LIMIT); + grpc.ready().await.map_err(|error| { + AppError::native(format!("Android emulator gRPC is not ready: {error}")) + })?; + let path = PathAndQuery::from_static( + "/android.emulation.control.EmulatorController/streamScreenshot", + ); + let mut request = tonic::Request::new(format); + if let Some(token) = emulator_grpc_token(port) { + let value = MetadataValue::try_from(format!("Bearer {token}")).map_err(|error| { + AppError::native(format!("Invalid Android emulator gRPC token: {error}")) + })?; + request.metadata_mut().insert("authorization", value); + } + let response = grpc + .server_streaming(request, path, tonic::codec::ProstCodec::default()) + .await + .map_err(|error| { + AppError::native(format!( + "Android emulator screenshot stream failed: {error}" + )) + })?; + Ok(AndroidGrpcFrameStream { + inner: response.into_inner(), + target, + }) + } + + pub fn accessibility_tree( + &self, + id: &str, + max_depth: Option, + ) -> Result { + let serial = self.serial_for_id(id)?; + let raw = self.run_adb_shell( + &serial, + "uiautomator dump /sdcard/simdeck_ui.xml >/dev/null && cat /sdcard/simdeck_ui.xml", + )?; + let xml = extract_xml(&raw); + let document = roxmltree::Document::parse(xml).map_err(|error| { + AppError::native(format!("Unable to parse UIAutomator XML: {error}")) + })?; + let mut roots = Vec::new(); + let root = document.root_element(); + let max_depth = max_depth.unwrap_or(80).min(80); + for child in root.children().filter(|node| node.has_tag_name("node")) { + roots.push(android_node_value(child, 0, max_depth)); + } + let (width, height) = self.screen_size_for_serial(&serial)?; + if roots.is_empty() { + roots.push(json!({ + "type": "screen", + "role": "screen", + "frame": frame_value(0.0, 0.0, width, height), + "children": [], + })); + } + Ok(json!({ + "source": "android-uiautomator", + "availableSources": ["android-uiautomator"], + "roots": roots, + })) + } + + fn device_value(&self, device: AndroidDevice) -> Value { + let id = id_for_avd(&device.avd_name); + let private_display = if let Some(serial) = device.serial.as_deref() { + let metrics = + self.display_metrics_for_serial(serial) + .unwrap_or(AndroidDisplayMetrics { + width: 0.0, + height: 0.0, + rotation_quarter_turns: 0, + corner_radii: AndroidCornerRadii::ZERO, + }); + json!({ + "displayReady": metrics.width > 0.0 && metrics.height > 0.0, + "displayStatus": "Ready", + "displayWidth": metrics.width, + "displayHeight": metrics.height, + "frameSequence": 0, + "rotationQuarterTurns": metrics.rotation_quarter_turns, + }) + } else { + json!({ + "displayReady": false, + "displayStatus": "Boot required", + "displayWidth": 0, + "displayHeight": 0, + "frameSequence": 0, + "rotationQuarterTurns": 0, + }) + }; + json!({ + "udid": id, + "id": id, + "platform": "android-emulator", + "name": device.avd_name, + "state": if device.is_booted { "Booted" } else { "Shutdown" }, + "isBooted": device.is_booted, + "isAvailable": true, + "lastBootedAt": Value::Null, + "dataPath": self.avd_dir(&device.avd_name), + "logPath": Value::Null, + "deviceTypeIdentifier": "android-emulator", + "deviceTypeName": "Android Emulator", + "runtimeIdentifier": "android", + "runtimeName": "Android", + "android": { + "avdName": device.avd_name, + "serial": device.serial, + "grpcPort": device.grpc_port, + }, + "privateDisplay": private_display, + }) + } + + fn serial_for_id(&self, id: &str) -> Result { + self.resolve_serial(&avd_from_id(id)?) + } + + fn resolve_serial(&self, avd_name: &str) -> Result { + self.running_emulators()?.remove(avd_name).ok_or_else(|| { + AppError::native(format!("Android emulator `{avd_name}` is not running.")) + }) + } + + fn running_emulators(&self) -> Result, AppError> { + static CACHE: OnceLock>> = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(None)); + if let Some((updated_at, running)) = cache.lock().unwrap().as_ref() { + if updated_at.elapsed() < RUNNING_EMULATOR_CACHE_TTL { + return Ok(running.clone()); + } + } + if !self.adb_path().exists() { + return Ok(HashMap::new()); + } + let output = self.run_adb(["devices"])?; + let mut result = HashMap::new(); + for line in output.lines().skip(1) { + let mut parts = line.split_whitespace(); + let Some(serial) = parts.next() else { continue }; + let Some(state) = parts.next() else { continue }; + if state != "device" || !serial.starts_with("emulator-") { + continue; + } + if let Some(name) = self.avd_name_for_serial(serial) { + result.insert(name, serial.to_owned()); + } + } + *cache.lock().unwrap() = Some((Instant::now(), result.clone())); + Ok(result) + } + + fn avd_name_for_serial(&self, serial: &str) -> Option { + for property in ["ro.boot.qemu.avd_name", "ro.kernel.qemu.avd_name"] { + if let Ok(output) = self.run_adb(["-s", serial, "shell", "getprop", property]) { + let name = output.trim(); + if !name.is_empty() { + return Some(name.to_owned()); + } + } + } + self.run_adb(["-s", serial, "emu", "avd", "name"]) + .ok() + .and_then(|output| { + output + .lines() + .map(str::trim) + .find(|line| !line.is_empty() && *line != "OK") + .map(ToOwned::to_owned) + }) + } + + fn grpc_port_for_avd(&self, avd_name: &str) -> Result { + static CACHE: OnceLock>> = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(None)); + if let Some((updated_at, ports)) = cache.lock().unwrap().as_ref() { + if updated_at.elapsed() < AVD_GRPC_PORT_CACHE_TTL { + if let Some(port) = ports.get(avd_name) { + return Ok(*port); + } + } + } + + let ports = self + .run_emulator(["-list-avds"])? + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .enumerate() + .map(|(index, name)| (name.to_owned(), DEFAULT_GRPC_PORT_BASE + index as u16)) + .collect::>(); + let port = ports + .get(avd_name) + .copied() + .ok_or_else(|| AppError::not_found(format!("Unknown Android AVD `{avd_name}`.")))?; + *cache.lock().unwrap() = Some((Instant::now(), ports)); + Ok(port) + } + + fn screen_size_for_serial(&self, serial: &str) -> Result<(f64, f64), AppError> { + let metrics = self.display_metrics_for_serial(serial)?; + Ok((metrics.width, metrics.height)) + } + + fn display_metrics_for_serial(&self, serial: &str) -> Result { + let cache = android_display_metrics_cache(); + if let Some((updated_at, metrics)) = cache.lock().unwrap().get(serial) { + if updated_at.elapsed() < SCREEN_SIZE_CACHE_TTL { + return Ok(*metrics); + } + } + let output = self.run_adb(["-s", serial, "shell", "dumpsys", "display"])?; + let metrics = parse_android_display_metrics(&output) + .or_else(|| self.wm_display_metrics_for_serial(serial).ok()) + .ok_or_else(|| AppError::native("Android emulator did not report display metrics."))?; + cache + .lock() + .unwrap() + .insert(serial.to_owned(), (Instant::now(), metrics)); + Ok(metrics) + } + + fn wm_display_metrics_for_serial( + &self, + serial: &str, + ) -> Result { + let output = self.run_adb(["-s", serial, "shell", "wm", "size"])?; + let size = output + .split_whitespace() + .find(|part| part.contains('x')) + .ok_or_else(|| AppError::native("Android emulator did not report a screen size."))?; + let (width, height) = size + .split_once('x') + .ok_or_else(|| AppError::native("Android emulator reported an invalid screen size."))?; + let width = width + .parse::() + .map_err(|_| AppError::native("Android emulator reported an invalid width."))?; + let height = height + .parse::() + .map_err(|_| AppError::native("Android emulator reported an invalid height."))?; + Ok(AndroidDisplayMetrics { + width, + height, + rotation_quarter_turns: 0, + corner_radii: AndroidCornerRadii::ZERO, + }) + } + + fn invalidate_display_metrics_for_serial(&self, serial: &str) { + android_display_metrics_cache() + .lock() + .unwrap() + .remove(serial); + } + + fn run_adb_shell(&self, serial: &str, script: &str) -> Result { + self.run_adb(["-s", serial, "shell", script]) + } + + fn run_adb(&self, args: [&str; N]) -> Result { + run_command_text(self.adb_path(), args) + } + + fn run_adb_bytes(&self, args: [&str; N]) -> Result, AppError> { + run_command_bytes(self.adb_path(), args) + } + + fn run_emulator(&self, args: [&str; N]) -> Result { + run_command_text(self.emulator_path(), args) + } + + fn adb_path(&self) -> PathBuf { + sdk_root().join("platform-tools/adb") + } + + fn emulator_path(&self) -> PathBuf { + sdk_root().join("emulator/emulator") + } + + fn avd_dir(&self, avd_name: &str) -> PathBuf { + home_dir().join(format!(".android/avd/{avd_name}.avd")) + } +} + +impl AndroidGrpcFrameStream { + pub async fn next_frame(&mut self) -> Result, AppError> { + let Some(image) = self.inner.message().await.map_err(|error| { + AppError::native(format!( + "Android emulator screenshot stream failed: {error}" + )) + })? + else { + return Ok(None); + }; + let format = image.format.ok_or_else(|| { + AppError::native("Android emulator screenshot did not include an image format.") + })?; + let width = if format.width > 0 { + format.width + } else { + image.width + }; + let height = if format.height > 0 { + format.height + } else { + image.height + }; + if width == 0 || height == 0 { + return Err(AppError::native( + "Android emulator screenshot did not include dimensions.", + )); + } + let rgba = rgba_display_order( + &image.image, + width, + height, + grpc::image_format::ImgFormat::try_from(format.format) + .unwrap_or(grpc::image_format::ImgFormat::Rgba8888), + )?; + let (width, height, mut rgba) = + normalize_android_frame_orientation(width, height, rgba, self.target); + flatten_android_frame_alpha(&mut rgba, width, height); + Ok(Some(AndroidFrame { + width, + height, + timestamp_us: image.timestamp_us, + rgba, + })) + } +} + +fn normalize_android_frame_orientation( + width: u32, + height: u32, + mut rgba: Vec, + target: Option, +) -> (u32, u32, Vec) { + let Some(target) = target else { + return (width, height, rgba); + }; + if width == 0 || height == 0 || target.width == 0 || target.height == 0 { + return (width, height, rgba); + } + + let (width, height) = if (width > height) == (target.width > target.height) { + (width, height) + } else { + rgba = if target.rotation_quarter_turns == 3 { + rotate_rgba_counterclockwise(&rgba, width, height) + } else { + rotate_rgba_clockwise(&rgba, width, height) + }; + (height, width) + }; + + if target.width > target.height { + rotate_rgba_180_in_place(&mut rgba, width, height); + } + + (width, height, rgba) +} + +fn rotate_rgba_clockwise(rgba: &[u8], width: u32, height: u32) -> Vec { + let width = width as usize; + let height = height as usize; + let mut out = vec![0; rgba.len()]; + for y in 0..height { + for x in 0..width { + let src = (y * width + x) * 4; + let dst_x = height - 1 - y; + let dst_y = x; + let dst = (dst_y * height + dst_x) * 4; + out[dst..dst + 4].copy_from_slice(&rgba[src..src + 4]); + } + } + out +} + +fn rotate_rgba_180_in_place(rgba: &mut [u8], width: u32, height: u32) { + let pixel_count = width as usize * height as usize; + for pixel in 0..(pixel_count / 2) { + let opposite = pixel_count - 1 - pixel; + for channel in 0..4 { + rgba.swap(pixel * 4 + channel, opposite * 4 + channel); + } + } +} + +fn rotate_rgba_counterclockwise(rgba: &[u8], width: u32, height: u32) -> Vec { + let width = width as usize; + let height = height as usize; + let mut out = vec![0; rgba.len()]; + for y in 0..height { + for x in 0..width { + let src = (y * width + x) * 4; + let dst_x = y; + let dst_y = width - 1 - x; + let dst = (dst_y * height + dst_x) * 4; + out[dst..dst + 4].copy_from_slice(&rgba[src..src + 4]); + } + } + out +} + +fn flatten_android_frame_alpha(rgba: &mut [u8], width: u32, height: u32) { + if !rgba.chunks_exact(4).any(|pixel| pixel[3] != 255) { + return; + } + + let width = width as usize; + let height = height as usize; + let Some(default_fill) = first_opaque_rgb(rgba) else { + for pixel in rgba.chunks_exact_mut(4) { + pixel[3] = 255; + } + return; + }; + + for y in 0..height { + let row_start = y * width * 4; + let row = &mut rgba[row_start..row_start + width * 4]; + let mut fill = first_opaque_rgb(row).unwrap_or(default_fill); + for pixel in row.chunks_exact_mut(4) { + if pixel[3] == 255 { + fill = [pixel[0], pixel[1], pixel[2]]; + continue; + } + composite_pixel_over_rgb(pixel, fill); + } + } +} + +fn first_opaque_rgb(rgba: &[u8]) -> Option<[u8; 3]> { + rgba.chunks_exact(4) + .find(|pixel| pixel[3] == 255) + .map(|pixel| [pixel[0], pixel[1], pixel[2]]) +} + +fn composite_pixel_over_rgb(pixel: &mut [u8], background: [u8; 3]) { + let alpha = u32::from(pixel[3]); + if alpha == 0 { + pixel[0] = background[0]; + pixel[1] = background[1]; + pixel[2] = background[2]; + pixel[3] = 255; + return; + } + + for channel in 0..3 { + pixel[channel] = ((u32::from(pixel[channel]) * alpha + + u32::from(background[channel]) * (255 - alpha) + + 127) + / 255) as u8; + } + pixel[3] = 255; +} + +fn run_command_text(program: PathBuf, args: [&str; N]) -> Result { + let output = run_command(program, args)?; + String::from_utf8(output) + .map_err(|error| AppError::native(format!("Command returned non-UTF8 output: {error}"))) +} + +fn run_command_bytes( + program: PathBuf, + args: [&str; N], +) -> Result, AppError> { + run_command(program, args) +} + +fn run_command(program: PathBuf, args: [&str; N]) -> Result, AppError> { + if !program.exists() { + return Err(AppError::native(format!( + "Android SDK binary not found at {}.", + program.display() + ))); + } + let mut child = Command::new(&program) + .args(args) + .env("ANDROID_HOME", sdk_root()) + .env("ANDROID_SDK_ROOT", sdk_root()) + .env("JAVA_HOME", java_home()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| { + AppError::native(format!("Unable to run {}: {error}", program.display())) + })?; + let stdout = child.stdout.take().ok_or_else(|| { + AppError::native(format!( + "Unable to capture stdout from {}.", + program.display() + )) + })?; + let stderr = child.stderr.take().ok_or_else(|| { + AppError::native(format!( + "Unable to capture stderr from {}.", + program.display() + )) + })?; + let stdout_reader = thread::spawn(move || read_command_stream(stdout)); + let stderr_reader = thread::spawn(move || read_command_stream(stderr)); + let deadline = Instant::now() + ANDROID_COMMAND_TIMEOUT; + let status = loop { + match child.try_wait().map_err(|error| { + AppError::native(format!("Unable to wait for {}: {error}", program.display())) + })? { + Some(status) => break status, + None if Instant::now() >= deadline => { + let _ = child.kill(); + let _ = child.wait().map_err(|error| { + AppError::native(format!( + "Unable to wait for timed-out {}: {error}", + program.display() + )) + })?; + let stdout = join_command_reader(stdout_reader, &program, "stdout")?; + let stderr = join_command_reader(stderr_reader, &program, "stderr")?; + let stderr_detail = command_stream_summary(&stderr) + .map(|summary| format!(": {summary}")) + .unwrap_or_default(); + return Err(AppError::native(format!( + "{} timed out after {}s{} (stdout {} bytes, stderr {} bytes)", + command_name(&program), + ANDROID_COMMAND_TIMEOUT.as_secs(), + stderr_detail, + stdout.len(), + stderr.len() + ))); + } + None => thread::sleep(Duration::from_millis(25)), + } + }; + let stdout = join_command_reader(stdout_reader, &program, "stdout")?; + let stderr = join_command_reader(stderr_reader, &program, "stderr")?; + if status.success() { + let stderr_text = String::from_utf8_lossy(&stderr); + if stderr_text.contains("No shell command implementation") { + return Err(AppError::native(stderr_text.trim().to_owned())); + } + return Ok(stdout); + } + let stderr = command_stream_summary(&stderr).unwrap_or_default(); + let stdout = command_stream_summary(&stdout); + Err(AppError::native(format!( + "{} failed: {}{}", + command_name(&program), + stderr, + stdout.map(|value| format!(" {value}")).unwrap_or_default() + ))) +} + +fn read_command_stream(mut stream: impl Read) -> std::io::Result> { + let mut buffer = Vec::new(); + stream.read_to_end(&mut buffer)?; + Ok(buffer) +} + +fn join_command_reader( + reader: thread::JoinHandle>>, + program: &Path, + stream_name: &str, +) -> Result, AppError> { + reader + .join() + .map_err(|_| { + AppError::native(format!( + "Unable to read {stream_name} from {}.", + program.display() + )) + })? + .map_err(|error| { + AppError::native(format!( + "Unable to read {stream_name} from {}: {error}", + program.display() + )) + }) +} + +fn command_name(program: &Path) -> &str { + program + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("Android command") +} + +fn command_stream_summary(bytes: &[u8]) -> Option { + let text = String::from_utf8_lossy(bytes); + let trimmed = text.trim(); + if trimmed.is_empty() { + return None; + } + let mut summary = trimmed.chars().take(2000).collect::(); + if trimmed.chars().nth(2000).is_some() { + summary.push_str("..."); + } + Some(summary) +} + +fn sdk_root() -> PathBuf { + env::var_os("ANDROID_HOME") + .or_else(|| env::var_os("ANDROID_SDK_ROOT")) + .map(PathBuf::from) + .filter(|path| path.exists()) + .unwrap_or_else(|| home_dir().join("Library/Android/sdk")) +} + +fn java_home() -> OsString { + env::var_os("JAVA_HOME").unwrap_or_else(|| OsString::from("/opt/homebrew/opt/openjdk")) +} + +fn home_dir() -> PathBuf { + env::var_os("HOME") + .map(PathBuf::from) + .unwrap_or_else(|| Path::new("/").to_path_buf()) +} + +fn emulator_grpc_token(port: u16) -> Option { + per_instance_grpc_token(port).or_else(global_grpc_token) +} + +fn per_instance_grpc_token(port: u16) -> Option { + let running_dir = home_dir().join("Library/Caches/TemporaryItems/avd/running"); + let entries = std::fs::read_dir(running_dir).ok()?; + let port_value = port.to_string(); + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("ini") { + continue; + } + let contents = std::fs::read_to_string(path).ok()?; + let fields = parse_ini(&contents); + if fields.get("grpc.port") == Some(&port_value) { + if let Some(token) = fields.get("grpc.token").filter(|token| !token.is_empty()) { + return Some(token.to_owned()); + } + } + } + None +} + +fn global_grpc_token() -> Option { + std::fs::read_to_string(home_dir().join(".emulator_console_auth_token")) + .ok() + .map(|token| token.trim().to_owned()) + .filter(|token| !token.is_empty()) +} + +fn parse_ini(contents: &str) -> HashMap { + contents + .lines() + .filter_map(|line| { + let line = line.trim(); + let (key, value) = line.split_once('=')?; + Some((key.trim().to_owned(), value.trim().to_owned())) + }) + .collect() +} + +fn rgba_display_order( + image: &[u8], + width: u32, + height: u32, + format: grpc::image_format::ImgFormat, +) -> Result, AppError> { + let width = width as usize; + let height = height as usize; + match format { + grpc::image_format::ImgFormat::Rgba8888 => { + let row_len = width * 4; + if image.len() < row_len * height { + return Err(AppError::native( + "Android emulator returned a truncated RGBA frame.", + )); + } + Ok(image[..row_len * height].to_vec()) + } + grpc::image_format::ImgFormat::Rgb888 => { + let src_row_len = width * 3; + if image.len() < src_row_len * height { + return Err(AppError::native( + "Android emulator returned a truncated RGB frame.", + )); + } + let mut out = BytesMut::with_capacity(width * height * 4); + out.resize(width * height * 4, 255); + for y in 0..height { + let src_row = y * src_row_len; + let dst_row = y * width * 4; + for x in 0..width { + let src = src_row + x * 3; + let dst = dst_row + x * 4; + out[dst] = image[src]; + out[dst + 1] = image[src + 1]; + out[dst + 2] = image[src + 2]; + out[dst + 3] = 255; + } + } + Ok(out.to_vec()) + } + grpc::image_format::ImgFormat::Png => Err(AppError::native( + "Android emulator gRPC returned PNG instead of raw pixels.", + )), + } +} + +fn extract_xml(output: &str) -> &str { + output + .find(", depth: usize, max_depth: usize) -> Value { + let bounds = parse_bounds(node.attribute("bounds").unwrap_or("")); + let class_name = node.attribute("class").unwrap_or(""); + let short_class = class_name.rsplit('.').next().unwrap_or(class_name); + let text = node.attribute("text").unwrap_or(""); + let content_desc = node.attribute("content-desc").unwrap_or(""); + let label = if !text.is_empty() { text } else { content_desc }; + let resource_id = node.attribute("resource-id").unwrap_or(""); + let role = android_role(node, short_class); + let mut children = Vec::new(); + if depth < max_depth { + for child in node.children().filter(|child| child.has_tag_name("node")) { + children.push(android_node_value(child, depth + 1, max_depth)); + } + } + json!({ + "source": "android-uiautomator", + "type": android_type(short_class, class_name), + "role": role, + "className": class_name, + "AXIdentifier": resource_id, + "AXLabel": label, + "AXValue": text, + "androidClass": class_name, + "androidPackage": node.attribute("package").unwrap_or(""), + "androidResourceId": resource_id, + "checkable": bool_attr(node, "checkable"), + "checked": bool_attr(node, "checked"), + "clickable": bool_attr(node, "clickable"), + "focusable": bool_attr(node, "focusable"), + "focused": bool_attr(node, "focused"), + "longClickable": bool_attr(node, "long-clickable"), + "password": bool_attr(node, "password"), + "scrollable": bool_attr(node, "scrollable"), + "selected": bool_attr(node, "selected"), + "text": text, + "title": label, + "enabled": bool_attr(node, "enabled"), + "isHidden": node.attribute("visible-to-user") == Some("false"), + "frame": frame_value(bounds.0, bounds.1, bounds.2, bounds.3), + "frameInScreen": frame_value(bounds.0, bounds.1, bounds.2, bounds.3), + "children": children, + }) +} + +fn parse_bounds(value: &str) -> (f64, f64, f64, f64) { + let numbers = value + .replace("][", ",") + .replace(['[', ']'], "") + .split(',') + .filter_map(|part| part.parse::().ok()) + .collect::>(); + if numbers.len() != 4 { + return (0.0, 0.0, 0.0, 0.0); + } + ( + numbers[0], + numbers[1], + (numbers[2] - numbers[0]).max(0.0), + (numbers[3] - numbers[1]).max(0.0), + ) +} + +fn frame_value(x: f64, y: f64, width: f64, height: f64) -> Value { + json!({ "x": x, "y": y, "width": width, "height": height }) +} + +fn bool_attr(node: roxmltree::Node<'_, '_>, name: &str) -> bool { + node.attribute(name) == Some("true") +} + +fn android_type(short_class: &str, class_name: &str) -> String { + let fallback = if short_class.is_empty() { + class_name + } else { + short_class + }; + if fallback.is_empty() { + "View".to_owned() + } else { + fallback.to_owned() + } +} + +fn android_display_metrics_cache() -> &'static Mutex { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn parse_android_display_metrics(output: &str) -> Option { + let rotation = parse_android_display_rotation(output).unwrap_or(0); + let corner_radii = parse_android_rounded_corners(output).unwrap_or(AndroidCornerRadii::ZERO); + if let Some(line) = output + .lines() + .find(|line| line.contains("mOverrideDisplayInfo=DisplayInfo")) + { + if let Some((width, height)) = parse_display_info_app_size(line) { + return Some(AndroidDisplayMetrics { + width, + height, + rotation_quarter_turns: rotation, + corner_radii, + }); + } + } + if let Some((width, height)) = output.lines().find_map(parse_logical_frame_size) { + return Some(AndroidDisplayMetrics { + width, + height, + rotation_quarter_turns: rotation, + corner_radii, + }); + } + None +} + +fn parse_android_rounded_corners(output: &str) -> Option { + let mut radii = AndroidCornerRadii::ZERO; + let mut found = false; + + for chunk in output.split("RoundedCorner{").skip(1) { + let section = chunk.split('}').next().unwrap_or(chunk); + let Some(position) = parse_named_value(section, "position=") else { + continue; + }; + let Some(radius) = + parse_named_value(section, "radius=").and_then(|value| value.parse::().ok()) + else { + continue; + }; + match position { + "TopLeft" => radii.top_left = radius, + "TopRight" => radii.top_right = radius, + "BottomRight" => radii.bottom_right = radius, + "BottomLeft" => radii.bottom_left = radius, + _ => continue, + } + found = true; + } + + found.then_some(radii) +} + +fn parse_named_value<'a>(input: &'a str, key: &str) -> Option<&'a str> { + let (_, value) = input.split_once(key)?; + Some(value.split([',', '}', ']']).next().unwrap_or(value).trim()) +} + +fn parse_android_display_rotation(output: &str) -> Option { + output + .lines() + .find(|line| line.contains("mOverrideDisplayInfo=DisplayInfo")) + .and_then(parse_display_info_rotation) + .or_else(|| { + output + .lines() + .find_map(|line| { + line.split_once("mCurrentOrientation=") + .map(|(_, value)| value) + }) + .and_then(parse_rotation_token) + }) +} + +fn parse_display_info_app_size(line: &str) -> Option<(f64, f64)> { + let (_, value) = line.rsplit_once(", app ")?; + parse_size_prefix(value) +} + +fn parse_display_info_rotation(line: &str) -> Option { + let (_, value) = line.rsplit_once(", rotation ")?; + parse_rotation_token(value) +} + +fn parse_rotation_token(value: &str) -> Option { + let digits = value + .trim() + .chars() + .take_while(|character| character.is_ascii_digit()) + .collect::(); + let rotation = digits.parse::().ok()?; + Some(rotation % 4) +} + +fn parse_size_prefix(value: &str) -> Option<(f64, f64)> { + let mut parts = value.split_whitespace(); + let width = parts.next()?.trim_end_matches(',').parse::().ok()?; + if parts.next()? != "x" { + return None; + } + let height = parts.next()?.trim_end_matches(',').parse::().ok()?; + Some((width, height)) +} + +fn parse_logical_frame_size(line: &str) -> Option<(f64, f64)> { + let (_, value) = line.split_once("logicalFrame=Rect(")?; + let (frame, _) = value.split_once(')')?; + let (_, max_values) = frame.split_once(" - ")?; + let (width, height) = max_values.split_once(',')?; + Some((width.trim().parse().ok()?, height.trim().parse().ok()?)) +} + +fn android_role(node: roxmltree::Node<'_, '_>, class_name: &str) -> &'static str { + let clickable = bool_attr(node, "clickable"); + let scrollable = bool_attr(node, "scrollable"); + match class_name { + "Button" | "ImageButton" | "FloatingActionButton" => "button", + "EditText" => "textField", + "TextView" => "staticText", + "ImageView" => "image", + "CheckBox" => "checkBox", + "RadioButton" => "radioButton", + "Switch" | "ToggleButton" => "switch", + "SeekBar" => "slider", + "RecyclerView" | "ListView" | "GridView" => "collection", + "ScrollView" | "HorizontalScrollView" | "NestedScrollView" | "ViewPager" => "scrollView", + "WebView" => "webView", + "ProgressBar" => "progressIndicator", + "Spinner" => "popUpButton", + "TabWidget" | "TabLayout" => "tabGroup", + "Toolbar" | "ActionBar" => "toolbar", + "ViewGroup" | "FrameLayout" | "LinearLayout" | "RelativeLayout" | "ConstraintLayout" + | "CoordinatorLayout" | "DrawerLayout" => "container", + _ if scrollable => "scrollView", + _ if clickable => "button", + _ => "view", + } +} + +fn android_key_code(hid: u16) -> u16 { + match hid { + 4..=29 => 29 + (hid - 4), + 30 => 8, + 31 => 9, + 32 => 10, + 33 => 11, + 34 => 12, + 35 => 13, + 36 => 14, + 37 => 15, + 38 => 16, + 39 => 7, + 40 => 66, + 41 => 111, + 42 => 67, + 43 => 61, + 44 => 62, + 45 => 69, + 46 => 70, + 47 => 71, + 48 => 72, + 49 => 73, + 51 => 74, + 52 => 75, + 53 => 68, + 54 => 55, + 55 => 56, + 56 => 76, + 57 => 115, + 58..=69 => 131 + (hid - 58), + 73 => 124, + 74 => 122, + 75 => 92, + 76 => 112, + 77 => 123, + 78 => 93, + 79 => 22, + 80 => 21, + 81 => 20, + 82 => 19, + _ => hid, + } +} + +fn hid_text_for_key(hid: u16, modifiers: u32) -> Option { + if modifiers & (MODIFIER_CONTROL | MODIFIER_OPTION | MODIFIER_COMMAND) != 0 { + return None; + } + + let shifted = modifiers & MODIFIER_SHIFT != 0; + let caps_locked = modifiers & MODIFIER_CAPS_LOCK != 0; + + if (4..=29).contains(&hid) { + let offset = (hid - 4) as u8; + let base = if shifted ^ caps_locked { b'A' } else { b'a' }; + return Some(char::from(base + offset).to_string()); + } + + let text = match (hid, shifted) { + (30, false) => "1", + (30, true) => "!", + (31, false) => "2", + (31, true) => "@", + (32, false) => "3", + (32, true) => "#", + (33, false) => "4", + (33, true) => "$", + (34, false) => "5", + (34, true) => "%", + (35, false) => "6", + (35, true) => "^", + (36, false) => "7", + (36, true) => "&", + (37, false) => "8", + (37, true) => "*", + (38, false) => "9", + (38, true) => "(", + (39, false) => "0", + (39, true) => ")", + (44, _) => " ", + (45, false) => "-", + (45, true) => "_", + (46, false) => "=", + (46, true) => "+", + (47, false) => "[", + (47, true) => "{", + (48, false) => "]", + (48, true) => "}", + (49, false) => "\\", + (49, true) => "|", + (51, false) => ";", + (51, true) => ":", + (52, false) => "'", + (52, true) => "\"", + (53, false) => "`", + (53, true) => "~", + (54, false) => ",", + (54, true) => "<", + (55, false) => ".", + (55, true) => ">", + (56, false) => "/", + (56, true) => "?", + _ => return None, + }; + Some(text.to_owned()) +} + +fn android_input_text_arg(text: &str) -> String { + let mut escaped = String::new(); + for character in text.chars() { + match character { + ' ' => escaped.push_str("%s"), + '%' => escaped.push_str("%25"), + '&' | '(' | ')' | '<' | '>' | ';' | '|' | '*' | '\\' | '"' | '\'' | '`' | '$' => { + escaped.push('\\'); + escaped.push(character); + } + _ => escaped.push(character), + } + } + escaped +} + +fn has_android_key_modifiers(modifiers: u32) -> bool { + modifiers & (MODIFIER_SHIFT | MODIFIER_CONTROL | MODIFIER_OPTION | MODIFIER_COMMAND) != 0 +} + +fn android_modifier_key_codes(modifiers: u32) -> Vec { + let mut keys = Vec::new(); + if modifiers & MODIFIER_CONTROL != 0 { + keys.push(113); + } + if modifiers & MODIFIER_OPTION != 0 { + keys.push(57); + } + if modifiers & MODIFIER_SHIFT != 0 { + keys.push(59); + } + if modifiers & MODIFIER_COMMAND != 0 { + keys.push(117); + } + keys +} + +fn android_log_level(line: &str) -> &'static str { + if line.contains(" E ") { + "error" + } else if line.contains(" W ") { + "warning" + } else if line.contains(" D ") { + "debug" + } else { + "info" + } +} + +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +fn ensure_android_clipboard_available(output: &str) -> Result<(), AppError> { + if output.contains("No shell command implementation") { + return Err(AppError::native( + "Android clipboard shell service is not implemented on this emulator image.", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn android_nodes_keep_class_type_and_semantic_role() { + let document = roxmltree::Document::parse( + r#""#, + ) + .unwrap(); + + let value = android_node_value(document.root_element(), 0, 10); + + assert_eq!(value["type"], "ViewGroup"); + assert_eq!(value["role"], "container"); + assert_eq!(value["AXIdentifier"], "com.example:id/hotseat"); + assert_eq!(value["androidClass"], "android.view.ViewGroup"); + assert_eq!(value["androidResourceId"], "com.example:id/hotseat"); + assert_eq!(value["enabled"], true); + } + + #[test] + fn clickable_unknown_android_nodes_are_buttons() { + let document = roxmltree::Document::parse( + r#""#, + ) + .unwrap(); + + let value = android_node_value(document.root_element(), 0, 10); + + assert_eq!(value["type"], "CustomTile"); + assert_eq!(value["role"], "button"); + } + + #[test] + fn android_touch_gesture_resolves_tap_on_end() { + let mut active = None; + + assert_eq!( + update_touch_gesture(&mut active, 0.4, 0.6, "began").unwrap(), + AndroidTouchAction::None + ); + assert_eq!( + update_touch_gesture(&mut active, 0.41, 0.6, "ended").unwrap(), + AndroidTouchAction::Tap { x: 0.41, y: 0.6 } + ); + } + + #[test] + fn android_touch_gesture_resolves_swipe_on_end() { + let mut active = None; + + assert_eq!( + update_touch_gesture(&mut active, 0.1, 0.2, "began").unwrap(), + AndroidTouchAction::None + ); + assert_eq!( + update_touch_gesture(&mut active, 0.8, 0.2, "ended").unwrap(), + AndroidTouchAction::Swipe { + start_x: 0.1, + start_y: 0.2, + end_x: 0.8, + end_y: 0.2, + duration_ms: 80, + } + ); + } + + #[test] + fn android_key_code_maps_usb_hid_keyboard_usages() { + assert_eq!(android_key_code(4), 29); + assert_eq!(android_key_code(29), 54); + assert_eq!(android_key_code(30), 8); + assert_eq!(android_key_code(39), 7); + assert_eq!(android_key_code(40), 66); + assert_eq!(android_key_code(42), 67); + assert_eq!(android_key_code(58), 131); + assert_eq!(android_key_code(69), 142); + assert_eq!(android_key_code(73), 124); + assert_eq!(android_key_code(79), 22); + } + + #[test] + fn hid_text_for_key_uses_shift_and_caps_for_printable_input() { + assert_eq!(hid_text_for_key(4, 0).as_deref(), Some("a")); + assert_eq!(hid_text_for_key(4, MODIFIER_SHIFT).as_deref(), Some("A")); + assert_eq!( + hid_text_for_key(4, MODIFIER_CAPS_LOCK).as_deref(), + Some("A") + ); + assert_eq!( + hid_text_for_key(4, MODIFIER_SHIFT | MODIFIER_CAPS_LOCK).as_deref(), + Some("a") + ); + assert_eq!(hid_text_for_key(30, 0).as_deref(), Some("1")); + assert_eq!(hid_text_for_key(30, MODIFIER_SHIFT).as_deref(), Some("!")); + assert_eq!(hid_text_for_key(56, MODIFIER_SHIFT).as_deref(), Some("?")); + assert_eq!(hid_text_for_key(80, 0), None); + assert_eq!(hid_text_for_key(4, MODIFIER_COMMAND), None); + } + + #[test] + fn android_input_text_arg_escapes_adb_shell_text() { + assert_eq!(android_input_text_arg("hello world"), "hello%sworld"); + assert_eq!(android_input_text_arg("100%"), "100%25"); + assert_eq!(android_input_text_arg("a&b"), "a\\&b"); + } + + #[test] + fn android_modifier_key_codes_match_android_meta_keys() { + assert_eq!(android_modifier_key_codes(MODIFIER_CONTROL), vec![113]); + assert_eq!(android_modifier_key_codes(MODIFIER_OPTION), vec![57]); + assert_eq!(android_modifier_key_codes(MODIFIER_SHIFT), vec![59]); + assert_eq!(android_modifier_key_codes(MODIFIER_COMMAND), vec![117]); + assert_eq!( + android_modifier_key_codes(MODIFIER_CONTROL | MODIFIER_SHIFT), + vec![113, 59] + ); + } + + #[test] + fn parse_android_display_metrics_prefers_current_app_size() { + let output = r#" + mViewports=[DisplayViewport{type=INTERNAL, logicalFrame=Rect(0, 0 - 2400, 1080), physicalFrame=Rect(0, 0 - 2400, 1080)}] + mCurrentOrientation=3 + mOverrideDisplayInfo=DisplayInfo{"Built-in Screen", real 2400 x 1080, largest app 2400 x 2400, smallest app 1080 x 1080, rotation 3, state ON, app 2400 x 1080, density 420} +"#; + + assert_eq!( + parse_android_display_metrics(output), + Some(AndroidDisplayMetrics { + width: 2400.0, + height: 1080.0, + rotation_quarter_turns: 3, + corner_radii: AndroidCornerRadii::ZERO, + }) + ); + } + + #[test] + fn parse_android_display_metrics_falls_back_to_logical_frame() { + let output = r#" + mViewports=[DisplayViewport{type=INTERNAL, logicalFrame=Rect(0, 0 - 1080, 2400), physicalFrame=Rect(0, 0 - 1080, 2400)}] + mCurrentOrientation=0 +"#; + + assert_eq!( + parse_android_display_metrics(output), + Some(AndroidDisplayMetrics { + width: 1080.0, + height: 2400.0, + rotation_quarter_turns: 0, + corner_radii: AndroidCornerRadii::ZERO, + }) + ); + } + + #[test] + fn parse_android_display_metrics_reads_rounded_corners() { + let output = r#" +DisplayDeviceInfo{"Built-in Screen", 1080 x 2400, roundedCorners RoundedCorners{[RoundedCorner{position=TopLeft, radius=104, center=Point(104, 104)}, RoundedCorner{position=TopRight, radius=104, center=Point(976, 104)}, RoundedCorner{position=BottomRight, radius=102, center=Point(978, 2298)}, RoundedCorner{position=BottomLeft, radius=102, center=Point(102, 2298)}]}} + mViewports=[DisplayViewport{type=INTERNAL, logicalFrame=Rect(0, 0 - 1080, 2400), physicalFrame=Rect(0, 0 - 1080, 2400)}] + mCurrentOrientation=0 +"#; + + assert_eq!( + parse_android_display_metrics(output), + Some(AndroidDisplayMetrics { + width: 1080.0, + height: 2400.0, + rotation_quarter_turns: 0, + corner_radii: AndroidCornerRadii { + top_left: 104.0, + top_right: 104.0, + bottom_right: 102.0, + bottom_left: 102.0, + }, + }) + ); + } + + #[test] + fn android_frame_orientation_rotates_when_stream_aspect_is_swapped() { + let rgba = vec![ + 1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, // + 4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255, + ]; + + let (width, height, rotated) = normalize_android_frame_orientation( + 3, + 2, + rgba, + Some(AndroidFrameTarget { + width: 2, + height: 3, + rotation_quarter_turns: 0, + }), + ); + + assert_eq!((width, height), (2, 3)); + assert_eq!( + rotated, + vec![ + 4, 0, 0, 255, 1, 0, 0, 255, // + 5, 0, 0, 255, 2, 0, 0, 255, // + 6, 0, 0, 255, 3, 0, 0, 255, + ] + ); + } + + #[test] + fn android_frame_orientation_keeps_matching_aspect() { + let rgba = vec![1, 2, 3, 255, 5, 6, 7, 255]; + + let (width, height, out) = normalize_android_frame_orientation( + 1, + 2, + rgba.clone(), + Some(AndroidFrameTarget { + width: 1080, + height: 2400, + rotation_quarter_turns: 0, + }), + ); + + assert_eq!((width, height), (1, 2)); + assert_eq!(out, rgba); + } + + #[test] + fn android_frame_orientation_flips_landscape_streams() { + let rgba = vec![ + 1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, // + 4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255, + ]; + + let (width, height, rotated) = normalize_android_frame_orientation( + 3, + 2, + rgba, + Some(AndroidFrameTarget { + width: 3, + height: 2, + rotation_quarter_turns: 1, + }), + ); + + assert_eq!((width, height), (3, 2)); + assert_eq!( + rotated, + vec![ + 6, 0, 0, 255, 5, 0, 0, 255, 4, 0, 0, 255, // + 3, 0, 0, 255, 2, 0, 0, 255, 1, 0, 0, 255, + ] + ); + } + + #[test] + fn android_frame_alpha_flattens_transparent_edges_to_nearest_row_pixel() { + let mut rgba = vec![0, 0, 0, 0, 10, 20, 30, 255, 40, 50, 60, 255, 0, 0, 0, 0]; + + flatten_android_frame_alpha(&mut rgba, 4, 1); + + assert_eq!( + rgba, + vec![10, 20, 30, 255, 10, 20, 30, 255, 40, 50, 60, 255, 40, 50, 60, 255,] + ); + } + + #[test] + fn android_frame_alpha_composites_partial_alpha() { + let mut rgba = vec![100, 50, 0, 255, 200, 200, 200, 128]; + + flatten_android_frame_alpha(&mut rgba, 2, 1); + + assert_eq!(rgba, vec![100, 50, 0, 255, 150, 125, 100, 255]); + } + + #[test] + fn android_frame_orientation_rotates_counterclockwise_then_flips_reverse_landscape() { + let rgba = vec![ + 1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, // + 4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255, + ]; + + let (width, height, rotated) = normalize_android_frame_orientation( + 2, + 3, + rgba, + Some(AndroidFrameTarget { + width: 3, + height: 2, + rotation_quarter_turns: 3, + }), + ); + + assert_eq!((width, height), (3, 2)); + assert_eq!( + rotated, + vec![ + 5, 0, 0, 255, 3, 0, 0, 255, 1, 0, 0, 255, // + 6, 0, 0, 255, 4, 0, 0, 255, 2, 0, 0, 255, + ] + ); + } +} + +mod grpc { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ImageTransport { + #[prost(enumeration = "image_transport::TransportChannel", tag = "1")] + pub channel: i32, + #[prost(string, tag = "2")] + pub handle: String, + } + + pub mod image_transport { + #[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration, + )] + #[repr(i32)] + pub enum TransportChannel { + Unspecified = 0, + Mmap = 1, + } + } + + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ImageFormat { + #[prost(enumeration = "image_format::ImgFormat", tag = "1")] + pub format: i32, + #[prost(uint32, tag = "3")] + pub width: u32, + #[prost(uint32, tag = "4")] + pub height: u32, + #[prost(uint32, tag = "5")] + pub display: u32, + #[prost(message, optional, tag = "6")] + pub transport: Option, + } + + pub mod image_format { + #[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration, + )] + #[repr(i32)] + pub enum ImgFormat { + Png = 0, + Rgba8888 = 1, + Rgb888 = 2, + } + } + + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Image { + #[prost(message, optional, tag = "1")] + pub format: Option, + #[prost(uint32, tag = "2")] + pub width: u32, + #[prost(uint32, tag = "3")] + pub height: u32, + #[prost(bytes = "vec", tag = "4")] + pub image: Vec, + #[prost(uint32, tag = "5")] + pub seq: u32, + #[prost(uint64, tag = "6")] + pub timestamp_us: u64, + } +} diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index bb2944b1..9d767552 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -1,3 +1,4 @@ +use crate::android::{self, AndroidBridge}; use crate::api::json::json; use crate::auth; use crate::config::Config; @@ -11,6 +12,7 @@ use crate::simulators::registry::SessionRegistry; use crate::simulators::session::SimulatorSession; use crate::static_files; use crate::transport::packet::FramePacket; +use crate::transport::webrtc::AndroidWebRtcSource; use crate::webkit; use axum::body::Body; use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; @@ -33,13 +35,14 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::net::TcpStream; use tokio::process::Command; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{broadcast, mpsc, Mutex}; use tokio::task; use tokio::time::timeout; use tower_http::trace::{DefaultMakeSpan, DefaultOnFailure, TraceLayer}; use tracing::Level; const SIMULATOR_INVENTORY_CACHE_TTL: Duration = Duration::from_secs(5); +const SIMULATOR_INVENTORY_TIMEOUT: Duration = Duration::from_secs(8); const H264_WS_MAGIC: &[u8; 4] = b"SDH1"; const H264_WS_HEADER_LEN: usize = 40; const H264_WS_FLAG_KEYFRAME: u8 = 1 << 0; @@ -55,6 +58,7 @@ pub struct AppState { pub inspectors: InspectorHub, pub metrics: Arc, pub simulator_inventory: SimulatorInventoryCache, + pub android: AndroidBridge, } #[derive(Clone, Default)] @@ -1341,9 +1345,9 @@ async fn inspector_response( } async fn list_simulators(State(state): State) -> Result, AppError> { - let simulators = list_simulators_cached(state.clone(), false).await?; + let simulators = all_device_values(state.clone(), false).await?; Ok(json(json_value!({ - "simulators": state.registry.enrich_simulators(simulators), + "simulators": simulators, }))) } @@ -1351,6 +1355,16 @@ async fn boot_simulator( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state.clone(), move |android| { + android.boot(&action_udid)?; + android.wait_until_booted(&action_udid, Duration::from_secs(240))?; + Ok(()) + }) + .await?; + return android_simulator_payload(state, udid).await; + } forget_lifecycle_session(&state, &udid); let action_udid = udid.clone(); run_bridge_action(state.clone(), move |bridge| { @@ -1364,6 +1378,11 @@ async fn shutdown_simulator( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state.clone(), move |android| android.shutdown(&action_udid)).await?; + return android_simulator_payload(state, udid).await; + } forget_lifecycle_session(&state, &udid); let action_udid = udid.clone(); run_bridge_action(state.clone(), move |bridge| { @@ -1377,6 +1396,11 @@ async fn erase_simulator( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| android.erase(&action_udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } forget_lifecycle_session(&state, &udid); let action_udid = udid.clone(); run_bridge_action(state, move |bridge| bridge.erase_simulator(&action_udid)).await?; @@ -1401,6 +1425,14 @@ async fn install_app( "Request body must include `appPath`.", )); } + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| { + android.install_app(&action_udid, &payload.app_path) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| { bridge.install_app(&action_udid, &payload.app_path) @@ -1419,6 +1451,14 @@ async fn uninstall_app( "Request body must include `bundleId`.", )); } + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| { + android.uninstall_app(&action_udid, &payload.bundle_id) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| { bridge.uninstall_app(&action_udid, &payload.bundle_id) @@ -1431,6 +1471,10 @@ async fn get_pasteboard( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let text = run_android_action(state, move |android| android.pasteboard_text(&udid)).await?; + return Ok(json(json_value!({ "text": text }))); + } let text = run_bridge_action(state, move |bridge| bridge.pasteboard_text(&udid)).await?; Ok(json(json_value!({ "text": text }))) } @@ -1440,6 +1484,13 @@ async fn set_pasteboard( Path(udid): Path, Json(payload): Json, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.set_pasteboard_text(&udid, &payload.text) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { bridge.set_pasteboard_text(&udid, &payload.text) }) @@ -1451,7 +1502,11 @@ async fn screenshot_png( State(state): State, Path(udid): Path, ) -> Result<(StatusCode, HeaderMap, Vec), AppError> { - let png = run_bridge_action(state, move |bridge| bridge.screenshot_png(&udid)).await?; + let png = if android::is_android_id(&udid) { + run_android_action(state, move |android| android.screenshot_png(&udid)).await? + } else { + run_bridge_action(state, move |bridge| bridge.screenshot_png(&udid)).await? + }; let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, "image/png".parse().unwrap()); headers.insert( @@ -1465,6 +1520,10 @@ async fn toggle_appearance( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.toggle_appearance(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| bridge.toggle_appearance(&action_udid)).await?; Ok(json(json_value!({ "ok": true }))) @@ -1474,6 +1533,9 @@ async fn refresh_stream( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + return Ok(json(json_value!({ "ok": true, "stream": "screenshot" }))); + } let session = state.registry.get_or_create_async(&udid).await?; if let Err(error) = session.ensure_started_async().await { state.registry.remove(&udid); @@ -1491,6 +1553,14 @@ async fn open_url( if payload.url.trim().is_empty() { return Err(AppError::bad_request("Request body must include `url`.")); } + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| { + android.open_url(&action_udid, &payload.url) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| { bridge.open_url(&action_udid, &payload.url) @@ -1509,6 +1579,14 @@ async fn launch_bundle( "Request body must include `bundleId`.", )); } + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| { + android.launch_package(&action_udid, &payload.bundle_id) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| { bridge.launch_bundle(&action_udid, &payload.bundle_id) @@ -1639,6 +1717,13 @@ async fn send_touch( let x = payload.x.clamp(0.0, 1.0); let y = payload.y.clamp(0.0, 1.0); let phase = payload.phase; + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_touch(&udid, x, y, &phase) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; input.send_touch(x, y, &phase) @@ -1669,6 +1754,24 @@ async fn send_touch_sequence( )); } } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + for event in payload.events { + android.send_touch( + &udid, + event.x.clamp(0.0, 1.0), + event.y.clamp(0.0, 1.0), + &event.phase, + )?; + if let Some(delay_ms) = event.delay_ms_after.filter(|delay_ms| *delay_ms > 0) { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; for event in payload.events { @@ -1692,6 +1795,10 @@ async fn control_socket( Path(udid): Path, websocket: WebSocketUpgrade, ) -> impl IntoResponse { + if android::is_android_id(&udid) { + return websocket + .on_upgrade(move |socket| handle_android_control_socket(state, udid, socket)); + } websocket.on_upgrade(move |socket| handle_control_socket(state, udid, socket)) } @@ -1704,12 +1811,109 @@ async fn h264_socket( websocket.on_upgrade(move |socket| handle_h264_socket(state, udid, query, socket)) } +async fn handle_android_control_socket(state: AppState, udid: String, socket: WebSocket) { + let (mut sender, mut receiver) = socket.split(); + let mut active_touch: Option = None; + let _ = sender + .send(Message::Text( + json_value!({ "type": "ready", "udid": udid, "platform": "android-emulator" }) + .to_string() + .into(), + )) + .await; + while let Some(message) = receiver.next().await { + let text = match message { + Ok(Message::Text(text)) => text, + Ok(Message::Binary(bytes)) => match String::from_utf8(bytes.to_vec()) { + Ok(text) => text.into(), + Err(_) => continue, + }, + Ok(Message::Close(_)) => break, + Ok(Message::Ping(_)) | Ok(Message::Pong(_)) => continue, + Err(_) => break, + }; + let Ok(control_message) = serde_json::from_str::(&text) else { + continue; + }; + let state = state.clone(); + let udid = udid.clone(); + let _ = run_android_control_message(state, udid, control_message, &mut active_touch).await; + } +} + +async fn run_android_control_message( + state: AppState, + udid: String, + message: ControlMessage, + active_touch: &mut Option, +) -> Result<(), AppError> { + match message { + ControlMessage::Touch { x, y, phase } => { + handle_android_control_touch(state, udid, x, y, phase, active_touch).await + } + ControlMessage::EdgeTouch { x, y, phase, .. } => { + handle_android_control_touch(state, udid, x, y, phase, active_touch).await + } + ControlMessage::MultiTouch { x1, y1, phase, .. } => { + handle_android_control_touch(state, udid, x1, y1, phase, active_touch).await + } + other => { + run_android_action(state, move |android| match other { + ControlMessage::Key { + key_code, + modifiers, + } => android.send_key(&udid, key_code, modifiers.unwrap_or(0)), + ControlMessage::Button { + button, + duration_ms, + phase, + .. + } => match phase.as_deref() { + Some("down" | "began") => Ok(()), + Some("up" | "ended" | "cancelled") | None => { + android.press_button(&udid, &button, duration_ms.unwrap_or(0)) + } + Some(_) => Err(AppError::bad_request( + "`phase` must be `down`, `up`, `began`, `ended`, or `cancelled`.", + )), + }, + ControlMessage::DismissKeyboard => android.dismiss_keyboard(&udid), + ControlMessage::Home => android.press_home(&udid), + ControlMessage::AppSwitcher => android.open_app_switcher(&udid), + ControlMessage::RotateLeft => android.rotate_left(&udid), + ControlMessage::RotateRight => android.rotate_right(&udid), + ControlMessage::ToggleAppearance => android.toggle_appearance(&udid), + ControlMessage::Touch { .. } + | ControlMessage::EdgeTouch { .. } + | ControlMessage::MultiTouch { .. } => Ok(()), + }) + .await + } + } +} + +async fn handle_android_control_touch( + state: AppState, + udid: String, + x: f64, + y: f64, + phase: String, + active_touch: &mut Option, +) -> Result<(), AppError> { + let action = android::update_touch_gesture(active_touch, x, y, &phase)?; + if matches!(action, android::AndroidTouchAction::None) { + return Ok(()); + } + run_android_action(state, move |android| action.perform(&android, &udid)).await +} + async fn webrtc_offer( State(state): State, + ConnectInfo(address): ConnectInfo, Path(udid): Path, Json(payload): Json, ) -> Result, AppError> { - crate::transport::webrtc::create_answer(state, udid, payload) + crate::transport::webrtc::create_answer(state, udid, payload, address.ip().is_loopback()) .await .map(Json) } @@ -1720,6 +1924,11 @@ async fn handle_h264_socket( initial_quality: StreamQualityPayload, socket: WebSocket, ) { + if android::is_android_id(&udid) { + handle_android_h264_socket(state, udid, initial_quality, socket).await; + return; + } + if initial_quality.has_any_value() { if let Err(error) = apply_stream_quality_payload(&state, &initial_quality) { tracing::debug!("Failed to apply H264 WebSocket stream quality for {udid}: {error}"); @@ -1831,6 +2040,115 @@ async fn handle_h264_socket( } } +async fn handle_android_h264_socket( + state: AppState, + udid: String, + initial_quality: StreamQualityPayload, + socket: WebSocket, +) { + let source = match AndroidWebRtcSource::start( + state.android.clone(), + state.metrics.clone(), + udid.clone(), + initial_quality.max_edge, + true, + ) + .await + { + Ok(source) => source, + Err(error) => { + tracing::debug!("Failed to create Android H264 WebSocket source for {udid}: {error}"); + return; + } + }; + + let mut subscription = source.subscribe(); + let (mut sender, mut receiver) = socket.split(); + let mut decoder_synced = false; + let mut last_sent_sequence: Option = None; + + let initial_keyframe = source + .wait_for_keyframe(H264_WS_KEYFRAME_WAIT_TIMEOUT) + .await + .filter(|frame| h264_ws_frame_is_decoder_sync(frame)); + + if let Some(keyframe) = initial_keyframe { + if h264_ws_frame_is_supported(&keyframe) { + let message_bytes = h264_ws_frame_message(&keyframe); + let message = Message::Binary(message_bytes); + if timeout(H264_WS_SEND_TIMEOUT, sender.send(message)) + .await + .ok() + .and_then(Result::ok) + .is_none() + { + return; + } + last_sent_sequence = Some(keyframe.frame_sequence); + decoder_synced = true; + } + } else { + source.request_keyframe(); + } + + loop { + tokio::select! { + received = receiver.next() => { + let Some(received) = received else { break }; + let message = match received { + Ok(message) => message, + Err(error) => { + tracing::debug!("Android H264 WebSocket closed for {udid}: {error}"); + break; + } + }; + if !handle_android_h264_socket_message(&state, &source, &message) { + break; + } + } + frame = subscription.recv() => { + let frame = match frame { + Ok(frame) => frame, + Err(broadcast::error::RecvError::Lagged(_)) => { + decoder_synced = false; + source.request_keyframe(); + continue; + } + Err(broadcast::error::RecvError::Closed) => break, + }; + if !h264_ws_frame_is_supported(&frame) { + continue; + } + if last_sent_sequence + .map(|sequence| frame.frame_sequence <= sequence) + .unwrap_or(false) + { + continue; + } + if !decoder_synced && !frame.is_keyframe { + source.request_keyframe(); + continue; + } + let is_keyframe = frame.is_keyframe; + let message_bytes = h264_ws_frame_message(&frame); + let message = Message::Binary(message_bytes); + if timeout(H264_WS_SEND_TIMEOUT, sender.send(message)) + .await + .ok() + .and_then(Result::ok) + .is_none() + { + break; + } + last_sent_sequence = Some(frame.frame_sequence); + if is_keyframe { + decoder_synced = true; + } + } + } + } +} + fn handle_h264_socket_message( state: &AppState, session: &SimulatorSession, @@ -1876,6 +2194,47 @@ fn handle_h264_socket_message( true } +fn handle_android_h264_socket_message( + state: &AppState, + source: &AndroidWebRtcSource, + message: &Message, +) -> bool { + let text = match message { + Message::Text(text) => text.as_str(), + Message::Binary(bytes) => match std::str::from_utf8(bytes) { + Ok(text) => text, + Err(_) => return true, + }, + Message::Close(_) => return false, + Message::Ping(_) | Message::Pong(_) => return true, + }; + let Ok(message) = serde_json::from_str::(text) else { + return true; + }; + match message { + H264SocketMessage::ClientStats { stats } => { + if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { + state.metrics.record_client_stream_stats(*stats); + } + } + H264SocketMessage::StreamControl { + force_keyframe, + snapshot, + } => { + if force_keyframe.unwrap_or(false) { + source.request_keyframe(); + } + if snapshot.unwrap_or(false) { + source.request_refresh(); + } + } + H264SocketMessage::StreamQuality { config: _ } => { + source.request_keyframe(); + } + } + true +} + #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "camelCase")] enum H264SocketMessage { @@ -2150,6 +2509,13 @@ async fn send_key( Path(udid): Path, Json(payload): Json, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_key(&udid, payload.key_code, payload.modifiers.unwrap_or(0)) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { bridge.send_key(&udid, payload.key_code, payload.modifiers.unwrap_or(0)) }) @@ -2172,6 +2538,21 @@ async fn send_key_sequence( "Key sequence cannot contain more than 512 key codes.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + let delay_ms = payload.delay_ms.unwrap_or(0); + let key_count = payload.key_codes.len(); + for (index, key_code) in payload.key_codes.into_iter().enumerate() { + android.send_key(&udid, key_code, 0)?; + if delay_ms > 0 && index + 1 < key_count { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; let delay_ms = payload.delay_ms.unwrap_or(0); @@ -2192,6 +2573,10 @@ async fn dismiss_keyboard( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.dismiss_keyboard(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.send_key(&udid, 41, 0)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2204,6 +2589,13 @@ async fn press_button( if payload.button.trim().is_empty() { return Err(AppError::bad_request("Request body must include `button`.")); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.press_button(&udid, &payload.button, payload.duration_ms.unwrap_or(0)) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } if let Some(phase) = payload.phase.as_deref() { let pressed = match phase { "down" | "began" => true, @@ -2237,6 +2629,10 @@ async fn press_home( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.press_home(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.press_home(&udid)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2245,6 +2641,10 @@ async fn open_app_switcher( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.open_app_switcher(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.open_app_switcher(&udid)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2253,6 +2653,10 @@ async fn rotate_right( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.rotate_right(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.rotate_right(&udid)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2261,6 +2665,10 @@ async fn rotate_left( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.rotate_left(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.rotate_left(&udid)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2269,6 +2677,11 @@ async fn chrome_profile( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let profile = + run_android_action(state, move |android| android.chrome_profile(&udid)).await?; + return Ok(json(profile)); + } let profile = run_bridge_action(state, move |bridge| bridge.chrome_profile(&udid)).await?; Ok(json(json_value!(profile))) } @@ -2278,6 +2691,11 @@ async fn chrome_png( Path(udid): Path, Query(query): Query, ) -> Result<(StatusCode, HeaderMap, Vec), AppError> { + if android::is_android_id(&udid) { + return Err(AppError::not_found( + "Android emulators do not expose device chrome assets.", + )); + } let include_buttons = query .buttons .as_deref() @@ -2349,6 +2767,11 @@ async fn screen_mask_png( State(state): State, Path(udid): Path, ) -> Result<(StatusCode, HeaderMap, Vec), AppError> { + if android::is_android_id(&udid) { + return Err(AppError::not_found( + "Android emulators do not expose screen mask assets.", + )); + } let png = run_bridge_action(state, move |bridge| bridge.screen_mask_png(&udid)).await?; let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, "image/png".parse().unwrap()); @@ -2383,6 +2806,22 @@ async fn accessibility_tree_value( max_depth: Option, include_hidden: bool, ) -> Result { + if android::is_android_id(&udid) { + let requested_source = source + .filter(|source| *source != "auto") + .map(|source| source.to_owned()); + return run_android_action(state, move |android| { + let mut tree = android.accessibility_tree(&udid, max_depth)?; + if include_hidden { + tree["includeHidden"] = Value::Bool(true); + } + if let Some(source) = requested_source { + tree["requestedSource"] = Value::String(source); + } + Ok(tree) + }) + .await; + } let requested_source = AccessibilityHierarchySource::parse(source)?; let max_depth = max_depth.map(|depth| depth.min(80)); @@ -2521,6 +2960,15 @@ async fn accessibility_point( )); } + if android::is_android_id(&udid) { + let snapshot = run_android_action(state, move |android| { + android.accessibility_tree(&udid, None) + }) + .await?; + return Ok(json(accessibility_point_snapshot( + &snapshot, query.x, query.y, + )?)); + } let snapshot = accessibility_snapshot(state, udid, Some((query.x, query.y)), None).await?; Ok(json(snapshot)) } @@ -2569,6 +3017,17 @@ async fn perform_tap_payload( tap_point_from_snapshot(&snapshot, &payload.selector)? }; + if android::is_android_id(&udid) { + return run_android_action(state, move |android| { + android.send_touch(&udid, x, y, "began")?; + if duration_ms > 0 { + std::thread::sleep(Duration::from_millis(duration_ms)); + } + android.send_touch(&udid, x, y, "ended") + }) + .await; + } + run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; input.send_touch(x, y, "began")?; @@ -2651,6 +3110,13 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul key_code, modifiers, } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_key(&udid, key_code, modifiers.unwrap_or(0)) + }) + .await?; + return Ok(json_value!({ "action": "key" })); + } run_bridge_action(state, move |bridge| { bridge.send_key(&udid, key_code, modifiers.unwrap_or(0)) }) @@ -2669,6 +3135,21 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul "keySequence cannot contain more than 512 key codes.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + let delay_ms = delay_ms.unwrap_or(0); + let key_count = key_codes.len(); + for (index, key_code) in key_codes.into_iter().enumerate() { + android.send_key(&udid, key_code, 0)?; + if delay_ms > 0 && index + 1 < key_count { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + }) + .await?; + return Ok(json_value!({ "action": "keySequence" })); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; let delay_ms = delay_ms.unwrap_or(0); @@ -2697,6 +3178,28 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul "touch requires finite normalized x and y.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + let x = x.clamp(0.0, 1.0); + let y = y.clamp(0.0, 1.0); + if down.unwrap_or(false) || up.unwrap_or(false) { + if down.unwrap_or(false) { + android.send_touch(&udid, x, y, "began")?; + } + if down.unwrap_or(false) && up.unwrap_or(false) { + std::thread::sleep(Duration::from_millis(delay_ms.unwrap_or(100))); + } + if up.unwrap_or(false) { + android.send_touch(&udid, x, y, "ended")?; + } + } else { + android.send_touch(&udid, x, y, phase.as_deref().unwrap_or("began"))?; + } + Ok(()) + }) + .await?; + return Ok(json_value!({ "action": "touch" })); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; let x = x.clamp(0.0, 1.0); @@ -2728,6 +3231,31 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul "touchSequence cannot contain more than 64 events.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + for event in events { + if !event.x.is_finite() || !event.y.is_finite() { + return Err(AppError::bad_request( + "touchSequence requires finite normalized x and y.", + )); + } + android.send_touch( + &udid, + event.x.clamp(0.0, 1.0), + event.y.clamp(0.0, 1.0), + &event.phase, + )?; + if let Some(delay_ms) = + event.delay_ms_after.filter(|delay_ms| *delay_ms > 0) + { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + }) + .await?; + return Ok(json_value!({ "action": "touchSequence" })); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; for event in events { @@ -2767,6 +3295,20 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul "swipe requires finite normalized coordinates.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_swipe( + &udid, + start_x, + start_y, + end_x, + end_y, + duration_ms.unwrap_or(350), + ) + }) + .await?; + return Ok(json_value!({ "action": "swipe" })); + } run_bridge_action(state, move |bridge| { let step_count = steps.unwrap_or(12).max(1); let delay = @@ -2799,6 +3341,20 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul } => { let (start_x, start_y, end_x, end_y, default_duration_ms) = normalized_gesture_coordinates(&preset, delta)?; + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_swipe( + &udid, + start_x, + start_y, + end_x, + end_y, + duration_ms.unwrap_or(default_duration_ms), + ) + }) + .await?; + return Ok(json_value!({ "action": "gesture", "preset": preset })); + } run_bridge_action(state, move |bridge| { let step_count = steps.unwrap_or(12).max(1); let delay = Duration::from_millis( @@ -2821,6 +3377,23 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul Ok(json_value!({ "action": "gesture", "preset": preset })) } BatchStep::Type { text, delay_ms } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + if delay_ms.is_some() { + for character in text.chars() { + android.type_text(&udid, &character.to_string())?; + if let Some(delay_ms) = delay_ms.filter(|delay_ms| *delay_ms > 0) { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + } else { + android.type_text(&udid, &text) + } + }) + .await?; + return Ok(json_value!({ "action": "type" })); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; for character in text.chars() { @@ -2843,6 +3416,13 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul button, duration_ms, } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.press_button(&udid, &button, duration_ms.unwrap_or(0)) + }) + .await?; + return Ok(json_value!({ "action": "button" })); + } run_bridge_action(state, move |bridge| { bridge.press_button(&udid, &button, duration_ms.unwrap_or(0)) }) @@ -2850,34 +3430,69 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul Ok(json_value!({ "action": "button" })) } BatchStep::Launch { bundle_id } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.launch_package(&udid, &bundle_id) + }) + .await?; + return Ok(json_value!({ "action": "launch" })); + } run_bridge_action(state, move |bridge| bridge.launch_bundle(&udid, &bundle_id)).await?; Ok(json_value!({ "action": "launch" })) } BatchStep::OpenUrl { url } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.open_url(&udid, &url)).await?; + return Ok(json_value!({ "action": "openUrl" })); + } run_bridge_action(state, move |bridge| bridge.open_url(&udid, &url)).await?; Ok(json_value!({ "action": "openUrl" })) } BatchStep::Home => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.press_home(&udid)).await?; + return Ok(json_value!({ "action": "home" })); + } run_bridge_action(state, move |bridge| bridge.press_home(&udid)).await?; Ok(json_value!({ "action": "home" })) } BatchStep::DismissKeyboard => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.dismiss_keyboard(&udid)).await?; + return Ok(json_value!({ "action": "dismissKeyboard" })); + } run_bridge_action(state, move |bridge| bridge.send_key(&udid, 41, 0)).await?; Ok(json_value!({ "action": "dismissKeyboard" })) } BatchStep::AppSwitcher => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.open_app_switcher(&udid)).await?; + return Ok(json_value!({ "action": "appSwitcher" })); + } run_bridge_action(state, move |bridge| bridge.open_app_switcher(&udid)).await?; Ok(json_value!({ "action": "appSwitcher" })) } BatchStep::RotateLeft => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.rotate_left(&udid)).await?; + return Ok(json_value!({ "action": "rotateLeft" })); + } run_bridge_action(state, move |bridge| bridge.rotate_left(&udid)).await?; Ok(json_value!({ "action": "rotateLeft" })) } BatchStep::RotateRight => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.rotate_right(&udid)).await?; + return Ok(json_value!({ "action": "rotateRight" })); + } run_bridge_action(state, move |bridge| bridge.rotate_right(&udid)).await?; Ok(json_value!({ "action": "rotateRight" })) } BatchStep::ToggleAppearance => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.toggle_appearance(&udid)).await?; + return Ok(json_value!({ "action": "toggleAppearance" })); + } run_bridge_action(state, move |bridge| bridge.toggle_appearance(&udid)).await?; Ok(json_value!({ "action": "toggleAppearance" })) } @@ -3072,6 +3687,77 @@ fn normalize_screen_point_from_snapshot( Ok(((x / width).clamp(0.0, 1.0), (y / height).clamp(0.0, 1.0))) } +fn accessibility_point_snapshot(snapshot: &Value, x: f64, y: f64) -> Result { + let roots = snapshot + .get("roots") + .and_then(Value::as_array) + .ok_or_else(|| AppError::not_found("Accessibility snapshot does not contain roots."))?; + let node = roots + .iter() + .rev() + .find_map(|root| deepest_node_at_point(root, x, y)) + .ok_or_else(|| AppError::not_found("No accessibility element contains the point."))?; + let mut node = node.clone(); + if let Some(object) = node.as_object_mut() { + object.remove("children"); + } + + let mut response = Map::new(); + for key in [ + "source", + "availableSources", + "requestedSource", + "fallbackReason", + "inspector", + "includeHidden", + ] { + if let Some(value) = snapshot.get(key) { + response.insert(key.to_owned(), value.clone()); + } + } + response.insert("roots".to_owned(), Value::Array(vec![node])); + Ok(Value::Object(response)) +} + +fn deepest_node_at_point(node: &Value, x: f64, y: f64) -> Option<&Value> { + let has_frame = node + .get("frame") + .or_else(|| node.get("frameInScreen")) + .is_some(); + if has_frame && !node_frame_contains_point(node, x, y).unwrap_or(false) { + return None; + } + for child in node + .get("children") + .and_then(Value::as_array) + .into_iter() + .flatten() + .rev() + { + if let Some(found) = deepest_node_at_point(child, x, y) { + return Some(found); + } + } + has_frame.then_some(node) +} + +fn node_frame_contains_point(node: &Value, x: f64, y: f64) -> Result { + let frame = node + .get("frame") + .or_else(|| node.get("frameInScreen")) + .ok_or_else(|| AppError::not_found("Accessibility node does not expose a frame."))?; + let frame_x = number_field(frame, "x")?; + let frame_y = number_field(frame, "y")?; + let width = number_field(frame, "width")?; + let height = number_field(frame, "height")?; + Ok(width > 0.0 + && height > 0.0 + && x >= frame_x + && y >= frame_y + && x < frame_x + width + && y < frame_y + height) +} + fn number_field(value: &Value, field: &str) -> Result { value .get(field) @@ -3314,6 +4000,10 @@ async fn simulator_logs( Query(query): Query, ) -> Result, AppError> { let limit = query.limit.unwrap_or(250).clamp(1, 1000); + if android::is_android_id(&udid) { + let entries = run_android_action(state, move |android| android.logs(&udid, limit)).await?; + return Ok(json(json_value!({ "entries": entries }))); + } let filters = LogFilters::new( split_filter_values(query.levels.as_deref()), split_filter_values(query.processes.as_deref()), @@ -4648,25 +5338,87 @@ where })? } +async fn run_android_action(state: AppState, action: F) -> Result +where + F: FnOnce(AndroidBridge) -> Result + Send + 'static, + T: Send + 'static, +{ + let android = state.android.clone(); + task::spawn_blocking(move || action(android)) + .await + .map_err(|error| { + AppError::internal(format!("Failed to join Android bridge task: {error}")) + })? +} + +async fn all_device_values(state: AppState, force_refresh: bool) -> Result, AppError> { + let ios = list_simulators_cached(state.clone(), force_refresh).await?; + let mut values = state.registry.enrich_simulators(ios); + let android_devices = + run_android_action(state.clone(), |android| android.list_devices()).await?; + values.extend(state.android.enrich_devices(android_devices)); + Ok(values) +} + async fn list_simulators_cached( state: AppState, force_refresh: bool, ) -> Result, AppError> { - let mut guard = state.simulator_inventory.inner.lock().await; - if !force_refresh { - if let (Some(simulators), Some(updated_at)) = (&guard.simulators, guard.updated_at) { - if updated_at.elapsed() <= SIMULATOR_INVENTORY_CACHE_TTL { - return Ok(simulators.clone()); + { + let guard = state.simulator_inventory.inner.lock().await; + if !force_refresh { + if let (Some(simulators), Some(updated_at)) = (&guard.simulators, guard.updated_at) { + if updated_at.elapsed() <= SIMULATOR_INVENTORY_CACHE_TTL { + return Ok(simulators.clone()); + } } } } - let simulators = run_bridge_action(state.clone(), |bridge| bridge.list_simulators()).await?; + let simulators = match timeout( + SIMULATOR_INVENTORY_TIMEOUT, + run_bridge_action(state.clone(), |bridge| bridge.list_simulators()), + ) + .await + { + Ok(result) => result?, + Err(_) => { + tracing::warn!("Timed out listing iOS simulators; returning cached inventory."); + let guard = state.simulator_inventory.inner.lock().await; + return Ok(guard.simulators.clone().unwrap_or_default()); + } + }; + + let mut guard = state.simulator_inventory.inner.lock().await; guard.simulators = Some(simulators.clone()); guard.updated_at = Some(Instant::now()); Ok(simulators) } +async fn android_simulator_payload(state: AppState, udid: String) -> Result, AppError> { + let android_devices = + run_android_action(state.clone(), |android| android.list_devices()).await?; + let simulator = state + .android + .enrich_devices(android_devices) + .into_iter() + .find(|entry| entry.get("udid").and_then(Value::as_str) == Some(udid.as_str())) + .ok_or_else(|| AppError::not_found(format!("Unknown Android emulator {udid}")))?; + Ok(json(json_value!({ "simulator": simulator }))) +} + +async fn simulator_payload(state: AppState, udid: String) -> Result, AppError> { + if android::is_android_id(&udid) { + return android_simulator_payload(state, udid).await; + } + let enriched = all_device_values(state.clone(), true).await?; + let simulator = enriched + .into_iter() + .find(|entry| entry.get("udid").and_then(Value::as_str) == Some(udid.as_str())) + .ok_or_else(|| AppError::not_found(format!("Unknown simulator {udid}")))?; + Ok(json(json_value!({ "simulator": simulator }))) +} + async fn accessibility_snapshot( state: AppState, udid: String, @@ -4679,23 +5431,14 @@ async fn accessibility_snapshot( .await } -async fn simulator_payload(state: AppState, udid: String) -> Result, AppError> { - let simulators = list_simulators_cached(state.clone(), true).await?; - let enriched = state.registry.enrich_simulators(simulators); - let simulator = enriched - .into_iter() - .find(|entry| entry.get("udid").and_then(Value::as_str) == Some(udid.as_str())) - .ok_or_else(|| AppError::not_found(format!("Unknown simulator {udid}")))?; - Ok(json(json_value!({ "simulator": simulator }))) -} - #[cfg(test)] mod tests { use super::{ - attach_tree_metadata, available_sources_for_snapshot, best_inspector_session, - compact_accessibility_snapshot, element_matches_selector, first_matching_element, - inspector_available_sources, inspector_metadata, inspector_session_from_published, - inspector_session_score, is_inspector_agent_transport_path, normalize_inspector_node, + accessibility_point_snapshot, attach_tree_metadata, available_sources_for_snapshot, + best_inspector_session, compact_accessibility_snapshot, element_matches_selector, + first_matching_element, inspector_available_sources, inspector_metadata, + inspector_session_from_published, inspector_session_score, + is_inspector_agent_transport_path, normalize_inspector_node, normalize_screen_point_from_snapshot, normalized_gesture_coordinates, parse_lsof_tcp_listener, resolved_stream_quality_limits, split_filter_values, stream_quality_profile, suppress_native_ax_translation_error, tap_point_from_snapshot, @@ -4795,6 +5538,35 @@ mod tests { assert_eq!(point, (1.0, 0.0)); } + #[test] + fn accessibility_point_snapshot_returns_deepest_node() { + let snapshot = json!({ + "source": "android-uiautomator", + "availableSources": ["android-uiautomator"], + "roots": [{ + "type": "FrameLayout", + "frame": { "x": 0.0, "y": 0.0, "width": 400.0, "height": 800.0 }, + "children": [{ + "type": "ViewGroup", + "AXIdentifier": "container", + "frame": { "x": 0.0, "y": 100.0, "width": 400.0, "height": 300.0 }, + "children": [{ + "type": "Button", + "AXIdentifier": "child-button", + "frame": { "x": 120.0, "y": 140.0, "width": 80.0, "height": 60.0 }, + "children": [] + }] + }] + }] + }); + + let point = accessibility_point_snapshot(&snapshot, 150.0, 160.0).unwrap(); + + assert_eq!(point["source"], "android-uiautomator"); + assert_eq!(point["roots"][0]["AXIdentifier"], "child-button"); + assert!(point["roots"][0].get("children").is_none()); + } + #[test] fn gesture_presets_clamp_delta_and_reject_unknown_names() { assert_eq!( diff --git a/server/src/main.rs b/server/src/main.rs index 6b0f6b99..523c23fc 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,3 +1,4 @@ +mod android; mod api; mod auth; mod config; @@ -635,6 +636,7 @@ enum DescribeUiSource { Flutter, Uikit, NativeAx, + AndroidUiautomator, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -815,8 +817,9 @@ fn studio_stream_quality_profile( }) } -fn command_service_url(explicit: Option) -> anyhow::Result { +fn command_service_url(explicit: Option<&str>) -> anyhow::Result { if let Some(url) = explicit + .map(ToOwned::to_owned) .or_else(|| env::var("SIMDECK_SERVER_URL").ok()) .filter(|value| !value.trim().is_empty()) { @@ -825,6 +828,18 @@ fn command_service_url(explicit: Option) -> anyhow::Result { Ok(ensure_project_daemon(DaemonLaunchOptions::default())?.http_url) } +fn command_service_url_for_udid( + udid: &str, + explicit: &Option, + service_url: &Option, +) -> anyhow::Result> { + if android::is_android_id(udid) { + Ok(Some(command_service_url(explicit.as_deref())?)) + } else { + Ok(service_url.clone()) + } +} + impl Default for DaemonLaunchOptions { fn default() -> Self { Self { @@ -2031,7 +2046,7 @@ fn main() -> anyhow::Result<()> { CoreSimulatorCommand::Restart => core_simulator::restart(), }, Command::List => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let simulators = service_get_json(&service_url, "/api/simulators")? .get("simulators") .cloned() @@ -2043,7 +2058,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Boot { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok(&service_url, &udid, "boot", &Value::Null)?; println!( "{}", @@ -2054,7 +2069,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Shutdown { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok(&service_url, &udid, "shutdown", &Value::Null)?; println!( "{}", @@ -2065,7 +2080,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::OpenUrl { udid, url } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_open_url(&service_url, &udid, &url)?; println!( "{}", @@ -2076,7 +2091,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Launch { udid, bundle_id } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_launch(&service_url, &udid, &bundle_id)?; println!( "{}", @@ -2087,7 +2102,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::ToggleAppearance { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok(&service_url, &udid, "toggle-appearance", &Value::Null)?; println_json( &serde_json::json!({ "ok": true, "udid": udid, "action": "toggle-appearance" }), @@ -2095,13 +2110,13 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Erase { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok(&service_url, &udid, "erase", &Value::Null)?; println_json(&serde_json::json!({ "ok": true, "udid": udid, "action": "erase" }))?; Ok(()) } Command::Install { udid, app_path } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok( &service_url, &udid, @@ -2114,7 +2129,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Uninstall { udid, bundle_id } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok( &service_url, &udid, @@ -2128,7 +2143,7 @@ fn main() -> anyhow::Result<()> { } Command::Pasteboard { command } => match command { PasteboardCommand::Get { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let text = service_get_json( &service_url, &format!("/api/simulators/{}/pasteboard", url_path_component(&udid)), @@ -2146,7 +2161,7 @@ fn main() -> anyhow::Result<()> { stdin, file, } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let text = read_text_input(text, stdin, file)?; service_post_ok( &service_url, @@ -2165,7 +2180,7 @@ fn main() -> anyhow::Result<()> { seconds, limit, } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let filters = native::bridge::LogFilters::new(Vec::new(), Vec::new(), String::new()); let _ = filters; let entries = service_get_json( @@ -2186,7 +2201,7 @@ fn main() -> anyhow::Result<()> { output, stdout, } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let png = service_get_bytes( &service_url, &format!( @@ -2221,7 +2236,7 @@ fn main() -> anyhow::Result<()> { include_hidden, direct, } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let snapshot = describe_ui_snapshot( &bridge, &udid, @@ -2245,7 +2260,13 @@ fn main() -> anyhow::Result<()> { up, delay_ms, } => { - if let Some(server_url) = service_url.as_deref().filter(|_| normalized) { + let android_device = android::is_android_id(&udid); + if android_device && !normalized { + anyhow::bail!("Android touch coordinates require --normalized."); + } + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref().filter(|_| normalized) { if down || up { let mut events = Vec::new(); if down { @@ -2300,8 +2321,10 @@ fn main() -> anyhow::Result<()> { pre_delay_ms, post_delay_ms, } => { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; if let (Some(server_url), Some(x), Some(y), true, None, None, None, None) = ( - service_url.as_deref(), + command_server_url.as_deref(), x, y, normalized, @@ -2313,7 +2336,7 @@ fn main() -> anyhow::Result<()> { sleep_ms(pre_delay_ms); service_tap(server_url, &udid, x, y, duration_ms)?; sleep_ms(post_delay_ms); - } else if let Some(server_url) = service_url.as_deref() { + } else if let Some(server_url) = command_server_url.as_deref() { sleep_ms(pre_delay_ms); service_tap_element( server_url, @@ -2375,18 +2398,41 @@ fn main() -> anyhow::Result<()> { pre_delay_ms, post_delay_ms, } => { - if let Some(server_url) = service_url.as_deref().filter(|_| normalized) { + let android_device = android::is_android_id(&udid); + if android_device && !normalized { + anyhow::bail!("Android swipe coordinates require --normalized."); + } + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref().filter(|_| normalized) { sleep_ms(pre_delay_ms); - service_swipe( - server_url, - &udid, - start_x, - start_y, - end_x, - end_y, - duration_ms, - steps, - )?; + if android_device { + service_batch( + server_url, + &udid, + vec![serde_json::json!({ + "action": "swipe", + "startX": start_x, + "startY": start_y, + "endX": end_x, + "endY": end_y, + "durationMs": duration_ms, + "steps": steps, + })], + false, + )?; + } else { + service_swipe( + server_url, + &udid, + start_x, + start_y, + end_x, + end_y, + duration_ms, + steps, + )?; + } sleep_ms(post_delay_ms); } else { let (start_x, start_y) = @@ -2421,7 +2467,33 @@ fn main() -> anyhow::Result<()> { pre_delay_ms, post_delay_ms, } => { - if let Some(server_url) = service_url.as_deref().filter(|_| normalized) { + let android_device = android::is_android_id(&udid); + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if android_device { + let server_url = command_server_url + .as_deref() + .ok_or_else(|| anyhow::anyhow!("Android command requires SimDeck daemon."))?; + sleep_ms(pre_delay_ms); + service_batch( + server_url, + &udid, + vec![serde_json::json!({ + "action": "gesture", + "preset": preset, + "durationMs": duration_ms, + "delta": delta, + "steps": 4, + })], + false, + )?; + sleep_ms(post_delay_ms); + println_json( + &serde_json::json!({ "ok": true, "udid": udid, "action": "gesture", "preset": preset }), + )?; + return Ok(()); + } + if let Some(server_url) = command_server_url.as_deref().filter(|_| normalized) { let gesture = gesture_coordinates( &bridge, &udid, @@ -2484,6 +2556,9 @@ fn main() -> anyhow::Result<()> { duration_ms, steps, } => { + if android::is_android_id(&udid) { + anyhow::bail!("Android pinch gestures are not supported by the ADB input bridge."); + } let frames = pinch_frames( &bridge, &udid, @@ -2509,6 +2584,9 @@ fn main() -> anyhow::Result<()> { duration_ms, steps, } => { + if android::is_android_id(&udid) { + anyhow::bail!("Android rotate gestures are not supported by the ADB input bridge."); + } let frames = rotate_gesture_frames( &bridge, &udid, @@ -2537,7 +2615,9 @@ fn main() -> anyhow::Result<()> { } => { let key_code = parse_hid_key(&key)?; sleep_ms(pre_delay_ms); - if let Some(server_url) = service_url.as_deref().filter(|_| duration_ms == 0) { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref().filter(|_| duration_ms == 0) { service_key(server_url, &udid, key_code, modifiers)?; } else if duration_ms > 0 && modifiers == 0 { let input = bridge.create_input_session(&udid)?; @@ -2557,7 +2637,9 @@ fn main() -> anyhow::Result<()> { delay_ms, } => { let keys = parse_key_list(&keycodes)?; - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_key_sequence(server_url, &udid, &keys, delay_ms)?; } else { let input = bridge.create_input_session(&udid)?; @@ -2580,7 +2662,9 @@ fn main() -> anyhow::Result<()> { } => { let modifier_mask = parse_modifier_mask(&modifiers)?; let key_code = parse_hid_key(&key)?; - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_key(server_url, &udid, key_code, modifier_mask)?; } else { bridge.send_key(&udid, key_code, modifier_mask)?; @@ -2596,7 +2680,21 @@ fn main() -> anyhow::Result<()> { delay_ms, } => { let text = read_text_input(text, stdin, file)?; - type_text(&bridge, &udid, &text, delay_ms)?; + if android::is_android_id(&udid) { + let server_url = command_service_url(explicit_server_url.as_deref())?; + service_batch( + &server_url, + &udid, + vec![serde_json::json!({ + "action": "type", + "text": text, + "delayMs": delay_ms, + })], + false, + )?; + } else { + type_text(&bridge, &udid, &text, delay_ms)?; + } println_json(&serde_json::json!({ "ok": true, "udid": udid, "action": "type" }))?; Ok(()) } @@ -2605,7 +2703,9 @@ fn main() -> anyhow::Result<()> { button, duration_ms, } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_button(server_url, &udid, &button, duration_ms)?; } else { bridge.press_button(&udid, &button, duration_ms)?; @@ -2622,7 +2722,9 @@ fn main() -> anyhow::Result<()> { stdin, continue_on_error, } => { - let report = if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + let report = if let Some(server_url) = command_server_url.as_deref() { let step_lines = read_batch_steps(steps, file, stdin)?; service_batch( server_url, @@ -2637,7 +2739,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::DismissKeyboard { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "dismiss-keyboard", &Value::Null)?; } else { bridge.send_key(&udid, 41, 0)?; @@ -2651,7 +2755,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Home { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "home", &Value::Null)?; } else { bridge.press_home(&udid)?; @@ -2660,7 +2766,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::AppSwitcher { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "app-switcher", &Value::Null)?; } else { bridge.open_app_switcher(&udid)?; @@ -2671,7 +2779,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::RotateLeft { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "rotate-left", &Value::Null)?; } else { bridge.rotate_left(&udid)?; @@ -2682,7 +2792,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::RotateRight { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "rotate-right", &Value::Null)?; } else { bridge.rotate_right(&udid)?; @@ -2693,7 +2805,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::ChromeProfile { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let profile = service_get_json( &service_url, &format!( @@ -3128,12 +3240,32 @@ fn describe_ui_snapshot( direct: bool, server_url: &str, ) -> anyhow::Result { - if point.is_none() && !direct { - match fetch_service_accessibility_tree(udid, source, max_depth, include_hidden, server_url) - { - Ok(snapshot) => return Ok(snapshot), - Err(error) if source != DescribeUiSource::Auto => return Err(error), - Err(_) => {} + if !direct { + if let Some((x, y)) = point { + if matches!( + source, + DescribeUiSource::Auto + | DescribeUiSource::NativeAx + | DescribeUiSource::AndroidUiautomator + ) { + match fetch_service_accessibility_point(udid, x, y, server_url) { + Ok(snapshot) => return Ok(snapshot), + Err(error) if source != DescribeUiSource::Auto => return Err(error), + Err(_) => {} + } + } + } else { + match fetch_service_accessibility_tree( + udid, + source, + max_depth, + include_hidden, + server_url, + ) { + Ok(snapshot) => return Ok(snapshot), + Err(error) if source != DescribeUiSource::Auto => return Err(error), + Err(_) => {} + } } } @@ -3169,6 +3301,19 @@ fn fetch_service_accessibility_tree( http_get_json(server_url, &path) } +fn fetch_service_accessibility_point( + udid: &str, + x: f64, + y: f64, + server_url: &str, +) -> anyhow::Result { + let path = format!( + "/api/simulators/{}/accessibility-point?x={x}&y={y}", + url_path_component(udid) + ); + http_get_json(server_url, &path) +} + fn http_get_json(server_url: &str, path: &str) -> anyhow::Result { http_request_json(server_url, "GET", path, None) } @@ -3559,6 +3704,7 @@ impl DescribeUiSource { Self::Flutter => "flutter", Self::Uikit => "uikit", Self::NativeAx => "native-ax", + Self::AndroidUiautomator => "android-uiautomator", } } } @@ -5179,6 +5325,7 @@ async fn serve( inspectors, metrics, simulator_inventory: Default::default(), + android: Default::default(), }; let http_router = app_router( diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs index 2d0659fb..9d6b2df1 100644 --- a/server/src/native/ffi.rs +++ b/server/src/native/ffi.rs @@ -268,6 +268,23 @@ unsafe extern "C" { error_message: *mut *mut c_char, ) -> bool; + pub fn xcw_native_h264_encoder_create( + callback: Option, + user_data: *mut c_void, + error_message: *mut *mut c_char, + ) -> *mut c_void; + pub fn xcw_native_h264_encoder_destroy(handle: *mut c_void); + pub fn xcw_native_h264_encoder_encode_rgba( + handle: *mut c_void, + rgba: *const u8, + length: usize, + width: u32, + height: u32, + timestamp_us: u64, + error_message: *mut *mut c_char, + ) -> bool; + pub fn xcw_native_h264_encoder_request_keyframe(handle: *mut c_void); + pub fn xcw_native_free_string(value: *mut c_char); pub fn xcw_native_free_bytes(bytes: xcw_native_owned_bytes); pub fn xcw_native_release_shared_bytes(bytes: xcw_native_shared_bytes); diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index e9caff8b..1aac81e6 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -1,23 +1,29 @@ +use crate::android; use crate::api::routes::{ apply_stream_quality_payload, run_control_message, run_toggle_appearance_control, AppState, ControlMessage, StreamQualityPayload, }; use crate::error::AppError; use crate::metrics::counters::ClientStreamStats; -use bytes::Bytes; +use crate::native::ffi; +use crate::transport::packet::{FramePacket, SharedFrame}; +use bytes::{BufMut, Bytes, BytesMut}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, VecDeque}; -use std::sync::atomic::Ordering; -use std::sync::{Arc, Mutex, OnceLock}; +use std::ffi::{c_void, CStr}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak}; use std::time::Duration; use tokio::sync::{broadcast, mpsc}; use tokio::task; -use tokio::time; +use tokio::time::{self, Instant}; use tracing::{info, warn}; use webrtc::api::interceptor_registry::register_default_interceptors; use webrtc::api::media_engine::{MediaEngine, MIME_TYPE_H264}; use webrtc::api::APIBuilder; +use webrtc::data_channel::data_channel_init::RTCDataChannelInit; use webrtc::data_channel::data_channel_message::DataChannelMessage; +use webrtc::data_channel::data_channel_state::RTCDataChannelState; use webrtc::data_channel::RTCDataChannel; use webrtc::ice_transport::ice_server::RTCIceServer; use webrtc::interceptor::registry::Registry; @@ -43,6 +49,7 @@ const ANNEX_B_START_CODE: &[u8] = &[0, 0, 0, 1]; const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_TELEMETRY_CHANNEL_LABEL: &str = "simdeck-telemetry"; +const WEBRTC_RGBA_CHANNEL_LABEL: &str = "simdeck-rgba"; const WEBRTC_DEFAULT_LOCAL_STREAM_FPS: u32 = 60; const WEBRTC_MAX_LOCAL_STREAM_FPS: u32 = 240; const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); @@ -53,6 +60,15 @@ const WEBRTC_FAST_ICE_GATHER_TIMEOUT: Duration = Duration::from_millis(250); const WEBRTC_FULL_ICE_GATHER_TIMEOUT: Duration = Duration::from_secs(3); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; const WEBRTC_PEER_DISCONNECTED_TIMEOUT: Duration = Duration::from_secs(12); +const ANDROID_WEBRTC_FRAME_BROADCAST_CAPACITY: usize = 128; +const ANDROID_WEBRTC_RAW_FRAME_BROADCAST_CAPACITY: usize = 8; +const ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES: usize = 48; +const ANDROID_WEBRTC_RGBA_CHUNK_BYTES: usize = 256 * 1024; +const ANDROID_WEBRTC_RGBA_CHUNK_MAGIC: u32 = 0x5344_5243; // "SDRC" +const ANDROID_WEBRTC_RGBA_VERSION: u8 = 1; +const ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888: u8 = 1; +const ANDROID_WEBRTC_RGBA_BUFFERED_FRAME_LIMIT: usize = 2; +const ANDROID_WEBRTC_FPS: u64 = 30; static WEBRTC_MEDIA_STREAMS: OnceLock>>> = OnceLock::new(); const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 16; @@ -81,6 +97,14 @@ pub struct WebRtcAnswerPayload { pub sdp: String, #[serde(rename = "type")] pub kind: String, + pub video: WebRtcVideoMetadata, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebRtcVideoMetadata { + pub width: u32, + pub height: u32, } #[derive(Debug, Clone, Serialize)] @@ -97,26 +121,63 @@ pub async fn create_answer( state: AppState, udid: String, payload: WebRtcOfferPayload, + peer_is_loopback: bool, ) -> Result { if payload.kind != "offer" { return Err(AppError::bad_request( "WebRTC payload must include type `offer`.", )); } - - let session = state.registry.get_or_create_async(&udid).await?; - if let Err(error) = session.ensure_started_async().await { - state.registry.remove(&udid); - return Err(error); + let is_android = android::is_android_id(&udid); + let transport = payload + .transport + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()); + if transport.is_some_and(|value| value.eq_ignore_ascii_case("rgba")) { + if !is_android { + return Err(AppError::bad_request( + "RGBA WebRTC transport is only supported for Android emulators.", + )); + } + if !peer_is_loopback { + return Err(AppError::bad_request( + "RGBA WebRTC transport is only available to loopback clients.", + )); + } + return create_android_rgba_answer(state, udid, payload).await; } if payload.transport.is_some() { return Err(AppError::bad_request( - "WebRTC preview supports media tracks only.", + "Unsupported WebRTC transport. Supported transports are media tracks and Android loopback RGBA.", )); } - if let Some(stream_config) = payload.stream_config.as_ref() { - apply_stream_quality_payload(&state, stream_config)?; + if !is_android { + if let Some(stream_config) = payload.stream_config.as_ref() { + apply_stream_quality_payload(&state, stream_config)?; + } } + + let source = if is_android { + WebRtcVideoSource::Android( + AndroidWebRtcSource::start( + state.android.clone(), + state.metrics.clone(), + udid.clone(), + None, + true, + ) + .await?, + ) + } else { + let session = state.registry.get_or_create_async(&udid).await?; + if let Err(error) = session.ensure_started_async().await { + state.registry.remove(&udid); + return Err(error); + } + WebRtcVideoSource::Simulator(session) + }; + info!( "WebRTC offer for {udid}: remote_candidates={} remote_candidate_types={} ice_servers={} ice_transport_policy={}", count_sdp_candidates(&payload.sdp), @@ -128,9 +189,9 @@ pub async fn create_answer( ice_transport_policy_label() ); - let first_frame = wait_for_h264_sync_keyframe(&session, WEBRTC_INITIAL_KEYFRAME_TIMEOUT) + let first_frame = wait_for_h264_sync_keyframe(&source, WEBRTC_INITIAL_KEYFRAME_TIMEOUT) .await - .ok_or_else(|| AppError::native("Timed out waiting for a simulator H.264 keyframe."))?; + .ok_or_else(|| AppError::native("Timed out waiting for a device H.264 keyframe."))?; let codec = first_frame .codec .as_deref() @@ -179,13 +240,22 @@ pub async fn create_answer( ); register_diagnostics(&peer_connection, &udid); let (stream_control_tx, stream_control_rx) = mpsc::unbounded_channel(); - register_control_data_channel( - &peer_connection, - session.clone(), - state.clone(), - udid.clone(), - stream_control_tx, - ); + match &source { + WebRtcVideoSource::Simulator(session) => register_control_data_channel( + &peer_connection, + session.clone(), + state.clone(), + udid.clone(), + stream_control_tx, + ), + WebRtcVideoSource::Android(source) => register_android_data_channel( + &peer_connection, + source.clone(), + state.clone(), + udid.clone(), + stream_control_tx, + ), + } let video_track = Arc::new(TrackLocalStaticRTP::new( RTCRtpCodecCapability { @@ -203,7 +273,7 @@ pub async fn create_answer( .add_track(video_track.clone() as Arc) .await .map_err(|error| AppError::internal(format!("add WebRTC video track: {error}")))?; - let rtcp_session = session.clone(); + let rtcp_source = source.clone(); let rtcp_udid = udid.clone(); tokio::spawn(async move { while let Ok((packets, _attributes)) = rtp_sender.read_rtcp().await { @@ -212,7 +282,7 @@ pub async fn create_answer( .any(|packet| rtcp_packet_requests_keyframe(packet.as_ref())) { info!("WebRTC RTCP requested keyframe for {rtcp_udid}"); - rtcp_session.request_keyframe(); + rtcp_source.request_keyframe(); } } }); @@ -258,13 +328,15 @@ pub async fn create_answer( summarize_sdp_candidate_types(&local_description.sdp) ); + let first_frame_width = first_frame.width; + let first_frame_height = first_frame.height; let (cancellation_token, cancellation) = register_webrtc_media_stream(&udid, payload.client_id.as_deref(), true); tokio::spawn( WebRtcMediaStream { state, udid, - session, + source, first_frame, peer_connection, video_track, @@ -278,6 +350,131 @@ pub async fn create_answer( Ok(WebRtcAnswerPayload { sdp: local_description.sdp, kind: "answer".to_owned(), + video: WebRtcVideoMetadata { + width: first_frame_width, + height: first_frame_height, + }, + }) +} + +async fn create_android_rgba_answer( + state: AppState, + udid: String, + payload: WebRtcOfferPayload, +) -> Result { + let source = AndroidWebRtcSource::start( + state.android.clone(), + state.metrics.clone(), + udid.clone(), + None, + false, + ) + .await?; + info!( + "Android RGBA WebRTC offer for {udid}: remote_candidates={} remote_candidate_types={} ice_servers={} ice_transport_policy={}", + count_sdp_candidates(&payload.sdp), + summarize_sdp_candidate_types(&payload.sdp), + std::env::var("SIMDECK_WEBRTC_ICE_SERVERS") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_STUN_URL.to_owned()), + ice_transport_policy_label() + ); + + let api = APIBuilder::new().build(); + let peer_connection = Arc::new( + api.new_peer_connection(RTCConfiguration { + ice_servers: ice_servers(), + ice_transport_policy: ice_transport_policy(), + ..Default::default() + }) + .await + .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, + ); + register_diagnostics(&peer_connection, &udid); + let (stream_control_tx, stream_control_rx) = mpsc::unbounded_channel(); + register_android_data_channel( + &peer_connection, + source.clone(), + state.clone(), + udid.clone(), + stream_control_tx, + ); + let rgba_channel = peer_connection + .create_data_channel( + WEBRTC_RGBA_CHANNEL_LABEL, + Some(RTCDataChannelInit { + ordered: Some(false), + max_retransmits: Some(0), + ..Default::default() + }), + ) + .await + .map_err(|error| AppError::internal(format!("create RGBA WebRTC data channel: {error}")))?; + + let fast_gather = + has_sdp_candidate_type(&payload.sdp, "host") && ice_transport_policy_label() == "all"; + let offer = RTCSessionDescription::offer(payload.sdp) + .map_err(|error| AppError::bad_request(format!("invalid WebRTC offer: {error}")))?; + peer_connection + .set_remote_description(offer) + .await + .map_err(|error| AppError::bad_request(format!("set remote WebRTC offer: {error}")))?; + let answer = peer_connection + .create_answer(None) + .await + .map_err(|error| AppError::internal(format!("create WebRTC answer: {error}")))?; + let mut gather_complete = peer_connection.gathering_complete_promise().await; + peer_connection + .set_local_description(answer) + .await + .map_err(|error| AppError::internal(format!("set WebRTC answer: {error}")))?; + let gather_timeout = if fast_gather { + WEBRTC_FAST_ICE_GATHER_TIMEOUT + } else { + WEBRTC_FULL_ICE_GATHER_TIMEOUT + }; + let gather_result = time::timeout(gather_timeout, gather_complete.recv()).await; + let mut local_description = peer_connection + .local_description() + .await + .ok_or_else(|| AppError::internal("WebRTC local description was not set."))?; + if gather_result.is_err() && count_sdp_candidates(&local_description.sdp) == 0 { + let _ = time::timeout(WEBRTC_FULL_ICE_GATHER_TIMEOUT, gather_complete.recv()).await; + local_description = peer_connection + .local_description() + .await + .ok_or_else(|| AppError::internal("WebRTC local description was not set."))?; + } + info!( + "Android RGBA WebRTC answer for {udid}: local_candidates={} local_candidate_types={}", + count_sdp_candidates(&local_description.sdp), + summarize_sdp_candidate_types(&local_description.sdp) + ); + + let (cancellation_token, cancellation) = + register_webrtc_media_stream(&udid, payload.client_id.as_deref(), true); + tokio::spawn( + WebRtcRgbaStream { + state, + udid, + source, + peer_connection, + rgba_channel, + cancellation_token, + cancellation, + stream_control_rx, + } + .run(), + ); + + Ok(WebRtcAnswerPayload { + sdp: local_description.sdp, + kind: "answer".to_owned(), + video: WebRtcVideoMetadata { + width: 0, + height: 0, + }, }) } @@ -481,6 +678,247 @@ fn attach_control_data_channel( })); } +fn register_android_data_channel( + peer_connection: &Arc, + source: AndroidWebRtcSource, + state: AppState, + udid: String, + stream_control_tx: mpsc::UnboundedSender, +) { + peer_connection.on_data_channel(Box::new(move |channel: Arc| { + let source = source.clone(); + let state = state.clone(); + let udid = udid.clone(); + let stream_control_tx = stream_control_tx.clone(); + Box::pin(async move { + let label = channel.label(); + if label != WEBRTC_CONTROL_CHANNEL_LABEL && label != WEBRTC_TELEMETRY_CHANNEL_LABEL { + return; + } + attach_android_data_channel(channel, source, state, udid, stream_control_tx); + }) + })); +} + +fn attach_android_data_channel( + channel: Arc, + source: AndroidWebRtcSource, + state: AppState, + udid: String, + stream_control_tx: mpsc::UnboundedSender, +) { + let (control_tx, control_rx) = mpsc::unbounded_channel::(); + task::spawn(run_android_webrtc_control_queue( + state.clone(), + udid.clone(), + control_rx, + )); + channel.on_message(Box::new(move |message: DataChannelMessage| { + let source = source.clone(); + let state = state.clone(); + let udid = udid.clone(); + let stream_control_tx = stream_control_tx.clone(); + let control_tx = control_tx.clone(); + Box::pin(async move { + let Ok(text) = std::str::from_utf8(&message.data) else { + warn!("Invalid Android WebRTC control message bytes for {udid}"); + return; + }; + if let Ok(message) = serde_json::from_str::(text) { + match message { + WebRtcDataChannelMessage::ClientStats { stats } => { + if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { + state.metrics.record_client_stream_stats(*stats); + } + } + WebRtcDataChannelMessage::StreamControl { + force_keyframe, + snapshot, + } => { + let command = WebRtcStreamCommand { + force_keyframe: force_keyframe.unwrap_or(false), + snapshot: snapshot.unwrap_or(false), + }; + if command.force_keyframe || command.snapshot { + source.request_keyframe(); + } + let _ = stream_control_tx.send(command); + } + WebRtcDataChannelMessage::StreamQuality { config } => { + let _ = config; + source.request_keyframe(); + } + } + return; + } + + let control_message = match serde_json::from_str::(text) { + Ok(message) => message, + Err(error) => { + warn!("Invalid Android WebRTC control message for {udid}: {error}"); + return; + } + }; + if control_tx.send(control_message).is_err() { + warn!("Android WebRTC control queue closed for {udid}"); + } + }) + })); +} + +async fn run_android_webrtc_control_queue( + state: AppState, + udid: String, + mut receiver: mpsc::UnboundedReceiver, +) { + let mut pending = VecDeque::new(); + let mut active_touch: Option = None; + loop { + let mut message = match pending.pop_front() { + Some(message) => message, + None => match receiver.recv().await { + Some(message) => message, + None => break, + }, + }; + if webrtc_control_message_is_move(&message) { + while let Ok(next_message) = receiver.try_recv() { + if webrtc_control_message_is_move(&next_message) { + message = next_message; + } else { + pending.push_back(next_message); + break; + } + } + } + + if let Err(error) = run_android_webrtc_control_message( + state.clone(), + udid.clone(), + message, + &mut active_touch, + ) + .await + { + warn!("Android WebRTC control message failed for {udid}: {error}"); + } + } +} + +async fn run_android_webrtc_control_message( + state: AppState, + udid: String, + message: ControlMessage, + active_touch: &mut Option, +) -> Result<(), AppError> { + match message { + ControlMessage::Touch { x, y, phase } => { + if !x.is_finite() || !y.is_finite() { + return Err(AppError::bad_request( + "`x` and `y` must be finite normalized numbers.", + )); + } + return handle_android_webrtc_touch( + state, + udid, + x.clamp(0.0, 1.0), + y.clamp(0.0, 1.0), + phase, + active_touch, + ) + .await; + } + ControlMessage::EdgeTouch { x, y, phase, .. } => { + if !x.is_finite() || !y.is_finite() { + return Err(AppError::bad_request( + "`x` and `y` must be finite normalized numbers.", + )); + } + return handle_android_webrtc_touch( + state, + udid, + x.clamp(0.0, 1.0), + y.clamp(0.0, 1.0), + phase, + active_touch, + ) + .await; + } + ControlMessage::MultiTouch { x1, y1, phase, .. } => { + if !x1.is_finite() || !y1.is_finite() { + return Err(AppError::bad_request( + "`x1` and `y1` must be finite normalized numbers.", + )); + } + return handle_android_webrtc_touch( + state, + udid, + x1.clamp(0.0, 1.0), + y1.clamp(0.0, 1.0), + phase, + active_touch, + ) + .await; + } + _ => {} + } + + task::spawn_blocking(move || match message { + ControlMessage::Key { + key_code, + modifiers, + } => state + .android + .send_key(&udid, key_code, modifiers.unwrap_or(0)), + ControlMessage::Button { + button, + duration_ms, + phase, + .. + } => match phase.as_deref() { + Some("down" | "began") => Ok(()), + Some("up" | "ended" | "cancelled") | None => { + state + .android + .press_button(&udid, &button, duration_ms.unwrap_or(0)) + } + Some(_) => Err(AppError::bad_request( + "`phase` must be `down`, `up`, `began`, `ended`, or `cancelled`.", + )), + }, + ControlMessage::DismissKeyboard => state.android.dismiss_keyboard(&udid), + ControlMessage::Home => state.android.press_home(&udid), + ControlMessage::AppSwitcher => state.android.open_app_switcher(&udid), + ControlMessage::RotateLeft => state.android.rotate_left(&udid), + ControlMessage::RotateRight => state.android.rotate_right(&udid), + ControlMessage::ToggleAppearance => state.android.toggle_appearance(&udid), + ControlMessage::Touch { .. } + | ControlMessage::EdgeTouch { .. } + | ControlMessage::MultiTouch { .. } => Ok(()), + }) + .await + .map_err(|error| AppError::internal(format!("Failed to join Android control task: {error}")))? +} + +async fn handle_android_webrtc_touch( + state: AppState, + udid: String, + x: f64, + y: f64, + phase: String, + active_touch: &mut Option, +) -> Result<(), AppError> { + let action = android::update_touch_gesture(active_touch, x, y, &phase)?; + if matches!(action, android::AndroidTouchAction::None) { + return Ok(()); + } + task::spawn_blocking(move || action.perform(&state.android, &udid)) + .await + .map_err(|error| { + AppError::internal(format!("Failed to join Android touch task: {error}")) + })? +} + async fn run_webrtc_control_queue( session: crate::simulators::session::SimulatorSession, state: AppState, @@ -842,32 +1280,476 @@ fn ice_transport_policy() -> RTCIceTransportPolicy { } } -async fn wait_for_h264_sync_keyframe( - session: &crate::simulators::session::SimulatorSession, - timeout_duration: Duration, -) -> Option { - if let Some(frame) = session.latest_keyframe() { - if h264_frame_is_decoder_sync(&frame) { - return Some(frame); +#[derive(Clone)] +pub(crate) struct AndroidWebRtcSource { + inner: Arc, +} + +struct AndroidWebRtcSourceInner { + udid: String, + encoder_handle: AtomicUsize, + callback_user_data: AtomicUsize, + shutdown_tx: broadcast::Sender<()>, + sender: broadcast::Sender, + raw_sender: broadcast::Sender>, + latest_keyframe: RwLock>, + metrics: Arc, +} + +unsafe impl Send for AndroidWebRtcSourceInner {} +unsafe impl Sync for AndroidWebRtcSourceInner {} + +impl AndroidWebRtcSource { + pub(crate) async fn start( + bridge: android::AndroidBridge, + metrics: Arc, + udid: String, + max_edge: Option, + encode_h264: bool, + ) -> Result { + let mut frame_stream = bridge.grpc_frame_stream(&udid, max_edge).await?; + let (sender, _) = broadcast::channel(ANDROID_WEBRTC_FRAME_BROADCAST_CAPACITY); + let (raw_sender, _) = broadcast::channel(ANDROID_WEBRTC_RAW_FRAME_BROADCAST_CAPACITY); + let (shutdown_tx, _) = broadcast::channel(1); + let inner = Arc::new(AndroidWebRtcSourceInner { + udid: udid.clone(), + encoder_handle: AtomicUsize::new(0), + callback_user_data: AtomicUsize::new(0), + shutdown_tx, + sender, + raw_sender, + latest_keyframe: RwLock::new(None), + metrics, + }); + if encode_h264 { + let user_data = Weak::into_raw(Arc::downgrade(&inner)) as *mut c_void; + let mut error = std::ptr::null_mut(); + let handle = unsafe { + ffi::xcw_native_h264_encoder_create( + Some(android_h264_encoder_frame_callback), + user_data, + &mut error, + ) + }; + if handle.is_null() { + unsafe { + let _ = Weak::from_raw(user_data as *const AndroidWebRtcSourceInner); + } + return Err(unsafe { take_native_error(error) }.unwrap_or_else(|| { + AppError::native("Unable to create Android H.264 encoder.") + })); + } + inner + .encoder_handle + .store(handle as usize, Ordering::Release); + inner + .callback_user_data + .store(user_data as usize, Ordering::Release); + } + + let source = Self { inner }; + let latest_frame = Arc::new(Mutex::new(None::>)); + let reader_inner = Arc::downgrade(&source.inner); + let reader_latest_frame = latest_frame.clone(); + let mut reader_shutdown_rx = source.inner.shutdown_tx.subscribe(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = reader_shutdown_rx.recv() => break, + frame = frame_stream.next_frame() => { + match frame { + Ok(Some(frame)) => { + let frame = Arc::new(frame); + *reader_latest_frame.lock().unwrap() = Some(frame); + } + Ok(None) => break, + Err(error) => { + let udid = reader_inner + .upgrade() + .map(|inner| inner.udid.clone()) + .unwrap_or_else(|| "android".to_owned()); + warn!("Android WebRTC raw frame stream failed for {udid}: {error}"); + break; + } + } + } + } + } + }); + + if encode_h264 { + let encoder_inner = Arc::downgrade(&source.inner); + let encoder_latest_frame = latest_frame; + let mut encoder_shutdown_rx = source.inner.shutdown_tx.subscribe(); + tokio::spawn(async move { + let min_frame_gap = android_webrtc_frame_interval(); + let mut ticker = time::interval(min_frame_gap); + ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + loop { + tokio::select! { + _ = encoder_shutdown_rx.recv() => break, + _ = ticker.tick() => { + let Some(inner) = encoder_inner.upgrade() else { + break; + }; + let frame = encoder_latest_frame.lock().unwrap().clone(); + let Some(frame) = frame else { + continue; + }; + if inner.latest_keyframe.read().unwrap().is_none() { + inner.request_keyframe(); + } + let handle = inner.encoder_handle.load(Ordering::Acquire); + let udid = inner.udid.clone(); + let encode_result = task::spawn_blocking(move || { + encode_android_rgba_frame(handle, &frame) + }) + .await + .map_err(|error| AppError::internal(format!("Failed to join Android encoder task: {error}"))) + .and_then(|result| result); + if let Err(error) = encode_result { + warn!("Android VideoToolbox encode failed for {udid}: {error}"); + } + } + } + } + }); + source.request_keyframe(); + } else { + let raw_inner = Arc::downgrade(&source.inner); + let raw_latest_frame = latest_frame; + let mut raw_shutdown_rx = source.inner.shutdown_tx.subscribe(); + tokio::spawn(async move { + let min_frame_gap = android_webrtc_frame_interval(); + let mut ticker = time::interval(min_frame_gap); + ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + loop { + tokio::select! { + _ = raw_shutdown_rx.recv() => break, + _ = ticker.tick() => { + let Some(inner) = raw_inner.upgrade() else { + break; + }; + let frame = raw_latest_frame.lock().unwrap().clone(); + let Some(frame) = frame else { + continue; + }; + let _ = inner.raw_sender.send(frame); + } + } + } + }); + } + Ok(source) + } + + pub(crate) fn subscribe(&self) -> broadcast::Receiver { + self.inner.sender.subscribe() + } + + fn subscribe_raw(&self) -> broadcast::Receiver> { + self.inner.raw_sender.subscribe() + } + + pub(crate) async fn wait_for_keyframe( + &self, + timeout_duration: Duration, + ) -> Option { + let deadline = Instant::now() + timeout_duration; + let baseline_sequence = self + .inner + .latest_keyframe + .read() + .unwrap() + .as_ref() + .map_or(0, |frame| frame.frame_sequence); + let mut receiver = self.inner.sender.subscribe(); + self.request_keyframe(); + + loop { + if let Some(frame) = self.inner.latest_keyframe.read().unwrap().clone() { + if frame.frame_sequence > baseline_sequence { + return Some(frame); + } + } + let remaining = deadline.checked_duration_since(Instant::now())?; + match time::timeout(remaining, receiver.recv()).await { + Ok(Ok(frame)) if frame.is_keyframe && frame.frame_sequence > baseline_sequence => { + return Some(frame) + } + Ok(Ok(_)) | Ok(Err(broadcast::error::RecvError::Lagged(_))) => { + self.request_keyframe(); + } + Ok(Err(_)) | Err(_) => return None, + } + } + } + + pub(crate) fn request_refresh(&self) {} + + pub(crate) fn request_keyframe(&self) { + self.inner.request_keyframe(); + } +} + +impl Drop for AndroidWebRtcSourceInner { + fn drop(&mut self) { + let _ = self.shutdown_tx.send(()); + let encoder_handle = self.encoder_handle.load(Ordering::Acquire); + let callback_user_data = self.callback_user_data.load(Ordering::Acquire); + unsafe { + if encoder_handle != 0 { + ffi::xcw_native_h264_encoder_destroy(encoder_handle as *mut c_void); + } + if callback_user_data != 0 { + let _ = Weak::from_raw(callback_user_data as *const AndroidWebRtcSourceInner); + } + } + } +} + +unsafe extern "C" fn android_h264_encoder_frame_callback( + frame: *const ffi::xcw_native_frame, + user_data: *mut c_void, +) { + if frame.is_null() || user_data.is_null() { + return; + } + + let weak = unsafe { Weak::from_raw(user_data as *const AndroidWebRtcSourceInner) }; + if let Some(inner) = weak.upgrade() { + unsafe { + inner.handle_encoded_frame(&*frame); + } + } + let _ = Weak::into_raw(weak); +} + +impl AndroidWebRtcSourceInner { + fn request_keyframe(&self) { + self.metrics + .keyframe_requests + .fetch_add(1, Ordering::Relaxed); + let encoder_handle = self.encoder_handle.load(Ordering::Acquire); + if encoder_handle == 0 { + return; + } + unsafe { + ffi::xcw_native_h264_encoder_request_keyframe(encoder_handle as *mut c_void); + } + } + + fn handle_encoded_frame(&self, frame: &ffi::xcw_native_frame) { + let description = unsafe { copy_native_shared_bytes(frame.description) }; + let Some(data) = (unsafe { copy_native_shared_bytes(frame.data) }) else { + return; + }; + let packet = Arc::new(FramePacket { + frame_sequence: frame.frame_sequence, + timestamp_us: frame.timestamp_us, + is_keyframe: frame.is_keyframe, + width: frame.width, + height: frame.height, + codec: native_c_string(frame.codec), + description, + data, + }); + self.metrics.frames_encoded.fetch_add(1, Ordering::Relaxed); + if packet.is_keyframe { + self.metrics + .keyframes_encoded + .fetch_add(1, Ordering::Relaxed); + *self.latest_keyframe.write().unwrap() = Some(packet.clone()); + } + let _ = self.sender.send(packet); + } +} + +fn encode_android_rgba_frame( + encoder_handle: usize, + frame: &android::AndroidFrame, +) -> Result<(), AppError> { + unsafe { + let mut error = std::ptr::null_mut(); + let ok = ffi::xcw_native_h264_encoder_encode_rgba( + encoder_handle as *mut c_void, + frame.rgba.as_ptr(), + frame.rgba.len(), + frame.width, + frame.height, + frame.timestamp_us, + &mut error, + ); + if ok { + Ok(()) + } else { + Err(take_native_error(error) + .unwrap_or_else(|| AppError::native("Android VideoToolbox encode failed."))) + } + } +} + +fn android_rgba_webrtc_frame_chunks( + sequence: u64, + frame: &android::AndroidFrame, +) -> Option> { + let expected_bytes = frame.width as usize * frame.height as usize * 4; + if expected_bytes == 0 || frame.rgba.len() != expected_bytes { + return None; + } + let mut chunks = Vec::with_capacity(frame.rgba.len().div_ceil(ANDROID_WEBRTC_RGBA_CHUNK_BYTES)); + for (chunk_index, chunk) in frame + .rgba + .chunks(ANDROID_WEBRTC_RGBA_CHUNK_BYTES) + .enumerate() + { + let chunk_offset = chunk_index * ANDROID_WEBRTC_RGBA_CHUNK_BYTES; + let mut bytes = + BytesMut::with_capacity(ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES + chunk.len()); + bytes.put_u32(ANDROID_WEBRTC_RGBA_CHUNK_MAGIC); + bytes.put_u8(ANDROID_WEBRTC_RGBA_VERSION); + bytes.put_u8(ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888); + bytes.put_u16(ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES as u16); + bytes.put_u64(sequence); + bytes.put_u64(frame.timestamp_us); + bytes.put_u32(frame.width); + bytes.put_u32(frame.height); + bytes.put_u32(frame.rgba.len() as u32); + bytes.put_u32(chunk_offset as u32); + bytes.put_u32(chunk.len() as u32); + bytes.put_u8(if chunk_offset + chunk.len() >= frame.rgba.len() { + 1 + } else { + 0 + }); + bytes.put_u8(0); + bytes.put_u16(0); + bytes.extend_from_slice(chunk); + chunks.push(bytes.freeze()); + } + Some(chunks) +} + +unsafe fn copy_native_shared_bytes(bytes: ffi::xcw_native_shared_bytes) -> Option { + if bytes.data.is_null() || bytes.length == 0 { + if !bytes.owner.is_null() { + unsafe { + ffi::xcw_native_release_shared_bytes(bytes); + } + } + return None; + } + + let copied = + unsafe { Bytes::copy_from_slice(std::slice::from_raw_parts(bytes.data, bytes.length)) }; + unsafe { + ffi::xcw_native_release_shared_bytes(bytes); + } + Some(copied) +} + +fn native_c_string(ptr: *const i8) -> Option { + if ptr.is_null() { + return None; + } + let value = unsafe { CStr::from_ptr(ptr) } + .to_string_lossy() + .trim() + .to_owned(); + if value.is_empty() { + None + } else { + Some(value) + } +} + +unsafe fn take_native_error(raw: *mut i8) -> Option { + if raw.is_null() { + return None; + } + let message = unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned(); + unsafe { + ffi::xcw_native_free_string(raw); + } + Some(AppError::native(message)) +} + +fn android_webrtc_frame_interval() -> Duration { + Duration::from_micros(1_000_000 / ANDROID_WEBRTC_FPS) +} + +#[derive(Clone)] +enum WebRtcVideoSource { + Simulator(crate::simulators::session::SimulatorSession), + Android(AndroidWebRtcSource), +} + +impl WebRtcVideoSource { + fn subscribe(&self) -> WebRtcFrameReceiver { + match self { + Self::Simulator(session) => WebRtcFrameReceiver::Simulator(session.subscribe()), + Self::Android(source) => WebRtcFrameReceiver::Android(source.subscribe()), + } + } + + async fn wait_for_keyframe(&self, timeout_duration: Duration) -> Option { + match self { + Self::Simulator(session) => session.wait_for_keyframe(timeout_duration).await, + Self::Android(source) => source.wait_for_keyframe(timeout_duration).await, + } + } + + fn request_refresh(&self) { + match self { + Self::Simulator(session) => session.request_refresh(), + Self::Android(source) => source.request_refresh(), } } + fn request_keyframe(&self) { + match self { + Self::Simulator(session) => session.request_keyframe(), + Self::Android(source) => source.request_keyframe(), + } + } +} + +enum WebRtcFrameReceiver { + Simulator(crate::simulators::session::FrameSubscription), + Android(broadcast::Receiver), +} + +impl WebRtcFrameReceiver { + async fn recv(&mut self) -> Result { + match self { + Self::Simulator(receiver) => receiver.recv().await, + Self::Android(receiver) => receiver.recv().await, + } + } +} + +async fn wait_for_h264_sync_keyframe( + source: &WebRtcVideoSource, + timeout_duration: Duration, +) -> Option { let deadline = time::Instant::now() + timeout_duration; loop { let remaining = deadline.checked_duration_since(time::Instant::now())?; - let frame = session.wait_for_keyframe(remaining).await?; + let frame = source.wait_for_keyframe(remaining).await?; if h264_frame_is_decoder_sync(&frame) { return Some(frame); } - session.request_keyframe(); + source.request_keyframe(); } } struct WebRtcMediaStream { state: AppState, - session: crate::simulators::session::SimulatorSession, + source: WebRtcVideoSource, udid: String, - first_frame: crate::transport::packet::SharedFrame, + first_frame: SharedFrame, peer_connection: Arc, video_track: Arc, cancellation_token: broadcast::Sender<()>, @@ -875,11 +1757,140 @@ struct WebRtcMediaStream { stream_control_rx: mpsc::UnboundedReceiver, } +struct WebRtcRgbaStream { + state: AppState, + source: AndroidWebRtcSource, + udid: String, + peer_connection: Arc, + rgba_channel: Arc, + cancellation_token: broadcast::Sender<()>, + cancellation: broadcast::Receiver<()>, + stream_control_rx: mpsc::UnboundedReceiver, +} + +impl WebRtcRgbaStream { + async fn run(self) { + let Self { + state, + source, + udid, + peer_connection, + rgba_channel, + cancellation_token, + mut cancellation, + mut stream_control_rx, + } = self; + let mut rx = source.subscribe_raw(); + let mut peer_state_interval = time::interval(Duration::from_millis(250)); + let mut peer_disconnected_since: Option = None; + let mut sequence = 0u64; + let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); + rgba_channel.on_open(Box::new({ + let udid = udid.clone(); + move || { + let udid = udid.clone(); + Box::pin(async move { + info!("Android RGBA WebRTC data channel open for {udid}"); + }) + } + })); + + loop { + tokio::select! { + _ = cancellation.recv() => { + warn!("Android RGBA WebRTC stream replaced for {udid}"); + break; + } + _ = peer_state_interval.tick() => { + let peer_state = peer_connection.connection_state(); + if matches!(peer_state, RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed) { + warn!("Android RGBA WebRTC stream closing for {udid}: peer state {peer_state}"); + break; + } + if peer_state == RTCPeerConnectionState::Disconnected { + let disconnected_since = + peer_disconnected_since.get_or_insert_with(time::Instant::now); + if disconnected_since.elapsed() >= WEBRTC_PEER_DISCONNECTED_TIMEOUT { + warn!("Android RGBA WebRTC stream closing for {udid}: peer state {peer_state}"); + break; + } + } else { + peer_disconnected_since = None; + } + } + command = stream_control_rx.recv() => { + let Some(command) = command else { + continue; + }; + if command.force_keyframe || command.snapshot { + source.request_refresh(); + } + } + frame = rx.recv() => { + let frame = match frame { + Ok(frame) => frame, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + state + .metrics + .frames_dropped_server + .fetch_add(skipped, Ordering::Relaxed); + continue; + } + Err(broadcast::error::RecvError::Closed) => { + warn!("Android RGBA WebRTC stream closing for {udid}: raw frame channel closed"); + break; + } + }; + if rgba_channel.ready_state() != RTCDataChannelState::Open { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + continue; + } + let Some(chunks) = android_rgba_webrtc_frame_chunks(sequence, &frame) else { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + continue; + }; + let buffered_amount = rgba_channel.buffered_amount().await; + if buffered_amount > frame.rgba.len() * ANDROID_WEBRTC_RGBA_BUFFERED_FRAME_LIMIT { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + continue; + } + sequence = sequence.wrapping_add(1); + let mut sent_frame = true; + for chunk in chunks { + let send_result = time::timeout(WEBRTC_REALTIME_WRITE_TIMEOUT, rgba_channel.send(&chunk)).await; + match send_result { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + warn!("Android RGBA WebRTC data channel send failed for {udid}: {error}"); + sent_frame = false; + break; + } + Err(_) => { + sent_frame = false; + break; + } + } + } + if sent_frame { + state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); + } else { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + } + } + } + } + + warn!("Android RGBA WebRTC stream ended for {udid}"); + clear_webrtc_media_stream(&udid, &cancellation_token); + let _ = peer_connection.close().await; + } +} + impl WebRtcMediaStream { async fn run(self) { let Self { state, - session, + source, udid, first_frame, peer_connection, @@ -888,7 +1899,7 @@ impl WebRtcMediaStream { mut cancellation, mut stream_control_rx, } = self; - let mut rx = session.subscribe(); + let mut rx = source.subscribe(); let mut send_timing = WebRtcSendTiming::new(); let mut peer_state_interval = time::interval(Duration::from_millis(250)); let realtime_stream = realtime_stream_enabled(); @@ -925,10 +1936,10 @@ impl WebRtcMediaStream { if recovery_action_for_write_timeout(realtime_stream) == FrameRecoveryAction::Refresh { - session.request_refresh(); + source.request_refresh(); } else { waiting_for_keyframe = true; - session.request_keyframe(); + source.request_keyframe(); } } Err(error) => { @@ -967,9 +1978,9 @@ impl WebRtcMediaStream { }; if command.force_keyframe || command.snapshot { waiting_for_keyframe = true; - session.request_keyframe(); + source.request_keyframe(); } else { - session.request_refresh(); + source.request_refresh(); } } frame = rx.recv() => { @@ -981,7 +1992,7 @@ impl WebRtcMediaStream { .frames_dropped_server .fetch_add(skipped, Ordering::Relaxed); waiting_for_keyframe = true; - session.request_keyframe(); + source.request_keyframe(); continue; } Err(broadcast::error::RecvError::Closed) => { @@ -997,7 +2008,7 @@ impl WebRtcMediaStream { waiting_for_keyframe = false; } else if frame.is_keyframe { waiting_for_keyframe = true; - session.request_keyframe(); + source.request_keyframe(); state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); continue; } @@ -1022,9 +2033,9 @@ impl WebRtcMediaStream { let recovery_action = recovery_action_for_write_timeout(realtime_stream); waiting_for_keyframe = recovery_action == FrameRecoveryAction::Keyframe; if recovery_action == FrameRecoveryAction::Refresh { - session.request_refresh(); + source.request_refresh(); } else { - session.request_keyframe(); + source.request_keyframe(); } } Err(error) => { @@ -1464,11 +2475,14 @@ impl Drop for WebRtcMetricsGuard { #[cfg(test)] mod tests { use super::{ - append_avcc_parameter_sets, append_length_prefixed_nalus, h264_annex_b_sample, - h264_frame_has_idr, h264_frame_is_decoder_sync, h264_sdp_fmtp_line, is_annex_b, - is_h264_codec, rtcp_packet_requests_keyframe, rtp_packet_pacing, WebRtcMetricsGuard, - WebRtcSendTiming, ANNEX_B_START_CODE, + android_rgba_webrtc_frame_chunks, append_avcc_parameter_sets, append_length_prefixed_nalus, + h264_annex_b_sample, h264_frame_has_idr, h264_frame_is_decoder_sync, h264_sdp_fmtp_line, + is_annex_b, is_h264_codec, rtcp_packet_requests_keyframe, rtp_packet_pacing, + WebRtcMetricsGuard, WebRtcSendTiming, ANDROID_WEBRTC_RGBA_CHUNK_BYTES, + ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES, ANDROID_WEBRTC_RGBA_CHUNK_MAGIC, + ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888, ANDROID_WEBRTC_RGBA_VERSION, ANNEX_B_START_CODE, }; + use crate::android; use crate::metrics::counters::Metrics; use crate::transport::packet::FramePacket; use bytes::Bytes; @@ -1929,6 +2943,71 @@ mod tests { ); } + #[test] + fn android_rgba_webrtc_frame_chunks_use_fixed_binary_header() { + let rgba = (0..(320 * 240 * 4)) + .map(|value| (value % 251) as u8) + .collect::>(); + let frame = android::AndroidFrame { + width: 320, + height: 240, + timestamp_us: 123_456, + rgba: rgba.clone(), + }; + + let chunks = android_rgba_webrtc_frame_chunks(7, &frame).unwrap(); + + assert_eq!(chunks.len(), 2); + let first = &chunks[0]; + assert_eq!( + u32::from_be_bytes(first[0..4].try_into().unwrap()), + ANDROID_WEBRTC_RGBA_CHUNK_MAGIC + ); + assert_eq!(first[4], ANDROID_WEBRTC_RGBA_VERSION); + assert_eq!(first[5], ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888); + assert_eq!( + u16::from_be_bytes(first[6..8].try_into().unwrap()) as usize, + ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES + ); + assert_eq!(u64::from_be_bytes(first[8..16].try_into().unwrap()), 7); + assert_eq!( + u64::from_be_bytes(first[16..24].try_into().unwrap()), + 123_456 + ); + assert_eq!(u32::from_be_bytes(first[24..28].try_into().unwrap()), 320); + assert_eq!(u32::from_be_bytes(first[28..32].try_into().unwrap()), 240); + assert_eq!( + u32::from_be_bytes(first[32..36].try_into().unwrap()) as usize, + rgba.len() + ); + assert_eq!(u32::from_be_bytes(first[36..40].try_into().unwrap()), 0); + assert_eq!( + u32::from_be_bytes(first[40..44].try_into().unwrap()) as usize, + ANDROID_WEBRTC_RGBA_CHUNK_BYTES + ); + assert_eq!(first[44], 0); + + let second = &chunks[1]; + assert_eq!( + u32::from_be_bytes(second[36..40].try_into().unwrap()) as usize, + ANDROID_WEBRTC_RGBA_CHUNK_BYTES + ); + assert_eq!( + u32::from_be_bytes(second[40..44].try_into().unwrap()) as usize, + rgba.len() - ANDROID_WEBRTC_RGBA_CHUNK_BYTES + ); + assert_eq!(second[44], 1); + let reassembled = chunks + .iter() + .flat_map(|chunk| { + chunk[ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES..] + .iter() + .copied() + }) + .collect::>(); + assert_eq!(reassembled, rgba); + } + trait CloneFrameForTest { fn clone_for_test(&self, frame_sequence: u64) -> Self; } diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index c37ea448..cb6bfedd 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -5,7 +5,7 @@ description: Use for simulator lifecycle, app install/launch, live viewing, UI i # SimDeck Agent Guide -SimDeck automates iOS Simulators. Use the CLI for automation and the browser UI for live human visibility. Works with UIKit, SwiftUI, React Native, Expo, and NativeScript apps. +SimDeck automates iOS Simulators and Android emulators. Use the CLI for automation and the browser UI for live human visibility. iOS works with UIKit, SwiftUI, React Native, Expo, and NativeScript apps; Android works through ADB, emulator lifecycle, screenshots, logs, and UIAutomator hierarchy dumps. SimDeck uses one warm daemon per project. Check it with `simdeck daemon status`; start it or open the browser UI when needed: @@ -47,6 +47,7 @@ simdeck shutdown simdeck erase simdeck core-simulator restart simdeck install /path/to/App.app +simdeck install android: /path/to/app.apk simdeck launch com.example.App simdeck uninstall com.example.App simdeck open-url myapp://route @@ -56,6 +57,12 @@ simdeck toggle-appearance Build apps with project tooling. +Android devices use IDs like `android:Pixel_8_API_36`. `simdeck list` discovers +AVDs from the Android SDK, `boot` starts `emulator -avd ... -no-window`, and +live browser viewing uses WebRTC. Local loopback Android viewing sends raw RGBA +frames over a WebRTC data channel; non-loopback Android viewing uses +VideoToolbox-encoded H.264. `simdeck stream` is still iOS-only. + ## Fast Agent Inspection Use targeted checks for test loops. `describe` is a diagnostic snapshot of the whole hierarchy; it is useful for planning, but it is expensive. For verification, prefer the daemon APIs exposed by `simdeck/test`: `query`, `waitFor`, `assert`, selector `tap`, and `batch`. @@ -71,10 +78,13 @@ simdeck describe --source react-native simdeck describe --source flutter simdeck describe --source uikit simdeck describe --source native-ax +simdeck describe --source android-uiautomator simdeck describe --direct ``` -Use `--source auto` with the project daemon. Use `--direct` or `--source native-ax` for the private CoreSimulator accessibility bridge. NativeScript, React Native, and Flutter inspector runtimes can add richer hierarchy data. +Use `--source auto` with the project daemon. Use `--direct` or `--source native-ax` for the private CoreSimulator accessibility bridge. Use `--source android-uiautomator` for Android emulator UIAutomator hierarchies. NativeScript, React Native, and Flutter inspector runtimes can add richer hierarchy data. +For Android IDs, `describe` uses `uiautomator dump`; use `--format agent` or +`--format compact-json` the same way as iOS. Prefer selectors, coordinates only when needed. Selector taps go through the daemon and wait for the element server-side.