diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df73e307..4e97519e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: swift: - - '6.3.1' + - '6.3.2' os: - 'ubuntu-latest' #- 'macos-latest' @@ -24,7 +24,7 @@ jobs: - 'macos-14' steps: - name: Checkout skipstone.git - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: # skip.git submodule needed for SkipDriveExternal submodules: true @@ -127,6 +127,16 @@ jobs: steps: - uses: Homebrew/actions/setup-homebrew@main + # The transpiled (Lite) and Fuse framework `swift test` runs execute the Kotlin suite under + # Robolectric, and recent Android SDK levels (e.g. SDK 36) require Java 21. The runners default to + # Java 17, so without this the Robolectric run fails with: + # "Failed to create a Robolectric sandbox: Android SDK 36 requires Java 21 (have Java 17)". + - name: "Set up Java 21 (required by Robolectric for recent Android SDKs)" + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '21' + - name: "Install Host Toolchain prerequisites" if: runner.os == 'Linux' run: | @@ -175,7 +185,8 @@ jobs: # perform an export of the app skip export --appindex --validate-export --show-tree --project some-app - - run: | + - name: "Install Swift SDK for Android" + run: | sdk_arg="" if [[ "${{ matrix.android-sdk }}" != "" ]]; then sdk_arg="--version ${{ matrix.android-sdk }}" @@ -263,22 +274,35 @@ jobs: ! skip android test --apk --arch ${{ matrix.android-arch }} - name: "Test new Skip Lite framework on emulator" - # Disabled on Linux: - # /home/runner/work/_temp/demo-lite-framework/Tests/DemoFrameworkTests/DemoFrameworkTests.swift:2:8: error: no such module 'OSLog' - if: runner.os != 'Linux' working-directory: ${{ runner.temp }} run: | skip init --no-build --transpiled-model --module-tests demo-lite-framework DemoFramework cd demo-lite-framework + # Robolectric test + swift test + # Emulator test ANDROID_SERIAL=emulator-5554 swift test - name: "Test new Skip Fuse framework on emulator" - # Disabled due to build error on Linux: - # /home/runner/work/_temp/demo-fuse-framework/.build/checkouts/skip-android-bridge/Sources/SkipAndroidBridge/AndroidBundle.swift:6:12: error: cannot inherit from class 'Bundle' (compiled with Swift 6.2.4) because it has overridable members that could not be loaded in Swift 5.10 + # compile errors on Linux with skip-android-bridge if: runner.os != 'Linux' working-directory: ${{ runner.temp }} run: | skip init --no-build --native-model --bridged --module-tests demo-fuse-framework DemoFramework cd demo-fuse-framework + # Robolectric test + swift test + # Emulator test ANDROID_SERIAL=emulator-5554 swift test + # now add a failing test to make sure it actually fails + echo '#if os(Android)' >> Tests/DemoFrameworkTests/DemoFrameworkTests.swift + echo '@Test func shouldFailOnAndroid() { #expect(1+1 == 3, "Test should fail on Android") }' >> Tests/DemoFrameworkTests/DemoFrameworkTests.swift + echo '#endif' >> Tests/DemoFrameworkTests/DemoFrameworkTests.swift + + swift build --build-tests + # Robolectric should still pass + swift test + # Emulator test should fail + ! ANDROID_SERIAL=emulator-5554 swift test + diff --git a/Sources/SkipBuild/Commands/AndroidTestCommand.swift b/Sources/SkipBuild/Commands/AndroidTestCommand.swift index 051b49b8..4cd5bd37 100644 --- a/Sources/SkipBuild/Commands/AndroidTestCommand.swift +++ b/Sources/SkipBuild/Commands/AndroidTestCommand.swift @@ -72,6 +72,12 @@ struct AndroidTestCommand: AndroidOperationCommand { @Flag(inversion: .prefixedNo, help: ArgumentHelp("Package and run tests as an Android APK")) var apk: Bool = false + @Option(help: ArgumentHelp("Build the native Swift Testing bundle + JNI harness into / (jniLibs layout) instead of running them; used by the Gradle connectedAndroidTest path for `mode: native` test modules", valueName: "dir")) + var buildTestLibs: String? = nil + + @Flag(help: ArgumentHelp("With --build-test-libs, build the test bundle + harness for the HOST (Robolectric / `testDebug`, no device) instead of Android")) + var robolectric: Bool = false + @Option(help: ArgumentHelp("Path to write the JSON event stream output", valueName: "path")) var eventStreamOutputPath: String? = nil @@ -80,7 +86,13 @@ struct AndroidTestCommand: AndroidOperationCommand { var args: [String] = [] func performCommand(with out: MessageQueue) async throws { - if apk { + if let buildTestLibs = buildTestLibs { + if robolectric { + try await buildLocalTestLibs(outputDir: buildTestLibs, with: out) + } else { + try await buildNativeTestLibs(outputDir: buildTestLibs, with: out) + } + } else if apk { try await runSwiftPMAsAPK(cleanup: cleanup, eventStreamOutputPath: eventStreamOutputPath, with: out) } else { try await runSwiftPM(cleanup: cleanup, commandEnvironment: env, defaultArch: .current, remoteFolder: remoteFolder, copy: copy, testingLibrary: testingLibrary, with: out) @@ -194,6 +206,28 @@ fileprivate extension AndroidOperationCommand { return signedAPK } + /// Build the native Swift Testing bundle + the Android JNI harness (`libtest_harness.so`) into + /// `/` (jniLibs layout) for the Gradle `connectedAndroidTest` path. The on-device + /// `org.swift.test.SwiftTestRunner` then `System.loadLibrary("test_harness")` and runs swt. + func buildNativeTestLibs(outputDir: String, with out: MessageQueue) async throws { + #if !canImport(SkipDriveExternal) + throw ToolLaunchError(errorDescription: "Cannot launch android command without SkipDriveExternal") + #else + try await stageTestLibs(try await androidTestLibsTarget(with: out), outputDir: outputDir, with: out) + #endif + } + + /// Build the native Swift Testing bundle + a pure-Swift host JNI harness (`libtest_harness.dylib`) + /// into `` (plus a `dyld-env.txt`) for the Robolectric (`testDebug`, no device) path. The + /// host `org.swift.test.SwiftTestRunner` then `System.load`s the harness and runs swt. + func buildLocalTestLibs(outputDir: String, with out: MessageQueue) async throws { + #if !canImport(SkipDriveExternal) + throw ToolLaunchError(errorDescription: "Cannot launch android command without SkipDriveExternal") + #else + try await stageTestLibs(try await hostTestLibsTarget(with: out), outputDir: outputDir, with: out) + #endif + } + /// Build Swift tests as a shared library, package into an APK with an Instrumentation runner, /// install, launch via `am instrument`, and stream structured test output. /// Uses `swt_abiv0_getEntryPoint` for Swift Testing integration. @@ -592,242 +626,225 @@ fileprivate extension AndroidOperationCommand { } } -private let testHarnessLib = "test_harness" -private let testPackage = "org.swift.test" -private let testClassName = "SwiftTestRunner" -private let testFullClass = "\(testPackage).\(testClassName)" - -/// Package.swift for the generated Swift test harness package. -/// Defines a dynamic library target that produces `libtest_harness.so` -private let harnessPackageSwift: String = """ -// swift-tools-version: 6.0 -import PackageDescription - -let package = Package( - name: "test-harness", - products: [ - .library(name: "\(testHarnessLib)", type: .dynamic, targets: ["TestHarness"]) - ], - targets: [ - .target( - name: "TestHarness", - linkerSettings: [ - .linkedLibrary("log"), - .linkedLibrary("dl"), - ] - ), - ] -) -""" - -/// Java source for the Android Instrumentation test runner. -/// Loads `libtest_harness.so`, calls `runTests()` via JNI, and uses -/// `sendStatus()`/`finish()` for structured output back to the host. -private let instrumentationJavaSource: String = """ -package \(testPackage); - -import android.app.Instrumentation; -import android.os.Bundle; - -public class \(testClassName) extends Instrumentation { - static { - android.util.Log.i("SwiftTest", "loading harness"); - System.loadLibrary("\(testHarnessLib)"); - android.util.Log.i("SwiftTest", "loaded harness"); - } - private native int runTests(); - - @Override - public void onCreate(Bundle arguments) { - android.util.Log.i("SwiftTest", "onCreate"); - super.onCreate(arguments); - // This triggers onStart() in a separate thread - start(); - android.util.Log.i("SwiftTest", "onCreate: started"); - } - @Override - public void onStart() { - super.onStart(); - Bundle result = new Bundle(); - try { - android.util.Log.i("SwiftTest", "onStart"); - super.onStart(); - android.util.Log.i("SwiftTest", "runTests"); - int exitCode = runTests(); - android.util.Log.i("SwiftTest", "runTests done"); - result.putString("status", exitCode == 0 ? "passed" : "failed"); - finish(exitCode == 0 ? -1 : exitCode, result); - } catch (Throwable t) { - android.util.Log.e("SwiftTest", "Test error", t); - finish(1, result); - } - } +// MARK: - Shared native test-libs staging (Android jniLibs / Darwin Robolectric) - public void reportTestOutput(String line) { - Bundle b = new Bundle(); - b.putString("stream", line + "\\n"); - sendStatus(0, b); - } -} -""" - -/// Swift source for the test harness. Implements JNI_OnLoad and the native `runTests` method. -/// Loads the test library via dlopen, invokes the Swift Testing entry point, and reports -/// test output back through JNI to the Java Instrumentation runner. -private func testHarnessSwiftSource(testLibName: String) -> String { - return """ -import Android -import Dispatch - -// MARK: - JNI type aliases - -typealias JNIEnvironment = UnsafeMutablePointer - -// MARK: - Global state - -nonisolated(unsafe) var g_jvm: UnsafeMutablePointer? = nil - -private func androidLog(_ priority: android_LogPriority, _ tag: String, _ message: String) { - __android_log_write(Int32(priority.rawValue), tag, message) -} - -// MARK: - JNI_OnLoad +#if canImport(SkipDriveExternal) -@_cdecl("JNI_OnLoad") -func JNI_OnLoad(_ vm: UnsafeMutablePointer?, _ reserved: UnsafeMutableRawPointer?) -> jint { - g_jvm = vm - androidLog(ANDROID_LOG_INFO, "SwiftTest", "JNI_OnLoad") - return jint(JNI_VERSION_1_6) +/// The platform-varying parts of building + staging the native Swift Testing bundle and its JNI harness. +/// `stageTestLibs` drives the shared pipeline; `androidTestLibsTarget` / `hostTestLibsTarget` supply the +/// Android-vs-Darwin specifics (toolchain build vs host `swift build`, `.so` vs `.dylib`, bundle layout, +/// dependency set, per-ABI vs flat staging, the harness package/source, and any post-staging). +fileprivate struct TestLibsTarget { + /// The Swift driver used to parse the package and build the harness. + let swiftCommand: String + /// The dynamic-library extension for staged artifacts (`so` / `dylib`). + let dylibExtension: String + /// Build the test bundle; returns its build-output dir, the environment for the harness build, and + /// any extra `swift build` arguments for the harness build (e.g. `--swift-sdk` on Android). + let buildTestBundle: () async throws -> (binDir: URL, harnessEnv: [String: String], harnessExtraArgs: [String]) + /// The loadable Mach-O within the build output (the `.so` itself on Android; the bundle's inner + /// executable on Darwin). + let loadableTestBundle: (_ binDir: URL, _ packageName: String) -> URL + /// The dynamic libraries to stage alongside the test bundle. + let dependencyLibraries: (_ binDir: URL) throws -> [URL] + /// The directory to stage the libraries into (per-ABI on Android, flat on Darwin). + let stagingDirectory: (_ outputDir: String) -> URL + /// The harness `Package.swift`, and its Swift source given the staged test-lib base name. + let harnessPackage: String + let harnessSource: (_ testLibBaseName: String) -> String + /// Fallback harness build-output dir if `swift build --show-bin-path` yields nothing (Android computes + /// it from the triple); `nil` to require `--show-bin-path`. + let harnessBinFallback: (_ harnessDir: URL) -> URL? + /// Any extra staging once the libraries are in place (Darwin writes `dyld-env.txt`). + let postStage: (_ libDir: URL) async throws -> Void } -// MARK: - Entry point type (ST-0002 JSON ABI) - -typealias EntryPoint = @convention(thin) @Sendable ( - _ configurationJSON: UnsafeRawBufferPointer?, - _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void -) async throws -> Bool - -// MARK: - JNI native method - -@_cdecl("Java_\(testFullClass.replacingOccurrences(of: ".", with: "_"))_runTests") -func runTests(_ env: JNIEnvironment, _ thisObj: jobject) -> jint { - let jni: JNINativeInterface = env.pointee!.pointee - - // Keep a global ref to the Instrumentation object for use from other threads - guard let globalThis: jobject = jni.NewGlobalRef(env, thisObj) else { - androidLog(ANDROID_LOG_ERROR, "SwiftTest", "Failed to create global ref") - return 1 - } - defer { jni.DeleteGlobalRef(env, globalThis) } - - // Load test library - androidLog(ANDROID_LOG_INFO, "SwiftTest", "Loading test library: \(testLibName)") - guard let handle = dlopen("\(testLibName)", RTLD_NOW) else { - let err = dlerror().flatMap({ String(cString: $0) }) ?? "" - androidLog(ANDROID_LOG_ERROR, "SwiftTest", "dlopen failed: \\(err)") - return 1 - } - - // Look up swt_abiv0_getEntryPoint - guard let sym = dlsym(handle, "swt_abiv0_getEntryPoint") else { - androidLog(ANDROID_LOG_ERROR, "SwiftTest", "swt_abiv0_getEntryPoint not found") - return 1 - } - typealias GetEntryPointFn = @convention(c) () -> UnsafeRawPointer? - let getEntryPoint = unsafeBitCast(sym, to: GetEntryPointFn.self) - - guard let rawEntryPoint = getEntryPoint() else { - androidLog(ANDROID_LOG_ERROR, "SwiftTest", "swt_abiv0_getEntryPoint returned NULL") - return 1 +fileprivate extension AndroidOperationCommand { + /// The last non-empty stdout line of a tool invocation (used for `--show-bin-path` / `xcode-select`). + func captureLine(_ tool: String, _ arguments: [String], env: [String: String]) async throws -> String { + guard let stream = try? await launchTool(tool, arguments: arguments, env: env) else { return "" } + var lines: [String] = [] + for try await line in stream { + let trimmed = line.line.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { lines.append(trimmed) } + } + return lines.last ?? "" } - let entryPoint = unsafeBitCast(rawEntryPoint, to: EntryPoint.self) - androidLog(ANDROID_LOG_INFO, "SwiftTest", "Running Swift Testing...") - - // wrap the jobject in a Sendable so it can be passed into the Task - struct SendableJobject: @unchecked Sendable { - let value: jobject - } + /// Shared pipeline: build the test bundle, stage it + its dependency libraries + the JNI harness into + /// the target's library directory (jniLibs/`` or flat), then run any per-target post-staging. + func stageTestLibs(_ target: TestLibsTarget, outputDir: String, with out: MessageQueue) async throws { + let buildConfig = toolchainOptions.configuration ?? BuildConfiguration.fromEnvironment() ?? .debug + let packageDir = toolchainOptions.packagePath ?? "." - let gThis = SendableJobject(value: globalThis) - // Record handler: report each JSON record back through JNI - let recordHandler: @Sendable (UnsafeRawBufferPointer) -> Void = { recordJSON in - guard let base = recordJSON.baseAddress, recordJSON.count > 0 else { return } - let json = String( - decoding: UnsafeBufferPointer(start: base.assumingMemoryBound(to: UInt8.self), count: recordJSON.count), - as: UTF8.self - ) - reportToJava(globalRef: gThis.value, line: json) - } + // 1. Build the test bundle (Android toolchain build, or host `swift build`). + let (binDir, harnessEnv, harnessExtraArgs) = try await target.buildTestBundle() + let packageName = try await parseSwiftPackage(with: out, at: packageDir, swift: target.swiftCommand).name - // Bridge sync → async via DispatchSemaphore - let semaphore = DispatchSemaphore(value: 0) - nonisolated(unsafe) var testSuccess = false - Task { - defer { semaphore.signal() } - do { - testSuccess = try await entryPoint(nil, recordHandler) - } catch { - androidLog(ANDROID_LOG_ERROR, "SwiftTest", "Entry point threw error: \\(error)") + // 2. Locate the loadable test bundle and the directory to stage into. + let loadable = target.loadableTestBundle(binDir, packageName) + guard FileManager.default.fileExists(atPath: loadable.path) else { + throw AndroidError(errorDescription: "Could not find native test bundle at: \(loadable.path)") } - } - semaphore.wait() - - let exitCode: Int32 = testSuccess ? 0 : 1 - return jint(exitCode) -} - -// MARK: - JNI callback to Java - -/// Calls `\(testClassName).reportTestOutput(String)` via JNI. -/// Handles thread attachment for cooperative pool threads. -private func reportToJava(globalRef: jobject, line: String) { - androidLog(ANDROID_LOG_INFO, "SwiftTest", "Test line: \\(line)") - - guard let jvm = g_jvm else { return } - let jii: JNIInvokeInterface = jvm.pointee!.pointee + let libDir = target.stagingDirectory(outputDir) + try? FileManager.default.removeItem(at: libDir) + try FileManager.default.createDirectory(at: libDir, withIntermediateDirectories: true) - var envPtr: UnsafeMutableRawPointer? = nil - let getResult = jii.GetEnv(jvm, &envPtr, jint(JNI_VERSION_1_6)) + // 3. Stage the test bundle (renamed to a loadable lib) + its dependency libraries. + let testLibName = "lib\(packageName)Test" + try FileManager.default.copyItem(at: loadable, to: libDir.appendingPathComponent("\(testLibName).\(target.dylibExtension)", isDirectory: false)) + for lib in try target.dependencyLibraries(binDir) { + let dest = libDir.appendingPathComponent(lib.lastPathComponent, isDirectory: false) + try? FileManager.default.removeItem(at: dest) + try FileManager.default.copyItem(at: lib, to: dest) + } - var needsDetach = false - if getResult == JNI_EDETACHED { - var attachedPtr: UnsafeMutablePointer? = nil - guard jii.AttachCurrentThread(jvm, &attachedPtr, nil) == JNI_OK else { - return + // 4. Build the JNI harness that dlopens the test bundle + drives swt, and stage it. + let stagingDir = try createTempDir() + defer { try? FileManager.default.removeItem(at: stagingDir) } + let harnessDir = stagingDir.appendingPathComponent("harness", isDirectory: true) + let harnessSourceDir = harnessDir.appendingPathComponent("Sources/TestHarness", isDirectory: true) + try FileManager.default.createDirectory(at: harnessSourceDir, withIntermediateDirectories: true) + try target.harnessPackage.write(to: harnessDir.appendingPathComponent("Package.swift", isDirectory: false), atomically: true, encoding: .utf8) + try target.harnessSource(testLibName).write(to: harnessSourceDir.appendingPathComponent("TestRunner.swift", isDirectory: false), atomically: true, encoding: .utf8) + + let harnessPkgArgs = ["--package-path", harnessDir.path, "--configuration", buildConfig.rawValue] + try await runCommand(command: [target.swiftCommand, "build"] + harnessPkgArgs + harnessExtraArgs, env: harnessEnv, with: out) + + let harnessBinOutput = try await captureLine(target.swiftCommand, ["build", "--show-bin-path"] + harnessPkgArgs + harnessExtraArgs, env: harnessEnv) + let harnessBinDir: URL + if !harnessBinOutput.isEmpty { + harnessBinDir = URL(fileURLWithPath: harnessBinOutput) + } else if let fallback = target.harnessBinFallback(harnessDir) { + harnessBinDir = fallback + } else { + throw AndroidError(errorDescription: "Could not resolve test harness build output path") } - if let attachedPtr { - envPtr = UnsafeMutableRawPointer(attachedPtr) + let harnessLibName = "lib\(testHarnessLib).\(target.dylibExtension)" + let harnessLib = harnessBinDir.appendingPathComponent(harnessLibName, isDirectory: false) + guard FileManager.default.fileExists(atPath: harnessLib.path) else { + throw AndroidError(errorDescription: "Expected test harness library at: \(harnessLib.path)") } - needsDetach = true - } else if getResult != JNI_OK { - return - } - defer { if needsDetach { _ = jii.DetachCurrentThread(jvm) } } - - guard let rawEnv = envPtr else { return } - let env = rawEnv.assumingMemoryBound(to: JNIEnv?.self) - let jni: JNINativeInterface = env.pointee!.pointee - - guard let cls: jclass = jni.GetObjectClass(env, globalRef) else { return } + try FileManager.default.copyItem(at: harnessLib, to: libDir.appendingPathComponent(harnessLibName, isDirectory: false)) - let methodName = "reportTestOutput" - let methodSig = "(Ljava/lang/String;)V" - guard let mid: jmethodID = methodName.withCString({ name in - methodSig.withCString({ sig in - jni.GetMethodID(env, cls, name, sig) - }) - }) else { return } + // 5. Per-target post-staging (Darwin writes the dynamic-loader env). + try await target.postStage(libDir) + } - guard let jstr = line.withCString({ cstr in - jni.NewStringUTF(env, cstr) - }) else { return } + /// Android target: builds via the Swift Android SDK toolchain and stages into `/` + /// with the Swift-runtime / NDK `.so` dependencies the androidTest APK must carry. + func androidTestLibsTarget(with out: MessageQueue) async throws -> TestLibsTarget { + let buildConfig = toolchainOptions.configuration ?? BuildConfiguration.fromEnvironment() ?? .debug + let packageDir = toolchainOptions.packagePath ?? "." + let archs = !toolchainOptions.arch.isEmpty ? toolchainOptions.arch : [AndroidArchArgument.current] + let architectures = archs.flatMap({ $0.architectures(configuration: buildConfig) }).uniqueElements() + guard let arch = architectures.first else { + throw AndroidError(errorDescription: "No target architecture specified") + } + let apiLevel = toolchainOptions.androidAPILevel + let tc = try buildToolchainConfiguration(for: arch) + let swiftCmd = tc.toolchainPath.appendingPathComponent("usr/bin/swift", isDirectory: false).path + let scratch = toolchainOptions.scratchPath ?? (packageDir + "/.build") + let buildSystemArgs = args + + return TestLibsTarget( + swiftCommand: swiftCmd, + dylibExtension: "so", + buildTestBundle: { + let (_, env, binPath) = try await self.runToolchainCommand(tc, executable: nil, testMode: .sharedObject, with: out) + let binDir: String + if let binPath = binPath, !binPath.isEmpty { + binDir = binPath + } else { + binDir = [scratch, arch.tripleKey(api: apiLevel, sdkVersion: tc.swiftSDKVersion), buildConfig.rawValue].joined(separator: "/") + } + var harnessExtra: [String] = [] + if let sdkName = tc.sdkName { harnessExtra += ["--swift-sdk", sdkName] } + if let bsIndex = buildSystemArgs.firstIndex(of: "--build-system"), bsIndex + 1 < buildSystemArgs.count { + harnessExtra += ["--build-system", buildSystemArgs[bsIndex + 1]] + } + return (URL(fileURLWithPath: binDir), env, harnessExtra) + }, + loadableTestBundle: { binDir, packageName in + binDir.appendingPathComponent("\(packageName)PackageTests.xctest", isDirectory: false) + }, + dependencyLibraries: { binDir in + let buildOutputLibraries = try self.files(at: binDir).filter({ $0.lastPathComponent.contains(".so") }) + let sdkLibraries = try self.files(at: tc.libPathDynamic, allowLinks: true) + .filter({ $0.lastPathComponent.contains(".so") }) + .filter({ !builtinLibraries.contains($0.lastPathComponent) }) + let cppShared = try self.files(at: tc.libSysrootArch, allowLinks: true) + .filter({ $0.lastPathComponent == "libc++_shared.so" }) + return buildOutputLibraries + sdkLibraries + cppShared + }, + stagingDirectory: { outputDir in + URL(fileURLWithPath: outputDir).appendingPathComponent(arch.abi, isDirectory: true) + }, + harnessPackage: harnessPackageSwift, + harnessSource: { testLibBaseName in testHarnessSwiftSource(testLibName: "\(testLibBaseName).so") }, + harnessBinFallback: { harnessDir in + URL(fileURLWithPath: [harnessDir.path + "/.build", arch.tripleKey(api: apiLevel, sdkVersion: tc.swiftSDKVersion), buildConfig.rawValue].joined(separator: "/")) + }, + postStage: { _ in } + ) + } - let args = [jvalue(l: jstr)] - args.withUnsafeBufferPointer { buf in - jni.CallVoidMethodA(env, globalRef, mid, buf.baseAddress) + /// Host target: builds the test bundle + a pure-Swift harness for the host (macOS) and stages them + /// flat with the Swift-runtime/bridge dylibs and a `dyld-env.txt` for the forked Robolectric JVM. + func hostTestLibsTarget(with out: MessageQueue) async throws -> TestLibsTarget { + let buildConfig = toolchainOptions.configuration ?? BuildConfiguration.fromEnvironment() ?? .debug + let packageDir = toolchainOptions.packagePath ?? "." + let scratch = toolchainOptions.scratchPath ?? (packageDir + "/.build") + let dylibSuffix = "dylib" // host build is currently macOS-only + + var buildEnv = ProcessInfo.processInfo.environment + buildEnv["SKIP_BRIDGE"] = "1" + let env = buildEnv + + var resolvedSwift = try await captureLine("/usr/bin/xcrun", ["--find", "swift"], env: env) + if resolvedSwift.isEmpty { resolvedSwift = "swift" } + let swift = resolvedSwift + + // -DROBOLECTRIC selects the host bridge code paths; -DSKIP_BRIDGE + SKIP_BRIDGE=1 keep bridged + // library products dynamic so they resolve at load time. + let testFlags = ["-Xcc", "-fPIC", "-Xswiftc", "-DSKIP_BRIDGE", "-Xswiftc", "-DROBOLECTRIC"] + let pkgArgs = ["--package-path", packageDir, "--scratch-path", scratch, "-c", buildConfig.rawValue] + + return TestLibsTarget( + swiftCommand: swift, + dylibExtension: dylibSuffix, + buildTestBundle: { + try await self.runCommand(command: [swift, "build", "--build-tests"] + pkgArgs + testFlags, env: env, with: out) + let binPath = try await self.captureLine(swift, ["build", "--show-bin-path", "--build-tests"] + pkgArgs + testFlags, env: env) + guard !binPath.isEmpty else { + throw AndroidError(errorDescription: "Could not resolve host test build bin path") + } + return (URL(fileURLWithPath: binPath), env, []) + }, + loadableTestBundle: { binDir, packageName in + binDir.appendingPathComponent("\(packageName)PackageTests.xctest/Contents/MacOS/\(packageName)PackageTests", isDirectory: false) + }, + dependencyLibraries: { binDir in + try self.files(at: binDir).filter({ $0.pathExtension == dylibSuffix }) + }, + stagingDirectory: { outputDir in URL(fileURLWithPath: outputDir) }, + harnessPackage: hostHarnessPackageSwift, + harnessSource: { testLibBaseName in hostTestHarnessSwiftSource(testLibName: testLibBaseName) }, + harnessBinFallback: { _ in nil }, + postStage: { libDir in + // Stage the dynamic-loader env so the forked test JVM resolves Testing.framework + the + // Swift runtime when the harness dlopens the test bundle (linked via @rpath). + let developerDir = try await self.captureLine("/usr/bin/xcode-select", ["-p"], env: env) + let platformDev = developerDir + "/Platforms/MacOSX.platform/Developer" + let dyldEnv = """ + DYLD_FRAMEWORK_PATH=\(platformDev)/Library/Frameworks:\(platformDev)/Library/PrivateFrameworks + DYLD_LIBRARY_PATH=\(platformDev)/usr/lib:\(libDir.path) + """ + try dyldEnv.write(to: libDir.appendingPathComponent("dyld-env.txt", isDirectory: false), atomically: true, encoding: .utf8) + } + ) } } -""" -} + +#endif diff --git a/Sources/SkipBuild/Commands/InitCommand.swift b/Sources/SkipBuild/Commands/InitCommand.swift index a0102ac5..fc390aea 100644 --- a/Sources/SkipBuild/Commands/InitCommand.swift +++ b/Sources/SkipBuild/Commands/InitCommand.swift @@ -152,8 +152,10 @@ This command will create a conventional Skip app or library project. let isApp = appid != nil || self.projectOptions.projectMode.contains(.nativeApp) || self.projectOptions.projectMode.contains(.transpiledApp) let moduleMode = self.moduleMode let nativeMode = self.nativeMode - // for now we default to creating tests only when non-native - let createTests = self.createOptions.moduleTests ?? nativeMode.isEmpty + // Default to creating a test module for both transpiled and native libraries: native test + // modules run their Swift Testing cases natively on Android via the generated JNI harness. + // (Native app targets still skip tests via the createTestModule guard in createSkipLibrary.) + let createTests = self.createOptions.moduleTests ?? true let options = createOptions.projectOptionValues(projectName: self.projectName) diff --git a/Sources/SkipBuild/Commands/SkipstoneCommand.swift b/Sources/SkipBuild/Commands/SkipstoneCommand.swift index 64fa2e19..6cb53cb1 100644 --- a/Sources/SkipBuild/Commands/SkipstoneCommand.swift +++ b/Sources/SkipBuild/Commands/SkipstoneCommand.swift @@ -334,6 +334,10 @@ struct SkipstoneCommand: BuildPluginOptionsCommand, StreamingCommand { } let isNativeModule = moduleMode(for: nil) == .native + // A `mode: native` test module runs its Swift Testing cases natively on Android via a generated + // JNI harness (org.swift.test.SwiftTestRunner) driven by Gradle connectedAndroidTest, rather than + // transpiling them to Kotlin/JUnit. See SwiftTestingHarness.swift + nativeTestGradleBlock. + let isNativeTestModule = isNativeModule && primaryModuleName.hasSuffix("Tests") // also add any files in the skipFolderFile to the list of sources (including the skip.yml and other metadata files) let skipFolderPathContents = try FileManager.default.enumeratedURLs(of: skipFolderPath.asURL) @@ -387,8 +391,9 @@ struct SkipstoneCommand: BuildPluginOptionsCommand, StreamingCommand { try fs.createDirectory(kotlinOutputFolder, recursive: true) } - // now make a link from src/androidTest/kotlin to src/test/kotlin so the same tests will run against an Android emulator/device with the ANDROID_SERIAL environment - if primaryModuleName.hasSuffix("Tests") { + // now make a link from src/androidTest/kotlin to src/test/kotlin so the same tests will run against an Android emulator/device with the ANDROID_SERIAL environment. + // A native test module instead generates a dedicated src/androidTest with the swt JNI runner (below), so skip the link. + if primaryModuleName.hasSuffix("Tests") && !isNativeTestModule { let androidTestOutputFolder = try AbsolutePath(outputFolderPath, validating: "../androidTest") removePath(androidTestOutputFolder) // remove any existing link in order to re-create it try fs.createSymbolicLink(addOutputFile(androidTestOutputFolder), pointingAt: outputFolderPath, relative: true) @@ -434,6 +439,21 @@ struct SkipstoneCommand: BuildPluginOptionsCommand, StreamingCommand { try saveSkipBridgeCode() try saveTestHarness() + // For a `mode: native` test module, generate the on-device AndroidJUnit4 runner that loads the + // generated JNI harness (libtest_harness.so) and drives the native Swift Testing suite. The + // matching gradle wiring (buildAndroidSwiftTestLibs task + androidTest jniLibs) is appended to the + // under-test module's build.gradle.kts in generatePerModuleGradle(). + if isNativeTestModule { + // On-device (connectedDebugAndroidTest) runner. + let runnerDir = moduleRootPath.appending(components: "src", "androidTest", "kotlin", "org", "swift", "test") + try fs.createDirectory(runnerDir, recursive: true) + try writeChanges(tag: "native test runner", to: runnerDir.appending(component: "\(testClassName).kt"), contents: nativeTestRunnerKotlinSource.utf8Data, readOnly: true) + // Host-JVM (Robolectric / testDebug, no device) runner. + let robolectricRunnerDir = moduleRootPath.appending(components: "src", "test", "kotlin", "org", "swift", "test") + try fs.createDirectory(robolectricRunnerDir, recursive: true) + try writeChanges(tag: "native test runner (Robolectric)", to: robolectricRunnerDir.appending(component: "\(testClassName).kt"), contents: robolectricTestRunnerKotlinSource.utf8Data, readOnly: true) + } + let sourceModules = try linkDependentModuleSources() try linkResources() @@ -699,6 +719,10 @@ struct SkipstoneCommand: BuildPluginOptionsCommand, StreamingCommand { // Namespace 'hello.skip' is used in multiple modules and/or libraries: AndroidManifest.xml, :skipstone:HelloSkip. Please ensure that all modules and libraries have a unique namespace. For more information, See https://developer.android.com/studio/build/configure-app-module#set-namespace // note that the duplicate android { } overrides any previous android block + // For a native test module, append the swt test-libs build wiring: the androidTest jniLibs + // path (connectedDebugAndroidTest, on device) and the Robolectric host path (testDebug, no device). + let nativeTestSupport = isNativeTestModule ? "\n\n" + nativeTestGradleBlock + "\n\n" + nativeTestRobolectricGradleBlock : "" + let contents = """ // build.gradle.kts generated by Skip for \(primaryModuleName) @@ -707,6 +731,7 @@ struct SkipstoneCommand: BuildPluginOptionsCommand, StreamingCommand { android { namespace = if ("\(packageName)" == rootProject.name) "\(packageName).module" else "\(packageName)" } + \(nativeTestSupport) """ //trace("created gradle: \(contents.split(separator: "\n").map({ $0.trimmingCharacters(in: .whitespaces) }).joined(separator: "; "))") diff --git a/Sources/SkipBuild/Commands/SwiftTestingHarness.swift b/Sources/SkipBuild/Commands/SwiftTestingHarness.swift new file mode 100644 index 00000000..32295726 --- /dev/null +++ b/Sources/SkipBuild/Commands/SwiftTestingHarness.swift @@ -0,0 +1,647 @@ +// Copyright (c) 2023 - 2026 Skip +// Licensed under the GNU Affero General Public License v3.0 +// SPDX-License-Identifier: AGPL-3.0-only + +import Foundation + +// MARK: - Shared native Swift Testing harness sources +// +// These define the JNI bridge that loads a native Swift Testing test library (`*.xctest`/`*.so`, +// which exports `swt_abiv0_getEntryPoint`) and drives the Swift Testing ABI-v0 entry point, reporting +// each JSON event record back to a Java/Kotlin `Instrumentation` via `reportTestOutput(String)`. +// +// Two consumers share these sources: +// - `skip android test --apk` (AndroidTestCommand.runSwiftPMAsAPK): hand-assembles a bare APK. +// - the `skip test` native-harness path (NativeTestRun): packages the same runner via a generated +// Gradle test-harness module so the Skip Kotlin bridge libraries + merged assets are present +// (which the bare `--apk` APK lacks — that is why localization crashes there with `JNI.jni was unset`). +// +// The Gradle path needs the runner to additionally initialise the Skip bridge (load the bridge native +// libraries → `JNI_OnLoad` sets `JNI.jni`; establish the Android `Context`; run `initAndroidBridge`) +// before invoking the entry point — see `swiftTestRunnerJavaSource(initBridge:)`. + +/// The library name produced by the Swift test harness package (`libtest_harness.so`). +let testHarnessLib = "test_harness" +/// The Java package + class of the instrumentation runner. Its name fixes the JNI symbol the harness +/// exports (`Java___runTests`), so the harness Swift source is generated from it. +let testPackage = "org.swift.test" +let testClassName = "SwiftTestRunner" +let testFullClass = "\(testPackage).\(testClassName)" + +/// Package.swift for the generated Swift test harness package. +/// Defines a dynamic library target that produces `libtest_harness.so`. +let harnessPackageSwift: String = """ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "test-harness", + products: [ + .library(name: "\(testHarnessLib)", type: .dynamic, targets: ["TestHarness"]) + ], + targets: [ + .target( + name: "TestHarness", + linkerSettings: [ + .linkedLibrary("log"), + .linkedLibrary("dl"), + .linkedLibrary("android"), // for the NDK ALooper_* functions used by setupMainLooper + ] + ), + ] +) +""" + +/// Java source for the Android Instrumentation test runner. +/// Loads `libtest_harness.so`, calls `runTests()` via JNI, and uses +/// `sendStatus()`/`finish()` for structured output back to the host. +/// +/// - Parameter initBridge: when true, initialise the Skip Android bridge (so `JNI.jni`, the Android +/// `Context`, and Foundation bootstrap are established before the native test runs — required for +/// tests that touch the Kotlin bridge, e.g. `String(localized:bundle:.module)`). The `--apk` path +/// passes `false` (bare APK with no bridge); the Gradle-harness path passes `true`. +func swiftTestRunnerJavaSource(initBridge: Bool) -> String { + let bridgeInit = initBridge ? """ + // Initialise the Skip bridge so the native test can call into the Kotlin-side Skip + // libraries (JNI, Android Context, localized Bundle assets). This loads the bridge + // native libraries listed in the merged manifest's SKIP_BRIDGE_MODULES meta-data. + try { + android.util.Log.i("SwiftTest", "launching Skip bridge"); + skip.foundation.ProcessInfo.Companion.launch(getContext()); + } catch (Throwable t) { + android.util.Log.e("SwiftTest", "Skip bridge launch failed", t); + } + + """ : "" + + return """ + package \(testPackage); + + import android.app.Instrumentation; + import android.os.Bundle; + + public class \(testClassName) extends Instrumentation { + static { + android.util.Log.i("SwiftTest", "loading harness"); + System.loadLibrary("\(testHarnessLib)"); + android.util.Log.i("SwiftTest", "loaded harness"); + } + private native int runTests(); + private native boolean setupMainLooper(); + + @Override + public void onCreate(Bundle arguments) { + android.util.Log.i("SwiftTest", "onCreate"); + // Wire the Swift main DispatchQueue into THIS (main) thread's Android Looper before any + // test job is created, so MainActor/main-queue work scheduled by the run can drain. + boolean looper = setupMainLooper(); + android.util.Log.i("SwiftTest", "setupMainLooper=" + looper); + super.onCreate(arguments); + // This triggers onStart() in a separate thread + start(); + android.util.Log.i("SwiftTest", "onCreate: started"); + } + + @Override + public void onStart() { + super.onStart(); + Bundle result = new Bundle(); + try { + android.util.Log.i("SwiftTest", "onStart"); + super.onStart(); + \(bridgeInit) android.util.Log.i("SwiftTest", "runTests"); + int exitCode = runTests(); + android.util.Log.i("SwiftTest", "runTests done"); + result.putString("status", exitCode == 0 ? "passed" : "failed"); + finish(exitCode == 0 ? -1 : exitCode, result); + } catch (Throwable t) { + android.util.Log.e("SwiftTest", "Test error", t); + finish(1, result); + } + } + + public void reportTestOutput(String line) { + Bundle b = new Bundle(); + b.putString("stream", line + "\\n"); + sendStatus(0, b); + } + } + """ +} + +/// The bare-APK instrumentation runner (no bridge init), used by `skip android test --apk`. +let instrumentationJavaSource: String = swiftTestRunnerJavaSource(initBridge: false) + +// MARK: - Conventional `mode: native` test module (Gradle connectedAndroidTest) generation + +/// Kotlin source for the on-device `AndroidJUnit4` test generated into a `mode: native` test module's +/// `src/androidTest/kotlin/org/swift/test/SwiftTestRunner.kt`. Loads `libtest_harness.so`, drains the +/// Swift main queue on the main thread, runs the Swift Testing suite (which `dlopen`s the test bundle), +/// and asserts success — so AGP's `connectedDebugAndroidTest` produces a JUnit result for `skip test`. +let nativeTestRunnerKotlinSource: String = """ +// Auto-generated by Skip — do not edit +package \(testPackage) + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Test +import org.junit.Assert.assertEquals +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class \(testClassName) { + private external fun setupMainLooper(): Boolean + private external fun runTests(): Int + + // The native harness reports each Swift Testing JSON event record here (via reportToJava). Logging it + // under the "SwiftTest" tag lets the Gradle block recover the per-test event stream from logcat after + // the connected run (which survives AGP uninstalling the test APK), so `skip test` can report per-test + // results instead of the single aggregate case. + fun reportTestOutput(line: String) { android.util.Log.i("SwiftTest", line) } + + @Test fun nativeSwiftTests() { + System.loadLibrary("\(testHarnessLib)") + // Wire the Swift main DispatchQueue into the device main Looper before running the suite. + InstrumentationRegistry.getInstrumentation().runOnMainSync { setupMainLooper() } + assertEquals("native Swift Testing cases should all pass", 0, runTests()) + } +} +""" + +/// Gradle DSL appended to a `mode: native` test module's under-test `build.gradle.kts`. Registers a +/// `buildAndroidSwiftTestLibs` task that runs `skip android test --build-test-libs` (building the test +/// bundle + `libtest_harness.so` + Swift-runtime/bridge `.so` deps into `/test-jni-libs/`), +/// adds that folder to the androidTest jniLibs, and hooks it before the androidTest native-lib merge. +/// Relies on `swiftBuildFolder()` / `swiftSourceFolder()` / `skipcmd` / `skipCommand` / `SkipBridgeExecOps` +/// defined by the native build block (skip-bridge's skip.yml) that the under-test module already carries. +let nativeTestGradleBlock: String = """ +// Native Swift Testing (`mode: native` test module) support generated by Skip +android { + sourceSets { + getByName("androidTest") { + jniLibs.srcDir("${swiftBuildFolder()}/test-jni-libs") + } + } + defaultConfig { + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } +} + +tasks.configureEach { + if ((name == "mergeDebugAndroidTestJniLibFolders" || name == "mergeReleaseAndroidTestJniLibFolders") && (System.getenv("SKIP_BRIDGE_ANDROID_BUILD_DISABLED") != "1")) { + dependsOn("buildAndroidSwiftTestLibs") + } +} + +tasks.register("buildAndroidSwiftTestLibs") { + doLast { + project.objects.newInstance().execOps.exec { + workingDir(layout.projectDirectory) + commandLine("sh", "-cx", "\\"${skipcmd}\\" android test --build-test-libs \\"${swiftBuildFolder()}/test-jni-libs\\" --package-path \\"${swiftSourceFolder()}\\" --configuration debug --scratch-path \\"${swiftBuildFolder()}/swift-test\\" --arch automatic --build-system native") + environment("SKIP_BRIDGE", "1") + environment("TARGET_OS_ANDROID", "1") + environment("DEVELOPER_DIR", "") + if (file(skipCommand).exists()) { environment("SKIP_COMMAND_OVERRIDE", skipCommand) } + } + } +} + +// Recover per-test results for a connected (device/emulator) androidTest run from logcat: the runner logs +// each swt event record under the "SwiftTest" tag, so we clear logcat before the run and dump it (raw, +// tag-filtered) afterward into the host connected-results dir, where `skip test` looks for the event +// stream. Using logcat (rather than a pulled on-device file) survives AGP uninstalling the test APK after +// the run, and needs no package name. Best-effort: on failure the parser finds no events and the aggregate +// row is used. `swtAdbArgs` prefixes `adb` (+ `-s ` when ANDROID_SERIAL is set) to the given args. +val swtAdb = (System.getenv("ANDROID_HOME") ?: System.getenv("ANDROID_SDK_ROOT"))?.let { "${it}/platform-tools/adb" } ?: "adb" +fun swtAdbArgs(vararg extra: String): List { + val args = mutableListOf(swtAdb) + val serial = System.getenv("ANDROID_SERIAL") + if (serial != null && serial.isNotEmpty()) { args.add("-s"); args.add(serial) } + args.addAll(extra) + return args +} +tasks.configureEach { + if (name == "connectedDebugAndroidTest" || name == "connectedReleaseAndroidTest") { + val swtConfig = if (name.contains("Release")) "release" else "debug" + doFirst { + try { + project.objects.newInstance().execOps.exec { + commandLine(swtAdbArgs("logcat", "-c")) + isIgnoreExitValue = true + } + } catch (e: Throwable) { } + } + doLast { + try { + val swtHostDir = layout.buildDirectory.dir("outputs/androidTest-results/connected/" + swtConfig).get().asFile + swtHostDir.mkdirs() + swtHostDir.resolve("swt-events.jsonl").outputStream().use { os -> + project.objects.newInstance().execOps.exec { + commandLine(swtAdbArgs("logcat", "-d", "-v", "raw", "-s", "SwiftTest:I")) + standardOutput = os + isIgnoreExitValue = true + } + } + } catch (e: Throwable) { } + } + } +} +""" + +/// Swift source for the test harness. Implements JNI_OnLoad and the native `runTests` method. +/// Loads the test library via dlopen, invokes the Swift Testing entry point, and reports +/// test output back through JNI to the Java Instrumentation runner. +func testHarnessSwiftSource(testLibName: String) -> String { + return """ +import Android +import Dispatch + +// MARK: - JNI type aliases + +typealias JNIEnvironment = UnsafeMutablePointer + +// MARK: - Global state + +nonisolated(unsafe) var g_jvm: UnsafeMutablePointer? = nil + +private func androidLog(_ priority: android_LogPriority, _ tag: String, _ message: String) { + __android_log_write(Int32(priority.rawValue), tag, message) +} + +// MARK: - JNI_OnLoad + +@_cdecl("JNI_OnLoad") +func JNI_OnLoad(_ vm: UnsafeMutablePointer?, _ reserved: UnsafeMutableRawPointer?) -> jint { + g_jvm = vm + androidLog(ANDROID_LOG_INFO, "SwiftTest", "JNI_OnLoad") + return jint(JNI_VERSION_1_6) +} + +// MARK: - Main looper setup (must run ON THE MAIN THREAD, before the test runs) + +// libdispatch internals (no CoreFoundation): the port of the Swift main DispatchQueue, and the +// callback that drains its pending work. These are what CFRunLoop's main-queue integration uses. +@_silgen_name("_dispatch_main_queue_callback_4CF") +func _dispatch_main_queue_callback_4CF() +@_silgen_name("_dispatch_get_main_queue_port_4CF") +func _dispatch_get_main_queue_port_4CF() -> Int32 + +// NDK android/looper.h functions (from libandroid.so), declared directly to avoid importing +// AndroidLooper (which pulls in CoreFoundation). ALOOPER_EVENT_INPUT == (1 << 0). +private let ALOOPER_EVENT_INPUT: CInt = 1 +@_silgen_name("ALooper_forThread") +func ALooper_forThread() -> OpaquePointer? +@_silgen_name("ALooper_acquire") +func ALooper_acquire(_ looper: OpaquePointer?) +@_silgen_name("ALooper_addFd") +func ALooper_addFd(_ looper: OpaquePointer?, _ fd: CInt, _ ident: CInt, _ events: CInt, _ callback: (@convention(c) (CInt, CInt, UnsafeMutableRawPointer?) -> CInt)?, _ data: UnsafeMutableRawPointer?) -> CInt + +// Android Looper callback: drain the Swift main queue whenever its port signals. +private func drainMainQueueCallback(_ fd: CInt, _ events: CInt, _ data: UnsafeMutableRawPointer?) -> CInt { + _dispatch_main_queue_callback_4CF() + return 1 // keep the fd registered +} + +// Wires the Swift main DispatchQueue into THIS thread's Android Looper, so MainActor / main-queue +// work scheduled by the Swift Testing run actually drains. Must run ON THE MAIN THREAD (which owns the +// Android main Looper). Without this the swt run hangs after `testStarted`. This mirrors what +// `AndroidLooper.installGlobalExecutor()` does in a real Fuse app, but implemented directly against the +// NDK ALooper + libdispatch to avoid pulling CoreFoundation into the standalone harness library. +@_cdecl("Java_\(testFullClass.replacingOccurrences(of: ".", with: "_"))_setupMainLooper") +func setupMainLooper(_ env: JNIEnvironment, _ thisObj: jobject) -> jboolean { + guard let looper = ALooper_forThread() else { + androidLog(ANDROID_LOG_ERROR, "SwiftTest", "setupMainLooper: no Android Looper on this thread") + return jboolean(0) + } + ALooper_acquire(looper) + let dispatchPort = _dispatch_get_main_queue_port_4CF() + let result = ALooper_addFd(looper, dispatchPort, 0, CInt(ALOOPER_EVENT_INPUT), drainMainQueueCallback, nil) + androidLog(ANDROID_LOG_INFO, "SwiftTest", "setupMainLooper addFd -> \\(result)") + return result == 1 ? jboolean(1) : jboolean(0) +} + +// MARK: - Entry point type (ST-0002 JSON ABI) + +typealias EntryPoint = @convention(thin) @Sendable ( + _ configurationJSON: UnsafeRawBufferPointer?, + _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void +) async throws -> Bool + +// MARK: - JNI native method + +@_cdecl("Java_\(testFullClass.replacingOccurrences(of: ".", with: "_"))_runTests") +func runTests(_ env: JNIEnvironment, _ thisObj: jobject) -> jint { + let jni: JNINativeInterface = env.pointee!.pointee + + // Keep a global ref to the Instrumentation object for use from other threads + guard let globalThis: jobject = jni.NewGlobalRef(env, thisObj) else { + androidLog(ANDROID_LOG_ERROR, "SwiftTest", "Failed to create global ref") + return 1 + } + defer { jni.DeleteGlobalRef(env, globalThis) } + + // Load test library + androidLog(ANDROID_LOG_INFO, "SwiftTest", "Loading test library: \(testLibName)") + guard let handle = dlopen("\(testLibName)", RTLD_NOW) else { + let err = dlerror().flatMap({ String(cString: $0) }) ?? "" + androidLog(ANDROID_LOG_ERROR, "SwiftTest", "dlopen failed: \\(err)") + return 1 + } + + // dlopen() does NOT invoke JNI_OnLoad (only System.loadLibrary does), so the Skip bridge's + // JNI.jni global stays unset and the first bridged call (e.g. AndroidBundle.main) traps with + // "JNI.jni was unset". We must manually invoke the BRIDGE's JNI_OnLoad with the cached JavaVM. + // + // We dlopen libSkipBridge.so explicitly and resolve JNI_OnLoad from THAT handle, rather than + // dlsym(testBundleHandle, "JNI_OnLoad"): the harness shim itself exports a global JNI_OnLoad + // (System.loadLibrary loads it RTLD_GLOBAL) that only sets g_jvm, and a handle-relative lookup + // can resolve the harness's symbol instead of the bridge's. Both return JNI_VERSION_1_6, so the + // wrong one silently leaves JNI.jni unset. Resolving from the libSkipBridge.so handle gets its own. + if let bridgeHandle = dlopen("libSkipBridge.so", RTLD_NOW) { + if let onLoadSym = dlsym(bridgeHandle, "JNI_OnLoad") { + typealias JNIOnLoadFn = @convention(c) (UnsafeMutablePointer?, UnsafeMutableRawPointer?) -> jint + let jniOnLoad = unsafeBitCast(onLoadSym, to: JNIOnLoadFn.self) + let version = jniOnLoad(g_jvm, nil) + androidLog(ANDROID_LOG_INFO, "SwiftTest", "invoked libSkipBridge JNI_OnLoad -> \\(version)") + } else { + androidLog(ANDROID_LOG_ERROR, "SwiftTest", "libSkipBridge.so has no JNI_OnLoad symbol") + } + } else { + let err = dlerror().flatMap({ String(cString: $0) }) ?? "" + androidLog(ANDROID_LOG_ERROR, "SwiftTest", "could not dlopen libSkipBridge.so: \\(err)") + } + + // Look up swt_abiv0_getEntryPoint + guard let sym = dlsym(handle, "swt_abiv0_getEntryPoint") else { + androidLog(ANDROID_LOG_ERROR, "SwiftTest", "swt_abiv0_getEntryPoint not found") + return 1 + } + typealias GetEntryPointFn = @convention(c) () -> UnsafeRawPointer? + let getEntryPoint = unsafeBitCast(sym, to: GetEntryPointFn.self) + + guard let rawEntryPoint = getEntryPoint() else { + androidLog(ANDROID_LOG_ERROR, "SwiftTest", "swt_abiv0_getEntryPoint returned NULL") + return 1 + } + let entryPoint = unsafeBitCast(rawEntryPoint, to: EntryPoint.self) + + androidLog(ANDROID_LOG_INFO, "SwiftTest", "Running Swift Testing...") + + // wrap the jobject in a Sendable so it can be passed into the Task + struct SendableJobject: @unchecked Sendable { + let value: jobject + } + + let gThis = SendableJobject(value: globalThis) + // Record handler: report each JSON record back through JNI + let recordHandler: @Sendable (UnsafeRawBufferPointer) -> Void = { recordJSON in + guard let base = recordJSON.baseAddress, recordJSON.count > 0 else { return } + let json = String( + decoding: UnsafeBufferPointer(start: base.assumingMemoryBound(to: UInt8.self), count: recordJSON.count), + as: UTF8.self + ) + reportToJava(globalRef: gThis.value, line: json) + } + + // Bridge sync → async via DispatchSemaphore + let semaphore = DispatchSemaphore(value: 0) + nonisolated(unsafe) var testSuccess = false + Task { + defer { semaphore.signal() } + do { + testSuccess = try await entryPoint(nil, recordHandler) + } catch { + androidLog(ANDROID_LOG_ERROR, "SwiftTest", "Entry point threw error: \\(error)") + } + } + semaphore.wait() + + let exitCode: Int32 = testSuccess ? 0 : 1 + return jint(exitCode) +} + +// MARK: - JNI callback to Java + +/// Calls `\(testClassName).reportTestOutput(String)` via JNI. +/// Handles thread attachment for cooperative pool threads. +private func reportToJava(globalRef: jobject, line: String) { + androidLog(ANDROID_LOG_INFO, "SwiftTest", "Test line: \\(line)") + + guard let jvm = g_jvm else { return } + let jii: JNIInvokeInterface = jvm.pointee!.pointee + + var envPtr: UnsafeMutableRawPointer? = nil + let getResult = jii.GetEnv(jvm, &envPtr, jint(JNI_VERSION_1_6)) + + var needsDetach = false + if getResult == JNI_EDETACHED { + var attachedPtr: UnsafeMutablePointer? = nil + guard jii.AttachCurrentThread(jvm, &attachedPtr, nil) == JNI_OK else { + return + } + if let attachedPtr { + envPtr = UnsafeMutableRawPointer(attachedPtr) + } + needsDetach = true + } else if getResult != JNI_OK { + return + } + defer { if needsDetach { _ = jii.DetachCurrentThread(jvm) } } + + guard let rawEnv = envPtr else { return } + let env = rawEnv.assumingMemoryBound(to: JNIEnv?.self) + let jni: JNINativeInterface = env.pointee!.pointee + + guard let cls: jclass = jni.GetObjectClass(env, globalRef) else { return } + + let methodName = "reportTestOutput" + let methodSig = "(Ljava/lang/String;)V" + guard let mid: jmethodID = methodName.withCString({ name in + methodSig.withCString({ sig in + jni.GetMethodID(env, cls, name, sig) + }) + }) else { return } + + guard let jstr = line.withCString({ cstr in + jni.NewStringUTF(env, cstr) + }) else { return } + + let args = [jvalue(l: jstr)] + args.withUnsafeBufferPointer { buf in + jni.CallVoidMethodA(env, globalRef, mid, buf.baseAddress) + } +} +""" +} + +// MARK: - Conventional `mode: native` test module (Robolectric / host JVM, no device) generation + +/// Package.swift for the *host* (Robolectric) Swift test harness package. Unlike the Android harness it +/// links nothing Android-specific (no `liblog`/`libandroid`), so it builds for the host platform. +let hostHarnessPackageSwift: String = """ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "test-harness", + platforms: [ .macOS(.v13) ], + products: [ + .library(name: "\(testHarnessLib)", type: .dynamic, targets: ["TestHarness"]) + ], + targets: [ + .target(name: "TestHarness") + ] +) +""" + +/// Swift source for the *host* test harness used by Robolectric (`testDebug`, no device). It is pure Swift +/// with raw-pointer JNI (the ABI passes opaque pointers, so no JNI headers / `import Android` are needed) +/// and NO Android NDK / ALooper: the JVM-hosted Swift runtime relaxes the MainActor assertion, so no main +/// queue drain is required, and the Swift Testing `Task` schedules because this is a separate `.dynamic` +/// library (the version-scripted `--build-tests` bundle has no concurrency workers, so an in-bundle Task +/// would never run). It `dlopen`s the test bundle + `libSkipBridge` (to set `JNI.jni`), runs the swt +/// ABI-v0 entry point, and returns 0/1; per-event JSON records go to stderr (captured by Gradle). +func hostTestHarnessSwiftSource(testLibName: String) -> String { + return """ +import Dispatch +#if canImport(Darwin) +import Darwin +private let dylibSuffix = "dylib" +#elseif canImport(Glibc) +import Glibc +private let dylibSuffix = "so" +#endif + +nonisolated(unsafe) var g_jvm: UnsafeMutableRawPointer? = nil + +@_cdecl("JNI_OnLoad") +public func JNI_OnLoad(_ vm: UnsafeMutableRawPointer?, _ reserved: UnsafeMutableRawPointer?) -> Int32 { + g_jvm = vm + return 0x00010006 // JNI_VERSION_1_6 +} + +@_cdecl("Java_\(testFullClass.replacingOccurrences(of: ".", with: "_"))_setupMainLooper") +public func setupMainLooper(_ env: UnsafeMutableRawPointer?, _ thisObj: UnsafeMutableRawPointer?) -> UInt8 { + return 1 // no-op on the host JVM (no Android Looper; MainActor assertion is relaxed in JVM-hosted mode) +} + +private typealias EntryPoint = @convention(thin) @Sendable ( + _ configurationJSON: UnsafeRawBufferPointer?, + _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void +) async throws -> Bool + +@_cdecl("Java_\(testFullClass.replacingOccurrences(of: ".", with: "_"))_runTests") +public func runTests(_ env: UnsafeMutableRawPointer?, _ thisObj: UnsafeMutableRawPointer?) -> Int32 { + // RTLD_GLOBAL so the statically-linked module's bridged Java_* symbols are JVM-resolvable. + guard let handle = dlopen("\(testLibName).\\(dylibSuffix)", RTLD_NOW | RTLD_GLOBAL) else { + fputs("HostTestHarness: dlopen \(testLibName).\\(dylibSuffix) failed: \\(dlerror().flatMap { String(cString: $0) } ?? "")\\n", stderr) + return 1 + } + // dlopen does not invoke JNI_OnLoad; set the Skip bridge's JNI.jni by invoking libSkipBridge's JNI_OnLoad. + if let h = dlopen("libSkipBridge.\\(dylibSuffix)", RTLD_NOW), let sym = dlsym(h, "JNI_OnLoad") { + typealias Fn = @convention(c) (UnsafeMutableRawPointer?, UnsafeMutableRawPointer?) -> Int32 + _ = unsafeBitCast(sym, to: Fn.self)(g_jvm, nil) + } + guard let sym = dlsym(handle, "swt_abiv0_getEntryPoint") else { + fputs("HostTestHarness: swt_abiv0_getEntryPoint not found\\n", stderr) + return 1 + } + typealias GetEntryPointFn = @convention(c) () -> UnsafeRawPointer? + guard let raw = unsafeBitCast(sym, to: GetEntryPointFn.self)() else { return 1 } + let entryPoint = unsafeBitCast(raw, to: EntryPoint.self) + // If SKIP_SWT_EVENTS is set (by the Gradle test block), persist the raw swt ABI-v0 JSON event stream + // (one record per line) so `skip test` can recover per-test results and report them individually + // instead of the single aggregate. fputs locks the FILE internally, so concurrent records are safe. + nonisolated(unsafe) let eventsFile: UnsafeMutablePointer? = getenv("SKIP_SWT_EVENTS").flatMap { fopen($0, "w") } + let recordHandler: @Sendable (UnsafeRawBufferPointer) -> Void = { rec in + guard let base = rec.baseAddress, rec.count > 0 else { return } + let json = String(decoding: UnsafeBufferPointer(start: base.assumingMemoryBound(to: UInt8.self), count: rec.count), as: UTF8.self) + if let eventsFile { fputs(json + "\\n", eventsFile); fflush(eventsFile) } + fputs("SwiftTest: " + json + "\\n", stderr) + } + let semaphore = DispatchSemaphore(value: 0) + nonisolated(unsafe) var success = false + Task { + defer { semaphore.signal() } + success = (try? await entryPoint(nil, recordHandler)) ?? false + } + semaphore.wait() + if let eventsFile { fclose(eventsFile) } + return success ? 0 : 1 +} +""" +} + +/// Kotlin source for the Robolectric (host JVM) test generated into a `mode: native` test module's +/// `src/test/kotlin/org/swift/test/SwiftTestRunner.kt`. Loads the host harness dylib (from the +/// `skip.test.libs` system property set by the Gradle block), caches the Robolectric `Context` for the +/// swt cooperative threads, and asserts the suite passes — so `testDebug` (no device) yields a JUnit result. +let robolectricTestRunnerKotlinSource: String = """ +// Auto-generated by Skip — do not edit +package \(testPackage) + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import org.junit.Test +import org.junit.Assert.assertEquals +import org.junit.runner.RunWith + +@RunWith(org.robolectric.RobolectricTestRunner::class) +class \(testClassName) { + private external fun runTests(): Int + + @Test fun nativeSwiftTests() { + val libs = System.getProperty("skip.test.libs") ?: error("skip.test.libs not set (buildLocalSwiftTestLibs did not run)") + System.load(libs + "/" + System.mapLibraryName("\(testHarnessLib)")) + // Cache the Robolectric Context so the native swt threads can resolve it. launch() sets the + // launch context before the (host-unavailable) initBridge call, so catch and continue. + val ctx = ApplicationProvider.getApplicationContext() + try { skip.foundation.ProcessInfo.launch(ctx) } catch (e: Throwable) { } + assertEquals("native Swift Testing cases should all pass", 0, runTests()) + } +} +""" + +/// Gradle DSL appended to a `mode: native` test module's under-test `build.gradle.kts` for the Robolectric +/// (`testDebug`, no device) path. Registers `buildLocalSwiftTestLibs` (runs `skip android test +/// --build-test-libs --robolectric`, building the host test bundle + `libtest_harness` + Swift-runtime/ +/// bridge dylibs into `/test-jni-libs-host` and writing a `dyld-env.txt`), then makes every `Test` +/// task depend on it, point `skip.test.libs` at that dir, and apply the staged `DYLD_*`/`LD_LIBRARY_PATH` +/// values (so `Testing.framework` + the Swift runtime resolve when the harness `dlopen`s the test bundle). +let nativeTestRobolectricGradleBlock: String = """ +// Native Swift Testing (`mode: native`) Robolectric (host JVM, no device) support generated by Skip +tasks.register("buildLocalSwiftTestLibs") { + doLast { + project.objects.newInstance().execOps.exec { + workingDir(layout.projectDirectory) + commandLine("sh", "-cx", "\\"${skipcmd}\\" android test --build-test-libs \\"${swiftBuildFolder()}/test-jni-libs-host\\" --robolectric --package-path \\"${swiftSourceFolder()}\\" --configuration debug --scratch-path \\"${swiftBuildFolder()}/swift-test-host\\"") + environment("SKIP_BRIDGE", "1") + if (file(skipCommand).exists()) { environment("SKIP_COMMAND_OVERRIDE", skipCommand) } + } + } +} + +tasks.withType().configureEach { + val libsDir = "${swiftBuildFolder()}/test-jni-libs-host" + if (System.getenv("SKIP_BRIDGE_ROBOLECTRIC_BUILD_DISABLED") != "1") { + dependsOn("buildLocalSwiftTestLibs") + } + systemProperty("skip.test.libs", libsDir) + doFirst { + // buildLocalSwiftTestLibs writes the dynamic-loader env (Testing.framework + Swift runtime paths) + // it resolved on the host; apply them to the forked test JVM before it loads the harness. + val dyldEnv = file(libsDir + "/dyld-env.txt") + if (dyldEnv.exists()) { + dyldEnv.readLines().forEach { line -> + val eq = line.indexOf('=') + if (eq > 0) { environment(line.substring(0, eq), line.substring(eq + 1)) } + } + } + // Have the native harness write the swt event stream into the junit output dir, so `skip test` + // can recover per-test results (otherwise only the aggregate SwiftTestRunner case is reported). + val eventsDir = reports.junitXml.outputLocation.get().asFile + eventsDir.mkdirs() + environment("SKIP_SWT_EVENTS", eventsDir.resolve("swt-events.jsonl").absolutePath) + } +} +""" diff --git a/Sources/SkipBuild/Commands/TestCommand.swift b/Sources/SkipBuild/Commands/TestCommand.swift index dc5a80cd..6974cace 100644 --- a/Sources/SkipBuild/Commands/TestCommand.swift +++ b/Sources/SkipBuild/Commands/TestCommand.swift @@ -13,6 +13,12 @@ fileprivate let testCommandEnabled = true fileprivate let testCommandEnabled = false #endif +/// The format for `skip test` result output: a human-readable `table` (default) or structured `json`. +enum TestOutputFormat: String, CaseIterable, ExpressibleByArgument { + case table + case json +} + @available(macOS 13, iOS 16, tvOS 16, watchOS 8, *) struct TestCommand: SkipCommand, StreamingCommand, ToolOptionsCommand { typealias Output = MessageBlock @@ -81,6 +87,12 @@ struct TestCommand: SkipCommand, StreamingCommand, ToolOptionsCommand { @Option(name: [.long], help: ArgumentHelp("Output summary table", valueName: "path")) var summaryFile: String? + @Option(name: [.long], help: ArgumentHelp("Test result output format: table (default) or json", valueName: "format")) + var testOutput: TestOutputFormat = .table + + @Option(name: [.long], help: ArgumentHelp("Write the test result output to this file instead of standard out", valueName: "path")) + var testOutputFile: String? + @Option(help: ArgumentHelp("Android device or emulator serial for instrumented tests (omit for local Robolectric testing)", valueName: "ANDROID_SERIAL")) var androidSerial: String? @@ -195,6 +207,10 @@ extension TestCommand { let skipModuleTests = xunitCasesAll.filter({ $0.name == "testSkipModule" }) let xunitCases = xunitCasesAll.filter({ $0.name != "testSkipModule" }) + // A `mode: native` Swift Testing test module is a CONVENTIONAL Skip test module (skipstone plugin + // + `SkipTest`), so it always produces a `testSkipModule` harness case and is driven through the + // Gradle path below (connectedDebugAndroidTest on device / testDebug on Robolectric) — see the + // generated runners + gradle blocks in SwiftTestingHarness.swift. if skipModuleTests.isEmpty { throw SkipDriveError(errorDescription: "Could not find Skip test testSkipModule in: \(xunitCases.map(\.name))") } @@ -208,6 +224,9 @@ extension TestCommand { var allXunitStats: [Stats] = [] var allJunitStats: [Stats] = [] + // per-module matched results, rendered together (table or json) after the loop + var moduleResults: [(module: String, darwin: String, android: String, cases: [(xunit: TestCaseInfo, junit: TestCaseInfo?)])] = [] + // load the junit result folders for skipModule in skipModules { //outputOptions.write("skipModule: \(skipModule)") @@ -221,7 +240,14 @@ extension TestCommand { let buildFolderBase = try AbsolutePath(validating: ".build", relativeTo: AbsolutePath(validating: project, relativeTo: AbsolutePath(validating: FileManager.default.currentDirectoryPath))) let testOutputBase = try buildPluginOutputFolder(forModule: skipModule + "Tests", inBuildFolder: buildFolderBase) - let testOutput = testOutputBase.appending(components: [skipModule.description, ".build", skipModule.description, "test-results", "test\(configuration.capitalized)UnitTest"]) + // connectedAndroidTest (emulator/device) writes results under outputs/androidTest-results/connected/; + // testDebugUnitTest (Robolectric, no device) writes under test-results/testUnitTest. + let testOutput: AbsolutePath + if additionalEnv["ANDROID_SERIAL"] != nil { + testOutput = testOutputBase.appending(components: [skipModule.description, ".build", skipModule.description, "outputs", "androidTest-results", "connected", configuration]) + } else { + testOutput = testOutputBase.appending(components: [skipModule.description, ".build", skipModule.description, "test-results", "test\(configuration.capitalized)UnitTest"]) + } junitFolder = testOutput.asURL } @@ -247,6 +273,15 @@ extension TestCommand { junitCases.append(contentsOf: junitResults.flatMap(\.testCases)) } + // A `mode: native` module runs its whole Swift Testing suite inside one JUnit case + // (`SwiftTestRunner.nativeSwiftTests`), so Gradle only reports that aggregate. The native test + // harness additionally writes the swt ABI-v0 event stream to `swt-events.jsonl` in the junit + // output dir; recover per-test results from it so each test is reported individually (matched + // against its host/Darwin counterpart) and the aggregate row is dropped. + let swtEventsURL = junitFolder.appendingPathComponent("swt-events.jsonl") + let nativeCases = FileManager.default.fileExists(atPath: swtEventsURL.path) ? parseNativeSwtEvents(swtEventsURL) : [] + junitCases.append(contentsOf: nativeCases.map { $0 as TestCaseInfo }) + // now we have all the test cases; for each xunit test, check for an equivalent JUnit test // note that xunit: classname="SkipZipTests.SkipZipTests" name="testDeflateInflate" // maps to junit: classname="skip.zip.SkipZipTests" name="testDeflateInflate$SkipZip_debugUnitTest" @@ -284,15 +319,36 @@ extension TestCommand { matchedCases.append((xunit: xunitCase, junit: cases.first)) } - let (testsTable, xunit, junit) = createTestSummaryTable(columnLength: maxColumnLength, matchedCases, testNameComparison) + // When per-test native results were recovered (above), they're matched individually, so the + // aggregate `SwiftTestRunner.nativeSwiftTests` case is redundant. Only fall back to reporting it + // directly (paired with itself) when no per-test stream was available, so its pass/fail still + // shows rather than being silently dropped. + if nativeCases.isEmpty { + let matchedJunitNames = Set(matchedCases.compactMap(\.junit).map(\.name)) + for junitCase in junitCases where junitCase.classname.hasSuffix(".SwiftTestRunner") && !matchedJunitNames.contains(junitCase.name) { + matchedCases.append((xunit: junitCase, junit: junitCase)) + } + } + + // Descriptive column titles: the host (XUnit) column is always Darwin/macOS; the Gradle (JUnit) + // column reflects the Android build mode — Fuse-compiled native (the run produced a + // SwiftTestRunner harness case) vs Lite-transpiled — and where it ran (a connected + // device/emulator named by ANDROID_SERIAL, else Robolectric on the host JVM). + let darwinTitle = "Darwin (macOS)" + let androidMode = (junitCases.contains(where: { $0.classname.hasSuffix(".SwiftTestRunner") }) || !nativeCases.isEmpty) ? "Fuse" : "Lite" + let androidTarget = additionalEnv["ANDROID_SERIAL"] ?? "Robolectric" + let androidTitle = "Android (\(androidMode) \(androidTarget))" + + let (xunit, junit) = computeStats(matchedCases) allXunitStats.append(xunit) allJunitStats.append(junit) - await out.write(status: nil, testsTable) + // collect for the post-loop render (table or json); the in-loop CI summary file stays markdown + moduleResults.append((module: String(skipModule), darwin: darwinTitle, android: androidTitle, cases: matchedCases)) // when we are running in CI, the "GITHUB_STEP_SUMMARY" contains the path of a file that can be used to write a markdown summary of the tests // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary if let summaryFile = self.summaryFile ?? ProcessInfo.processInfo.environment["GITHUB_STEP_SUMMARY"] ?? ProcessInfo.processInfo.environment["CI_STEP_SUMMARY"], !summaryFile.isEmpty { - let (summaryTable, _, _) = createTestSummaryTable(columnLength: 1024, matchedCases, testNameComparison) + let (summaryTable, _, _) = createTestSummaryTable(columnLength: 1024, darwinTitle: darwinTitle, androidTitle: androidTitle, matchedCases, testNameComparison) if !FileManager.default.fileExists(atPath: summaryFile) { _ = FileManager.default.createFile(atPath: summaryFile, contents: nil, attributes: nil) @@ -306,6 +362,21 @@ extension TestCommand { } } + // render the collected results in the requested format and route them to a file or standard out + let outputText: String + switch testOutput { + case .table: + outputText = moduleResults.map { createTestSummaryTable(columnLength: maxColumnLength, darwinTitle: $0.darwin, androidTitle: $0.android, $0.cases, testNameComparison).table }.joined(separator: "\n") + case .json: + outputText = try renderJSONReport(moduleResults, testNameComparison) + } + if let testOutputFile = self.testOutputFile, !testOutputFile.isEmpty { + try outputText.write(toFile: testOutputFile, atomically: true, encoding: .utf8) + await out.write(status: nil, "Wrote \(testOutput.rawValue) test results to \(testOutputFile)") + } else { + await out.write(status: nil, outputText) + } + let exitCode = try? testResult?.get().exitCode let aggregateStats = { ($0 as [Stats]).reduce(into: Stats()) { stats, result in @@ -335,7 +406,98 @@ extension TestCommand { #endif } - private func createTestSummaryTable(columnLength: Int, _ matchedCases: [(xunit: TestCaseInfo, junit: TestCaseInfo?)], _ testNameComparison: (TestCaseInfo, TestCaseInfo) -> Bool) -> (table: String, xunit: Stats, junit: Stats) { + /// Parse the native test harness's swt ABI-v0 JSON event stream (`swt-events.jsonl`, one record per + /// line) into per-test results. The `kind:"test"`/`kind:"function"` records catalog each test's + /// function-style `name` and its `id` (`//::`); the + /// `kind:"event"` `testEnded`/`issueRecorded`/`testSkipped` records (keyed by `testID`) give pass/ + /// fail/skip. Returns one `NativeTestCase` per function that ran, with `classname` = the suite id + /// (so it matches the host xunit case for the same test) and `name` = the function name. + private func parseNativeSwtEvents(_ url: URL) -> [NativeTestCase] { + guard let text = try? String(contentsOf: url, encoding: .utf8) else { return [] } + struct State { var failed = false; var skipped = false; var ran = false; var started: Double? = nil; var ended: Double? = nil } + var names: [String: String] = [:] // function id -> "demoFramework()" + var suites: [String: String] = [:] // function id -> "Module.Suite" + var states: [String: State] = [:] + func instant(_ payload: [String: Any]) -> Double? { (payload["instant"] as? [String: Any])?["absolute"] as? Double } + for line in text.split(separator: "\n") { + // Tolerate any leading text before the JSON object: the connected (device) path recovers the + // stream from logcat, where some records arrive raw and others carry a "Test line: " prefix. + guard let brace = line.firstIndex(of: "{") else { continue } + guard let obj = try? JSONSerialization.jsonObject(with: Data(line[brace...].utf8)) as? [String: Any], + let payload = obj["payload"] as? [String: Any] else { continue } + switch obj["kind"] as? String { + case "test": + if (payload["kind"] as? String) == "function", + let id = payload["id"] as? String, let name = payload["name"] as? String { + names[id] = name + suites[id] = String(id.prefix(while: { $0 != "/" })) // "Module.Suite" before the first '/' + } + case "event": + guard let testID = payload["testID"] as? String, names[testID] != nil else { break } + switch payload["kind"] as? String { + case "testStarted": + states[testID, default: State()].started = instant(payload) + case "issueRecorded": + if let issue = payload["issue"] as? [String: Any], (issue["isFailure"] as? Bool) == true { + states[testID, default: State()].failed = true + } + case "testSkipped": + states[testID, default: State()].skipped = true + states[testID, default: State()].ran = true + case "testEnded": + let symbol = (payload["messages"] as? [[String: Any]])?.first?["symbol"] as? String + if symbol == "fail" { states[testID, default: State()].failed = true } + states[testID, default: State()].ended = instant(payload) + states[testID, default: State()].ran = true + default: break + } + default: break + } + } + return names.compactMap { id, name in + guard let state = states[id], state.ran else { return nil } // only tests that actually ran + // duration from the testStarted/testEnded monotonic `absolute` instants (0 = unknown) + let duration: TimeInterval + if let started = state.started, let ended = state.ended { duration = max(0, ended - started) } else { duration = 0 } + return NativeTestCase(name: name, classname: suites[id] ?? "", time: duration, skipped: state.skipped, hasFailures: state.failed) + } + } + + /// The Darwin (xunit) and Android (junit) pass/fail/skip/missing tallies for a module's matched cases. + private func computeStats(_ matchedCases: [(xunit: TestCaseInfo, junit: TestCaseInfo?)]) -> (xunit: Stats, junit: Stats) { + var (xunitStats, junitStats) = (Stats(), Stats()) + for (xunit, junit) in matchedCases { + xunitStats.update(xunit) + junitStats.update(junit) + } + return (xunitStats, junitStats) + } + + /// Render the matched results across all modules as a structured JSON document (per-test status + + /// duration for each platform, plus per-platform tallies). Slashes are left unescaped for readability. + private func renderJSONReport(_ moduleResults: [(module: String, darwin: String, android: String, cases: [(xunit: TestCaseInfo, junit: TestCaseInfo?)])], _ testNameComparison: (TestCaseInfo, TestCaseInfo) -> Bool) throws -> String { + func result(_ test: TestCaseInfo?) -> TestReport.Result? { + guard let test = test else { return nil } // unmatched on this platform + let status = test.skipped ? "skip" : test.hasFailures ? "fail" : "pass" + return TestReport.Result(status: status, time: test.time > 0 ? test.time : nil) + } + var (allXunit, allJunit) = (Stats(), Stats()) + let modules: [TestReport.Module] = moduleResults.map { mr in + let cases: [TestReport.Case] = mr.cases.sorted(by: { testNameComparison($0.xunit, $1.xunit) }).map { (xunit, junit) in + allXunit.update(xunit) + allJunit.update(junit) + return TestReport.Case(suite: xunit.classname.split(separator: ".").last?.description ?? xunit.classname, name: xunit.name, darwin: result(xunit), android: result(junit)) + } + return TestReport.Module(module: mr.module, darwin: mr.darwin, android: mr.android, cases: cases) + } + func counts(_ s: Stats) -> TestReport.Counts { TestReport.Counts(passed: s.passed, failed: s.failed, skipped: s.skipped, missing: s.missing) } + let report = TestReport(modules: modules, summary: TestReport.Summary(darwin: counts(allXunit), android: counts(allJunit))) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + return String(decoding: try encoder.encode(report), as: UTF8.self) + } + + private func createTestSummaryTable(columnLength: Int, darwinTitle: String, androidTitle: String, _ matchedCases: [(xunit: TestCaseInfo, junit: TestCaseInfo?)], _ testNameComparison: (TestCaseInfo, TestCaseInfo) -> Bool) -> (table: String, xunit: Stats, junit: Stats) { // now output all of the test cases var outputColumns: [[String]] = [[], [], [], []] @@ -349,7 +511,7 @@ extension TestCommand { } //addSeparator() - addRow(["Test", "Case", "Swift", "Kotlin"]) + addRow(["Test", "Case", darwinTitle, androidTitle]) addSeparator() var (xunitStats, junitStats) = (Stats(), Stats()) @@ -365,10 +527,12 @@ extension TestCommand { guard let test = test else { return "????" // unmatched } - let result = (test.skipped == true ? "SKIP" : test.hasFailures ? "FAIL" : "PASS") - //result += " (" + ((round(test.time * 1000) / 1000).description) + ")" + var result = (test.skipped == true ? "SKIP" : test.hasFailures ? "FAIL" : "PASS") + // append the per-test duration when known (0 = unknown, e.g. an unmatched/aggregate case) + if test.time > 0 { + result += String(format: " (%.2fs)", test.time) + } return result - } outputColumns[2].append(desc(xunit)) @@ -383,7 +547,10 @@ extension TestCommand { // pad all the columns for nice output let lengths = outputColumns.map({ $0.reduce(0, { max($0, $1.count) })}) for (index, length) in lengths.enumerated() { - outputColumns[index] = outputColumns[index].map { $0.pad(min(length, columnLength), paddingCharacter: $0 == "-" ? "-" : " ") } + // Cap the variable-length Test/Case columns at columnLength; let the fixed Darwin/Android + // result-column headers size to their full width so the mode/device labels aren't truncated. + let width = index <= 1 ? min(length, columnLength) : length + outputColumns[index] = outputColumns[index].map { $0.pad(width, paddingCharacter: $0 == "-" ? "-" : " ") } } let rowCount = outputColumns.map({ $0.count }).min() ?? 0 @@ -411,6 +578,40 @@ extension TestCommand { } } +/// The structured `skip test` results emitted by `--test-output=json`: per-test status and duration on +/// each platform, plus per-platform tallies. A `null` per-platform result means the test was unmatched +/// there; a `null`/omitted `time` means the duration is unknown (e.g. an aggregate or unmatched case). +struct TestReport: Encodable { + struct Result: Encodable { + let status: String // "pass" / "fail" / "skip" + let time: TimeInterval? // seconds; omitted when unknown + } + struct Case: Encodable { + let suite: String + let name: String + let darwin: Result? + let android: Result? + } + struct Module: Encodable { + let module: String + let darwin: String // Darwin column title + let android: String // Android column title + let cases: [Case] + } + struct Counts: Encodable { + let passed: Int + let failed: Int + let skipped: Int + let missing: Int + } + struct Summary: Encodable { + let darwin: Counts + let android: Counts + } + let modules: [Module] + let summary: Summary +} + protocol TestCaseInfo { /// e.g.: someTestCaseThatAlwaysFails() var name: String { get } @@ -424,6 +625,16 @@ protocol TestCaseInfo { var hasFailures: Bool { get } } +/// A per-test result synthesized from a `mode: native` module's swt event stream, so each native Swift +/// Testing case can be reported (and matched against its host/Darwin counterpart) individually. +struct NativeTestCase: TestCaseInfo { + var name: String + var classname: String + var time: TimeInterval + var skipped: Bool + var hasFailures: Bool +} + #if canImport(SkipDriveExternal) // needed for GradleDriver.TestCase extension GradleDriver.TestCase : TestCaseInfo { var hasFailures: Bool { diff --git a/Sources/SkipBuild/SkipProject.swift b/Sources/SkipBuild/SkipProject.swift index 686cd622..4f2160d3 100644 --- a/Sources/SkipBuild/SkipProject.swift +++ b/Sources/SkipBuild/SkipProject.swift @@ -1059,46 +1059,25 @@ public class \(moduleName)Module { let rfolder = isNativeModule ? nil : resourceFolder var testCaseCode: String - if options.testCaseMode == .testing { + // Native test modules run via the Swift Testing ABI harness (swt_abiv0), which cannot + // execute XCTest cases, so always scaffold Swift Testing for them regardless of the option. + if options.testCaseMode == .testing || isNativeModule { testCaseCode = """ \(testSourceHeader)import Testing -import OSLog import Foundation """ - if isNativeModule { - testCaseCode += """ -import SkipBridge - -""" - } - testCaseCode += """ @testable import \(moduleName) -let logger: Logger = Logger(subsystem: "\(moduleName)", category: "Tests") - @Suite struct \(moduleName)Tests { """ - if isNativeModule { - testCaseCode += """ - init() { - #if SKIP - // needed to load the compiled bridge when the tests are transpiled - loadPeerLibrary(packageName: "\(projectName)", moduleName: "\(moduleName)") - #endif - } - -""" - } - testCaseCode += """ @Test func \(moduleName.prefix(1).lowercased() + moduleName.dropFirst())() throws { - logger.log("running test\(moduleName)") #expect(1 + 2 == 3, "basic test") } @@ -1186,13 +1165,6 @@ import Foundation """ - if isNativeModule { - testCaseCode += """ -import SkipBridge - -""" - } - testCaseCode += """ @testable import \(moduleName) @@ -1203,18 +1175,6 @@ final class \(moduleName)Tests: XCTestCase { """ - if isNativeModule { - testCaseCode += """ - override func setUp() { - #if SKIP - // needed to load the compiled bridge when the tests are transpiled - loadPeerLibrary(packageName: "\(projectName)", moduleName: "\(moduleName)") - #endif - } - -""" - } - testCaseCode += """ func test\(moduleName)() throws { @@ -1311,14 +1271,14 @@ struct TestData : Codable, Hashable { """ if moduleMode.isNative { - // The test target for a natively-compiled Skip Fuse module must itself be transpiled: its - // XCTest cases are transpiled to JUnit tests so the test harness can collect the results. - // A native test target would have its test classes dropped during bridging, leaving no - // tests to run, so it must explicitly opt into transpiled mode. + // The test target for a natively-compiled Skip Fuse module is itself a native test module: + // its Swift Testing cases run natively on Android (not transpiled to Kotlin/JUnit). The + // skipstone plugin generates the JNI harness glue and drives the cases on the emulator / + // Robolectric via the same testSkipModule → runGradleTests mechanism as transpiled tests. testSkipYaml += """ skip: - mode: 'transpiled' + mode: 'native' """ diff --git a/Tests/SkipBuildTests/SkipCommandTests.swift b/Tests/SkipBuildTests/SkipCommandTests.swift index 19d829bb..3467247d 100644 --- a/Tests/SkipBuildTests/SkipCommandTests.swift +++ b/Tests/SkipBuildTests/SkipCommandTests.swift @@ -191,16 +191,12 @@ final class SkipCommandTests: XCTestCase { let testCaseCode = try load("Tests/SomeModuleTests/SomeModuleTests.swift") XCTAssertEqual(testCaseCode, """ import Testing - import OSLog import Foundation @testable import SomeModule - let logger: Logger = Logger(subsystem: "SomeModule", category: "Tests") - @Suite struct SomeModuleTests { @Test func someModule() throws { - logger.log("running testSomeModule") #expect(1 + 2 == 3, "basic test") } @@ -736,23 +732,12 @@ final class SkipCommandTests: XCTestCase { let testCaseCode = try load("Tests/SomeModuleTests/SomeModuleTests.swift") XCTAssertEqual(testCaseCode, """ import Testing - import OSLog import Foundation - import SkipBridge @testable import SomeModule - let logger: Logger = Logger(subsystem: "SomeModule", category: "Tests") - @Suite struct SomeModuleTests { - init() { - #if SKIP - // needed to load the compiled bridge when the tests are transpiled - loadPeerLibrary(packageName: "basic-project", moduleName: "SomeModule") - #endif - } @Test func someModule() throws { - logger.log("running testSomeModule") #expect(1 + 2 == 3, "basic test") } @@ -794,7 +779,7 @@ final class SkipCommandTests: XCTestCase { # contents: skip: - mode: 'transpiled' + mode: 'native' """) @@ -835,13 +820,20 @@ final class SkipCommandTests: XCTestCase { . ├─ Package.swift ├─ README.md - └─ Sources - └─ SomeModule + ├─ Sources + │ └─ SomeModule + │ ├─ Resources + │ │ └─ Localizable.xcstrings + │ ├─ Skip + │ │ └─ skip.yml + │ └─ SomeModule.swift + └─ Tests + └─ SomeModuleTests ├─ Resources - │ └─ Localizable.xcstrings + │ └─ TestData.json ├─ Skip │ └─ skip.yml - └─ SomeModule.swift + └─ SomeModuleTests.swift """) @@ -909,6 +901,10 @@ final class SkipCommandTests: XCTestCase { .target(name: "SomeModule", dependencies: [ .product(name: "SkipFuse", package: "skip-fuse") ], resources: [.process("Resources")], plugins: [.plugin(name: "skipstone", package: "skip")]), + .testTarget(name: "SomeModuleTests", dependencies: [ + "SomeModule", + .product(name: "SkipTest", package: "skip") + ], resources: [.process("Resources")], plugins: [.plugin(name: "skipstone", package: "skip")]), ] ) @@ -1031,23 +1027,12 @@ final class SkipCommandTests: XCTestCase { let testCaseCode = try load("Tests/ModelModuleTests/ModelModuleTests.swift") XCTAssertEqual(testCaseCode, """ import Testing - import OSLog import Foundation - import SkipBridge @testable import ModelModule - let logger: Logger = Logger(subsystem: "ModelModule", category: "Tests") - @Suite struct ModelModuleTests { - init() { - #if SKIP - // needed to load the compiled bridge when the tests are transpiled - loadPeerLibrary(packageName: "cool-app", moduleName: "ModelModule") - #endif - } @Test func modelModule() throws { - logger.log("running testModelModule") #expect(1 + 2 == 3, "basic test") } @@ -1217,23 +1202,12 @@ final class SkipCommandTests: XCTestCase { let testCaseCode = try load("Tests/ModelModuleTests/ModelModuleTests.swift") XCTAssertEqual(testCaseCode, """ import Testing - import OSLog import Foundation - import SkipBridge @testable import ModelModule - let logger: Logger = Logger(subsystem: "ModelModule", category: "Tests") - @Suite struct ModelModuleTests { - init() { - #if SKIP - // needed to load the compiled bridge when the tests are transpiled - loadPeerLibrary(packageName: "cool-app", moduleName: "ModelModule") - #endif - } @Test func modelModule() throws { - logger.log("running testModelModule") #expect(1 + 2 == 3, "basic test") } @@ -2035,16 +2009,12 @@ final class SkipCommandTests: XCTestCase { let testCaseCode = try load("Tests/SomeModuleTests/SomeModuleTests.swift") XCTAssertEqual(testCaseCode, """ import Testing - import OSLog import Foundation @testable import SomeModule - let logger: Logger = Logger(subsystem: "SomeModule", category: "Tests") - @Suite struct SomeModuleTests { @Test func someModule() throws { - logger.log("running testSomeModule") #expect(1 + 2 == 3, "basic test") }