From 553ca795ba256470ec4c329061ca0282f3f4fa2b Mon Sep 17 00:00:00 2001 From: Abhishek Bedi Date: Tue, 23 Aug 2022 19:14:52 +0400 Subject: [PATCH 1/8] Added Support to run iOS Test Runners Tag --- README.md | 38 ++++++++++++++++++- .../config/vendor/VendorConfiguration.kt | 1 + .../kotlin/com/malinskiy/marathon/Marathon.kt | 5 ++- .../com/malinskiy/marathon/test/Test.kt | 3 +- sample/ios-app/Marathonfile | 3 +- .../sample-appUITests/StoryboardTests.swift | 31 +++++++-------- .../malinskiy/marathon/ios/IOSTestParser.kt | 14 ++++++- 7 files changed, 74 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 1272d04e7..2990100f8 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,49 @@ Cross-platform test runner written for Android and iOS projects ## Main focus -- **stability** of test execution adjusting for flakiness in the environment and in the tests. +- **stability** of test execution adjusting for flakiness in the environment and in the tests. - **performance** using high parallelization (handling dozens of devices) ## Documentation Please check the official [documentation](https://marathonlabs.github.io/marathon/) for installation, configuration and more +## [iOS Only] Added Support for XCtest UI Test Functions (with Tags) +You can now tag to your iOS UITests just like you would add tag to your feature file (if you are using [Cucumberish](https://cocoapods.org/pods/Cucumberish) or [XCTGherkin](https://cocoapods.org/pods/XCTest-Gherkin)) +_(Multiline tag support is not yet there. Pls make sure that tag is added just above the func signature)_ +> How to tag your UITests. See Sample below. +``` + // @Flowers @apple @mock-batch-1 + func testButton() { + let button = app.buttons.firstMatch + XCTAssertTrue(button.waitForExistence()) + XCTAssertTrue(button.isHittable) + button.tap() + let label = app.staticTexts.firstMatch + XCTAssertTrue(label.waitForExistence()) + } +``` + +> Tag reference in marathon file. (See last line) + +``` +vendorConfiguration: + type: "iOS" + derivedDataDir: "derived-data" + sourceRoot: "sample-appUITests" + knownHostsPath: ${HOME}/.ssh/known_hosts + remoteUsername: ${USER} + remotePrivateKey: ${HOME}/.ssh/marathon + xcTestRunnerTag: "Flowers" +``` + +Refer Branch: [`origin/ios-uitest-runner-via-tags`](https://github.com/abhishekbedi1432/Cross-Platform-Test-Runner-Marathon/tree/ios-uitest-runner-via-tags) + +> Pls Note: +> Tags are case-sensitive 😉 +> xcTestRunnerTag supports single param at the moment +> Multiline tag support is not yet there. Pls make sure that tag is added just above the func signature + License ------- diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt index b160fe427..403359b08 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt @@ -79,6 +79,7 @@ sealed class VendorConfiguration { @JsonProperty("hideRunnerOutput") val hideRunnerOutput: Boolean = false, @JsonProperty("compactOutput") val compactOutput: Boolean = false, @JsonProperty("keepAliveIntervalMillis") val keepAliveIntervalMillis: Long = 0L, + @JsonProperty("xcTestRunnerTag") val xcTestRunnerTag: String? = null, @JsonProperty("devices") val devicesFile: File? = null, ) : VendorConfiguration() { /** diff --git a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt index a2bc6e49d..e64a05c91 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt @@ -5,6 +5,7 @@ import com.malinskiy.marathon.analytics.internal.pub.Track import com.malinskiy.marathon.analytics.internal.sub.TrackerInternal import com.malinskiy.marathon.config.Configuration import com.malinskiy.marathon.config.LogicalConfigurationValidator +import com.malinskiy.marathon.config.vendor.VendorConfiguration import com.malinskiy.marathon.device.DeviceProvider import com.malinskiy.marathon.exceptions.NoDevicesException import com.malinskiy.marathon.exceptions.NoTestCasesFoundException @@ -146,7 +147,9 @@ class Marathon( } configuration.filteringConfiguration.allowlist.forEach { tests = it.toTestFilter().filter(tests) } configuration.filteringConfiguration.blocklist.forEach { tests = it.toTestFilter().filterNot(tests) } - return tests + val iosConfig = configuration.vendorConfiguration as? VendorConfiguration.IOSConfiguration + + return tests.filter { it.tags.contains(iosConfig?.xcTestRunnerTag) } } private fun prepareTestShard(tests: List, analytics: Analytics): TestShard { diff --git a/core/src/main/kotlin/com/malinskiy/marathon/test/Test.kt b/core/src/main/kotlin/com/malinskiy/marathon/test/Test.kt index 74faee1d5..59f48d4a5 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/test/Test.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/test/Test.kt @@ -6,7 +6,8 @@ data class Test( val pkg: String, val clazz: String, val method: String, - val metaProperties: Collection + val metaProperties: Collection, + val tags: Collection = emptyList() ) { override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/sample/ios-app/Marathonfile b/sample/ios-app/Marathonfile index a770a814a..e87602a8c 100644 --- a/sample/ios-app/Marathonfile +++ b/sample/ios-app/Marathonfile @@ -5,7 +5,7 @@ poolingStrategy: type: "omni" batchingStrategy: type: "fixed-size" - size: 5 + size: 1 debug: true filteringConfiguration: allowlist: @@ -18,3 +18,4 @@ vendorConfiguration: knownHostsPath: ${HOME}/.ssh/known_hosts remoteUsername: ${USER} remotePrivateKey: ${HOME}/.ssh/marathon + xcTestRunnerTag: "Flowers" diff --git a/sample/ios-app/sample-appUITests/StoryboardTests.swift b/sample/ios-app/sample-appUITests/StoryboardTests.swift index 66b007ee2..a097f094d 100644 --- a/sample/ios-app/sample-appUITests/StoryboardTests.swift +++ b/sample/ios-app/sample-appUITests/StoryboardTests.swift @@ -12,21 +12,7 @@ class StoryboardTests: XCTestCase { private var app: XCUIApplication! - override func setUp() { - super.setUp() - - continueAfterFailure = false - - app = XCUIApplication() - app.launch() - } - - override func tearDown() { - app = nil - - super.tearDown() - } - + // @Flowers @apple @mock-batch-1 func testButton() { let button = app.buttons.firstMatch XCTAssertTrue(button.waitForExistence()) @@ -44,6 +30,7 @@ class StoryboardTests: XCTestCase { XCTAssertTrue(button.isHittable) } + // @Flowers @mock-batch-1 @apple func testLabel() { let button = app.buttons.firstMatch button.waitForExistence() @@ -61,6 +48,20 @@ class StoryboardTests: XCTestCase { let label = app.staticTexts.firstMatch XCTAssertEqual(label.label, "Label") } + override func setUp() { + super.setUp() + + continueAfterFailure = false + + app = XCUIApplication() + app.launch() + } + + override func tearDown() { + app = nil + + super.tearDown() + } } private let standardTimeout: TimeInterval = 30.0 diff --git a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSTestParser.kt b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSTestParser.kt index d51b1b00f..9d230b7f9 100644 --- a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSTestParser.kt +++ b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSTestParser.kt @@ -10,6 +10,7 @@ import java.io.File class IOSTestParser(private val vendorConfiguration: VendorConfiguration.IOSConfiguration) : TestParser { private val swiftTestClassRegex = """class ([^:\s]+)\s*:\s*XCTestCase""".toRegex() private val swiftTestMethodRegex = """^.*func\s+(test[^(\s]*)\s*\(.*$""".toRegex() + private val swiftTagRegex = """@([^:\s]+)\w*""".toRegex() private val logger = MarathonLogging.logger(IOSTestParser::class.java.simpleName) @@ -20,6 +21,11 @@ class IOSTestParser(private val vendorConfiguration: VendorConfiguration.IOSConf * marked as skipped in `xctestrun` file. */ override suspend fun extract(): List { + + if (vendorConfiguration.xcTestRunnerTag.isNullOrEmpty()) { + logger.warn { "[iOS] Did not find `xcTestRunnerTag` in iOS Vendor Config. This will result in running all tests" } + } + if (!vendorConfiguration.sourceRoot.isDirectory) { throw IllegalArgumentException("Expected a directory at $vendorConfiguration.sourceRoot") } @@ -35,7 +41,8 @@ class IOSTestParser(private val vendorConfiguration: VendorConfiguration.IOSConf val implementedTests = mutableListOf() for (file in swiftFilesWithTests) { var testClassName: String? = null - for (line in file.readLines()) { + + for ((index, line) in file.readLines().withIndex()) { val className = line.firstMatchOrNull(swiftTestClassRegex) val methodName = line.firstMatchOrNull(swiftTestMethodRegex) @@ -44,7 +51,10 @@ class IOSTestParser(private val vendorConfiguration: VendorConfiguration.IOSConf } if (testClassName != null && methodName != null) { - implementedTests.add(Test(targetName, testClassName, methodName, emptyList())) + // Find & Update Tags here + val tagLine = file.readLines()[index - 1] + val tags = swiftTagRegex.findAll(tagLine).map { it.groupValues[1] }.toList() + implementedTests.add(Test(targetName, testClassName, methodName, emptyList(), tags)) } } } From 1f09d5904f9b2968c6e28ad3e256032f0f1f0cca Mon Sep 17 00:00:00 2001 From: Abhishek Bedi Date: Tue, 23 Aug 2022 19:18:51 +0400 Subject: [PATCH 2/8] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2990100f8..70da3427b 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Cross-platform test runner written for Android and iOS projects Please check the official [documentation](https://marathonlabs.github.io/marathon/) for installation, configuration and more -## [iOS Only] Added Support for XCtest UI Test Functions (with Tags) +## [iOS Only] Added Support to consume tags for test functions under XCTestCase Subclasses You can now tag to your iOS UITests just like you would add tag to your feature file (if you are using [Cucumberish](https://cocoapods.org/pods/Cucumberish) or [XCTGherkin](https://cocoapods.org/pods/XCTest-Gherkin)) _(Multiline tag support is not yet there. Pls make sure that tag is added just above the func signature)_ > How to tag your UITests. See Sample below. From 479429211a10110cd52794acd6a7ac1513133ea3 Mon Sep 17 00:00:00 2001 From: Abhishek Bedi Date: Sat, 17 Sep 2022 00:15:38 +0400 Subject: [PATCH 3/8] [iOS] Added Support for XCResult --- .../marathon/config/serialization/ConfigurationFactory.kt | 2 ++ .../serialization/yaml/VendorConfigurationDeserializer.kt | 5 ++++- .../malinskiy/marathon/config/vendor/VendorConfiguration.kt | 1 + .../config/serialization/ConfigurationFactoryTest.kt | 1 + core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt | 4 ++++ sample/ios-app/Marathonfile | 1 + .../src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt | 4 ++++ 7 files changed, 17 insertions(+), 1 deletion(-) diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt index 3e229a528..3d8146eae 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt @@ -55,6 +55,7 @@ class ConfigurationFactory( is VendorConfiguration.IOSConfiguration -> { // Any relative path specified in Marathonfile should be resolved against the directory Marathonfile is in val resolvedDerivedDataDir = marathonfileDir.resolve(configuration.vendorConfiguration.derivedDataDir) + val resolvedResultBundlePath = marathonfileDir.resolve(configuration.vendorConfiguration.xcResultBundlePath) val finalXCTestRunPath = configuration.vendorConfiguration.xctestrunPath?.resolveAgainst(marathonfileDir) ?: fileListProvider .fileList(resolvedDerivedDataDir) @@ -71,6 +72,7 @@ class ConfigurationFactory( sourceRoot = optionalSourceRoot, devicesFile = optionalDevices, knownHostsPath = optionalKnownHostsPath, + xcResultBundlePath = resolvedResultBundlePath ) } VendorConfiguration.StubVendorConfiguration -> configuration.vendorConfiguration diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/VendorConfigurationDeserializer.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/VendorConfigurationDeserializer.kt index 5a194b62e..547a2f7f5 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/VendorConfigurationDeserializer.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/VendorConfigurationDeserializer.kt @@ -34,6 +34,8 @@ class VendorConfigurationDeserializer( // Any relative path specified in Marathonfile should be resolved against the directory Marathonfile is in val resolvedDerivedDataDir = marathonfileDir.resolve(iosConfiguration.derivedDataDir) + val resolvedResultBundlePath = marathonfileDir.resolve(iosConfiguration.xcResultBundlePath) + val finalXCTestRunPath = iosConfiguration.xctestrunPath?.resolveAgainst(marathonfileDir) ?: fileListProvider .fileList(resolvedDerivedDataDir) @@ -50,7 +52,8 @@ class VendorConfigurationDeserializer( sourceRoot = optionalSourceRoot, devicesFile = optionalDevices, knownHostsPath = optionalKnownHostsPath, - ) + xcResultBundlePath = resolvedResultBundlePath, + ) } TYPE_ANDROID -> { (node as ObjectNode).remove("type") diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt index b160fe427..7103ee934 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt @@ -79,6 +79,7 @@ sealed class VendorConfiguration { @JsonProperty("hideRunnerOutput") val hideRunnerOutput: Boolean = false, @JsonProperty("compactOutput") val compactOutput: Boolean = false, @JsonProperty("keepAliveIntervalMillis") val keepAliveIntervalMillis: Long = 0L, + @JsonProperty("xcResultBundlePath") val xcResultBundlePath: File, @JsonProperty("devices") val devicesFile: File? = null, ) : VendorConfiguration() { /** diff --git a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt index 94bbeafd1..19407aab6 100644 --- a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt +++ b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt @@ -269,6 +269,7 @@ class ConfigurationFactoryTest { keepAliveIntervalMillis = 300000L, devicesFile = file.parentFile.resolve("Testdevices").canonicalFile, sourceRoot = file.parentFile.resolve(".").canonicalFile, + xcResultBundlePath = file.parentFile.resolve("a/resultBundlePath").canonicalFile, ) } diff --git a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt index a2bc6e49d..1704e17ca 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt @@ -79,6 +79,10 @@ class Marathon( val tests = applyTestFilters(parsedTests) val shard = prepareTestShard(tests, analytics) + log.info("\n\n\n **** Marathon File Params **** \n") + log.info("${configuration}") + log.info("\n\n\n") + log.info("Scheduling ${tests.size} tests") log.debug(tests.joinToString(", ") { it.toTestName() }) val currentCoroutineContext = coroutineContext diff --git a/sample/ios-app/Marathonfile b/sample/ios-app/Marathonfile index a770a814a..9dded8c0b 100644 --- a/sample/ios-app/Marathonfile +++ b/sample/ios-app/Marathonfile @@ -18,3 +18,4 @@ vendorConfiguration: knownHostsPath: ${HOME}/.ssh/known_hosts remoteUsername: ${USER} remotePrivateKey: ${HOME}/.ssh/marathon + xcResultBundlePath: "derived-data/resultBundlePath" \ No newline at end of file diff --git a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt index 374c6ba4f..b13c85f04 100644 --- a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt +++ b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt @@ -176,6 +176,9 @@ class IOSDevice( timer ) + val timestamp = System.currentTimeMillis().toString() + val resultBundlePath: String = "${iosConfiguration.xcResultBundlePath}_" + timestamp + val command = listOf( "cd '$remoteDir';", @@ -183,6 +186,7 @@ class IOSDevice( "xcodebuild test-without-building", "-xctestrun ${remoteXctestrunFile.path}", testBatch.toXcodebuildArguments(), + "-resultBundlePath $resultBundlePath", "-destination 'platform=iOS simulator,id=$udid' ;", "exit" ) From c0c66c3b761507c035e1e60270defea06c826ad6 Mon Sep 17 00:00:00 2001 From: Abhishek Bedi Date: Sat, 17 Sep 2022 01:02:59 +0400 Subject: [PATCH 4/8] Improved timestamp format --- .../kotlin/com/malinskiy/marathon/ios/IOSDevice.kt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt index b13c85f04..a1c6b0ab5 100644 --- a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt +++ b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/IOSDevice.kt @@ -44,6 +44,9 @@ import java.net.InetAddress import java.net.UnknownHostException import java.util.concurrent.TimeoutException import kotlin.coroutines.CoroutineContext +import java.text.SimpleDateFormat +import java.sql.Timestamp +import java.util.* class IOSDevice( val simulator: RemoteSimulator, @@ -176,7 +179,8 @@ class IOSDevice( timer ) - val timestamp = System.currentTimeMillis().toString() + // Adding timestamp to support multiple XCResult bundle creation + val timestamp = getTimezone() val resultBundlePath: String = "${iosConfiguration.xcResultBundlePath}_" + timestamp val command = @@ -386,3 +390,11 @@ private fun String.toInetAddressOrNull(): InetAddress? { private fun TestBatch.toXcodebuildArguments(): String = tests.joinToString(separator = " ") { "-only-testing:\"${it.pkg}/${it.clazz}/${it.method}\"" } + +private fun getTimezone(): String { + val stamp = Timestamp(System.currentTimeMillis()) + val date = Date(stamp.time) + val sdf = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss", Locale.getDefault()) + sdf.timeZone = TimeZone.getDefault() + return sdf.format(date) +} From e5f73c98bd3859c759a297e4e49b683f6df1e5e7 Mon Sep 17 00:00:00 2001 From: Abhishek Bedi Date: Mon, 26 Sep 2022 10:36:41 +0400 Subject: [PATCH 5/8] Modified XCTestRun to adapt to new format of xctestrun ( supporting test plan) --- .../marathon/ios/xctestrun/Xctestrun.kt | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/xctestrun/Xctestrun.kt b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/xctestrun/Xctestrun.kt index ba491e64d..31b14e0eb 100644 --- a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/xctestrun/Xctestrun.kt +++ b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/xctestrun/Xctestrun.kt @@ -24,41 +24,53 @@ class Xctestrun(inputStream: InputStream) { ?: throw IllegalArgumentException("could not parse xctestrun") // testable target properties - +/* private val target = PropertyListKey.TargetName( propertyList.keys.firstOrNull { it != PropertyListKey.__xctestrun_metadata__.toKeyString() } ?: throw IllegalArgumentException("xctestrun file does not define any testable targets") ) +*/ + + private val targetMap: Map = ( + ((((((propertyList.valueForKeypath("TestConfigurations") as? Array)?.first()) as Map<*, *>) + .filterKeys { it == "TestTargets" } )).values.first() as? Array)?.first() as Map + ) /** * Test target identifier. Used in test names specified with -onlyTesting: option passed to xcodebuild */ - val targetName = target.toKeyString() +// val targetName = target.toKeyString() + val targetName = targetMap.getValue("BlueprintName") as String /** * Testable product module name. Appears in testing logs as a test identifier prefix. */ - val productModuleName = propertyList.valueForKeypath(target, PropertyListKey.ProductModuleName) as String +// val productModuleName = propertyList.valueForKeypath(target, PropertyListKey.ProductModuleName) as String + val productModuleName = targetMap.getValue("ProductModuleName") as String /** * @see xcodebuild.xctestrun(5) */ - val isUITestBundle = propertyList.valueForKeypath(target, PropertyListKey.IsUITestBundle) as Boolean +// val isUITestBundle = propertyList.valueForKeypath(target, PropertyListKey.IsUITestBundle) as Boolean + val isUITestBundle = targetMap.getValue("IsUITestBundle") as Boolean /** * @see xcodebuild.xctestrun(5) */ - val environmentVariables = propertyList.valueForKeypath(target, PropertyListKey.EnvironmentVariables) as PropertyListMap +// val environmentVariables = propertyList.valueForKeypath(target, PropertyListKey.EnvironmentVariables) as PropertyListMap + val environmentVariables = targetMap.getValue("EnvironmentVariables") as PropertyListMap /** * @see xcodebuild.xctestrun(5) */ - val testingEnvironmentVariables = propertyList.valueForKeypath(target, PropertyListKey.TestingEnvironmentVariables) as PropertyListMap +// val testingEnvironmentVariables = propertyList.valueForKeypath(target, PropertyListKey.TestingEnvironmentVariables) as PropertyListMap + val testingEnvironmentVariables = targetMap.getValue("TestingEnvironmentVariables") as PropertyListMap /** * @see xcodebuild.xctestrun(5) */ - private val skipTestIdentifiers = propertyList.valueForKeypath(target, PropertyListKey.SkipTestIdentifiers) as Array +// private val skipTestIdentifiers = propertyList.valueForKeypath(target, PropertyListKey.SkipTestIdentifiers) as Array + private val skipTestIdentifiers = targetMap.getValue("SkipTestIdentifiers") as Array /** * Returns `true` if specified test should be excluded from the test run. From dcbb081cc1e25308cbbbc894f4e914b8955180dd Mon Sep 17 00:00:00 2001 From: Abhishek Bedi Date: Tue, 27 Sep 2022 08:53:35 +0400 Subject: [PATCH 6/8] Added Test Plan to Sample App --- sample/ios-app/Marathonfile | 11 +- sample/ios-app/UITesting.xctestplan | 34 +++++ sample/ios-app/UnitTesting.xctestplan | 24 ++++ .../sample-app.xcodeproj/project.pbxproj | 135 +++++++++++++++++- .../xcshareddata/xcschemes/UITesting.xcscheme | 37 ++--- .../sample-appTests/sample_appTests.swift | 29 ++++ .../ios-app/sample-appUITests/MoreTests.swift | 2 + .../ios-app/sample-appUITests/SlowTests.swift | 2 + 8 files changed, 251 insertions(+), 23 deletions(-) create mode 100644 sample/ios-app/UITesting.xctestplan create mode 100644 sample/ios-app/UnitTesting.xctestplan create mode 100644 sample/ios-app/sample-appTests/sample_appTests.swift diff --git a/sample/ios-app/Marathonfile b/sample/ios-app/Marathonfile index 2bde7a733..88f6bfe33 100644 --- a/sample/ios-app/Marathonfile +++ b/sample/ios-app/Marathonfile @@ -5,12 +5,12 @@ poolingStrategy: type: "omni" batchingStrategy: type: "fixed-size" - size: 5 + size: 2 debug: true -filteringConfiguration: - allowlist: - - type: "simple-class-name" - regex: "StoryboardTests" +#filteringConfiguration: +# allowlist: +# - type: "simple-class-name" +# regex: "StoryboardTests" vendorConfiguration: type: "iOS" derivedDataDir: "derived-data" @@ -20,3 +20,4 @@ vendorConfiguration: remotePrivateKey: ${HOME}/.ssh/marathon xcResultBundlePath: "derived-data/resultBundlePath" xcTestRunnerTag: "Flowers" + xcTestPlan: "UITesting" diff --git a/sample/ios-app/UITesting.xctestplan b/sample/ios-app/UITesting.xctestplan new file mode 100644 index 000000000..3dce9293a --- /dev/null +++ b/sample/ios-app/UITesting.xctestplan @@ -0,0 +1,34 @@ +{ + "configurations" : [ + { + "id" : "F89CABE9-488A-423F-9C0B-ABBA5860F98F", + "name" : "Configuration 1", + "options" : { + + } + } + ], + "defaultOptions" : { + "codeCoverage" : false, + "targetForVariableExpansion" : { + "containerPath" : "container:sample-app.xcodeproj", + "identifier" : "1271DC9521351C6D002B8D3E", + "name" : "sample-app" + } + }, + "testTargets" : [ + { + "skippedTests" : [ + "MoreTests\/testDismissModal()", + "SkippedSuite", + "StoryboardTests\/testDisabledButton()" + ], + "target" : { + "containerPath" : "container:sample-app.xcodeproj", + "identifier" : "1271DCB421351C6D002B8D3E", + "name" : "sample-appUITests" + } + } + ], + "version" : 1 +} diff --git a/sample/ios-app/UnitTesting.xctestplan b/sample/ios-app/UnitTesting.xctestplan new file mode 100644 index 000000000..2d137344e --- /dev/null +++ b/sample/ios-app/UnitTesting.xctestplan @@ -0,0 +1,24 @@ +{ + "configurations" : [ + { + "id" : "BD75184A-5BEE-4C42-A2E2-3B453AFFCC77", + "name" : "Configuration 1", + "options" : { + + } + } + ], + "defaultOptions" : { + + }, + "testTargets" : [ + { + "target" : { + "containerPath" : "container:sample-app.xcodeproj", + "identifier" : "686DC94628E1DAA800A49EC9", + "name" : "sample-appTests" + } + } + ], + "version" : 1 +} diff --git a/sample/ios-app/sample-app.xcodeproj/project.pbxproj b/sample/ios-app/sample-app.xcodeproj/project.pbxproj index c3524363f..927f53c77 100644 --- a/sample/ios-app/sample-app.xcodeproj/project.pbxproj +++ b/sample/ios-app/sample-app.xcodeproj/project.pbxproj @@ -15,6 +15,7 @@ 1271DCA421351C6D002B8D3E /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 1271DCA221351C6D002B8D3E /* LaunchScreen.storyboard */; }; 1271DCBA21351C6D002B8D3E /* StoryboardTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1271DCB921351C6D002B8D3E /* StoryboardTests.swift */; }; 12872C4D21A6A595005543BA /* CrashingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 12872C4C21A6A595005543BA /* CrashingTests.swift */; }; + 686DC94A28E1DAA800A49EC9 /* sample_appTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 686DC94928E1DAA800A49EC9 /* sample_appTests.swift */; }; DA32B580214D224E004D00BC /* FailingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA32B57F214D224E004D00BC /* FailingTests.swift */; }; DA387D99214C37FA00109B63 /* FlakyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA387D98214C37FA00109B63 /* FlakyTests.swift */; }; DA387D9B214C39AB00109B63 /* BooleanExtensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA387D9A214C39AB00109B63 /* BooleanExtensions.swift */; }; @@ -29,6 +30,13 @@ remoteGlobalIDString = 1271DC9521351C6D002B8D3E; remoteInfo = "sample-app"; }; + 686DC94B28E1DAA800A49EC9 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1271DC8E21351C6D002B8D3E /* Project object */; + proxyType = 1; + remoteGlobalIDString = 1271DC9521351C6D002B8D3E; + remoteInfo = "sample-app"; + }; /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ @@ -44,6 +52,10 @@ 1271DCB921351C6D002B8D3E /* StoryboardTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StoryboardTests.swift; sourceTree = ""; }; 1271DCBB21351C6D002B8D3E /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 12872C4C21A6A595005543BA /* CrashingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CrashingTests.swift; sourceTree = ""; }; + 686DC94128E1D2B600A49EC9 /* UITesting.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = UITesting.xctestplan; sourceTree = ""; }; + 686DC94228E1D53900A49EC9 /* UnitTesting.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = UnitTesting.xctestplan; sourceTree = ""; }; + 686DC94728E1DAA800A49EC9 /* sample-appTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "sample-appTests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; }; + 686DC94928E1DAA800A49EC9 /* sample_appTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = sample_appTests.swift; sourceTree = ""; }; DA32B57F214D224E004D00BC /* FailingTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FailingTests.swift; sourceTree = ""; }; DA387D98214C37FA00109B63 /* FlakyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlakyTests.swift; sourceTree = ""; }; DA387D9A214C39AB00109B63 /* BooleanExtensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BooleanExtensions.swift; sourceTree = ""; }; @@ -65,14 +77,24 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 686DC94428E1DAA800A49EC9 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ 1271DC8D21351C6D002B8D3E = { isa = PBXGroup; children = ( + 686DC94128E1D2B600A49EC9 /* UITesting.xctestplan */, + 686DC94228E1D53900A49EC9 /* UnitTesting.xctestplan */, 1271DC9821351C6D002B8D3E /* sample-app */, 1271DCB821351C6D002B8D3E /* sample-appUITests */, + 686DC94828E1DAA800A49EC9 /* sample-appTests */, 1271DC9721351C6D002B8D3E /* Products */, ); sourceTree = ""; @@ -82,6 +104,7 @@ children = ( 1271DC9621351C6D002B8D3E /* sample-app.app */, 1271DCB521351C6D002B8D3E /* sample-appUITests.xctest */, + 686DC94728E1DAA800A49EC9 /* sample-appTests.xctest */, ); name = Products; sourceTree = ""; @@ -114,6 +137,14 @@ path = "sample-appUITests"; sourceTree = ""; }; + 686DC94828E1DAA800A49EC9 /* sample-appTests */ = { + isa = PBXGroup; + children = ( + 686DC94928E1DAA800A49EC9 /* sample_appTests.swift */, + ); + path = "sample-appTests"; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -152,13 +183,31 @@ productReference = 1271DCB521351C6D002B8D3E /* sample-appUITests.xctest */; productType = "com.apple.product-type.bundle.ui-testing"; }; + 686DC94628E1DAA800A49EC9 /* sample-appTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 686DC94D28E1DAA800A49EC9 /* Build configuration list for PBXNativeTarget "sample-appTests" */; + buildPhases = ( + 686DC94328E1DAA800A49EC9 /* Sources */, + 686DC94428E1DAA800A49EC9 /* Frameworks */, + 686DC94528E1DAA800A49EC9 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 686DC94C28E1DAA800A49EC9 /* PBXTargetDependency */, + ); + name = "sample-appTests"; + productName = "sample-appTests"; + productReference = 686DC94728E1DAA800A49EC9 /* sample-appTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ 1271DC8E21351C6D002B8D3E /* Project object */ = { isa = PBXProject; attributes = { - LastSwiftUpdateCheck = 0920; + LastSwiftUpdateCheck = 1400; LastUpgradeCheck = 0920; ORGANIZATIONNAME = "Panforov, Yurii (Agoda)"; TargetAttributes = { @@ -171,6 +220,11 @@ ProvisioningStyle = Automatic; TestTargetID = 1271DC9521351C6D002B8D3E; }; + 686DC94628E1DAA800A49EC9 = { + CreatedOnToolsVersion = 14.0; + ProvisioningStyle = Automatic; + TestTargetID = 1271DC9521351C6D002B8D3E; + }; }; }; buildConfigurationList = 1271DC9121351C6D002B8D3E /* Build configuration list for PBXProject "sample-app" */; @@ -187,6 +241,7 @@ projectRoot = ""; targets = ( 1271DC9521351C6D002B8D3E /* sample-app */, + 686DC94628E1DAA800A49EC9 /* sample-appTests */, 1271DCB421351C6D002B8D3E /* sample-appUITests */, ); }; @@ -210,6 +265,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 686DC94528E1DAA800A49EC9 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -236,6 +298,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 686DC94328E1DAA800A49EC9 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 686DC94A28E1DAA800A49EC9 /* sample_appTests.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ @@ -244,6 +314,11 @@ target = 1271DC9521351C6D002B8D3E /* sample-app */; targetProxy = 1271DCB621351C6D002B8D3E /* PBXContainerItemProxy */; }; + 686DC94C28E1DAA800A49EC9 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 1271DC9521351C6D002B8D3E /* sample-app */; + targetProxy = 686DC94B28E1DAA800A49EC9 /* PBXContainerItemProxy */; + }; /* End PBXTargetDependency section */ /* Begin PBXVariantGroup section */ @@ -435,6 +510,55 @@ }; name = Release; }; + 686DC94E28E1DAA800A49EC9 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; + MARKETING_VERSION = 1.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + PRODUCT_BUNDLE_IDENTIFIER = "com.talabat.sample-appTests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/sample-app.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/sample-app"; + }; + name = Debug; + }; + 686DC94F28E1DAA800A49EC9 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; + MARKETING_VERSION = 1.0; + MTL_FAST_MATH = YES; + PRODUCT_BUNDLE_IDENTIFIER = "com.talabat.sample-appTests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/sample-app.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/sample-app"; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -465,6 +589,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + 686DC94D28E1DAA800A49EC9 /* Build configuration list for PBXNativeTarget "sample-appTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 686DC94E28E1DAA800A49EC9 /* Debug */, + 686DC94F28E1DAA800A49EC9 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; rootObject = 1271DC8E21351C6D002B8D3E /* Project object */; diff --git a/sample/ios-app/sample-app.xcodeproj/xcshareddata/xcschemes/UITesting.xcscheme b/sample/ios-app/sample-app.xcodeproj/xcshareddata/xcschemes/UITesting.xcscheme index 612c2bf5e..7dd5efd67 100644 --- a/sample/ios-app/sample-app.xcodeproj/xcshareddata/xcschemes/UITesting.xcscheme +++ b/sample/ios-app/sample-app.xcodeproj/xcshareddata/xcschemes/UITesting.xcscheme @@ -1,7 +1,7 @@ + version = "1.7"> @@ -28,7 +28,7 @@ buildForAnalyzing = "NO"> @@ -54,8 +54,25 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - language = "" shouldUseLaunchSchemeArgsEnv = "YES"> + + + + + + + + + + @@ -79,23 +96,11 @@ - - - - - - - - Date: Tue, 27 Sep 2022 08:55:58 +0400 Subject: [PATCH 7/8] Added XCTestPlan in iOS Vendor Config to support Test Plans --- .../serialization/ConfigurationFactory.kt | 19 +++++++++++++++---- .../config/vendor/VendorConfiguration.kt | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt index 3d8146eae..9490a5de9 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt @@ -56,11 +56,22 @@ class ConfigurationFactory( // Any relative path specified in Marathonfile should be resolved against the directory Marathonfile is in val resolvedDerivedDataDir = marathonfileDir.resolve(configuration.vendorConfiguration.derivedDataDir) val resolvedResultBundlePath = marathonfileDir.resolve(configuration.vendorConfiguration.xcResultBundlePath) - val finalXCTestRunPath = configuration.vendorConfiguration.xctestrunPath?.resolveAgainst(marathonfileDir) - ?: fileListProvider + + // Adding support for Test Plan + val testPlanName = configuration.vendorConfiguration.xcTestPlan + + var finalXCTestRunPath = if(!testPlanName.isNullOrEmpty()) { + fileListProvider .fileList(resolvedDerivedDataDir) - .firstOrNull { it.extension == "xctestrun" } - ?: throw ConfigurationException("Unable to find an xctestrun file in derived data folder") + .firstOrNull { it.extension == "xctestrun" && it.name.contains("$testPlanName") } ?: throw ConfigurationException("Unable to find matching TestPlan. Please recheck if testplan is enabled") + } else { + configuration.vendorConfiguration.xctestrunPath?.resolveAgainst(marathonfileDir) + ?: fileListProvider + .fileList(resolvedDerivedDataDir) + .firstOrNull { it.extension == "xctestrun" } + ?: throw ConfigurationException("Unable to find an xctestrun file in derived data folder") + } + val optionalSourceRoot = configuration.vendorConfiguration.sourceRoot.resolveAgainst(marathonfileDir) val optionalDevices = configuration.vendorConfiguration.devicesFile?.resolveAgainst(marathonfileDir) ?: marathonfileDir.resolve("Marathondevices") diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt index 5bdcc8d4e..f3a147b70 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt @@ -83,6 +83,7 @@ sealed class VendorConfiguration { @JsonProperty("keepAliveIntervalMillis") val keepAliveIntervalMillis: Long = 0L, @JsonProperty("xcResultBundlePath") val xcResultBundlePath: File, @JsonProperty("xcTestRunnerTag") val xcTestRunnerTag: String? = null, + @JsonProperty("xcTestPlan") val xcTestPlan: String? = null, @JsonProperty("devices") val devicesFile: File? = null, ) : VendorConfiguration() { /** From 8488d9fdce7c25f6ac625cc7b05202f0e7450f2e Mon Sep 17 00:00:00 2001 From: Abhishek Bedi Date: Tue, 25 Oct 2022 06:14:03 +0400 Subject: [PATCH 8/8] Fixed extracting testplan when Scheme & Target name are both same. --- .../marathon/config/serialization/ConfigurationFactory.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt index 9490a5de9..3ddcbf086 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt @@ -59,11 +59,12 @@ class ConfigurationFactory( // Adding support for Test Plan val testPlanName = configuration.vendorConfiguration.xcTestPlan + val testPlanWithUnderscore = "_" + testPlanName + "_" var finalXCTestRunPath = if(!testPlanName.isNullOrEmpty()) { fileListProvider .fileList(resolvedDerivedDataDir) - .firstOrNull { it.extension == "xctestrun" && it.name.contains("$testPlanName") } ?: throw ConfigurationException("Unable to find matching TestPlan. Please recheck if testplan is enabled") + .firstOrNull { it.extension == "xctestrun" && it.name.contains("$testPlanWithUnderscore") } ?: throw ConfigurationException("Unable to find matching TestPlan. Please recheck if testplan is enabled") } else { configuration.vendorConfiguration.xctestrunPath?.resolveAgainst(marathonfileDir) ?: fileListProvider