Speedup integration testing pipeline

docs/configuration-options.md

@@ -74,9 +74,10 @@ benchmark {
 ```
 
 ### Kotlin/JVM
-| Option                                      | Description                                                | Possible Values                        | Default Value  |
-|---------------------------------------------|------------------------------------------------------------|----------------------------------------|----------------|
-| `advanced("jvmForks", value)`               | Specifies the number of times the harness should fork.     | Non-negative Integer, `"definedByJmh"` | `1`            |
+| Option                                      | Description                                                                | Possible Values                        | Default Value |
+|---------------------------------------------|----------------------------------------------------------------------------|----------------------------------------|---------------|
+| `advanced("jvmForks", value)`               | Specifies the number of times the harness should fork.                     | Non-negative Integer, `"definedByJmh"` | `1`           |
+| `advanced("jmhIgnoreLock", value)`          | Sets a value of the `jmh.ignoreLock` property during benchmarks execution. | `true`, `false`                        | unset         |
 
 **Notes on "jvmForks":**
 - **0** - "no fork", i.e., no subprocesses are forked to run benchmarks.

integration/build.gradle.kts

@@ -38,4 +38,9 @@ tasks.test {
     }
     systemProperty("minSupportedGradleVersion", libs.versions.minSupportedGradle.get())
     systemProperty("minSupportedKotlinVersion", libs.versions.minSupportedKotlin.get())
+
+    val forks = project.providers.gradleProperty("testing.max.forks").orNull?.toInt()
+        ?: (Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(1)
+
+    maxParallelForks = forks
 }

integration/src/test/kotlin/kotlinx/benchmark/integration/AnnotationsValidationTest.kt

@@ -2,11 +2,11 @@ package kotlinx.benchmark.integration
 
 import kotlin.test.*
 
-class AnnotationsValidationTest : GradleTest() {
+abstract class AnnotationsValidationTest : GradleTest() {
 
-    private val platformBenchmarkTask = "nativeBenchmark"
+    protected val platformBenchmarkTask = "nativeBenchmark"
 
-    private fun executeBenchmark(
+    protected fun executeBenchmark(
         benchmarkFunction: String? = null,
         setupFunction: String? = null,
         teardownFunction: String? = null,
@@ -26,7 +26,7 @@ class AnnotationsValidationTest : GradleTest() {
         runJvmBenchmark(runner, jvmSpecificError)
     }
 
-    private fun runPlatformBenchmark(runner: Runner, error: String?) {
+    protected fun runPlatformBenchmark(runner: Runner, error: String?) {
         if (error != null) {
             runner.runAndFail(platformBenchmarkTask) {
                 assertOutputContains(error)
@@ -38,7 +38,7 @@ class AnnotationsValidationTest : GradleTest() {
         }
     }
 
-    private fun runJvmBenchmark(runner: Runner, jvmSpecificError: String?) {
+    protected fun runJvmBenchmark(runner: Runner, jvmSpecificError: String?) {
         if (jvmSpecificError != null) {
             runner.runAndFail("jvmBenchmark") {
                 assertOutputContains(jvmSpecificError)
@@ -49,7 +49,9 @@ class AnnotationsValidationTest : GradleTest() {
             }
         }
     }
+}
 
+class ParamAnnotationsValidationTest : AnnotationsValidationTest() {
     // @Param
 
     @Test
@@ -142,7 +144,9 @@ class AnnotationsValidationTest : GradleTest() {
             jvmSpecificError = "@Param can only be placed over the annotation-compatible types: primitives, primitive wrappers, Strings, or enums."
         )
     }
+}
 
+class BenchmarkAnnotationsValidationTest : AnnotationsValidationTest() {
     // @Benchmark
 
     @Test
@@ -204,7 +208,9 @@ class AnnotationsValidationTest : GradleTest() {
             jvmSpecificError = "Method parameters should be either @State classes"
         )
     }
+}
 
+class SetupAnnotationsValidationTest : AnnotationsValidationTest() {
     // @Setup
 
     @Test
@@ -271,7 +277,9 @@ class AnnotationsValidationTest : GradleTest() {
             jvmSpecificError = "Method parameters should be either @State classes"
         )
     }
+}
 
+class TearDownAnnotationsValidationTest : AnnotationsValidationTest() {
     // TearDown
 
     @Test
@@ -338,7 +346,9 @@ class AnnotationsValidationTest : GradleTest() {
             jvmSpecificError = "Method parameters should be either @State classes"
         )
     }
+}
 
+class MixedAnnotationsValidationTest : AnnotationsValidationTest() {
     // Mix
 
     @Test
@@ -365,4 +375,4 @@ class AnnotationsValidationTest : GradleTest() {
             teardownFunction = "plainFunction"
         )
     }
-}
+}

integration/src/test/kotlin/kotlinx/benchmark/integration/ConfigurationCacheTest.kt

@@ -12,6 +12,7 @@ class ConfigurationCacheTest : GradleTest() {
                 iterations = 1
                 iterationTime = 100
                 iterationTimeUnit = "ms"
+                advanced("jmhIgnoreLock", true)
             }
         }
 
@@ -36,7 +37,7 @@ class ConfigurationCacheTest : GradleTest() {
     fun testConfigurationCacheNative() = runConfigurationCacheTest(
         "kotlin-multiplatform",
         listOf(":nativeBenchmark"),
-        listOf(":compileKotlinNative", ":nativeBenchmarkGenerate", ":compileNativeBenchmarkKotlinNative", ":linkNativeBenchmarkReleaseExecutableNative")
+        listOf(":compileKotlinNative", ":nativeBenchmarkGenerate", ":compileNativeBenchmarkKotlinNative", ":linkNativeBenchmarkDebugExecutableNative")
     )
 
     @Test

integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsOverrideAnnotationsTest.kt

@@ -91,7 +91,7 @@ class OptionsOverrideAnnotationsTest : GradleTest() {
     @Test
     fun testOutputTimeUnit() {
         val expectedOutputTimeUnit = "ns"
-        val expectedCount = /*warmups*/3 + /*iterations*/5 + /*Success:*/1 + /*summary*/1
+        val expectedCount = /*warmups*/2 + /*iterations*/1 + /*Success:*/1 + /*summary*/1
 
         testConfiguration(
             setupBlock = {
@@ -110,7 +110,7 @@ class OptionsOverrideAnnotationsTest : GradleTest() {
     @Test
     fun testMode() {
         val expectedMode = "avgt"
-        val expectedCount = /*warmups*/3 + /*iterations*/5 + /*Success:*/1 + /*summary*/1
+        val expectedCount = /*warmups*/2 + /*iterations*/1 + /*Success:*/1 + /*summary*/1
 
         testConfiguration(
             setupBlock = {

integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt

@@ -240,6 +240,13 @@ class OptionsValidationTest : GradleTest() {
                 iterationTimeUnit = "ms"
                 advanced("jsUseBridge", "x")
             }
+
+            configuration("invalidJmhIgnoreLock") {
+                iterations = 1
+                iterationTime = 100
+                iterationTimeUnit = "ms"
+                advanced("jmhIgnoreLock", "x")
+            }
         }
 
         runner.runAndFail("blankAdvancedConfigNameBenchmark") {
@@ -260,6 +267,9 @@ class OptionsValidationTest : GradleTest() {
         runner.runAndFail("invalidJsUseBridgeBenchmark") {
             assertOutputContains("Invalid value for 'jsUseBridge': 'x'. Expected a Boolean value.")
         }
+        runner.runAndFail("invalidJmhIgnoreLock") {
+            assertOutputContains("Invalid value for 'jmhIgnoreLock': 'x'. Expected a Boolean value.")
+        }
     }
 }
 
@@ -274,4 +284,4 @@ private object ValidOptions {
     )
     val modes = setOf("thrpt", "avgt", "Throughput", "AverageTime")
     val nativeForks = setOf("perBenchmark", "perIteration")
-}
+}

integration/src/test/kotlin/kotlinx/benchmark/integration/ReportFormatTest.kt

@@ -17,6 +17,7 @@ class ReportFormatTest : GradleTest() {
                     iterationTime = 100
                     iterationTimeUnit = "ms"
                     reportFormat = format
+                    advanced("jmhIgnoreLock", true)
                 }
             }
         }

integration/src/test/kotlin/kotlinx/benchmark/integration/SourceSetAsBenchmarkTargetTest.kt

@@ -16,6 +16,7 @@ class SourceSetAsBenchmarkTargetTest : GradleTest() {
                     iterationTime = 100
                     iterationTimeUnit = "ms"
                     reportFormat = "csv"
+                    advanced("jmhIgnoreLock", true)
                 }
             }
 

integration/src/test/kotlin/kotlinx/benchmark/integration/SuiteSourceGeneratorTest.kt

@@ -28,7 +28,7 @@ class SuiteSourceGeneratorTest : GradleTest() {
                 }
             },
             checkBlock = {
-                val parameters = "iterations = 12, warmups = 5, iterationTime = IterationTime\\(200, BenchmarkTimeUnit.MILLISECONDS\\)"
+                val parameters = "iterations = 12, warmups = 2, iterationTime = IterationTime\\(200, BenchmarkTimeUnit.MILLISECONDS\\)"
                 assertGeneratedDescriptorContainsCode(parameters)
             }
         )
@@ -44,7 +44,7 @@ class SuiteSourceGeneratorTest : GradleTest() {
             },
             checkBlock = {
                 // time and timeUnit of @Warmup are ignored: https://github.com/Kotlin/kotlinx-benchmark/issues/74
-                val parameters = "iterations = 3, warmups = 12, iterationTime = IterationTime\\(1, BenchmarkTimeUnit.SECONDS\\)"
+                val parameters = "iterations = 1, warmups = 12, iterationTime = IterationTime\\(1, BenchmarkTimeUnit.SECONDS\\)"
                 assertGeneratedDescriptorContainsCode(parameters)
             }
         )
@@ -183,4 +183,4 @@ class SuiteSourceGeneratorTest : GradleTest() {
             runner.runAndSucceed(":compile${capitalizedTarget}BenchmarkKotlin${capitalizedTarget}")
         }
     }
-}
+}

integration/src/test/resources/templates/annotations-validation/build.gradle

@@ -1,3 +1,4 @@
+import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.HostManager
 
@@ -17,6 +18,15 @@ benchmark {
         register("jvm")
         register("js")
         register("wasmJs")
-        register("native")
+        register("native") {
+            // don't optimize binaries to speedup test execution
+            buildType = NativeBuildType.DEBUG
+        }
+    }
+
+    configurations {
+        main {
+            advanced("jmhIgnoreLock", true)
+        }
     }
 }

integration/src/test/resources/templates/annotations-validation/src/commonMain/kotlin/CommonBenchmark.kt

@@ -4,8 +4,8 @@ import kotlinx.benchmark.*
 import kotlin.math.*
 
 @State(Scope.Benchmark)
-@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
-@Warmup(iterations = 5, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
+@Measurement(iterations = 1, time = 1500, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
+@Warmup(iterations = 1, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
 @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
 @BenchmarkMode(Mode.Throughput)
 open class CommonBenchmark {

integration/src/test/resources/templates/config-options/build.gradle

@@ -1,3 +1,4 @@
+import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.HostManager
 
@@ -17,6 +18,15 @@ benchmark {
         register("jvm")
         register("js")
         register("wasmJs")
-        register("native")
+        register("native") {
+            // don't optimize binaries to speedup test execution
+            buildType = NativeBuildType.DEBUG
+        }
+    }
+
+    configurations {
+        main {
+            advanced("jmhIgnoreLock", true)
+        }
     }
 }

integration/src/test/resources/templates/config-options/src/commonMain/kotlin/CommonBenchmark.kt

@@ -4,8 +4,8 @@ import kotlinx.benchmark.*
 import kotlin.math.*
 
 @State(Scope.Benchmark)
-@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
-@Warmup(iterations = 5, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
+@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
+@Warmup(iterations = 2, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
 @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
 @BenchmarkMode(Mode.Throughput)
 open class CommonBenchmark {

integration/src/test/resources/templates/invalid-target/wasm-nodejs/src/commonMain/kotlin/CommonBenchmark.kt

@@ -4,7 +4,8 @@ import kotlinx.benchmark.*
 import kotlin.math.*
 
 @State(Scope.Benchmark)
-@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
+@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
+@Warmup(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
 @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
 @BenchmarkMode(Mode.Throughput)
 open class CommonBenchmark {

integration/src/test/resources/templates/kmp-with-toolchain/higher-version-than-in-gradle/build.gradle

@@ -6,4 +6,10 @@ benchmark {
     targets {
         register("jvm")
     }
+
+    configurations {
+        main {
+            advanced("jmhIgnoreLock", true)
+        }
+    }
 }

integration/src/test/resources/templates/kmp-with-toolchain/min-supported-version/build.gradle

@@ -6,4 +6,10 @@ benchmark {
     targets {
         register("jvm")
     }
+
+    configurations {
+        main {
+            advanced("jmhIgnoreLock", true)
+        }
+    }
 }

integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/build.gradle

@@ -13,4 +13,10 @@ benchmark {
         register("jvmCustom")
         register("jsCustom")
     }
+
+    configurations {
+        main {
+            advanced("jmhIgnoreLock", true)
+        }
+    }
 }

integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jsCustom/kotlin/JsCustomBenchmark.kt

@@ -4,13 +4,13 @@ import kotlinx.benchmark.*
 import kotlin.math.*
 
 @State(Scope.Benchmark)
-@Warmup(iterations = 2, time = 100, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
-@Measurement(iterations = 3, time = 200, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
+@Warmup(iterations = 1, time = 100, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
+@Measurement(iterations = 1, time = 200, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
 @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
 open class JsCustomBenchmark {
     @Benchmark
     open fun hashCodeBenchmark(): Int {
         val value = log(sqrt(3.0) * cos(3.0), 2.0)
         return JsTestData(CommonTestData(value)).hashCode()
     }
-}
+}

integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jvmCustom/kotlin/JvmCustomBenchmark.kt

@@ -4,13 +4,13 @@ import kotlinx.benchmark.*
 import kotlin.math.*
 
 @State(Scope.Benchmark)
-@Warmup(iterations = 2, time = 100, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
-@Measurement(iterations = 3, time = 200, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
+@Warmup(iterations = 1, time = 100, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
+@Measurement(iterations = 1, time = 200, timeUnit = BenchmarkTimeUnit.MILLISECONDS)
 @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
 open class JvmCustomBenchmark {
     @Benchmark
     open fun hashCodeBenchmark(): Int {
         val value = log(sqrt(3.0) * cos(3.0), 2.0)
         return JvmTestData(CommonTestData(value)).hashCode()
     }
-}
+}

integration/src/test/resources/templates/kotlin-multiplatform/build.gradle

@@ -1,3 +1,4 @@
+import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.HostManager
 
@@ -17,6 +18,15 @@ benchmark {
         register("jvm")
         register("js")
         register("wasmJs")
-        register("native")
+        register("native") {
+            // don't optimize binaries to speedup test execution
+            buildType = NativeBuildType.DEBUG
+        }
+    }
+
+    configurations {
+        main {
+            advanced("jmhIgnoreLock", true)
+        }
     }
 }

integration/src/test/resources/templates/kotlin-multiplatform/src/commonMain/kotlin/CommonBenchmark.kt

@@ -4,7 +4,7 @@ import kotlinx.benchmark.*
 import kotlin.math.*
 
 @State(Scope.Benchmark)
-@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
+@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
 @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
 @BenchmarkMode(Mode.Throughput)
 open class CommonBenchmark {