diff --git a/docs/configuration-options.md b/docs/configuration-options.md index 6cce4c3f..381ee075 100644 --- a/docs/configuration-options.md +++ b/docs/configuration-options.md @@ -74,9 +74,10 @@ benchmark { ``` ### Kotlin/JVM -| Option | Description | Possible Values | Default Value | -|---------------------------------------------|------------------------------------------------------------|----------------------------------------|----------------| -| `advanced("jvmForks", value)` | Specifies the number of times the harness should fork. | Non-negative Integer, `"definedByJmh"` | `1` | +| Option | Description | Possible Values | Default Value | +|---------------------------------------------|----------------------------------------------------------------------------|----------------------------------------|---------------| +| `advanced("jvmForks", value)` | Specifies the number of times the harness should fork. | Non-negative Integer, `"definedByJmh"` | `1` | +| `advanced("jmhIgnoreLock", value)` | Sets a value of the `jmh.ignoreLock` property during benchmarks execution. | `true`, `false` | unset | **Notes on "jvmForks":** - **0** - "no fork", i.e., no subprocesses are forked to run benchmarks. diff --git a/integration/build.gradle.kts b/integration/build.gradle.kts index 44c9acfd..73867d55 100644 --- a/integration/build.gradle.kts +++ b/integration/build.gradle.kts @@ -38,4 +38,9 @@ tasks.test { } systemProperty("minSupportedGradleVersion", libs.versions.minSupportedGradle.get()) systemProperty("minSupportedKotlinVersion", libs.versions.minSupportedKotlin.get()) + + val forks = project.providers.gradleProperty("testing.max.forks").orNull?.toInt() + ?: (Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(1) + + maxParallelForks = forks } diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/AnnotationsValidationTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/AnnotationsValidationTest.kt index c0b785d8..630a07f5 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/AnnotationsValidationTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/AnnotationsValidationTest.kt @@ -2,11 +2,11 @@ package kotlinx.benchmark.integration import kotlin.test.* -class AnnotationsValidationTest : GradleTest() { +abstract class AnnotationsValidationTest : GradleTest() { - private val platformBenchmarkTask = "nativeBenchmark" + protected val platformBenchmarkTask = "nativeBenchmark" - private fun executeBenchmark( + protected fun executeBenchmark( benchmarkFunction: String? = null, setupFunction: String? = null, teardownFunction: String? = null, @@ -26,7 +26,7 @@ class AnnotationsValidationTest : GradleTest() { runJvmBenchmark(runner, jvmSpecificError) } - private fun runPlatformBenchmark(runner: Runner, error: String?) { + protected fun runPlatformBenchmark(runner: Runner, error: String?) { if (error != null) { runner.runAndFail(platformBenchmarkTask) { assertOutputContains(error) @@ -38,7 +38,7 @@ class AnnotationsValidationTest : GradleTest() { } } - private fun runJvmBenchmark(runner: Runner, jvmSpecificError: String?) { + protected fun runJvmBenchmark(runner: Runner, jvmSpecificError: String?) { if (jvmSpecificError != null) { runner.runAndFail("jvmBenchmark") { assertOutputContains(jvmSpecificError) @@ -49,7 +49,9 @@ class AnnotationsValidationTest : GradleTest() { } } } +} +class ParamAnnotationsValidationTest : AnnotationsValidationTest() { // @Param @Test @@ -142,7 +144,9 @@ class AnnotationsValidationTest : GradleTest() { jvmSpecificError = "@Param can only be placed over the annotation-compatible types: primitives, primitive wrappers, Strings, or enums." ) } +} +class BenchmarkAnnotationsValidationTest : AnnotationsValidationTest() { // @Benchmark @Test @@ -204,7 +208,9 @@ class AnnotationsValidationTest : GradleTest() { jvmSpecificError = "Method parameters should be either @State classes" ) } +} +class SetupAnnotationsValidationTest : AnnotationsValidationTest() { // @Setup @Test @@ -271,7 +277,9 @@ class AnnotationsValidationTest : GradleTest() { jvmSpecificError = "Method parameters should be either @State classes" ) } +} +class TearDownAnnotationsValidationTest : AnnotationsValidationTest() { // TearDown @Test @@ -338,7 +346,9 @@ class AnnotationsValidationTest : GradleTest() { jvmSpecificError = "Method parameters should be either @State classes" ) } +} +class MixedAnnotationsValidationTest : AnnotationsValidationTest() { // Mix @Test @@ -365,4 +375,4 @@ class AnnotationsValidationTest : GradleTest() { teardownFunction = "plainFunction" ) } -} \ No newline at end of file +} diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/ConfigurationCacheTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/ConfigurationCacheTest.kt index fa1ee3a7..23b06e75 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/ConfigurationCacheTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/ConfigurationCacheTest.kt @@ -12,6 +12,7 @@ class ConfigurationCacheTest : GradleTest() { iterations = 1 iterationTime = 100 iterationTimeUnit = "ms" + advanced("jmhIgnoreLock", true) } } @@ -36,7 +37,7 @@ class ConfigurationCacheTest : GradleTest() { fun testConfigurationCacheNative() = runConfigurationCacheTest( "kotlin-multiplatform", listOf(":nativeBenchmark"), - listOf(":compileKotlinNative", ":nativeBenchmarkGenerate", ":compileNativeBenchmarkKotlinNative", ":linkNativeBenchmarkReleaseExecutableNative") + listOf(":compileKotlinNative", ":nativeBenchmarkGenerate", ":compileNativeBenchmarkKotlinNative", ":linkNativeBenchmarkDebugExecutableNative") ) @Test diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsOverrideAnnotationsTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsOverrideAnnotationsTest.kt index 720ee53f..bd5c7966 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsOverrideAnnotationsTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsOverrideAnnotationsTest.kt @@ -91,7 +91,7 @@ class OptionsOverrideAnnotationsTest : GradleTest() { @Test fun testOutputTimeUnit() { val expectedOutputTimeUnit = "ns" - val expectedCount = /*warmups*/3 + /*iterations*/5 + /*Success:*/1 + /*summary*/1 + val expectedCount = /*warmups*/2 + /*iterations*/1 + /*Success:*/1 + /*summary*/1 testConfiguration( setupBlock = { @@ -110,7 +110,7 @@ class OptionsOverrideAnnotationsTest : GradleTest() { @Test fun testMode() { val expectedMode = "avgt" - val expectedCount = /*warmups*/3 + /*iterations*/5 + /*Success:*/1 + /*summary*/1 + val expectedCount = /*warmups*/2 + /*iterations*/1 + /*Success:*/1 + /*summary*/1 testConfiguration( setupBlock = { diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt index 1965ecd5..1c6ac4d4 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt @@ -240,6 +240,13 @@ class OptionsValidationTest : GradleTest() { iterationTimeUnit = "ms" advanced("jsUseBridge", "x") } + + configuration("invalidJmhIgnoreLock") { + iterations = 1 + iterationTime = 100 + iterationTimeUnit = "ms" + advanced("jmhIgnoreLock", "x") + } } runner.runAndFail("blankAdvancedConfigNameBenchmark") { @@ -260,6 +267,9 @@ class OptionsValidationTest : GradleTest() { runner.runAndFail("invalidJsUseBridgeBenchmark") { assertOutputContains("Invalid value for 'jsUseBridge': 'x'. Expected a Boolean value.") } + runner.runAndFail("invalidJmhIgnoreLock") { + assertOutputContains("Invalid value for 'jmhIgnoreLock': 'x'. Expected a Boolean value.") + } } } @@ -274,4 +284,4 @@ private object ValidOptions { ) val modes = setOf("thrpt", "avgt", "Throughput", "AverageTime") val nativeForks = setOf("perBenchmark", "perIteration") -} \ No newline at end of file +} diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/ReportFormatTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/ReportFormatTest.kt index de36c108..0a16a620 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/ReportFormatTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/ReportFormatTest.kt @@ -17,6 +17,7 @@ class ReportFormatTest : GradleTest() { iterationTime = 100 iterationTimeUnit = "ms" reportFormat = format + advanced("jmhIgnoreLock", true) } } } diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/SourceSetAsBenchmarkTargetTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/SourceSetAsBenchmarkTargetTest.kt index 38684cdf..5de6cf76 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/SourceSetAsBenchmarkTargetTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/SourceSetAsBenchmarkTargetTest.kt @@ -16,6 +16,7 @@ class SourceSetAsBenchmarkTargetTest : GradleTest() { iterationTime = 100 iterationTimeUnit = "ms" reportFormat = "csv" + advanced("jmhIgnoreLock", true) } } diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/SuiteSourceGeneratorTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/SuiteSourceGeneratorTest.kt index 58a35544..d2414e75 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/SuiteSourceGeneratorTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/SuiteSourceGeneratorTest.kt @@ -28,7 +28,7 @@ class SuiteSourceGeneratorTest : GradleTest() { } }, checkBlock = { - val parameters = "iterations = 12, warmups = 5, iterationTime = IterationTime\\(200, BenchmarkTimeUnit.MILLISECONDS\\)" + val parameters = "iterations = 12, warmups = 2, iterationTime = IterationTime\\(200, BenchmarkTimeUnit.MILLISECONDS\\)" assertGeneratedDescriptorContainsCode(parameters) } ) @@ -44,7 +44,7 @@ class SuiteSourceGeneratorTest : GradleTest() { }, checkBlock = { // time and timeUnit of @Warmup are ignored: https://github.com/Kotlin/kotlinx-benchmark/issues/74 - val parameters = "iterations = 3, warmups = 12, iterationTime = IterationTime\\(1, BenchmarkTimeUnit.SECONDS\\)" + val parameters = "iterations = 1, warmups = 12, iterationTime = IterationTime\\(1, BenchmarkTimeUnit.SECONDS\\)" assertGeneratedDescriptorContainsCode(parameters) } ) @@ -183,4 +183,4 @@ class SuiteSourceGeneratorTest : GradleTest() { runner.runAndSucceed(":compile${capitalizedTarget}BenchmarkKotlin${capitalizedTarget}") } } -} \ No newline at end of file +} diff --git a/integration/src/test/resources/templates/annotations-validation/build.gradle b/integration/src/test/resources/templates/annotations-validation/build.gradle index 41ba62d1..68d5a0c2 100644 --- a/integration/src/test/resources/templates/annotations-validation/build.gradle +++ b/integration/src/test/resources/templates/annotations-validation/build.gradle @@ -1,3 +1,4 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.HostManager @@ -17,6 +18,15 @@ benchmark { register("jvm") register("js") register("wasmJs") - register("native") + register("native") { + // don't optimize binaries to speedup test execution + buildType = NativeBuildType.DEBUG + } + } + + configurations { + main { + advanced("jmhIgnoreLock", true) + } } } diff --git a/integration/src/test/resources/templates/annotations-validation/src/commonMain/kotlin/CommonBenchmark.kt b/integration/src/test/resources/templates/annotations-validation/src/commonMain/kotlin/CommonBenchmark.kt index 83bf4915..3e146173 100644 --- a/integration/src/test/resources/templates/annotations-validation/src/commonMain/kotlin/CommonBenchmark.kt +++ b/integration/src/test/resources/templates/annotations-validation/src/commonMain/kotlin/CommonBenchmark.kt @@ -4,8 +4,8 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) -@Warmup(iterations = 5, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1500, timeUnit = BenchmarkTimeUnit.MILLISECONDS) +@Warmup(iterations = 1, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class CommonBenchmark { diff --git a/integration/src/test/resources/templates/config-options/build.gradle b/integration/src/test/resources/templates/config-options/build.gradle index 41ba62d1..68d5a0c2 100644 --- a/integration/src/test/resources/templates/config-options/build.gradle +++ b/integration/src/test/resources/templates/config-options/build.gradle @@ -1,3 +1,4 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.HostManager @@ -17,6 +18,15 @@ benchmark { register("jvm") register("js") register("wasmJs") - register("native") + register("native") { + // don't optimize binaries to speedup test execution + buildType = NativeBuildType.DEBUG + } + } + + configurations { + main { + advanced("jmhIgnoreLock", true) + } } } diff --git a/integration/src/test/resources/templates/config-options/src/commonMain/kotlin/CommonBenchmark.kt b/integration/src/test/resources/templates/config-options/src/commonMain/kotlin/CommonBenchmark.kt index b1dcc978..c6c40f9e 100644 --- a/integration/src/test/resources/templates/config-options/src/commonMain/kotlin/CommonBenchmark.kt +++ b/integration/src/test/resources/templates/config-options/src/commonMain/kotlin/CommonBenchmark.kt @@ -4,8 +4,8 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) -@Warmup(iterations = 5, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Warmup(iterations = 2, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class CommonBenchmark { diff --git a/integration/src/test/resources/templates/invalid-target/wasm-nodejs/src/commonMain/kotlin/CommonBenchmark.kt b/integration/src/test/resources/templates/invalid-target/wasm-nodejs/src/commonMain/kotlin/CommonBenchmark.kt index 6367fc74..9bc800c2 100644 --- a/integration/src/test/resources/templates/invalid-target/wasm-nodejs/src/commonMain/kotlin/CommonBenchmark.kt +++ b/integration/src/test/resources/templates/invalid-target/wasm-nodejs/src/commonMain/kotlin/CommonBenchmark.kt @@ -4,7 +4,8 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class CommonBenchmark { diff --git a/integration/src/test/resources/templates/kmp-with-toolchain/higher-version-than-in-gradle/build.gradle b/integration/src/test/resources/templates/kmp-with-toolchain/higher-version-than-in-gradle/build.gradle index 2f1368fe..00012e41 100644 --- a/integration/src/test/resources/templates/kmp-with-toolchain/higher-version-than-in-gradle/build.gradle +++ b/integration/src/test/resources/templates/kmp-with-toolchain/higher-version-than-in-gradle/build.gradle @@ -6,4 +6,10 @@ benchmark { targets { register("jvm") } + + configurations { + main { + advanced("jmhIgnoreLock", true) + } + } } diff --git a/integration/src/test/resources/templates/kmp-with-toolchain/min-supported-version/build.gradle b/integration/src/test/resources/templates/kmp-with-toolchain/min-supported-version/build.gradle index 2f1368fe..00012e41 100644 --- a/integration/src/test/resources/templates/kmp-with-toolchain/min-supported-version/build.gradle +++ b/integration/src/test/resources/templates/kmp-with-toolchain/min-supported-version/build.gradle @@ -6,4 +6,10 @@ benchmark { targets { register("jvm") } + + configurations { + main { + advanced("jmhIgnoreLock", true) + } + } } diff --git a/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/build.gradle b/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/build.gradle index 05a7a565..975d7480 100644 --- a/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/build.gradle +++ b/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/build.gradle @@ -13,4 +13,10 @@ benchmark { register("jvmCustom") register("jsCustom") } + + configurations { + main { + advanced("jmhIgnoreLock", true) + } + } } diff --git a/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jsCustom/kotlin/JsCustomBenchmark.kt b/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jsCustom/kotlin/JsCustomBenchmark.kt index e98c4e54..2742c37b 100644 --- a/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jsCustom/kotlin/JsCustomBenchmark.kt +++ b/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jsCustom/kotlin/JsCustomBenchmark.kt @@ -4,8 +4,8 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Warmup(iterations = 2, time = 100, timeUnit = BenchmarkTimeUnit.MILLISECONDS) -@Measurement(iterations = 3, time = 200, timeUnit = BenchmarkTimeUnit.MILLISECONDS) +@Warmup(iterations = 1, time = 100, timeUnit = BenchmarkTimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 200, timeUnit = BenchmarkTimeUnit.MILLISECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) open class JsCustomBenchmark { @Benchmark @@ -13,4 +13,4 @@ open class JsCustomBenchmark { val value = log(sqrt(3.0) * cos(3.0), 2.0) return JsTestData(CommonTestData(value)).hashCode() } -} \ No newline at end of file +} diff --git a/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jvmCustom/kotlin/JvmCustomBenchmark.kt b/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jvmCustom/kotlin/JvmCustomBenchmark.kt index d88348ee..ea415dee 100644 --- a/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jvmCustom/kotlin/JvmCustomBenchmark.kt +++ b/integration/src/test/resources/templates/kotlin-multiplatform-separate-source-set/src/jvmCustom/kotlin/JvmCustomBenchmark.kt @@ -4,8 +4,8 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Warmup(iterations = 2, time = 100, timeUnit = BenchmarkTimeUnit.MILLISECONDS) -@Measurement(iterations = 3, time = 200, timeUnit = BenchmarkTimeUnit.MILLISECONDS) +@Warmup(iterations = 1, time = 100, timeUnit = BenchmarkTimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 200, timeUnit = BenchmarkTimeUnit.MILLISECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) open class JvmCustomBenchmark { @Benchmark @@ -13,4 +13,4 @@ open class JvmCustomBenchmark { val value = log(sqrt(3.0) * cos(3.0), 2.0) return JvmTestData(CommonTestData(value)).hashCode() } -} \ No newline at end of file +} diff --git a/integration/src/test/resources/templates/kotlin-multiplatform/build.gradle b/integration/src/test/resources/templates/kotlin-multiplatform/build.gradle index bff52dfd..78757606 100644 --- a/integration/src/test/resources/templates/kotlin-multiplatform/build.gradle +++ b/integration/src/test/resources/templates/kotlin-multiplatform/build.gradle @@ -1,3 +1,4 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.HostManager @@ -17,6 +18,15 @@ benchmark { register("jvm") register("js") register("wasmJs") - register("native") + register("native") { + // don't optimize binaries to speedup test execution + buildType = NativeBuildType.DEBUG + } + } + + configurations { + main { + advanced("jmhIgnoreLock", true) + } } } diff --git a/integration/src/test/resources/templates/kotlin-multiplatform/src/commonMain/kotlin/CommonBenchmark.kt b/integration/src/test/resources/templates/kotlin-multiplatform/src/commonMain/kotlin/CommonBenchmark.kt index 6367fc74..685152b4 100644 --- a/integration/src/test/resources/templates/kotlin-multiplatform/src/commonMain/kotlin/CommonBenchmark.kt +++ b/integration/src/test/resources/templates/kotlin-multiplatform/src/commonMain/kotlin/CommonBenchmark.kt @@ -4,7 +4,7 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class CommonBenchmark { diff --git a/integration/src/test/resources/templates/kotlin-multiplatform/src/jsMain/kotlin/RootBenchmark.kt b/integration/src/test/resources/templates/kotlin-multiplatform/src/jsMain/kotlin/RootBenchmark.kt index 5deb4794..a43047cf 100644 --- a/integration/src/test/resources/templates/kotlin-multiplatform/src/jsMain/kotlin/RootBenchmark.kt +++ b/integration/src/test/resources/templates/kotlin-multiplatform/src/jsMain/kotlin/RootBenchmark.kt @@ -2,7 +2,7 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class RootBenchmark { diff --git a/integration/src/test/resources/templates/kotlin-multiplatform/src/nativeMain/kotlin/RootBenchmark.kt b/integration/src/test/resources/templates/kotlin-multiplatform/src/nativeMain/kotlin/RootBenchmark.kt index 5deb4794..a43047cf 100644 --- a/integration/src/test/resources/templates/kotlin-multiplatform/src/nativeMain/kotlin/RootBenchmark.kt +++ b/integration/src/test/resources/templates/kotlin-multiplatform/src/nativeMain/kotlin/RootBenchmark.kt @@ -2,7 +2,7 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class RootBenchmark { diff --git a/integration/src/test/resources/templates/kotlin-multiplatform/src/wasmJsMain/kotlin/RootBenchmark.kt b/integration/src/test/resources/templates/kotlin-multiplatform/src/wasmJsMain/kotlin/RootBenchmark.kt index 5deb4794..a43047cf 100644 --- a/integration/src/test/resources/templates/kotlin-multiplatform/src/wasmJsMain/kotlin/RootBenchmark.kt +++ b/integration/src/test/resources/templates/kotlin-multiplatform/src/wasmJsMain/kotlin/RootBenchmark.kt @@ -2,7 +2,7 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class RootBenchmark { diff --git a/integration/src/test/resources/templates/kotlin-native/src/commonMain/kotlin/CommonBenchmark.kt b/integration/src/test/resources/templates/kotlin-native/src/commonMain/kotlin/CommonBenchmark.kt index 6367fc74..9bc800c2 100644 --- a/integration/src/test/resources/templates/kotlin-native/src/commonMain/kotlin/CommonBenchmark.kt +++ b/integration/src/test/resources/templates/kotlin-native/src/commonMain/kotlin/CommonBenchmark.kt @@ -4,7 +4,8 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class CommonBenchmark { diff --git a/integration/src/test/resources/templates/kotlin-native/src/nativeMain/kotlin/DebugBenchmark.kt b/integration/src/test/resources/templates/kotlin-native/src/nativeMain/kotlin/DebugBenchmark.kt index 044800a1..16e540a1 100644 --- a/integration/src/test/resources/templates/kotlin-native/src/nativeMain/kotlin/DebugBenchmark.kt +++ b/integration/src/test/resources/templates/kotlin-native/src/nativeMain/kotlin/DebugBenchmark.kt @@ -4,7 +4,8 @@ import kotlinx.benchmark.* import kotlin.native.Platform @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class DebugBenchmark { diff --git a/integration/src/test/resources/templates/project-with-resources/build.gradle b/integration/src/test/resources/templates/project-with-resources/build.gradle index 6cabf895..9e022c6c 100644 --- a/integration/src/test/resources/templates/project-with-resources/build.gradle +++ b/integration/src/test/resources/templates/project-with-resources/build.gradle @@ -1,3 +1,4 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.HostManager @@ -17,7 +18,10 @@ kotlin { benchmark { targets { - register("native") + register("native") { + // don't optimize binaries to speedup test execution + buildType = NativeBuildType.DEBUG + } register("js") register("wasmJs") } diff --git a/integration/src/test/resources/templates/source-generation/build.gradle b/integration/src/test/resources/templates/source-generation/build.gradle index 41ba62d1..68d5a0c2 100644 --- a/integration/src/test/resources/templates/source-generation/build.gradle +++ b/integration/src/test/resources/templates/source-generation/build.gradle @@ -1,3 +1,4 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.HostManager @@ -17,6 +18,15 @@ benchmark { register("jvm") register("js") register("wasmJs") - register("native") + register("native") { + // don't optimize binaries to speedup test execution + buildType = NativeBuildType.DEBUG + } + } + + configurations { + main { + advanced("jmhIgnoreLock", true) + } } } diff --git a/integration/src/test/resources/templates/source-generation/src/commonMain/kotlin/CommonBenchmark.kt b/integration/src/test/resources/templates/source-generation/src/commonMain/kotlin/CommonBenchmark.kt index 7ec9ebe1..d2677dc2 100644 --- a/integration/src/test/resources/templates/source-generation/src/commonMain/kotlin/CommonBenchmark.kt +++ b/integration/src/test/resources/templates/source-generation/src/commonMain/kotlin/CommonBenchmark.kt @@ -4,8 +4,8 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) -@Warmup(iterations = 5, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Warmup(iterations = 2, time = 500, timeUnit = BenchmarkTimeUnit.MILLISECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class CommonBenchmark { diff --git a/integration/src/test/resources/templates/transitive-dependencies-resolution/build.gradle b/integration/src/test/resources/templates/transitive-dependencies-resolution/build.gradle index cdc301de..a0554a6c 100644 --- a/integration/src/test/resources/templates/transitive-dependencies-resolution/build.gradle +++ b/integration/src/test/resources/templates/transitive-dependencies-resolution/build.gradle @@ -1,3 +1,4 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType import org.jetbrains.kotlin.konan.target.HostManager import org.jetbrains.kotlin.konan.target.KonanTarget @@ -34,7 +35,10 @@ kotlin { benchmark { targets { - register("native") + register("native") { + // don't optimize binaries to speedup test execution + buildType = NativeBuildType.DEBUG + } register("js") register("wasmJs") } diff --git a/integration/src/test/resources/templates/wasm-gc-non-experimental/wasm-d8/src/commonMain/kotlin/CommonBenchmark.kt b/integration/src/test/resources/templates/wasm-gc-non-experimental/wasm-d8/src/commonMain/kotlin/CommonBenchmark.kt index 6367fc74..9bc800c2 100644 --- a/integration/src/test/resources/templates/wasm-gc-non-experimental/wasm-d8/src/commonMain/kotlin/CommonBenchmark.kt +++ b/integration/src/test/resources/templates/wasm-gc-non-experimental/wasm-d8/src/commonMain/kotlin/CommonBenchmark.kt @@ -4,7 +4,8 @@ import kotlinx.benchmark.* import kotlin.math.* @State(Scope.Benchmark) -@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) @OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @BenchmarkMode(Mode.Throughput) open class CommonBenchmark { diff --git a/plugin/main/src/kotlinx/benchmark/gradle/JvmTasks.kt b/plugin/main/src/kotlinx/benchmark/gradle/JvmTasks.kt index 74e9de09..d813ec29 100644 --- a/plugin/main/src/kotlinx/benchmark/gradle/JvmTasks.kt +++ b/plugin/main/src/kotlinx/benchmark/gradle/JvmTasks.kt @@ -7,6 +7,7 @@ import org.gradle.api.file.* import org.gradle.api.tasks.* import org.gradle.api.tasks.compile.* import org.gradle.jvm.tasks.* +import org.jetbrains.kotlin.cli.common.toBooleanLenient import java.io.* @KotlinxBenchmarkPluginInternalApi @@ -132,6 +133,10 @@ fun Project.createJvmBenchmarkExecTask( val reportFile = setupReporting(target, config) args(writeParameters(target.name, reportFile, traceFormat(), config)) + when (config.advanced["jmhIgnoreLock"]) { + true -> jvmArgs("-Djmh.ignoreLock=true") + false -> jvmArgs("-Djmh.ignoreLock=false") + } javaLauncher.set(javaLauncherProvider()) } } diff --git a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt index acb88f18..5d3485a7 100644 --- a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt +++ b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt @@ -241,6 +241,10 @@ private fun validateConfig(config: BenchmarkConfiguration) { } } + "jmhIgnoreLock" -> require(value is Boolean) { + "Invalid value for 'jmhIgnoreLock': '$value'. Expected a Boolean value." + } + "jsUseBridge" -> require(value is Boolean) { "Invalid value for 'jsUseBridge': '$value'. Expected a Boolean value." }