From 4161a33a0ba63ddd2c0aceffe8c64bc4226313a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Wed, 15 Jan 2025 14:38:58 +0100 Subject: [PATCH] Finishing touches --- .../workflows/performance_score_director.yml | 17 +++++------------ scoredirector-benchmark.properties | 10 +++++----- .../solver/benchmarks/micro/coldstart/Main.java | 5 +++-- .../benchmarks/micro/common/AbstractMain.java | 2 ++ .../micro/common/ResultCapturingJMHRunner.java | 6 ++++++ 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index a46096dc..2b0b605a 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -127,7 +127,7 @@ jobs: benchmark: needs: build - runs-on: ubuntu-latest # We need a stable machine to actually run the benchmarks. + runs-on: self-hosted # We need a stable machine to actually run the benchmarks. strategy: fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue. matrix: @@ -174,9 +174,9 @@ jobs: working-directory: ./timefold-solver-benchmarks shell: bash run: | - echo "forks=2" > scoredirector-benchmark.properties - echo "warmup_iterations=1" >> scoredirector-benchmark.properties - echo "measurement_iterations=1" >> scoredirector-benchmark.properties + echo "forks=20" > scoredirector-benchmark.properties + echo "warmup_iterations=5" >> scoredirector-benchmark.properties + echo "measurement_iterations=5" >> scoredirector-benchmark.properties echo "relative_score_error_threshold=0.02" >> scoredirector-benchmark.properties echo "score_director_type=cs" >> scoredirector-benchmark.properties echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties @@ -216,14 +216,7 @@ jobs: name: results-${{ matrix.example }}-${{ github.event.inputs.baseline }}_vs_${{ github.event.inputs.branch }} path: | ./timefold-solver-benchmarks/scoredirector-benchmark.properties - ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/*.log - ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/*.jfr - ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/*.html - ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/*.json - ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/*.log - ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/*.jfr - ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/*.html - ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/*.json + ./timefold-solver-benchmarks/results/scoredirector - name: Report results working-directory: ./timefold-solver-benchmarks diff --git a/scoredirector-benchmark.properties b/scoredirector-benchmark.properties index 463df87c..1eed5967 100644 --- a/scoredirector-benchmark.properties +++ b/scoredirector-benchmark.properties @@ -3,25 +3,25 @@ # Possible values: cs, cs_justified, easy, incremental # Not including either will cause all examples to not be executed for that score director type. # If included, example may still not be executed because it has no support for that score director type. -score_director_type=cs +score_director_type=cs,cs_justified,easy,incremental # Possible values: cloud_balancing, conference_scheduling, curriculum_course, examination, machine_reassignment, # meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning, # traveling_tournament, tsp, vehicle_routing # Not including either will cause all score director types to not be executed for that example. -example=cloud_balancing +example=cloud_balancing,conference_scheduling,curriculum_course,examination,machine_reassignment,meeting_scheduling,nurse_rostering,patient_admission_scheduling,task_assigning,traveling_tournament,tsp,vehicle_routing # How many forks to run each example with. # Default is 10. -forks=2 +#forks=10 # How many warmup iterations to run each example with. # Default is 5. -warmup_iterations=2 +#warmup_iterations=5 # How many measurement iterations to run each example with. # Default is 5. -measurement_iterations=2 +#measurement_iterations=5 # Ratio of benchmark score error to benchmark score above which warnings will be output. # The higher the threshold, the less reliable the benchmark score is. diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/coldstart/Main.java b/src/main/java/ai/timefold/solver/benchmarks/micro/coldstart/Main.java index 63db61c5..66e293dc 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/coldstart/Main.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/coldstart/Main.java @@ -37,8 +37,8 @@ import ai.timefold.solver.benchmarks.micro.coldstart.jmh.TimeToFirstScoreBenchmark; import ai.timefold.solver.benchmarks.micro.coldstart.jmh.TimeToSolverFactoryBenchmark; import ai.timefold.solver.benchmarks.micro.common.AbstractMain; +import ai.timefold.solver.benchmarks.micro.common.ResultCapturingJMHRunner; -import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; import org.slf4j.Logger; @@ -69,7 +69,8 @@ public static void main(String[] args) throws RunnerException, IOException { options = processBenchmark(options, configuration); options = main.initAsyncProfiler(options); - var runResults = new Runner(options.build()).run(); + var runner = new ResultCapturingJMHRunner(main.resultsDirectory, options.build()); + var runResults = runner.run(); main.convertJfrToFlameGraphs(); var relativeScoreErrorThreshold = configuration.getRelativeScoreErrorThreshold(); diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java b/src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java index 0a14aa7d..bf52a6d9 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java @@ -120,6 +120,7 @@ protected void convertJfrToFlameGraphs() { var combinedJfr = resultsDirectory.resolve("combined.jfr"); var jfrAssembleLog = resultsDirectory.resolve("jfr-assemble.log"); try { + // Merge all the JFR files that were generated by JMH and copied over by our custom JMH runner. var process = new ProcessBuilder() .command("jfr", "assemble", resultsDirectory.toAbsolutePath().toString(), combinedJfr.toAbsolutePath().toString()) @@ -129,6 +130,7 @@ protected void convertJfrToFlameGraphs() { LOGGER.error("Failed combining JFR files. See '{}' for details.", jfrAssembleLog); return; } + // From the combined JMH file, create flame graphs. generateFlameGraphsFromJfr(combinedJfr, null); generateFlameGraphsFromJfr(combinedJfr, "alloc"); } catch (Exception e) { diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/common/ResultCapturingJMHRunner.java b/src/main/java/ai/timefold/solver/benchmarks/micro/common/ResultCapturingJMHRunner.java index c86e5aac..d4d5ff60 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/common/ResultCapturingJMHRunner.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/common/ResultCapturingJMHRunner.java @@ -22,6 +22,12 @@ import org.openjdk.jmh.util.UnCloseablePrintStream; import org.openjdk.jmh.util.Utils; +/** + * The JFR file from one fork will overwrite JFR files from previous forks. + * But we want all of them to be saved. + * Therefore we need to break into JMH, and save the JFR file after each fork. + * Because JMH doesn't expose when a fork is finished, we need to count iterations. + */ public final class ResultCapturingJMHRunner extends Runner { public ResultCapturingJMHRunner(Path resultsDirectory, Options options) {