Finishing touches

.github/workflows/performance_score_director.yml

@@ -127,7 +127,7 @@ jobs:
 
   benchmark:
     needs: build
-    runs-on: ubuntu-latest # We need a stable machine to actually run the benchmarks.
+    runs-on: self-hosted # We need a stable machine to actually run the benchmarks.
     strategy:
       fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue.
       matrix:
@@ -174,9 +174,9 @@ jobs:
         working-directory: ./timefold-solver-benchmarks
         shell: bash
         run: |
-          echo "forks=2" > scoredirector-benchmark.properties
-          echo "warmup_iterations=1" >> scoredirector-benchmark.properties
-          echo "measurement_iterations=1" >> scoredirector-benchmark.properties
+          echo "forks=20" > scoredirector-benchmark.properties
+          echo "warmup_iterations=5" >> scoredirector-benchmark.properties
+          echo "measurement_iterations=5" >> scoredirector-benchmark.properties
           echo "relative_score_error_threshold=0.02" >> scoredirector-benchmark.properties
           echo "score_director_type=cs" >> scoredirector-benchmark.properties
           echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties
@@ -216,14 +216,7 @@ jobs:
           name: results-${{ matrix.example }}-${{ github.event.inputs.baseline }}_vs_${{ github.event.inputs.branch }}
           path: |
             ./timefold-solver-benchmarks/scoredirector-benchmark.properties
-            ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/*.log
-            ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/*.jfr
-            ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/*.html
-            ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.baseline }}/*.json
-            ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/*.log
-            ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/*.jfr
-            ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/*.html
-            ./timefold-solver-benchmarks/results/scoredirector/${{ github.event.inputs.branch }}/*.json
+            ./timefold-solver-benchmarks/results/scoredirector
 
       - name: Report results
         working-directory: ./timefold-solver-benchmarks

scoredirector-benchmark.properties

@@ -3,25 +3,25 @@
 # Possible values: cs, cs_justified, easy, incremental
 # Not including either will cause all examples to not be executed for that score director type.
 # If included, example may still not be executed because it has no support for that score director type.
-score_director_type=cs
+score_director_type=cs,cs_justified,easy,incremental
 
 # Possible values: cloud_balancing, conference_scheduling, curriculum_course, examination, machine_reassignment,
 #                  meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning,
 #                  traveling_tournament, tsp, vehicle_routing
 # Not including either will cause all score director types to not be executed for that example.
-example=cloud_balancing
+example=cloud_balancing,conference_scheduling,curriculum_course,examination,machine_reassignment,meeting_scheduling,nurse_rostering,patient_admission_scheduling,task_assigning,traveling_tournament,tsp,vehicle_routing
 
 # How many forks to run each example with.
 # Default is 10.
-forks=2
+#forks=10
 
 # How many warmup iterations to run each example with.
 # Default is 5.
-warmup_iterations=2
+#warmup_iterations=5
 
 # How many measurement iterations to run each example with.
 # Default is 5.
-measurement_iterations=2
+#measurement_iterations=5
 
 # Ratio of benchmark score error to benchmark score above which warnings will be output.
 # The higher the threshold, the less reliable the benchmark score is.

src/main/java/ai/timefold/solver/benchmarks/micro/coldstart/Main.java

@@ -37,8 +37,8 @@
 import ai.timefold.solver.benchmarks.micro.coldstart.jmh.TimeToFirstScoreBenchmark;
 import ai.timefold.solver.benchmarks.micro.coldstart.jmh.TimeToSolverFactoryBenchmark;
 import ai.timefold.solver.benchmarks.micro.common.AbstractMain;
+import ai.timefold.solver.benchmarks.micro.common.ResultCapturingJMHRunner;
 
-import org.openjdk.jmh.runner.Runner;
 import org.openjdk.jmh.runner.RunnerException;
 import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
 import org.slf4j.Logger;
@@ -69,7 +69,8 @@ public static void main(String[] args) throws RunnerException, IOException {
         options = processBenchmark(options, configuration);
         options = main.initAsyncProfiler(options);
 
-        var runResults = new Runner(options.build()).run();
+        var runner = new ResultCapturingJMHRunner(main.resultsDirectory, options.build());
+        var runResults = runner.run();
         main.convertJfrToFlameGraphs();
 
         var relativeScoreErrorThreshold = configuration.getRelativeScoreErrorThreshold();

src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java

@@ -120,6 +120,7 @@ protected void convertJfrToFlameGraphs() {
             var combinedJfr = resultsDirectory.resolve("combined.jfr");
             var jfrAssembleLog = resultsDirectory.resolve("jfr-assemble.log");
             try {
+                // Merge all the JFR files that were generated by JMH and copied over by our custom JMH runner.
                 var process = new ProcessBuilder()
                         .command("jfr", "assemble", resultsDirectory.toAbsolutePath().toString(),
                                 combinedJfr.toAbsolutePath().toString())
@@ -129,6 +130,7 @@ protected void convertJfrToFlameGraphs() {
                     LOGGER.error("Failed combining JFR files. See '{}' for details.", jfrAssembleLog);
                     return;
                 }
+                // From the combined JMH file, create flame graphs.
                 generateFlameGraphsFromJfr(combinedJfr, null);
                 generateFlameGraphsFromJfr(combinedJfr, "alloc");
             } catch (Exception e) {

src/main/java/ai/timefold/solver/benchmarks/micro/common/ResultCapturingJMHRunner.java

@@ -22,6 +22,12 @@
 import org.openjdk.jmh.util.UnCloseablePrintStream;
 import org.openjdk.jmh.util.Utils;
 
+/**
+ * The JFR file from one fork will overwrite JFR files from previous forks.
+ * But we want all of them to be saved.
+ * Therefore we need to break into JMH, and save the JFR file after each fork.
+ * Because JMH doesn't expose when a fork is finished, we need to count iterations.
+ */
 public final class ResultCapturingJMHRunner extends Runner {
 
     public ResultCapturingJMHRunner(Path resultsDirectory, Options options) {