Add machine comparison list for published results

LICENSE.rst

@@ -1,4 +1,5 @@
 Copyright (c) 2011-2018, Michael Droettboom, Space Telescope Science Institute, Pauli Virtanen
+Copyright (c) 2022, Oracle and/or its affiliates.
 
 All rights reserved.
 

asv/commands/publish.py

@@ -58,6 +58,14 @@ def setup_arguments(cls, subparsers):
             '--html-dir', '-o', default=None, help=(
                 "Optional output directory. Default is 'html_dir' "
                 "from asv config"))
+        parser.add_argument(
+            '--baseline-machine', default=None, help=(
+                "Optional baseline comparisons between machines. Provide "
+                "machine name"))
+        parser.add_argument(
+            '--generate-markdown', action='store_true', dest='generate_markdown',
+                help=("Optional output a generated markdown file comparisons "
+                "between machines in the 'html_dir'."))
 
         parser.set_defaults(func=cls.run_from_args)
 
@@ -67,6 +75,9 @@ def setup_arguments(cls, subparsers):
     def run_from_conf_args(cls, conf, args):
         if args.html_dir is not None:
             conf.html_dir = args.html_dir
+        if args.baseline_machine is not None:
+            conf.baseline_machine = args.baseline_machine
+        conf.generate_markdown = args.generate_markdown
         return cls.run(conf=conf, range_spec=args.range, pull=not args.no_pull)
 
     @staticmethod

asv/config.py

@@ -42,6 +42,8 @@ def __init__(self):
         self.build_command = None
         self.install_command = None
         self.uninstall_command = None
+        self.baseline_machine = None
+        self.generate_markdown = False
 
     @classmethod
     def load(cls, path=None):

asv/plugins/comparisonlist.py

@@ -0,0 +1,210 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+import os
+import itertools
+
+from ..console import log
+from ..publishing import OutputPublisher
+from .. import util
+
+
+def benchmark_param_iter(benchmark):
+    """
+    Iterate over all combinations of parameterized benchmark parameters.
+
+    Yields
+    ------
+    idx : int
+        Combination flat index. `None` if benchmark not parameterized.
+    params : tuple
+        Tuple of parameter values.
+
+    """
+    if not benchmark['params']:
+        yield None, ()
+    else:
+        for item in enumerate(itertools.product(*benchmark['params'])):
+            yield item
+
+time_units = [
+    ['`ps`', 'picoseconds', 0.000000000001],
+    ['`ns`', 'nanoseconds', 0.000000001],
+    ['`μs`', 'microseconds', 0.000001],
+    ['`ms`', 'milliseconds', 0.001],
+    ['`s`', 'seconds', 1],
+    ['`m`', 'minutes', 60],
+    ['`h`', 'hours', 60 * 60],
+    ['`d`', 'days', 60 * 60 * 24],
+    ['`w`', 'weeks', 60 * 60 * 24 * 7],
+    ['`y`', 'years', 60 * 60 * 24 * 7 * 52],
+    ['`C`', 'centuries', 60 * 60 * 24 * 7 * 52 * 100]
+]
+
+def pretty_time_unit(x, unit):
+    if unit == 'seconds':
+        for i in range(len(time_units) - 1):
+            if abs(x) < time_units[i+1][2]:
+                return '%.3f' % (x / time_units[i][2]) + time_units[i][0]
+        return 'inf'
+    else:
+        return '%.3f' % x + unit
+
+class ComparisonList(OutputPublisher):
+    name = "comparisonlist"
+    button_label = "List view"
+    description = "Display as a list"
+    order = 1
+
+    @classmethod
+    def publish(cls, conf, repo, benchmarks, graphs, revisions):
+        machines = list(graphs.get_params()["machine"])
+        num_machines = len(machines)
+        baseline_machine_idx = -1
+        if conf.baseline_machine:
+            baseline_machine_idx = machines.index(conf.baseline_machine)
+
+        result = {
+            "machines" : machines,
+            "benchmarks" : None,
+        }
+        benchmarks_result = []
+        warned = False
+
+        # Investigate all benchmarks
+        for benchmark_name, benchmark in sorted(benchmarks.items()):
+            log.dot()
+
+            benchmark_graphs = graphs.get_graph_group(benchmark_name)
+
+            # For parameterized benchmarks, consider each combination separately
+            for idx, benchmark_param in benchmark_param_iter(benchmark):
+                pretty_name = benchmark_name
+
+                if benchmark.get('pretty_name'):
+                    pretty_name = benchmark['pretty_name']
+
+                if idx is not None:
+                    pretty_name = '{0}({1})'.format(pretty_name,
+                                                    ", ".join(benchmark_param))
+
+                # Each environment parameter combination is reported
+                # separately on the comparisonlist page
+                benchmark_graphs = graphs.get_graph_group(benchmark_name)
+                benchmark_data = None
+                best_val = None
+                worst_val = None
+                for graph in benchmark_graphs:
+                    machine_idx = machines.index(graph.params["machine"])
+                    if not benchmark_data:
+                        benchmark_data = {
+                            "name" : benchmark_name,
+                            "pretty_name" : pretty_name,
+                            "idx" : idx,
+                            "best" : -1,
+                            "worst" : -1,
+                            "last_rev" : [None] * num_machines,
+                            "last_value" : [None] * num_machines,
+                            "last_err" : [None] * num_machines,
+                            "cmp_percent" : [0.] * num_machines,
+                        }
+
+                    # Produce interesting information, based on
+                    # stepwise fit on the benchmark data (reduces noise)
+                    steps = graph.get_steps()
+                    if idx is not None and steps:
+                        steps = graph.get_steps()[idx]
+
+                    last_value = None
+                    last_err = None
+                    last_rev = None
+
+                    if not steps:
+                        # No data
+                        pass
+                    else:
+                        last_piece = steps[-1]
+                        last_value = last_piece[2]
+                        if best_val is None or last_value < best_val:
+                            benchmark_data["best"] = machine_idx
+                            best_val = last_value
+                        if worst_val is None or last_value > worst_val:
+                            benchmark_data["worst"] = machine_idx
+                            worst_val = last_value
+                        last_err = last_piece[4]
+                        last_rev = last_piece[1] - 1
+                        if not warned and benchmark_data["last_value"][machine_idx]:
+                            warned = True
+                            log.warning("There are two machines that has the same name '%s'" % machines[machine_idx])
+                        benchmark_data["last_value"][machine_idx] = last_value
+                        benchmark_data["last_err"][machine_idx] = last_err
+                        benchmark_data["last_rev"][machine_idx] = last_rev
+                if benchmark_data and best_val:
+                    benchmarks_result.append(benchmark_data)
+
+        if baseline_machine_idx != -1:
+            num_benchmarks = len(benchmarks_result)
+            cmp_list = [0.] * num_machines
+            for bench_idx in range(num_benchmarks):
+                values = benchmarks_result[bench_idx]["last_value"]
+                b = values[baseline_machine_idx]
+                if b:
+                    for machine_idx in range(num_machines):
+                        v = values[machine_idx]
+                        if v:
+                            p = (v - b) / b * 100
+                            cmp_list[machine_idx] += p
+                            benchmarks_result[bench_idx]["cmp_percent"][machine_idx] = p
+
+            benchmarks_average_cmp = [None] * num_machines
+            for machine_idx in range(num_machines):
+                benchmarks_average_cmp[machine_idx] = cmp_list[machine_idx]/num_benchmarks
+            result["average"] = benchmarks_average_cmp
+            result["baseline"] = baseline_machine_idx
+
+
+        def machine_idx_sort(row):
+            idx = row['best']
+            if idx == -1:
+                return 9999
+            if baseline_machine_idx != -1:
+                if idx == baseline_machine_idx:
+                    v = max(row["cmp_percent"])/100
+                    return idx - v
+                else:
+                    v = row["cmp_percent"][idx]/100
+                    return idx + v
+
+            return idx
+        result["benchmarks"] = sorted(benchmarks_result, key=machine_idx_sort)
+        # Write results to file
+        util.write_json(os.path.join(conf.html_dir, "comparison.json"), result, compact=True)
+
+        if conf.generate_markdown:
+            # Generate a markdown page
+            with open(os.path.join(conf.html_dir, "comparison.md"), "w") as fp:
+                machines = result["machines"]
+                num_machines = len(machines)
+                fp.write('# Benchmark Machine\n')
+                fp.write('* CPU: %s\n' % list(graphs.get_params()["cpu"])[0])
+                fp.write('* CPU Cores: %s\n' % list(graphs.get_params()["num_cpu"])[0])
+                fp.write('* OS: %s\n' % list(graphs.get_params()["os"])[0])
+                fp.write('* RAM: %dGB\n' % (int(list(graphs.get_params()["ram"])[0])//1000000))
+                fp.write('\n\n')
+                fp.write('# Results\n')
+                fp.write('| No. |' + '|'.join(machines + ["Benchmarks"]) + '|\n')
+                fp.write('| :-- |' + '|'.join([":--"] * (num_machines + 1)) + '|\n')
+                if baseline_machine_idx != -1:
+                    avg = ['%.2f%%' % v for v in result["average"]]
+                    fp.write('| - |' + '|'.join(avg + ["Average"]) + '|\n')
+                count = 1
+                for benchmark in result["benchmarks"]:
+                    if None in benchmark["last_value"]:
+                        continue
+                    unit = benchmarks[benchmark["name"]]["unit"]
+                    row = '| %d ' % count
+                    count += 1
+                    for machine_idx in range(num_machines):
+                        row += '|' + pretty_time_unit(benchmark["last_value"][machine_idx], unit)
+                        if baseline_machine_idx != -1 and baseline_machine_idx != machine_idx:
+                            row += ' `%.2f%%`' % benchmark["cmp_percent"][machine_idx]
+                    row += '|' + benchmark["pretty_name"] + '|\n'
+                    fp.write(row)

asv/www/asv.js

@@ -382,6 +382,7 @@ $(document).ready(function() {
             $("#graph-display").hide();
             $("#summarygrid-display").hide();
             $("#summarylist-display").hide();
+            $("#comparisonlist-display").hide();
             $('#regressions-display').hide();
             $('.tooltip').remove();
             loaded_pages[name](params);
@@ -461,6 +462,7 @@ $(document).ready(function() {
             $('#regressions-display').hide();
             $('#summarygrid-display').hide();
             $('#summarylist-display').hide();
+            $('#comparisonlist-display').hide();
 
             hashchange();
         }).fail(function () {

asv/www/comparisonlist.css

@@ -0,0 +1,68 @@
+#comparisonlist-body {
+    padding-left: 2em;
+    padding-right: 2em;
+    padding-top: 1em;
+    padding-bottom: 2em;
+}
+
+#comparisonlist-body table thead th {
+    background: white;
+    cursor: pointer;
+    white-space: nowrap;
+    position: sticky;
+    top: 0; /* Don't forget this, required for the stickiness */
+}
+
+#comparisonlist-body table thead th.desc:after {
+    content: ' \2191';
+}
+
+#comparisonlist-body table thead th.asc:after {
+    content: ' \2193';
+}
+
+#comparisonlist-body table.ignored {
+    padding-top: 1em;
+    color: #ccc;
+    background-color: #eee;
+}
+
+#comparisonlist-body table.ignored a {
+    color: #82abda;
+}
+
+#comparisonlist-body table tbody td.positive-change {
+    background-color: #fdd;
+}
+
+#comparisonlist-body table tbody td.negative-change {
+    background-color: #dfd;
+}
+
+#comparisonlist-body table tbody td.stats {
+    white-space: nowrap;
+    font-weight: bold;
+}
+
+#comparisonlist-body table tbody td.value {
+    white-space: nowrap;
+}
+
+#comparisonlist-body table tbody td.value-best {
+    white-space: nowrap;
+    color: green;
+}
+
+#comparisonlist-body table tbody td.value-worst {
+    white-space: nowrap;
+    color: red;
+}
+
+#comparisonlist-body table tbody td.change a {
+    color: black;
+    white-space: nowrap;
+}
+
+#comparisonlist-body table tbody td.change-date {
+    white-space: nowrap;
+}