From 62fc903f17d42b715dfc9b7bc389ad7ff97a998e Mon Sep 17 00:00:00 2001
From: Ryan Levick <ryan.levick@gmail.com>
Date: Thu, 29 Jul 2021 15:03:45 +0200
Subject: [PATCH 1/2] Add more figures to the summary

---
 site/static/compare.html | 129 ++++++++++++++++++++++++---------------
 1 file changed, 80 insertions(+), 49 deletions(-)

diff --git a/site/static/compare.html b/site/static/compare.html
index 0e0080ed3..b3fa95e71 100644
--- a/site/static/compare.html
+++ b/site/static/compare.html
@@ -210,7 +210,17 @@
 
         .summary {
             display: flex;
+            justify-content: center;
             align-items: center;
+            width: 20%;
+        }
+
+        #summary .improvement {
+            border: 1px solid green;
+        }
+
+        #summary .regression {
+            border: 1px solid red;
         }
     </style>
     <script src="https://cdn.jsdelivr.net/npm/vue@2/dist/vue.js"></script>
@@ -322,7 +332,8 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                         </li>
                         <li>
                             <label>
-                                <input type="checkbox" id="build-incremental-unchanged" v-model="filter.cache.incrUnchanged" />
+                                <input type="checkbox" id="build-incremental-unchanged"
+                                    v-model="filter.cache.incrUnchanged" />
                                 <span class="cache-label">incr-unchanged</span>
                             </label>
                             <div class="tooltip">?
@@ -334,7 +345,8 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                         </li>
                         <li>
                             <label>
-                                <input type="checkbox" id="build-incremental-patched" v-model="filter.cache.incrPatched" />
+                                <input type="checkbox" id="build-incremental-patched"
+                                    v-model="filter.cache.incrPatched" />
                                 <span class="cache-label">incr-patched</span>
                             </label>
                             <div class="tooltip">?
@@ -361,31 +373,36 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
             </div>
         </fieldset>
         <div v-if="data" id="content" style="margin-top: 15px">
-            <div style="display: flex; justify-content: space-evenly;">
-                <span style="font-weight: bold;">Summary:</span>
-                <span class="summary positive">
-                    {{summary.numRegressed}}
-                    <svg style="width:18px;height:18px" viewBox="0 0 24 24">
-                        <path
-                            d="M16,18L18.29,15.71L13.41,10.83L9.41,14.83L2,7.41L3.41,6L9.41,12L13.41,8L19.71,14.29L22,12V18H16Z">
-                        </path>
-                    </svg>
-                </span>
-                <span class="summary">
-                    {{summary.numUnchanged}}
-                    <svg style="width:18px;height:18px" viewBox="0 0 24 24">
-                        <path d="M22,12L18,8V11H3V13H18V16L22,12Z"></path>
-                    </svg>
-                </span>
-                <span class="summary negative">
-                    {{summary.numImproved}}
-                    <svg style="width:18px;height:18px" viewBox="0 0 24 24">
-                        <path
-                            d="M16,6L18.29,8.29L13.41,13.17L9.41,9.17L2,16.59L3.41,18L9.41,12L13.41,16L19.71,9.71L22,12V6H16Z">
-                        </path>
-                    </svg>
-                </span>
-            </div>
+            <template v-for="summaryPair in Object.entries(summary)">
+                <div id="summary" style="display: flex;">
+                    <span style="font-weight: bold;width: 20%;">{{ summaryPair[0] }}:</span>
+                    <div style="display: flex; justify-content: center; width: 80%;"
+                        v-bind="{ class: summaryClass(summaryPair[1]) }">
+                        <span class="summary positive">
+                            {{summaryPair[1].regressions.toString().padStart(3, "&nbsp;")}}
+                            <svg style="width:18px;height:18px" viewBox="0 0 24 24">
+                                <path
+                                    d="M16,18L18.29,15.71L13.41,10.83L9.41,14.83L2,7.41L3.41,6L9.41,12L13.41,8L19.71,14.29L22,12V18H16Z">
+                                </path>
+                            </svg>
+                        </span>
+                        <span class="summary">
+                            {{summaryPair[1].unchanged.toString().padStart(3, "&nbsp;")}}
+                            <svg style="width:18px;height:18px" viewBox="0 0 24 24">
+                                <path d="M22,12L18,8V11H3V13H18V16L22,12Z"></path>
+                            </svg>
+                        </span>
+                        <span class="summary negative">
+                            {{summaryPair[1].improvements.toString().padStart(3, "&nbsp;")}}
+                            <svg style="width:18px;height:18px" viewBox="0 0 24 24">
+                                <path
+                                    d="M16,6L18.29,8.29L13.41,13.17L9.41,9.17L2,16.59L3.41,18L9.41,12L13.41,16L19.71,9.71L22,12V6H16Z">
+                                </path>
+                            </svg>
+                        </span>
+                    </div>
+                </div>
+            </template>
             <table id="benches" class="compare">
                 <template v-for="bench in benches">
                     <tbody>
@@ -626,34 +643,38 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                     return findQueryParam("stat") || "instructions:u";
                 },
                 summary() {
-                    let numRegressed = 0;
-                    let numImproved = 0;
-                    let numUnchanged = 0;
-                    for (let name of Object.keys(this.data.a.data)) {
-                        for (let d of this.data.a.data[name]) {
-                            const run = d[0];
+                    const filtered = Object.fromEntries(this.benches.map(b => [b.name, Object.fromEntries(b.variants.map(v => [v.casename, true]))]));
+                    const newCount = { regressions: 0, improvements: 0, unchanged: 0 }
+                    let result = { all: { ...newCount }, filtered: { ...newCount } }
+                    for (let benchmarkAndProfile of Object.keys(this.data.a.data)) {
+                        for (let d of this.data.a.data[benchmarkAndProfile]) {
+                            const scenario = d[0];
                             const datumA = d[1];
-                            const datumB = this.data.b.data[name]?.find(x => x[0] == run)?.[1];
+                            const datumB = this.data.b.data[benchmarkAndProfile]?.find(x => x[0] == scenario)?.[1];
                             if (!datumB) {
                                 continue;
                             }
+                            let scenarioKind = scenario.substr(0, (scenario.indexOf(":") + 1 || scenario.length + 1) - 1);
+                            result[scenarioKind] ||= { ...newCount }
                             let percent = 100 * ((datumB - datumA) / datumA);
-                            const isDodgy = this.isDodgy(name, run);
+                            const isDodgy = this.isDodgy(benchmarkAndProfile, scenario);
                             const threshold = isDodgy ? 1.0 : 0.2;
                             if (percent > threshold) {
-                                numRegressed += 1;
+                                result.all.regressions += 1;
+                                result.filtered.regressions += filtered[benchmarkAndProfile]?.[scenario] || 0;
+                                result[scenarioKind].regressions += 1;
                             } else if (percent < -threshold) {
-                                numImproved += 1;
+                                result.all.improvements += 1;
+                                result.filtered.improvements += filtered[benchmarkAndProfile]?.[scenario] || 0;
+                                result[scenarioKind].improvements += 1;
                             } else {
-                                numUnchanged += 1
+                                result.all.unchanged += 1;
+                                result.filtered.unchanged += filtered[benchmarkAndProfile]?.[scenario] || 0;
+                                result[scenarioKind].unchanged += 1;
                             }
                         }
                     }
-                    return {
-                        numRegressed,
-                        numImproved,
-                        numUnchanged,
-                    }
+                    return result;
 
                 }
             },
@@ -668,11 +689,11 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                     let klass = "";
                     if (pct > 1) {
                         klass = 'positive';
-                    } else if (pct > 0.1) {
+                    } else if (pct >= 0.2) {
                         klass = 'slightly-positive';
                     } else if (pct < -1) {
                         klass = 'negative';
-                    } else if (pct < -0.1) {
+                    } else if (pct <= -0.2) {
                         klass = 'slightly-negative';
                     }
                     return klass;
@@ -713,13 +734,23 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                     }
                     return result;
                 },
-                isDodgy(name, run) {
+                isDodgy(benchmarkAndProfile, scenario) {
                     let variance = this.data.variance;
                     if (!variance) {
                         return false;
                     }
-                    variance = this.data.variance[name + "-" + run];
+                    variance = this.data.variance[benchmarkAndProfile + "-" + scenario];
                     return (variance?.description?.type ?? "Normal") != "Normal";
+                },
+                summaryClass(summary) {
+                    if (summary.unchanged > Math.max(summary.improvements, summary.regressions)) {
+                        return '';
+                    }
+                    if (summary.improvements > summary.regressions) {
+                        return 'improvement';
+                    } else {
+                        return 'regression';
+                    }
                 }
             },
         });
@@ -729,7 +760,7 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
             if (variant.isDodgy) {
                 return percent > 1.0;
             } else {
-                return percent > 0.2;
+                return percent >= 0.2;
             }
 
         }
@@ -790,4 +821,4 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
     </script>
 </body>
 
-</html>
+</html>
\ No newline at end of file

From 1dcf3df729d548c23f80408e7d2cdd262dee83de Mon Sep 17 00:00:00 2001
From: Ryan Levick <me@ryanlevick.com>
Date: Wed, 4 Aug 2021 10:42:45 +0200
Subject: [PATCH 2/2] Improve summary

---
 site/static/compare.html | 72 ++++++++++++++--------------------------
 1 file changed, 24 insertions(+), 48 deletions(-)

diff --git a/site/static/compare.html b/site/static/compare.html
index b3fa95e71..7b8a9cbee 100644
--- a/site/static/compare.html
+++ b/site/static/compare.html
@@ -208,20 +208,18 @@
             text-align: center;
         }
 
+        #summary {
+            border: 1px dashed;
+            padding: 4px;
+            border-radius: 6px;
+        }
+
         .summary {
             display: flex;
             justify-content: center;
             align-items: center;
             width: 20%;
         }
-
-        #summary .improvement {
-            border: 1px solid green;
-        }
-
-        #summary .regression {
-            border: 1px solid red;
-        }
     </style>
     <script src="https://cdn.jsdelivr.net/npm/vue@2/dist/vue.js"></script>
 </head>
@@ -364,7 +362,8 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                         <span class="tooltip">?
                             <span class="tooltiptext">
                                 Whether to filter out all benchmarks that do not show significant changes. A significant
-                                change is any change above 0.2% for non-noisy benchmarks and 1.0% for noisy ones.
+                                change is any change greater than or equal to 0.2% for non-noisy benchmarks and above
+                                1.0% for noisy ones.
                             </span>
                         </span>
                     </div>
@@ -373,11 +372,11 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
             </div>
         </fieldset>
         <div v-if="data" id="content" style="margin-top: 15px">
-            <template v-for="summaryPair in Object.entries(summary)">
-                <div id="summary" style="display: flex;">
-                    <span style="font-weight: bold;width: 20%;">{{ summaryPair[0] }}:</span>
-                    <div style="display: flex; justify-content: center; width: 80%;"
-                        v-bind="{ class: summaryClass(summaryPair[1]) }">
+            <div id="summary">
+                <div v-for="summaryPair in Object.entries(summary)" style="display: flex;">
+                    <span style="font-weight: bold; width: 30%; margin-left: 15%; text-transform: capitalize;">{{
+                        summaryPair[0] }}:</span>
+                    <div style="display: flex; justify-content: flex-end; width: 80%; margin-right: 15%;">
                         <span class="summary positive">
                             {{summaryPair[1].regressions.toString().padStart(3, "&nbsp;")}}
                             <svg style="width:18px;height:18px" viewBox="0 0 24 24">
@@ -402,7 +401,7 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                         </span>
                     </div>
                 </div>
-            </template>
+            </div>
             <table id="benches" class="compare">
                 <template v-for="bench in benches">
                     <tbody>
@@ -424,7 +423,7 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                                     <a
                                         v-bind:href="percentLink(data.b.commit, data.a.commit, bench.name, run.casename)">
                                         <span v-bind:class="percentClass(run.percent)">
-                                            {{ run.percent }}%{{run.isDodgy ? "?" : ""}}
+                                            {{ run.percent.toFixed(2) }}%{{run.isDodgy ? "?" : ""}}
                                         </span>
                                     </a>
                                 </td>
@@ -537,11 +536,7 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                             if (!datumB) {
                                 continue;
                             }
-                            let percent = (100 * (datumB - datumA) / datumA).toFixed(1);
-                            if (percent === "-0.0") {
-                                percent = "0.0";
-                            }
-
+                            let percent = (100 * (datumB - datumA) / datumA);
                             let isDodgy = false;
                             if (data.variance) {
                                 let variance = data.variance[name + "-" + key];
@@ -565,11 +560,10 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                         Object.keys(data.a.data).
                             filter(n => filter.name && filter.name.trim() ? n.includes(filter.name.trim()) : true).
                             map(name => {
-                                const variants = toVariants(name).filter(v => filter.showOnlySignificant ? isSignificant(v) : true);
+                                const variants = toVariants(name).filter(v => filter.showOnlySignificant ? isSignificant(v.percent, v.isDodgy) : true);
                                 const pcts = variants.map(field => parseFloat(field.percent));
                                 const maxPct = Math.max(...pcts).toFixed(1);
                                 const minPct = Math.min(...pcts).toFixed(1);
-                                const maxCasenameLen = Math.max(...variants.map(f => f.casename.length));
                                 if (variants.length > 0) {
                                     variants[0].first = true;
                                 }
@@ -579,7 +573,6 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                                     variants,
                                     maxPct,
                                     minPct,
-                                    maxCasenameLen,
                                 };
                             }).
                             filter(b => b.variants.length > 0);
@@ -654,23 +647,17 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                             if (!datumB) {
                                 continue;
                             }
-                            let scenarioKind = scenario.substr(0, (scenario.indexOf(":") + 1 || scenario.length + 1) - 1);
-                            result[scenarioKind] ||= { ...newCount }
                             let percent = 100 * ((datumB - datumA) / datumA);
                             const isDodgy = this.isDodgy(benchmarkAndProfile, scenario);
-                            const threshold = isDodgy ? 1.0 : 0.2;
-                            if (percent > threshold) {
+                            if (percent > 0 && isSignificant(percent, isDodgy)) {
                                 result.all.regressions += 1;
                                 result.filtered.regressions += filtered[benchmarkAndProfile]?.[scenario] || 0;
-                                result[scenarioKind].regressions += 1;
-                            } else if (percent < -threshold) {
+                            } else if (percent < 0 && isSignificant(percent, isDodgy)) {
                                 result.all.improvements += 1;
                                 result.filtered.improvements += filtered[benchmarkAndProfile]?.[scenario] || 0;
-                                result[scenarioKind].improvements += 1;
                             } else {
                                 result.all.unchanged += 1;
                                 result.filtered.unchanged += filtered[benchmarkAndProfile]?.[scenario] || 0;
-                                result[scenarioKind].unchanged += 1;
                             }
                         }
                     }
@@ -742,27 +729,16 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                     variance = this.data.variance[benchmarkAndProfile + "-" + scenario];
                     return (variance?.description?.type ?? "Normal") != "Normal";
                 },
-                summaryClass(summary) {
-                    if (summary.unchanged > Math.max(summary.improvements, summary.regressions)) {
-                        return '';
-                    }
-                    if (summary.improvements > summary.regressions) {
-                        return 'improvement';
-                    } else {
-                        return 'regression';
-                    }
-                }
             },
         });
 
-        function isSignificant(variant) {
-            const percent = Math.abs(variant.percent);
-            if (variant.isDodgy) {
-                return percent > 1.0;
+        function isSignificant(percent, isNoisy) {
+            const perAbs = Math.abs(percent);
+            if (isNoisy) {
+                return perAbs > 1.0;
             } else {
-                return percent >= 0.2;
+                return perAbs >= 0.2;
             }
-
         }
 
         function toggleFilters(id, toggle) {