Skip to content

Commit 9695fd5

Browse files
aduh95aduh95
committed
tools: use lighter shell to aggregate benchmark results
Signed-off-by: Antoine du Hamel <duhamelantoine1995@gmail.com>
1 parent ede6d6a commit 9695fd5

4 files changed

Lines changed: 257 additions & 217 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 72 additions & 213 deletions
Original file line numberDiff line numberDiff line change
@@ -1,196 +1,29 @@
11
name: Benchmark
22

33
on:
4-
workflow_dispatch:
5-
inputs:
6-
repo:
7-
type: string
8-
description: GitHub repository to fetch from (default to the current repo)
9-
pr_id:
10-
type: number
11-
required: true
12-
description: The PR to test
13-
commit:
14-
required: true
15-
type: string
16-
description: The expect HEAD of the PR
17-
category:
18-
required: true
19-
type: string
20-
description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark
21-
filter:
22-
type: string
23-
description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c`
24-
runs:
25-
type: number
26-
default: 30
27-
description: How many times to repeat each benchmark
28-
post-comment:
29-
type: boolean
30-
description: Post a comment linking to the run, with the aggregated result once known.
31-
token:
32-
type: string
33-
description: A GitHub token to post comments cross repository (not recommended)
4+
push:
345

356
permissions:
367
contents: read
378

389
jobs:
39-
post-comment:
40-
if: inputs.post-comment
41-
outputs:
42-
body: ${{ steps.comment.outputs.COMMENT_BODY }}
43-
url: ${{ steps.comment.outputs.COMMENT_URL }}
44-
runs-on: ubuntu-slim
45-
permissions:
46-
pull-requests: write
47-
steps:
48-
- name: Mark token input as sensitive
49-
if: inputs.token != ''
50-
run: echo "::add-mask::${{ inputs.token }}"
51-
- name: Add link to the current run
52-
id: comment
53-
run: |
54-
FILTER_IF_SET=
55-
[ -z "$FILTER" ] || FILTER_IF_SET=" / $FILTER"
56-
COMMENT_BODY="Benchmark GHA (${CATEGORIES}${FILTER_IF_SET}): ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
57-
echo "COMMENT_BODY=$COMMENT_BODY" >> "$GITHUB_OUTPUT"
58-
echo "COMMENT_URL=$(gh pr comment -R "${REPO:-$GITHUB_REPOSITORY}" "$PR_ID" --body "$COMMENT_BODY")" >> "$GITHUB_OUTPUT"
59-
env:
60-
CATEGORIES: ${{ inputs.category }}
61-
FILTER: ${{ inputs.filter }}
62-
GH_TOKEN: ${{ inputs.token || github.token }}
63-
REPO: ${{ inputs.repo }}
64-
PR_ID: ${{ inputs.pr_id }}
65-
build:
66-
strategy:
67-
fail-fast: true
68-
matrix:
69-
include:
70-
- runner: ubuntu-24.04
71-
system: x86_64-linux
72-
- runner: ubuntu-24.04-arm
73-
system: aarch64-linux
74-
- runner: macos-15-intel
75-
system: x86_64-darwin
76-
- runner: macos-latest
77-
system: aarch64-darwin
78-
name: '${{ matrix.system }}: with shared libraries'
79-
runs-on: ${{ matrix.runner }}
80-
steps:
81-
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
82-
with:
83-
repository: ${{ inputs.repo || github.repository }}
84-
ref: refs/pull/${{ inputs.pr_id }}/merge
85-
persist-credentials: false
86-
fetch-depth: 2
87-
88-
- name: Validate PR head and roll back to base commit
89-
run: |
90-
[ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ]
91-
git reset HEAD^ --hard
92-
env:
93-
EXPECTED_SHA: ${{ inputs.commit }}
94-
95-
- uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6
96-
with:
97-
extra_nix_config: sandbox = true
98-
99-
- uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
100-
with:
101-
# We do not pass any `authToken` to avoid polluting the cache with potentially untrusted code.
102-
name: nodejs
103-
104-
- name: Configure sccache
105-
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
106-
with:
107-
script: |
108-
core.exportVariable('SCCACHE_GHA_VERSION', 'on');
109-
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
110-
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
111-
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
112-
113-
- name: Build Node.js on the base commit
114-
run: |
115-
nix-shell \
116-
-I nixpkgs=./tools/nix/pkgs.nix \
117-
--pure --keep TAR_DIR --keep FLAKY_TESTS \
118-
--keep SCCACHE_GHA_ENABLED --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
119-
--arg useSeparateDerivationForV8 true \
120-
--arg loadJSBuiltinsDynamically false \
121-
--arg ccache "${NIX_SCCACHE:-null}" \
122-
--arg devTools '[]' \
123-
--arg benchmarkTools '[]' \
124-
--run '
125-
make build-ci -j4 V=1
126-
'
127-
mv out/Release/node base_node
128-
129-
- name: Checkout the merge commit
130-
run: git reset FETCH_HEAD --hard
131-
132-
- name: Re-build Node.js on the merge commit
133-
# ccache is disabled here to avoid polluting the cache. Local build outputs should make this build relatively quick anyway.
134-
run: |
135-
nix-shell \
136-
-I nixpkgs=./tools/nix/pkgs.nix \
137-
--pure \
138-
--arg useSeparateDerivationForV8 true \
139-
--arg loadJSBuiltinsDynamically false \
140-
--arg ccache 'null' \
141-
--arg devTools '[]' \
142-
--arg benchmarkTools '[]' \
143-
--run '
144-
make -j4 V=1
145-
'
146-
147-
- name: Run benchmark
148-
run: |
149-
nix-shell \
150-
-I nixpkgs=./tools/nix/pkgs.nix \
151-
--pure --keep FILTER --keep LC_ALL --keep LANG \
152-
--arg loadJSBuiltinsDynamically false \
153-
--arg ccache 'null' \
154-
--arg icu 'null' \
155-
--arg sharedLibDeps '{}' \
156-
--arg devTools '[]' \
157-
--run '
158-
set -o pipefail
159-
./base_node benchmark/compare.js \
160-
--filter "$FILTER" \
161-
--runs ${{ inputs.runs }} \
162-
--old ./base_node --new ./node \
163-
-- ${{ inputs.category }} \
164-
| tee /dev/stderr \
165-
> ${{ matrix.system }}.csv
166-
echo "> [!WARNING] "
167-
echo "> Do not take GHA benchmark results as face value, always confirm them"
168-
echo "> using a dedicated machine, e.g. Jenkins CI."
169-
echo
170-
echo "Benchmark results:"
171-
echo
172-
echo '"'"'```'"'"'
173-
Rscript benchmark/compare.R < ${{ matrix.system }}.csv
174-
echo '"'"'```'"'"'
175-
echo
176-
echo "> [!WARNING] "
177-
echo "> Do not take GHA benchmark results as face value, always confirm them"
178-
echo "> using a dedicated machine, e.g. Jenkins CI."
179-
' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"
180-
env:
181-
FILTER: ${{ inputs.filter }}
182-
183-
- name: Upload raw benchmark results
184-
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
185-
with:
186-
name: csv-${{ matrix.system }}
187-
path: ${{ matrix.system }}.csv
188-
18910
aggregate-results:
190-
needs: [build, post-comment]
191-
if: ${{ always() && needs.build.result == 'success' && needs.post-comment.result == (inputs.post-comment && 'success' || 'skipped') }}
11+
# needs: [build, post-comment]
12+
# if: ${{ always() && needs.build.result == 'success' && needs.post-comment.result == (inputs.post-comment && 'success' || 'skipped') }}
19213
name: Aggregate benchmark results
193-
runs-on: ubuntu-slim
14+
# strategy:
15+
# fail-fast: false
16+
# matrix:
17+
# include:
18+
# - runner: ubuntu-24.04
19+
# system: x86_64-linux
20+
# - runner: ubuntu-24.04-arm
21+
# system: aarch64-linux
22+
# # - runner: macos-15-intel
23+
# # system: x86_64-darwin
24+
# - runner: macos-latest
25+
# system: aarch64-darwin
26+
runs-on: ubuntu-slim # ${{ matrix.runner }}
19427
permissions:
19528
contents: read
19629
pull-requests: write
@@ -199,28 +32,80 @@ jobs:
19932
with:
20033
persist-credentials: false
20134
sparse-checkout: |
35+
why.sh
20236
benchmark/*.R
20337
tools/nix/*.nix
20438
*.nix
20539
sparse-checkout-cone-mode: false
20640

20741
- name: Download benchmark raw results
42+
if: false
20843
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
20944
with:
21045
pattern: csv-*
21146
merge-multiple: true
21247
path: raw-results
48+
repository: nodejs/node
49+
github-token: ${{ secrets.GH_USER_TOKEN }}
50+
run-id: 29243127788
21351

21452
- uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6
21553
with:
216-
extra_nix_config: sandbox = true
54+
extra_nix_config: |
55+
sandbox = true
56+
trusted-public-keys = nodejs-test.cachix.org-1:WvIzKH8IKBPG+VBNgA33RHVP11pL42HiqYSfvLYjwBc= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
57+
substituters = https://nodejs-test.cachix.org https://cache.nixos.org/
58+
59+
# - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
60+
# with:
61+
# name: nodejs-test
62+
# authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
21763

21864
- name: Benchmark results
21965
run: |
220-
nix-shell \
66+
R=/nix/store/mxx091hng2bd5q8pi0gqpjc35qrg4pn7-R-4.6.1
67+
68+
while IFS= read -r drv; do
69+
nix why-depends --precise "$R" "$drv"
70+
sleep 1
71+
done <<'EOF'
72+
/nix/store/38n0ganzvrlk08c8l3wbwxzsqmxgqi6j-acl-2.4.0
73+
/nix/store/r88gpfhsd1bww3bz9cl5crrxnwg5kiap-attr-2.6.0
74+
/nix/store/ffv6vz8kps2qz63xq85ksfz0iwvvvq7w-blas-3
75+
/nix/store/ahlj2fipymncli1pmb8y5bbyjzqrw4zk-bzip2-1.0.8-bin
76+
/nix/store/di26b1kkbammy0sj70nq5qzvfrh78wxl-coreutils-9.11
77+
/nix/store/2fj4q3rg23fikdb43m039c967scsr5d9-diffutils-3.12
78+
/nix/store/xsaclnvars896fzd98jgfsz784jmzlr3-ed-1.22.5
79+
/nix/store/784nd0cwrm99bzqwkx4fzd9j6vz4kxpm-file-5.48
80+
/nix/store/fcqbwp5hx5wh3lzf0457wmf0divknz7y-findutils-4.11.0
81+
/nix/store/l0pfzsyrxmgfbzxxzxpwi6j6mhnkyrcr-gawk-5.4.1
82+
/nix/store/6z90l28w0aqfqgb34gzh3linnci8k1id-gmp-with-cxx-6.3.0
83+
/nix/store/qlpiji0zybs4hl1dw00nfvl60fnlg90j-gnu-config-2024-01-01
84+
/nix/store/aak8d9mrdv9sgn0lcg7xss7wxdg9sqh3-gnugrep-3.12
85+
/nix/store/pqqnhh0960k1a10jm41lgnq5vrxzs05x-gnumake-4.4.1
86+
/nix/store/7mfsmbhsd3arnypipwm251rcd0b45riy-gnused-4.10
87+
/nix/store/iwfv3zbkmmphvwqp05kd7r55n3vfcfyd-gnutar-1.35
88+
/nix/store/8dlps0306xac4yf2m85iqk0k8ppbd5yx-gzip-1.14
89+
/nix/store/7gzli4v63mrj9ai0ypsbwmpvyjmp6h0c-lapack-3
90+
/nix/store/7svf4fa139ajvhfaxry3smvlbhrnraq1-libidn2-2.3.8
91+
/nix/store/sy2whqnl8z8aakhh0n0sgrflj27hc6in-libunistring-1.4.2
92+
/nix/store/w11kc8r5ixj3ir37q8ndrv82gimr5vjd-ncurses-6.6
93+
/nix/store/h1spxd2iij2l2ln2jk33mmx7kk7xhvph-openblas-0.3.33
94+
/nix/store/gvn2w8kxsxdjh1nsw88gp9fjyrcxwmkj-pcre2-10.47
95+
/nix/store/rrqlx2czgin1x4w9j8lsvmsmq6v4zkb0-readline-8.3p3
96+
/nix/store/2nndxyf3phkb5aggxm3c16sapa0f49kz-tzdata-2026c
97+
/nix/store/ya8cl7vzwcaf32zwhkgs1xzrwwlk9g93-update-autotools-gnu-config-scripts-hook
98+
/nix/store/i03xl4gzrfywjvs1y47908m302zd94sn-which-2.25
99+
/nix/store/75jq54q1qfv5n6hxxp8bd9yb6kc1yiz4-xz-5.8.3-bin
100+
EOF
101+
102+
false && nix-shell \
221103
-I nixpkgs=./tools/nix/pkgs.nix \
222104
--pure \
223-
-E '(import <nixpkgs> {}).mkShell { buildInputs = import ./tools/nix/benchmarkTools.nix { withHttpBenchmarkDeps = false; }; }' \
105+
-E '(import <nixpkgs> {}).mkShellNoCC {
106+
buildInputs = import ./tools/nix/benchmarkTools.nix {
107+
withHttpBenchmarkDeps = false;
108+
}; }' \
224109
--run '
225110
export LC_ALL=C.UTF-8
226111
echo "> [!WARNING] "
@@ -236,30 +121,4 @@ jobs:
236121
echo "> [!WARNING] "
237122
echo "> Do not take GHA benchmark results as face value, always confirm them"
238123
echo "> using a dedicated machine, e.g. Jenkins CI."
239-
' | tee /dev/stderr ${{ inputs.post-comment && 'body.txt' || '' }} >> "$GITHUB_STEP_SUMMARY"
240-
- name: Mark token input as sensitive
241-
if: inputs.token != ''
242-
run: echo "::add-mask::${{ inputs.token }}"
243-
- name: Edit comment
244-
if: inputs.post-comment
245-
run: |
246-
{
247-
echo "$COMMENT_BODY"
248-
echo
249-
echo '<details><summary>Results</summary>'
250-
echo
251-
cat body.txt
252-
echo
253-
echo '</details>'
254-
} | jq -Rrcs '{ body: . }' | gh api \
255-
--method PATCH \
256-
-H "Accept: application/vnd.github+json" \
257-
-H "X-GitHub-Api-Version: 2026-03-10" \
258-
"/repos/${REPO:-$GITHUB_REPOSITORY}/issues/comments/${COMMENT_URL##*-}" \
259-
--input -
260-
env:
261-
GH_TOKEN: ${{ inputs.token || github.token }}
262-
REPO: ${{ inputs.repo }}
263-
COMMENT_BODY: ${{ needs.post-comment.outputs.body }}
264-
COMMENT_URL: ${{ needs.post-comment.outputs.url }}
265-
PR_ID: ${{ inputs.pr_id }}
124+
' | tee /dev/stderr ${{ inputs.post-comment && 'body.txt' || '' }} >> "$GITHUB_STEP_SUMMARY"

0 commit comments

Comments
 (0)