Skip to content

Commit 3ca0d2e

Browse files
vouillonvouillon
authored
More benchmarks (#1899)
* Improve timing debug information * Benchmarks: dockerfile clean-up * Benchmark: fiat-crypto * Benchmark: ocamlc * Benchmark: Bonsai's partial render table * Benchmark: Camlboy * Benchmarks: combined view of both JavaSCript and Wasm
1 parent 62e8cdd commit 3ca0d2e

File tree

18 files changed

+270
-29
lines changed

18 files changed

+270
-29
lines changed

‎.dockerignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
bench.Dockerfile
2+
_build
3+
.git
4+
compiler/tests-*
5+
manual
6+
examples

‎bench.Dockerfile

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,59 @@
1-
FROM ocaml/opam:debian-ocaml-5.3
1+
FROM ocaml/opam:debian-ocaml-5.2
22
WORKDIR /bench-dir
3-
RUN sudo apt-get update && \
4-
sudo apt-get install -qq -yy --no-install-recommends pkg-config libgmp-dev \
5-
wget
3+
4+
RUN sudo apt-get update \
5+
&& sudo apt-get install -qq -yy --no-install-recommends \
6+
pkg-config libgmp-dev jq time
7+
68
RUN sudo ln -sf /usr/bin/opam-2.3 /usr/bin/opam
7-
RUN opam remote add origin https://github.com/ocaml/opam-repository.git && \
8-
opam update
9-
RUN wget -q https://nodejs.org/download/v8-canary/v24.0.0-v8-canary2025030537242e55ac/node-v24.0.0-v8-canary2025030537242e55ac-linux-x64.tar.xz && \
10-
tar xf node-v24.0.0-v8-canary2025030537242e55ac-linux-x64.tar.xz
11-
ENV PATH="/bench-dir/node-v24.0.0-v8-canary2025030537242e55ac-linux-x64/bin:$PATH"
12-
RUN wget -q https://github.com/WebAssembly/binaryen/releases/download/version_122/binaryen-version_122-x86_64-linux.tar.gz && \
13-
tar xf binaryen-version_122-x86_64-linux.tar.gz
14-
ENV PATH="/bench-dir/binaryen-version_122/bin:$PATH"
9+
RUN opam remote add origin https://github.com/ocaml/opam-repository.git \
10+
&& opam update
11+
12+
# Install node
13+
ENV NODE_VERSION=v24.0.0-v8-canary2025030537242e55ac
14+
ENV NODE=node-$NODE_VERSION-linux-x64
15+
RUN curl -q https://nodejs.org/download/v8-canary/$NODE_VERSION/$NODE.tar.xz \
16+
| tar xJf -
17+
ENV PATH="/bench-dir/$NODE/bin:$PATH"
18+
19+
# Install binaryen
20+
ENV BINARYEN_VERSION=version_122
21+
ENV BINARYEN=binaryen-$BINARYEN_VERSION
22+
RUN curl -Lq https://github.com/WebAssembly/binaryen/releases/download/$BINARYEN_VERSION/$BINARYEN-x86_64-linux.tar.gz \
23+
| tar zxf -
24+
ENV PATH="/bench-dir/$BINARYEN/bin:$PATH"
1525
RUN opam install --fake binaryen-bin
26+
27+
# Jane Street opam packages
28+
RUN mkdir janestreet \
29+
&& cd janestreet \
30+
&& git clone --depth 20 https://github.com/janestreet/opam-repository \
31+
&& cd opam-repository \
32+
&& git checkout 41c89c7824533f6b63cc5b6d75e6ddb1441d1520 \
33+
&& opam remote add js .
34+
35+
# Install dependencies
1636
COPY --chown=opam:opam ./*.opam ./
1737
RUN opam pin -yn --with-version=dev .
1838
RUN opam install -y --deps-only js_of_ocaml-compiler
39+
40+
# Install js_of_ocaml / wasm_of_ocaml
1941
COPY --chown=opam:opam . ./
2042
RUN opam install -y wasm_of_ocaml-compiler
43+
44+
# Compile partial render table benchmark
45+
RUN opam install opam-format stringext uucp cstruct
46+
RUN opam exec -- dune exec tools/ci_setup.exe janestreet .
47+
RUN opam install ppxlib.0.35.0 # temporary workaround
48+
RUN cd janestreet/lib/bonsai_web_components && git config pull.rebase true && git pull
49+
RUN eval $(opam env) \
50+
&& dune build --root janestreet --profile release lib/bonsai_web_components/partial_render_table/bench/bin/main.bc.wasm.js lib/bonsai_web_components/partial_render_table/bench/bin/main.bc.js
51+
RUN cp -r janestreet/_build/default/lib/bonsai_web_components/partial_render_table/bench/bin/main.bc.* ./benchmarks/benchmark-partial-render-table
52+
53+
# CAMLBOY
54+
RUN opam install brr \
55+
&& git clone --depth 1 https://github.com/ocaml-wasm/CAMLBOY -b node \
56+
&& cd CAMLBOY \
57+
&& opam exec -- dune build --root . --profile release ./bin/web
58+
2159
WORKDIR ./benchmarks

‎benchmarks/Makefile

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,21 @@ all: _perf graphsnopr graphspr graphseff _noperf
1212

1313
# This target is the one run by `current-bench`,
1414
# see https://github.com/ocurrent/current-bench
15-
.PHONY: bench
15+
.PHONY: bench microbenchmarks
1616
bench:
17+
$(MAKE) microbenchmarks
18+
$(MAKE) -C benchmark-fiat-crypto bench
19+
$(MAKE) -C benchmark-ocamlc bench
20+
$(MAKE) -C benchmark-partial-render-table bench
21+
$(MAKE) -C benchmark-camlboy bench
22+
23+
microbenchmarks:
1724
make _noprecomp
1825
$(RUN) ./report-wasm-cb.config ./report-jsoo-cb.config
1926
$(REPORT) -format current-bench -config report-wasm-cb.config \
20-
-ylabel "Wasm_of_ocaml"
27+
-ylabel "Wasm_of_ocaml" | sh utils/aggregate.sh wasm
2128
$(REPORT) -format current-bench -config report-jsoo-cb.config \
22-
-ylabel "Js_of_ocaml"
23-
29+
-ylabel "Js_of_ocaml" | sh utils/aggregate.sh js
2430

2531
graphsnopr: _noprecomp $(GRAPHSNOPR)
2632

‎benchmarks/benchmark-camlboy/Makefile

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
.PHONY: bench perform
2+
3+
export NAME=Camlboy
4+
5+
SHELL=/bin/bash -o pipefail
6+
7+
SCRIPT=../../CAMLBOY/_build/default/bin/web/bench_node.bc
8+
ROM=../../CAMLBOY/resource/games/tobu.gb
9+
10+
bench:
11+
$(MAKE) perform COMPILER=Js_of_ocaml SUFFIX=.js KIND=js
12+
$(MAKE) perform COMPILER=Wasm_of_ocaml SUFFIX=.wasm.js KIND=wasm
13+
14+
perform:
15+
node $(SCRIPT)$(SUFFIX) $(ROM) | \
16+
tee /dev/stderr | \
17+
ocaml output_results.ml $(COMPILER) | \
18+
sh ../utils/aggregate.sh $(KIND)

‎benchmarks/benchmark-camlboy/output_results.ml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
let () =
2+
Format.printf
3+
{|{ "name": "%s",
4+
"results":
5+
[ { "name": "Camlboy",
6+
"metrics":
7+
[ { "name": "Frames per second",
8+
"units": "Hz",
9+
"value": %s } ] } ] }@.|}
10+
Sys.argv.(1)
11+
(read_line ())

‎benchmarks/benchmark-fiat-crypto/Makefile

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
.PHONY: bench perform
2+
3+
export NAME=Fiat-Crypto
4+
5+
SHELL=/bin/bash -o pipefail
6+
7+
bench:
8+
$(MAKE) perform COMPILER=js_of_ocaml EXTRA_ARGS="--enable effects" KIND=js
9+
$(MAKE) perform COMPILER=wasm_of_ocaml EXTRA_ARGS="" KIND=wasm
10+
11+
perform: bedrock2_fiat_crypto.byte
12+
/usr/bin/time -f "%E %R" $(COMPILER) --debug times --source-map $(EXTRA_ARGS) $< 2>&1 | \
13+
ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) $(NAME) | \
14+
sh ../utils/aggregate.sh $(KIND)
15+
16+
bedrock2_fiat_crypto.byte: bedrock2_fiat_crypto.ml
17+
ocamlfind ocamlc -w -20 -g -linkpkg -package js_of_ocaml -g $< -o $@
18+
19+
bedrock2_fiat_crypto.ml: bedrock2_fiat_crypto.ml.xz
20+
unxz --keep $<

‎benchmarks/benchmark-fiat-crypto/bedrock2_fiat_crypto.ml.xz

691 KB
Binary file not shown.

‎benchmarks/benchmark-ocamlc/Makefile

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
.PHONY: bench perform
2+
3+
export NAME=Ocamlc
4+
5+
SHELL=/bin/bash -o pipefail
6+
7+
bench:
8+
cp -r ../sources/ml .
9+
$(MAKE) perform COMPILER=js_of_ocaml SCRIPT=ocamlc.js KIND=js
10+
$(MAKE) perform COMPILER=wasm_of_ocaml SCRIPT=ocamlc.wasm.js KIND=wasm
11+
12+
ARGS=ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml
13+
14+
perform:
15+
/usr/bin/time -f "%E %R" $(COMPILER) --debug times --opt 2 --pretty `which ocamlc.byte` -o $(SCRIPT) 2>&1 | \
16+
ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) $(NAME) | \
17+
sh ../utils/aggregate.sh $(KIND)
18+
/usr/bin/time -f '{"compiler": "$(COMPILER)", "time":"%E"}' node $(SCRIPT) -c $(ARGS) 2>&1 | \
19+
sh ../utils/format_metrics.sh exec | \
20+
sh ../utils/aggregate.sh $(KIND)
21+

‎benchmarks/benchmark-partial-render-table/Makefile

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
.PHONY: bench perform
2+
3+
export NAME=Partial Render Table
4+
5+
SHELL=/bin/bash -o pipefail
6+
7+
bench:
8+
$(MAKE) perform COMPILER=js_of_ocaml SCRIPT=main.bc.js KIND=js
9+
$(MAKE) perform COMPILER=wasm_of_ocaml SCRIPT=main.bc.wasm.js KIND=wasm
10+
11+
BYTE_FILE=../../janestreet/_build/default/lib/bonsai_web_components/partial_render_table/bench/bin/main.bc-for-jsoo
12+
13+
perform:
14+
/usr/bin/time -f "%E %R" $(COMPILER) --debug times --opt 2 --pretty $(BYTE_FILE) -o out.js 2>&1 | \
15+
tee /dev/stderr | \
16+
ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) "$(NAME)" | \
17+
sh ../utils/aggregate.sh $(KIND)
18+
node $(SCRIPT) | \
19+
tee /dev/stderr | \
20+
ocaml -I +str str.cma summarize_results.ml $(COMPILER) | \
21+
sh ../utils/aggregate.sh $(KIND)

‎benchmarks/benchmark-partial-render-table/summarize_results.ml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
let delim = Str.regexp_string "│"
2+
3+
let geometric_mean l =
4+
exp (List.fold_left ( +. ) 0. (List.map log l) /. float (List.length l))
5+
6+
let parse s =
7+
Scanf.sscanf (String.trim s) "%f%s"
8+
@@ fun f u ->
9+
match u with
10+
| "ns" -> f *. 1e-6
11+
| "us" -> f *. 1e-3
12+
| "ms" -> f
13+
| "s" -> f *. 1e3
14+
| _ -> assert false
15+
16+
let () =
17+
let measures = ref [] in
18+
(try
19+
while true do
20+
let l = read_line () in
21+
if String.starts_with ~prefix:"├" l
22+
then
23+
let l = read_line () |> Str.split delim |> List.tl |> List.map parse in
24+
measures := l @ !measures
25+
done
26+
with End_of_file -> ());
27+
Format.printf
28+
{|{ "name": "%s",
29+
"results":
30+
[ { "name": "Partial Render Table",
31+
"metrics":
32+
[ { "name": "Execution time (geometric mean)",
33+
"units": "ms",
34+
"value": %f } ] } ] }@.|}
35+
(String.capitalize_ascii Sys.argv.(1))
36+
(geometric_mean !measures)

‎benchmarks/run.ml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,13 @@ module Param = struct
3434
{ warm_up_time = 1.0
3535
; min_measures = 10
3636
; max_confidence = 0.03
37-
; max_duration = 20.
37+
; max_duration = 120.
3838
; verbose = false
3939
}
4040

41-
let fast x = { x with min_measures = 5; max_confidence = 0.15 }
41+
let fast x = { x with min_measures = 5; max_confidence = 0.15; max_duration = 20. }
4242

43-
let ffast x = { x with min_measures = 2; max_confidence = 42. }
43+
let ffast x = { x with min_measures = 2; max_confidence = 42.; max_duration = 20. }
4444

4545
let verbose x = { x with verbose = true }
4646
end

‎benchmarks/utils/aggregate.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/sh
2+
TEMP=$(mktemp)
3+
4+
tee $TEMP | \
5+
jq '.name |= "Js vs Wasm"' | \
6+
jq '.results[] |= select(.name | test("Code size") | not)' |\
7+
jq '.results[].metrics[].name |= if test("/") then .+"-'$1'" else .+"/'$1'" end'
8+
cat $TEMP
9+
rm $TEMP

‎benchmarks/utils/compilation_metrics.ml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
let time_re = Str.regexp "^ \\([^ ].*\\): \\([0-9.]+\\)$"
2+
3+
let () =
4+
let times = Hashtbl.create 16 in
5+
let last_line = ref "" in
6+
(try
7+
while true do
8+
let l = read_line () in
9+
(if Str.string_match time_re l 0
10+
then
11+
let nm = Str.matched_group 1 l in
12+
let t = float_of_string (Str.matched_group 2 l) in
13+
Hashtbl.replace times nm (t +. try Hashtbl.find times nm with Not_found -> 0.));
14+
last_line := l
15+
done
16+
with End_of_file -> ());
17+
let l = Hashtbl.fold (fun nm v rem -> (nm, v) :: rem) times [] in
18+
let l = List.filter (fun (_, v) -> v > 0.2) l in
19+
let l = List.map (fun (nm, v) -> "Compilation phases/" ^ nm, "s", v) l in
20+
let l' =
21+
Scanf.sscanf !last_line "%f:%f %f" (fun m s mem ->
22+
[ "Compilation time", "s", (m *. 60.) +. s
23+
; "Compilation memory usage", "KiB", mem
24+
])
25+
in
26+
Format.printf
27+
{|{ "name": "%s",
28+
"results":
29+
[ { "name": "%s",
30+
"metrics":@.|}
31+
(String.capitalize_ascii Sys.argv.(1))
32+
Sys.argv.(2);
33+
Format.printf " [ @[";
34+
List.iteri
35+
(fun i (nm, u, v) ->
36+
if i > 0 then Format.printf ",@ ";
37+
Format.printf {|{ "name": "%s", "units": "%s", "value": %f }|} nm u v)
38+
(l' @ l);
39+
Format.printf "@] ] } ] }@."

‎benchmarks/utils/format_metrics.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/sh
2+
3+
if [ "$1" = "exec" ]; then
4+
jq '{name: ((.compiler[:1] | ascii_upcase) + .compiler[1:]), results: [{name: "'$NAME'", metrics: [{name: "Execution time", "units":"s", value: (.time | split(":") | map(tonumber) | .[0] * 60 + .[1])}]}]}'
5+
else
6+
jq '{name: ((.compiler[:1] | ascii_upcase) + .compiler[1:]), results: [{name: "'$NAME'", metrics: [{name: "Compilation time", "units":"s", value: (.time | split(":") | map(tonumber) | .[0] * 60 + .[1])}, {name: "Memory usage", "units":"KiB", value:(.mem)}]}]}'
7+
fi

‎compiler/lib-wasm/generate.ml

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ open Code
2121
module W = Wasm_ast
2222
open Code_generation
2323

24+
let times = Debug.find "times"
25+
2426
let effects_cps () =
2527
match Config.effects () with
2628
| `Cps | `Double_translation -> true
@@ -1233,9 +1235,12 @@ let fix_switch_branches p =
12331235
let start () = make_context ~value_type:Gc_target.Value.value
12341236

12351237
let f ~context ~unit_name p ~live_vars ~in_cps ~deadcode_sentinal ~debug =
1238+
let t = Timer.make () in
12361239
let p = if effects_cps () then fix_switch_branches p else p in
12371240
let module G = Generate (Gc_target) in
1238-
G.f ~context ~unit_name ~live_vars ~in_cps ~deadcode_sentinal ~debug p
1241+
let res = G.f ~context ~unit_name ~live_vars ~in_cps ~deadcode_sentinal ~debug p in
1242+
if times () then Format.eprintf " code gen.: %a@." Timer.print t;
1243+
res
12391244

12401245
let add_start_function =
12411246
let module G = Generate (Gc_target) in
@@ -1246,11 +1251,15 @@ let add_init_function =
12461251
G.add_init_function
12471252

12481253
let output ch ~context =
1254+
let t = Timer.make () in
12491255
let module G = Generate (Gc_target) in
12501256
let fields = G.output ~context in
1251-
Wat_output.f ch fields
1257+
Wat_output.f ch fields;
1258+
if times () then Format.eprintf " output: %a@." Timer.print t
12521259

12531260
let wasm_output ch ~context =
1261+
let t = Timer.make () in
12541262
let module G = Generate (Gc_target) in
12551263
let fields = G.output ~context in
1256-
Wasm_output.f ch fields
1264+
Wasm_output.f ch fields;
1265+
if times () then Format.eprintf " output: %a@." Timer.print t

‎compiler/lib/generate.ml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2188,13 +2188,13 @@ let f
21882188
~warn_on_unhandled_effect
21892189
~deadcode_sentinal
21902190
debug =
2191+
let mutated_vars = Freevars.f_mutable p in
2192+
let freevars = Freevars.f p in
21912193
let t' = Timer.make () in
21922194
let share = Share.get ~trampolined_calls ~in_cps ~alias_prims:exported_runtime p in
21932195
let exported_runtime =
21942196
if exported_runtime then Some (Code.Var.fresh_n "runtime", ref false) else None
21952197
in
2196-
let mutated_vars = Freevars.f_mutable p in
2197-
let freevars = Freevars.f p in
21982198
let ctx =
21992199
Ctx.initial
22002200
~warn_on_unhandled_effect

‎compiler/lib/global_flow.ml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -654,12 +654,11 @@ let f ~fast p =
654654
in
655655
program_deps st p;
656656
if times ()
657-
then Format.eprintf " global flow analysis (initialize): %a@." Timer.print t1;
657+
then Format.eprintf " global flow analysis (initialize): %a@." Timer.print t1;
658658
let t2 = Timer.make () in
659659
let approximation = solver st in
660-
if times ()
661-
then Format.eprintf " global flow analysis (solve): %a@." Timer.print t2;
662-
if times () then Format.eprintf " global flow analysis: %a@." Timer.print t;
660+
if times () then Format.eprintf " global flow analysis (solve): %a@." Timer.print t2;
661+
if times () then Format.eprintf " global flow analysis: %a@." Timer.print t;
663662
if debug ()
664663
then
665664
Var.ISet.iter

0 commit comments

Comments
 (0)