Skip to content

Commit 9cb4f47

Browse files
jmidjmid
authored
Merge pull request #366 from shym/dune-onetest
Teach `dune` to focus on one particular test
2 parents 7262577 + dc39965 commit 9cb4f47

File tree

4 files changed

+103
-89
lines changed

4 files changed

+103
-89
lines changed

‎.github/workflows/common.yml

Lines changed: 9 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -28,26 +28,10 @@ on:
2828
description: 'OCAMLRUNPARAM to use'
2929
type: string
3030
default: ''
31-
only_test:
32-
description: 'Only test to run (eg “src/array/lin_tests.exe”); whole suite is run if empty'
31+
dune_alias:
32+
description: 'dune alias that should be built in the main step'
3333
type: string
34-
default: ''
35-
seed:
36-
description: 'Seed for the only test'
37-
type: string
38-
default: ''
39-
repeats:
40-
description: 'Number of test attempts'
41-
type: string
42-
default: '10'
43-
repeats_failfast:
44-
description: 'When repeating a test, stop as soon as one test fails'
45-
type: boolean
46-
default: false
47-
subsuite:
48-
description: 'Directories which should be taken as part of the test suite'
49-
type: string
50-
default: 'src'
34+
default: 'runtest'
5135
compiler:
5236
description: 'Compiler to use'
5337
type: string
@@ -83,19 +67,15 @@ on:
8367
outputs:
8468
skippart2:
8569
description: 'Whether Part 2 (Cygwin workflow) should be skipped (because we are running only one test)'
86-
value: ${{ jobs.test.outputs.skippart2 }}
70+
value: ${{ jobs.test.outputs.skipnextjob }}
8771

8872
jobs:
8973
test:
9074
env:
9175
QCHECK_MSG_INTERVAL: '60'
9276
DUNE_PROFILE: ${{ inputs.dune_profile }}
9377
OCAMLRUNPARAM: ${{ inputs.runparam }}
94-
ONLY_TEST: ${{ inputs.only_test }}
95-
SEED: ${{ inputs.seed }}
96-
REPEATS: ${{ inputs.repeats }}
97-
REPEATS_FAILFAST: ${{ inputs.repeats_failfast }}
98-
SUBSUITE: ${{ inputs.subsuite }}
78+
DUNE_CI_ALIAS: ${{ inputs.dune_alias }}
9979
COMPILER: ${{ inputs.compiler }}
10080
OCAML_COMPILER_GIT_REF: ${{ inputs.compiler_git_ref }}
10181
CUSTOM_COMPILER_VERSION: ${{ inputs.custom_compiler_version }}
@@ -113,7 +93,7 @@ jobs:
11393
timeout-minutes: ${{ inputs.timeout }}
11494

11595
outputs:
116-
skippart2: ${{ steps.winonlyone.outputs.skippart2 }}
96+
skipnextjob: ${{ steps.mainstep.outputs.skipnextjob }}
11797

11898
steps:
11999
- name: Configure environment (Cygwin)
@@ -298,66 +278,10 @@ jobs:
298278

299279
- name: Build the test suite
300280
run: opam exec -- dune build
301-
if: env.ONLY_TEST == ''
302281

303282
- name: Run the internal package tests
304283
run: opam exec -- dune runtest -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice test/
305-
if: env.ONLY_TEST == ''
306284

307-
- name: Run the multicore test suite (Linux / macOS)
308-
run: opam exec -- dune runtest -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice $SUBSUITE
309-
if: "runner.os != 'Windows' && env.ONLY_TEST == ''"
310-
311-
- name: Run the multicore test suite (Windows / Cygwin)
312-
run: opam exec -- dune runtest -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice @(-Split $Env:SUBSUITE)
313-
if: "runner.os == 'Windows' && env.ONLY_TEST == ''"
314-
315-
- name: Run only one test (Linux / macOS)
316-
run: |
317-
failures=0
318-
for i in `seq "$REPEATS"`; do
319-
echo
320-
echo "Starting $i-th run"
321-
if [ -n "$SEED" ]; then
322-
if ! opam exec -- dune exec "$ONLY_TEST" -- -v -s "$SEED"; then
323-
if [ "$REPEATS_FAILFAST" = "true" ]; then
324-
exit 1
325-
else
326-
failures=$((failures + 1))
327-
fi
328-
fi
329-
else
330-
if ! opam exec -- dune exec "$ONLY_TEST" -- -v; then
331-
if [ "$REPEATS_FAILFAST" = "true" ]; then
332-
exit 1
333-
else
334-
failures=$((failures + 1))
335-
fi
336-
fi
337-
fi
338-
done
339-
echo "Test failed $failures times"
340-
[ "$failures" = 0 ]
341-
if: env.ONLY_TEST != '' && runner.os != 'Windows'
342-
343-
- name: Run only one test (Windows)
344-
id: winonlyone
345-
run: |
346-
echo "skippart2=true" >> "${env:GITHUB_OUTPUT}"
347-
if("${env:REPEATS_FAILFAST}" -eq "false") {
348-
$ErrorActionPreference = 'Continue'
349-
}
350-
$failures = 0
351-
for($i = 1; $i -le "${env:REPEATS}"; $i++) {
352-
echo ""
353-
echo "Starting $i-th run"
354-
if("${env:SEED}" -eq "") {
355-
opam exec -- dune exec "${env:ONLY_TEST}" -- -v
356-
if($? -eq 0) { $failures += 1 }
357-
} else {
358-
opam exec -- dune exec "${env:ONLY_TEST}" -- -v -s "${env:SEED}"
359-
if($? -eq 0) { $failures += 1 }
360-
}
361-
}
362-
echo "Test failed $failures times"
363-
if: env.ONLY_TEST != '' && runner.os == 'Windows'
285+
- name: Run the multicore test suite
286+
id: mainstep
287+
run: opam exec -- dune build "@ci" -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice

‎.github/workflows/cygwin-510.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
compiler: ocaml-variants.5.1.0~beta1+options+win
1111
cygwin: true
1212
timeout: 360
13-
subsuite: src/array src/atomic src/bigarray src/buffer src/bytes src/domain src/dynlink src/ephemeron src/floatarray src/hashtbl src/io
13+
dune_alias: 'ci1'
1414

1515
part2:
1616
needs: part1
@@ -21,4 +21,4 @@ jobs:
2121
compiler: ocaml-variants.5.1.0~beta1+options+win
2222
cygwin: true
2323
timeout: 360
24-
subsuite: src/lazy src/neg_tests src/queue src/semaphore src/stack src/statistics src/sys src/thread src/threadomain src/weak
24+
dune_alias: 'ci2'

‎.github/workflows/cygwin-520-trunk.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
cygwin: true
1212
compiler_git_ref: refs/heads/trunk
1313
timeout: 360
14-
subsuite: src/array src/atomic src/bigarray src/buffer src/bytes src/domain src/dynlink src/ephemeron src/floatarray src/hashtbl src/io
14+
dune_alias: 'ci1'
1515

1616
part2:
1717
needs: part1
@@ -23,4 +23,4 @@ jobs:
2323
cygwin: true
2424
compiler_git_ref: refs/heads/trunk
2525
timeout: 360
26-
subsuite: src/lazy src/neg_tests src/queue src/semaphore src/stack src/statistics src/sys src/thread src/threadomain src/weak
26+
dune_alias: 'ci2'

‎dune

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,93 @@
77
(name default)
88
(package multicoretests)
99
(deps (alias src/default)))
10+
11+
; The aliases to control what is run in CI
12+
; It can either be the full test suite, or focus on a single test
13+
; Aliases ci1 and ci2 are useful to split the test suite into
14+
; subparts, for platforms such as Cygwin which are too slow
15+
(alias
16+
(name ci)
17+
(package multicoretests)
18+
(deps
19+
(alias_rec %{env:DUNE_CI_ALIAS=runtest})))
20+
; (alias_rec focusedtest)))
21+
22+
(alias
23+
(name ci1)
24+
(package multicoretests)
25+
(deps
26+
(alias_rec src/array/runtest)
27+
(alias_rec src/atomic/runtest)
28+
(alias_rec src/bigarray/runtest)
29+
(alias_rec src/buffer/runtest)
30+
(alias_rec src/bytes/runtest)
31+
(alias_rec src/domain/runtest)
32+
(alias_rec src/dynlink/runtest)
33+
(alias_rec src/ephemeron/runtest)
34+
(alias_rec src/floatarray/runtest)
35+
(alias_rec src/hashtbl/runtest)
36+
(alias_rec src/io/runtest)))
37+
38+
(alias
39+
(name ci2)
40+
(package multicoretests)
41+
(deps
42+
(alias_rec src/lazy/runtest)
43+
(alias_rec src/neg_tests/runtest)
44+
(alias_rec src/queue/runtest)
45+
(alias_rec src/semaphore/runtest)
46+
(alias_rec src/stack/runtest)
47+
(alias_rec src/sys/runtest)
48+
(alias_rec src/thread/runtest)
49+
(alias_rec src/threadomain/runtest)
50+
(alias_rec src/weak/runtest)))
51+
52+
; @focusedtest
53+
; repeat a single test a couple of times
54+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
55+
56+
; A rule to repeat the test executable given as dependency a couple of
57+
; times and report at the end whether this worked
58+
59+
; To change the test to repeat, change the source of the `copy`:
60+
(rule
61+
(copy src/io/lin_tests_dsl_domain.exe focusedtest.exe))
62+
63+
(rule
64+
(alias focusedtest)
65+
(package multicoretests)
66+
(deps focusedtest.exe)
67+
(enabled_if (<> %{os_type} Win32))
68+
(action
69+
(no-infer
70+
(progn
71+
(system "echo skipnextjob=true >> \"%{env:GITHUB_OUTPUT=dummy_file}\"")
72+
; ^ this is to ensure only one job is run in multi-job GitHub
73+
; workflows (only Cygwin at the moment)
74+
75+
(write-file hoped "")
76+
(write-file failed-runs "")
77+
(bash
78+
"for i in `seq 20`; do echo Starting $i-th run; if ! ./focusedtest.exe -v ; then echo $i >> failed-runs; fi; done")
79+
; edit the previous line to focus on a particular seed
80+
(diff failed-runs hoped)))))
81+
82+
(rule
83+
(alias focusedtest)
84+
(package multicoretests)
85+
(deps focusedtest.exe)
86+
(enabled_if (= %{os_type} Win32))
87+
(action
88+
(no-infer
89+
(progn
90+
(system "echo skipnextjob=true >> %{env:GITHUB_OUTPUT=dummy_file}")
91+
; ^ this is to ensure only one job is run in multi-job GitHub
92+
; workflows (only Cygwin at the moment)
93+
94+
(write-file hoped "")
95+
(write-file failed-runs "")
96+
(run cmd /q /c
97+
"for %G in (1,2,3,4, 5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) do (echo Starting %G-th run && focusedtest.exe -v || echo %G >> failed-runs)")
98+
; edit the previous line to focus on a particular seed
99+
(diff failed-runs hoped)))))

0 commit comments

Comments
 (0)