Skip to content

Commit 01b5408

Browse files
zhanyong-wanzhanyongwan
zhanyong-wan
and
zhanyongwan
authored
Make 2 more run_tests.sh support test selection (#9192)
Co-authored-by: Zhanyong Wan <[email protected]>
1 parent edc1a88 commit 01b5408

File tree

7 files changed

+256
-181
lines changed

7 files changed

+256
-181
lines changed

‎benchmarks/run_benchmark.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ LOGFILE=/tmp/benchmark_test.log
55

66
# Note [Keep Going]
77
#
8-
# Set the `CONTINUE_ON_ERROR` flag to `true` to make the CI tests continue on error.
8+
# Set the `CONTINUE_ON_ERROR` flag to `1` to make the CI tests continue on error.
99
# This will allow you to see all the failures on your PR, not stopping with the first
1010
# test failure like the default behavior.
1111
CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-0}"

‎test/benchmarks/run_tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ export PYTHONPATH=$PYTHONPATH:$CDIR/../../benchmarks/
99

1010
# Note [Keep Going]
1111
#
12-
# Set the `CONTINUE_ON_ERROR` flag to `true` to make the CI tests continue on error.
12+
# Set the `CONTINUE_ON_ERROR` flag to `1` to make the CI tests continue on error.
1313
# This will allow you to see all the failures on your PR, not stopping with the first
1414
# test failure like the default behavior.
1515
CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-0}"

‎test/benchmarks/run_torchbench_tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ TORCHBENCH_DIR=$PYTORCH_DIR/benchmark
1414

1515
# Note [Keep Going]
1616
#
17-
# Set the `CONTINUE_ON_ERROR` flag to `true` to make the CI tests continue on error.
17+
# Set the `CONTINUE_ON_ERROR` flag to `1` to make the CI tests continue on error.
1818
# This will allow you to see all the failures on your PR, not stopping with the first
1919
# test failure like the default behavior.
2020
CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-0}"

‎test/neuron/run_tests.sh‎

Lines changed: 47 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,25 @@
11
#!/bin/bash
22
set -exo pipefail
33
CDIR="$(cd "$(dirname "$0")"/../ ; pwd -P)"
4-
LOGFILE=/tmp/pytorch_py_test.log
5-
MAX_GRAPH_SIZE=500
6-
GRAPH_CHECK_FREQUENCY=100
7-
VERBOSITY=2
84

95
# Utils file
106
source "${CDIR}/utils/run_tests_utils.sh"
117

8+
parse_options_to_vars $@
9+
10+
# Consume the parsed commandline arguments.
11+
shift $(($OPTIND - 1))
12+
1213
# Note [Keep Going]
1314
#
14-
# Set the `CONTINUE_ON_ERROR` flag to `true` to make the CI tests continue on error.
15+
# Set the `CONTINUE_ON_ERROR` flag to `1` to make the CI tests continue on error.
1516
# This will allow you to see all the failures on your PR, not stopping with the first
1617
# test failure like the default behavior.
1718
CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-0}"
1819
if [[ "$CONTINUE_ON_ERROR" == "1" ]]; then
1920
set +e
2021
fi
2122

22-
while getopts 'LM:C:V:' OPTION
23-
do
24-
case $OPTION in
25-
L)
26-
LOGFILE=
27-
;;
28-
M)
29-
MAX_GRAPH_SIZE=$OPTARG
30-
;;
31-
C)
32-
GRAPH_CHECK_FREQUENCY=$OPTARG
33-
;;
34-
V)
35-
VERBOSITY=$OPTARG
36-
;;
37-
esac
38-
done
39-
shift $(($OPTIND - 1))
40-
4123
export TRIM_GRAPH_SIZE=$MAX_GRAPH_SIZE
4224
export TRIM_GRAPH_CHECK_FREQUENCY=$GRAPH_CHECK_FREQUENCY
4325
export TORCH_TEST_DEVICES="$CDIR/pytorch_test_base.py"
@@ -49,6 +31,9 @@ TORCH_XLA_DIR=$(cd ~; dirname "$(python -c 'import torch_xla; print(torch_xla.__
4931
COVERAGE_FILE="$CDIR/../.coverage"
5032

5133
function run_coverage {
34+
if ! test_is_selected "$1"; then
35+
return
36+
fi
5237
if [ "${USE_COVERAGE:-0}" != "0" ]; then
5338
coverage run --source="$TORCH_XLA_DIR" -p "$@"
5439
else
@@ -57,61 +42,97 @@ function run_coverage {
5742
}
5843

5944
function run_test {
45+
if ! test_is_selected "$1"; then
46+
return
47+
fi
6048
echo "Running in PjRt runtime: $@"
6149
PJRT_DEVICE=NEURON NEURON_NUM_DEVICES=1 run_coverage "$@"
6250
}
6351

6452
function run_test_without_functionalization {
53+
if ! test_is_selected "$1"; then
54+
return
55+
fi
6556
echo "Running with XLA_DISABLE_FUNCTIONALIZATION: $@"
6657
XLA_DISABLE_FUNCTIONALIZATION=1 run_test "$@"
6758
}
6859

6960
function run_xla_ir_debug {
61+
if ! test_is_selected "$1"; then
62+
return
63+
fi
7064
echo "Running with XLA_IR_DEBUG: $@"
7165
XLA_IR_DEBUG=1 run_test "$@"
7266
}
7367

7468
function run_use_bf16 {
69+
if ! test_is_selected "$1"; then
70+
return
71+
fi
7572
echo "Running with XLA_USE_BF16: $@"
7673
XLA_USE_BF16=1 run_test "$@"
7774
}
7875

7976
function run_downcast_bf16 {
77+
if ! test_is_selected "$1"; then
78+
return
79+
fi
8080
echo "Running with XLA_DOWNCAST_BF16: $@"
8181
XLA_DOWNCAST_BF16=1 run_test "$@"
8282
}
8383

8484
function run_xla_hlo_debug {
85+
if ! test_is_selected "$1"; then
86+
return
87+
fi
8588
echo "Running with XLA_IR_DEBUG: $@"
8689
XLA_HLO_DEBUG=1 run_test "$@"
8790
}
8891

8992
function run_dynamic {
93+
if ! test_is_selected "$1"; then
94+
return
95+
fi
9096
echo "Running in DynamicShape mode: $@"
9197
XLA_EXPERIMENTAL="nonzero:masked_select:masked_scatter:nms" run_test "$@"
9298
}
9399

94100
function run_eager_debug {
101+
if ! test_is_selected "$1"; then
102+
return
103+
fi
95104
echo "Running in Eager Debug mode: $@"
96105
XLA_USE_EAGER_DEBUG_MODE=1 run_test "$@"
97106
}
98107

99108
function run_pt_xla_debug {
109+
if ! test_is_selected "$1"; then
110+
return
111+
fi
100112
echo "Running in save tensor file mode: $@"
101113
PT_XLA_DEBUG=1 PT_XLA_DEBUG_FILE="/tmp/pt_xla_debug.txt" run_test "$@"
102114
}
103115

104116
function run_pt_xla_debug_level1 {
117+
if ! test_is_selected "$1"; then
118+
return
119+
fi
105120
echo "Running in save tensor file mode: $@"
106121
PT_XLA_DEBUG_LEVEL=1 PT_XLA_DEBUG_FILE="/tmp/pt_xla_debug.txt" run_test "$@"
107122
}
108123

109124
function run_pt_xla_debug_level2 {
125+
if ! test_is_selected "$1"; then
126+
return
127+
fi
110128
echo "Running in save tensor file mode: $@"
111129
PT_XLA_DEBUG_LEVEL=2 PT_XLA_DEBUG_FILE="/tmp/pt_xla_debug.txt" run_test "$@"
112130
}
113131

114132
function run_torchrun {
133+
if ! test_is_selected "$1"; then
134+
return
135+
fi
115136
PJRT_DEVICE=NEURON torchrun --nnodes 1 --nproc-per-node 2 $@
116137
}
117138

@@ -325,6 +346,8 @@ function run_tests {
325346
fi
326347
}
327348

349+
set_test_filter $@
350+
328351
if [ "$LOGFILE" != "" ]; then
329352
run_tests 2>&1 | tee $LOGFILE
330353
else

‎test/run_tests.sh‎

Lines changed: 15 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -2,49 +2,17 @@
22
set -exo pipefail
33

44
# Absolute path to the directory of this script.
5-
CDIR="$(cd "$(dirname "$0")" ; pwd -P)"
5+
CDIR="$(
6+
cd "$(dirname "$0")"
7+
pwd -P
8+
)"
69

710
# Import utilities.
811
source "${CDIR}/utils/run_tests_utils.sh"
912

10-
# Default option values. Can be overridden via commandline flags.
11-
LOGFILE=/tmp/pytorch_py_test.log
12-
MAX_GRAPH_SIZE=500
13-
GRAPH_CHECK_FREQUENCY=100
14-
VERBOSITY=2
15-
16-
# Parse commandline flags:
17-
# -L
18-
# disable writing to the log file at $LOGFILE.
19-
# -M max_graph_size
20-
# -C graph_check_frequency
21-
# -V verbosity
22-
# -h
23-
# print the help string
24-
while getopts 'LM:C:V:h' OPTION
25-
do
26-
case $OPTION in
27-
L)
28-
LOGFILE=
29-
;;
30-
M)
31-
MAX_GRAPH_SIZE=$OPTARG
32-
;;
33-
C)
34-
GRAPH_CHECK_FREQUENCY=$OPTARG
35-
;;
36-
V)
37-
VERBOSITY=$OPTARG
38-
;;
39-
h)
40-
echo -e "Usage: $0 TEST_FILTER...\nwhere TEST_FILTERs are globs match .py test files. If no test filter is provided, runs all tests."
41-
exit 0
42-
;;
43-
\?) # This catches all invalid options.
44-
echo "ERROR: Invalid commandline flag."
45-
exit 1
46-
esac
47-
done
13+
parse_options_to_vars $@
14+
15+
# Consume the parsed commandline arguments.
4816
shift $(($OPTIND - 1))
4917

5018
# Set the `CONTINUE_ON_ERROR` flag to `1` to make the CI tests continue on error.
@@ -62,35 +30,12 @@ export PYTORCH_TEST_WITH_SLOW=1
6230
export XLA_DUMP_FATAL_STACK=1
6331
export CPU_NUM_DEVICES=4
6432

65-
TORCH_XLA_DIR=$(cd ~; dirname "$(python -c 'import torch_xla; print(torch_xla.__file__)')")
33+
TORCH_XLA_DIR=$(
34+
cd ~
35+
dirname "$(python -c 'import torch_xla; print(torch_xla.__file__)')"
36+
)
6637
COVERAGE_FILE="$CDIR/../.coverage"
6738

68-
# Given $1 as a (possibly not normalized) test filepath, returns successfully
69-
# if it matches any of the space-separated globs $_TEST_FILTER. If
70-
# $_TEST_FILTER is empty, returns successfully.
71-
function test_is_selected {
72-
if [[ -z "$_TEST_FILTER" ]]; then
73-
return 0 # success
74-
fi
75-
76-
# _TEST_FILTER is a space-separate list of globs. Loop through the
77-
# list elements.
78-
for _FILTER in $_TEST_FILTER; do
79-
# realpath normalizes the paths (e.g. resolving `..` and relative paths)
80-
# so that they can be compared.
81-
case `realpath $1` in
82-
`realpath $_FILTER`)
83-
return 0 # success
84-
;;
85-
*)
86-
# No match
87-
;;
88-
esac
89-
done
90-
91-
return 1 # failure
92-
}
93-
9439
function run_coverage {
9540
if ! test_is_selected "$1"; then
9641
return
@@ -230,7 +175,7 @@ function run_xla_op_tests1 {
230175
run_dynamic "$CDIR/test_operations.py" "$@" --verbosity=$VERBOSITY
231176
run_dynamic "$CDIR/ds/test_dynamic_shapes.py"
232177
run_dynamic "$CDIR/ds/test_dynamic_shape_models.py" "$@" --verbosity=$VERBOSITY
233-
run_eager_debug "$CDIR/test_operations.py" "$@" --verbosity=$VERBOSITY
178+
run_eager_debug "$CDIR/test_operations.py" "$@" --verbosity=$VERBOSITY
234179
run_test "$CDIR/test_operations.py" "$@" --verbosity=$VERBOSITY
235180
run_test "$CDIR/test_xla_graph_execution.py" "$@" --verbosity=$VERBOSITY
236181
run_pt_xla_debug_level2 "$CDIR/test_xla_graph_execution.py" "$@" --verbosity=$VERBOSITY
@@ -263,7 +208,7 @@ function run_xla_op_tests1 {
263208
run_test "$CDIR/test_python_ops.py"
264209
run_test "$CDIR/test_ops.py"
265210
run_test "$CDIR/test_metrics.py"
266-
if [ -f "/tmp/metrics.txt" ] ; then
211+
if [ -f "/tmp/metrics.txt" ]; then
267212
rm /tmp/metrics.txt
268213
fi
269214
XLA_METRICS_FILE=/tmp/metrics.txt run_test "$CDIR/test_metrics.py"
@@ -371,7 +316,7 @@ function run_xla_op_tests3 {
371316
# Please keep PJRT_DEVICE and GPU_NUM_DEVICES explicit in the following test commands.
372317
echo "single-host-single-process"
373318
PJRT_DEVICE=CUDA GPU_NUM_DEVICES=1 python3 test/test_train_mp_imagenet.py --fake_data --batch_size=16 --num_epochs=1 --num_cores=1 --num_steps=25 --model=resnet18
374-
PJRT_DEVICE=CUDA torchrun --nnodes=1 --node_rank=0 --nproc_per_node=1 test/test_train_mp_imagenet.py --fake_data --pjrt_distributed --batch_size=16 --num_epochs=1 --num_steps=25 --model=resnet18
319+
PJRT_DEVICE=CUDA torchrun --nnodes=1 --node_rank=0 --nproc_per_node=1 test/test_train_mp_imagenet.py --fake_data --pjrt_distributed --batch_size=16 --num_epochs=1 --num_steps=25 --model=resnet18
375320

376321
echo "single-host-multi-process"
377322
num_devices=$(nvidia-smi --list-gpus | wc -l)
@@ -488,16 +433,7 @@ function run_tests {
488433
fi
489434
}
490435

491-
if [[ $# -ge 1 ]]; then
492-
# There are positional arguments - set $_TEST_FILTER to them.
493-
_TEST_FILTER=$@
494-
# Sometimes a test may fail even if it doesn't match _TEST_FILTER. Therefore,
495-
# we need to set this to be able to get to the test(s) we want to run.
496-
CONTINUE_ON_ERROR=1
497-
else
498-
# No positional argument - run all tests.
499-
_TEST_FILTER=""
500-
fi
436+
set_test_filter $@
501437

502438
if [ "$LOGFILE" != "" ]; then
503439
run_tests 2>&1 | tee $LOGFILE

0 commit comments

Comments
 (0)