From d55aae1ecab887a738758abce8456622f0e3be15 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Tue, 17 Dec 2024 11:10:22 -0800 Subject: [PATCH] CI: Check that test expectations are up-to-date --- .circleci/config.yml | 76 +++++++----------------- .github/workflows/check-expectations.yml | 45 ++++++++++++++ requirements-dev.txt | 3 - tools/maint/rebaseline_tests.py | 37 ++++++++---- 4 files changed, 92 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/check-expectations.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 61d7cb912a278..e1822592b4350 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -732,6 +732,28 @@ jobs: core0.test_hello_argc core2.test_demangle_stacks_symbol_map" - upload-test-results + test-expectations: + executor: bionic + environment: + LANG: "C.UTF-8" + EMTEST_SKIP_V8: "1" + steps: + - checkout + - run: + name: submodule update + command: git submodule update --init + - pip-install + - install-emsdk + - run: git checkout origin/main + - run: + name: check test expectations are up-to-date + command: | + if ./tools/maint/rebaseline_tests.py --check; then + echo "Test expectations are out-of-date on the main branch." + echo "You can run `./tools/maint/rebaseline_tests.py --new-branch` + echo "and use it to create a seperate PR." + exit 1 + fi test-node-compat: # We don't use `bionic` here since its too old to run recent node versions: # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found` @@ -973,58 +995,4 @@ workflows: build-test: jobs: - ruff - - mypy - - eslint - - build-docs - - build-linux - - test-sanity: - requires: - - build-linux - - test-posixtest: - requires: - - build-linux - - test-core0: - requires: - - build-linux - - test-core2: - requires: - - build-linux - - test-core3: - requires: - - build-linux - - test-wasm64: - requires: - - build-linux - - test-wasm64-4gb: - requires: - - build-linux - - test-wasm2js1: - requires: - - build-linux - - test-other: - requires: - - build-linux - - test-browser-chrome: - requires: - - build-linux - - test-browser-chrome-2gb: - requires: - - build-linux - - test-browser-chrome-wasm64: - requires: - - build-linux - - test-browser-chrome-wasm64-4gb: - requires: - - build-linux - - test-browser-firefox: - requires: - - build-linux - - test-browser-firefox-wasm64 - - test-sockets-chrome: - requires: - - build-linux - - test-jsc - - test-spidermonkey - - test-node-compat - - test-windows - test-mac-arm64 diff --git a/.github/workflows/check-expectations.yml b/.github/workflows/check-expectations.yml new file mode 100644 index 0000000000000..a8c9587a3e33b --- /dev/null +++ b/.github/workflows/check-expectations.yml @@ -0,0 +1,45 @@ +name: CI + +on: + pull_request: + +permissions: + contents: read + +jobs: + check-expectations: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # We want access to other branches, specifically `main` + - name: pip install + run: | + which python3 + python3 --version + python3 -m pip install -r requirements-dev.txt + - name: Install emsdk + run: | + EM_CONFIG=$HOME/emsdk/.emscripten + echo $EM_CONFIG + echo "EM_CONFIG=$EM_CONFIG" >> $GITHUB_ENV + curl -# -L -o ~/emsdk-main.tar.gz https://github.com/emscripten-core/emsdk/archive/main.tar.gz + tar -C ~ -xf ~/emsdk-main.tar.gz + mv ~/emsdk-main ~/emsdk + cd ~/emsdk + ./emsdk install tot + ./emsdk activate tot + echo "JS_ENGINES = [NODE_JS]" >> $EM_CONFIG + echo "final config:" + cat $EM_CONFIG + - name: Check test expectaions on main + run: | + git checkout origin/main + git checkout - ./tools/maint/rebaseline_tests.py + if ! ./tools/maint/rebaseline_tests.py --check-only; then + echo "Test expectations are out-of-date on the main branch." + echo "You can run `./tools/maint/rebaseline_tests.py --new-branch`" + echo "and use it to create a seperate PR." + exit 1 + fi diff --git a/requirements-dev.txt b/requirements-dev.txt index 6fea74921d9df..84c824c5ee1b6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,9 +10,6 @@ ruff==0.8.2 types-requests==2.27.14 unittest-xml-reporting==3.1.0 -# See https://github.com/emscripten-core/emscripten/issues/19785 -lxml==4.9.2 - # This version is mentioned in `site/source/docs/site/about.rst`. # Please keep them in sync. sphinx==7.1.2 diff --git a/tools/maint/rebaseline_tests.py b/tools/maint/rebaseline_tests.py index 09c9455acf278..b1170b2579c00 100755 --- a/tools/maint/rebaseline_tests.py +++ b/tools/maint/rebaseline_tests.py @@ -63,24 +63,25 @@ def process_changed_file(filename): def main(argv): parser = argparse.ArgumentParser() - parser.add_argument('-s', '--skip-tests', action='store_true', help="don't actually run the tests, just analyze the existing results") - parser.add_argument('-b', '--new-branch', action='store_true', help='create a new branch containing the updates') - parser.add_argument('-c', '--clear-cache', action='store_true', help='clear the cache before rebaselining (useful when working with llvm changes)') + parser.add_argument('-s', '--skip-tests', action='store_true', help="Don't actually run the tests, just analyze the existing results") + parser.add_argument('-b', '--new-branch', action='store_true', help='Create a new branch containing the updates') + parser.add_argument('-c', '--clear-cache', action='store_true', help='Clear the cache before rebaselining (useful when working with llvm changes)') + parser.add_argument('-n', '--check-only', dest='check_only', action='store_true', help='Return non-zero if test expectations are out of date, and skip creating a git commit') args = parser.parse_args() if args.clear_cache: run(['emcc', '--clear-cache']) if not args.skip_tests: - if run(['git', 'status', '-uno', '--porcelain']).strip(): + if not args.check_only and run(['git', 'status', '-uno', '--porcelain']).strip(): print('tree is not clean') return 1 subprocess.check_call(['test/runner', '--rebaseline', '--browser=0'] + TESTS, cwd=root_dir) if not run(['git', 'status', '-uno', '--porcelain']): - print('no updates found') - return 1 + print('test expectations are up-to-date') + return 0 output = run(['git', 'status', '-uno', '--porcelain']) filenames = [] @@ -88,7 +89,16 @@ def main(argv): status, filename = line.strip().split(' ', 1) filenames.append(filename) - commit_message = f''' + if args.check_only: + message = f'''Test expectations are out-of-date + +The following ({len(filenames)}) test expectation files were updated by +running the tests with `--rebaseline`: + +``` +''' + else: + message = f''' Automatic rebaseline of codesize expectations. NFC This is an automatic change generated by tools/maint/rebaseline_tests.py. @@ -100,18 +110,21 @@ def main(argv): ''' for file in filenames: - commit_message += process_changed_file(file) + message += process_changed_file(file) - commit_message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n' + message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n' - commit_message += '```\n' + message += '```\n' + + print(message) + if args.check_only: + return 1 if args.new_branch: run(['git', 'checkout', '-b', 'rebaseline_tests']) run(['git', 'add', '-u', '.']) - run(['git', 'commit', '-F', '-'], input=commit_message) + run(['git', 'commit', '-F', '-'], input=message) - print(commit_message) return 0