CI: Check that test expectations are up-to-date

Author: sbc100

.circleci/config.yml

@@ -732,6 +732,28 @@ jobs:
             core0.test_hello_argc
             core2.test_demangle_stacks_symbol_map"
       - upload-test-results
+  test-expectations:
+    executor: bionic
+    environment:
+      LANG: "C.UTF-8"
+      EMTEST_SKIP_V8: "1"
+    steps:
+      - checkout
+      - run:
+          name: submodule update
+          command: git submodule update --init
+      - pip-install
+      - install-emsdk
+      - run: git checkout origin/main
+      - run:
+          name: check test expectations are up-to-date
+          command: |
+            if ./tools/maint/rebaseline_tests.py --check; then
+              echo "Test expectations are out-of-date on the main branch."
+              echo "You can run `./tools/maint/rebaseline_tests.py --new-branch`
+              echo "and use it to create a seperate PR."
+              exit 1
+            fi
   test-node-compat:
     # We don't use `bionic` here since its too old to run recent node versions:
     # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found`
@@ -973,58 +995,4 @@ workflows:
   build-test:
     jobs:
       - ruff
-      - mypy
-      - eslint
-      - build-docs
-      - build-linux
-      - test-sanity:
-          requires:
-            - build-linux
-      - test-posixtest:
-          requires:
-            - build-linux
-      - test-core0:
-          requires:
-            - build-linux
-      - test-core2:
-          requires:
-            - build-linux
-      - test-core3:
-          requires:
-            - build-linux
-      - test-wasm64:
-          requires:
-            - build-linux
-      - test-wasm64-4gb:
-          requires:
-            - build-linux
-      - test-wasm2js1:
-          requires:
-            - build-linux
-      - test-other:
-          requires:
-            - build-linux
-      - test-browser-chrome:
-          requires:
-            - build-linux
-      - test-browser-chrome-2gb:
-          requires:
-            - build-linux
-      - test-browser-chrome-wasm64:
-          requires:
-            - build-linux
-      - test-browser-chrome-wasm64-4gb:
-          requires:
-            - build-linux
-      - test-browser-firefox:
-          requires:
-            - build-linux
-      - test-browser-firefox-wasm64
-      - test-sockets-chrome:
-          requires:
-            - build-linux
-      - test-jsc
-      - test-spidermonkey
-      - test-node-compat
-      - test-windows
-      - test-mac-arm64
+

.github/workflows/check-expectations.yml

@@ -0,0 +1,42 @@
+name: CI
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  check-expectations:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # We want access to other branches, specifically `main`
+      - name: pip install
+        run: python3 -m pip install -r requirements-dev.txt
+      - name: Install emsdk
+        run: |
+          EM_CONFIG=$HOME/emsdk/.emscripten
+          echo $EM_CONFIG
+          echo "EM_CONFIG=$EM_CONFIG" >> $GITHUB_ENV
+          curl -# -L -o ~/emsdk-main.tar.gz https://github.com/emscripten-core/emsdk/archive/main.tar.gz
+          tar -C ~ -xf ~/emsdk-main.tar.gz
+          mv ~/emsdk-main ~/emsdk
+          cd ~/emsdk
+          ./emsdk install tot
+          ./emsdk activate tot
+          echo "JS_ENGINES = [NODE_JS]" >> $EM_CONFIG
+          echo "final config:"
+          cat $EM_CONFIG
+      - name: Check test expectaions on main
+        run: |
+          git checkout origin/main
+          git checkout - ./tools/maint/rebaseline_tests.py
+          if ! ./tools/maint/rebaseline_tests.py --check-only; then
+            echo "Test expectations are out-of-date on the main branch."
+            echo "You can run `./tools/maint/rebaseline_tests.py --new-branch`"
+            echo "and use it to create a seperate PR."
+            exit 1
+          fi

tools/maint/rebaseline_tests.py

@@ -63,32 +63,42 @@ def process_changed_file(filename):
 
 def main(argv):
   parser = argparse.ArgumentParser()
-  parser.add_argument('-s', '--skip-tests', action='store_true', help="don't actually run the tests, just analyze the existing results")
-  parser.add_argument('-b', '--new-branch', action='store_true', help='create a new branch containing the updates')
-  parser.add_argument('-c', '--clear-cache', action='store_true', help='clear the cache before rebaselining (useful when working with llvm changes)')
+  parser.add_argument('-s', '--skip-tests', action='store_true', help="Don't actually run the tests, just analyze the existing results")
+  parser.add_argument('-b', '--new-branch', action='store_true', help='Create a new branch containing the updates')
+  parser.add_argument('-c', '--clear-cache', action='store_true', help='Clear the cache before rebaselining (useful when working with llvm changes)')
+  parser.add_argument('-n', '--check-only', dest='check_only', action='store_true', help='Return non-zero if test expectations are out of date, and skip creating a git commit')
   args = parser.parse_args()
 
   if args.clear_cache:
     run(['emcc', '--clear-cache'])
 
   if not args.skip_tests:
-    if run(['git', 'status', '-uno', '--porcelain']).strip():
+    if not args.check_only and run(['git', 'status', '-uno', '--porcelain']).strip():
       print('tree is not clean')
       return 1
 
     subprocess.check_call(['test/runner', '--rebaseline', '--browser=0'] + TESTS, cwd=root_dir)
 
   if not run(['git', 'status', '-uno', '--porcelain']):
-    print('no updates found')
-    return 1
+    print('test expectations are up-to-date')
+    return 0
 
   output = run(['git', 'status', '-uno', '--porcelain'])
   filenames = []
   for line in output.splitlines():
     status, filename = line.strip().split(' ', 1)
     filenames.append(filename)
 
-  commit_message = f'''
+  if args.check_only:
+    message = f'''Test expectations are out-of-date
+
+The following ({len(filenames)}) test expectation files were updated by
+running the tests with `--rebaseline`:
+
+```
+'''
+  else:
+    message = f'''
 Automatic rebaseline of codesize expectations. NFC
 
 This is an automatic change generated by tools/maint/rebaseline_tests.py.
@@ -100,16 +110,20 @@ def main(argv):
 '''
 
   for file in filenames:
-    commit_message += process_changed_file(file)
+    message += process_changed_file(file)
 
-  commit_message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n'
+  message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n'
 
-  commit_message += '```\n'
+  message += '```\n'
+
+  if opts.check_only:
+    print(message)
+    return 1
 
   if args.new_branch:
     run(['git', 'checkout', '-b', 'rebaseline_tests'])
   run(['git', 'add', '-u', '.'])
-  run(['git', 'commit', '-F', '-'], input=commit_message)
+  run(['git', 'commit', '-F', '-'], input=message)
 
   print(commit_message)
   return 0