CI: Check that test expectations are up-to-date

Author: sbc100

.circleci/config.yml

@@ -732,6 +732,28 @@ jobs:
             core0.test_hello_argc
             core2.test_demangle_stacks_symbol_map"
       - upload-test-results
+  test-expectations:
+    executor: bionic
+    environment:
+      LANG: "C.UTF-8"
+      EMTEST_SKIP_V8: "1"
+    steps:
+      - checkout
+      - run:
+          name: submodule update
+          command: git submodule update --init
+      - pip-install
+      - install-emsdk
+      - run: git checkout origin/main
+      - run:
+          name: check test expectations are up-to-date
+          command: |
+            if ./tools/maint/rebaseline_tests.py --check; then
+              echo "Test expectations are out-of-date on the main branch."
+              echo "You can run `./tools/maint/rebaseline_tests.py --new-branch`
+              echo "and use it to create a seperate PR."
+              exit 1
+            fi
   test-node-compat:
     # We don't use `bionic` here since its too old to run recent node versions:
     # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found`
@@ -973,58 +995,4 @@ workflows:
   build-test:
     jobs:
       - ruff
-      - mypy
-      - eslint
-      - build-docs
-      - build-linux
-      - test-sanity:
-          requires:
-            - build-linux
-      - test-posixtest:
-          requires:
-            - build-linux
-      - test-core0:
-          requires:
-            - build-linux
-      - test-core2:
-          requires:
-            - build-linux
-      - test-core3:
-          requires:
-            - build-linux
-      - test-wasm64:
-          requires:
-            - build-linux
-      - test-wasm64-4gb:
-          requires:
-            - build-linux
-      - test-wasm2js1:
-          requires:
-            - build-linux
-      - test-other:
-          requires:
-            - build-linux
-      - test-browser-chrome:
-          requires:
-            - build-linux
-      - test-browser-chrome-2gb:
-          requires:
-            - build-linux
-      - test-browser-chrome-wasm64:
-          requires:
-            - build-linux
-      - test-browser-chrome-wasm64-4gb:
-          requires:
-            - build-linux
-      - test-browser-firefox:
-          requires:
-            - build-linux
-      - test-browser-firefox-wasm64
-      - test-sockets-chrome:
-          requires:
-            - build-linux
-      - test-jsc
-      - test-spidermonkey
-      - test-node-compat
-      - test-windows
-      - test-mac-arm64
+

.github/workflows/check-expectations.yml

@@ -0,0 +1,39 @@
+name: CI
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  check-expectations:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      - name: Check test expectaions on main
+        run: |
+          pwd
+          git branch -a
+          git checkout origin/main
+          if ./tools/maint/rebaseline_tests.py --check; then
+            echo "Test expectations are out-of-date on the main branch."
+            echo "You can run `./tools/maint/rebaseline_tests.py --new-branch`"
+            echo "and use it to create a seperate PR."
+            exit 1
+          fi
+      - name: Install emsdk
+        run: |
+          curl -# -L -o ~/emsdk-main.tar.gz https://github.com/emscripten-core/emsdk/archive/main.tar.gz
+          tar -C ~ -xf ~/emsdk-main.tar.gz
+          mv ~/emsdk-main ~/emsdk
+          cd ~/emsdk
+          ./emsdk install tot
+          ./emsdk activate tot
+          # Remove the emsdk version of emscripten to save space in the
+          # persistent workspace and to avoid any confusion with the version
+          # we are trying to test.
+          echo "JS_ENGINES = [NODE_JS]" >> .emscripten
+          echo "final .emscripten:"
+          cat ~/emsdk/.emscripten

tools/maint/rebaseline_tests.py

@@ -63,9 +63,10 @@ def process_changed_file(filename):
 
 def main(argv):
   parser = argparse.ArgumentParser()
-  parser.add_argument('-s', '--skip-tests', action='store_true', help="don't actually run the tests, just analyze the existing results")
-  parser.add_argument('-b', '--new-branch', action='store_true', help='create a new branch containing the updates')
-  parser.add_argument('-c', '--clear-cache', action='store_true', help='clear the cache before rebaselining (useful when working with llvm changes)')
+  parser.add_argument('-s', '--skip-tests', action='store_true', help="Don't actually run the tests, just analyze the existing results")
+  parser.add_argument('-b', '--new-branch', action='store_true', help='Create a new branch containing the updates')
+  parser.add_argument('-c', '--clear-cache', action='store_true', help='Clear the cache before rebaselining (useful when working with llvm changes)')
+  parser.add_argument('--check', action='store_true', help='Return non-zero if test expectations are out of date')
   args = parser.parse_args()
 
   if args.clear_cache:
@@ -79,16 +80,25 @@ def main(argv):
     subprocess.check_call(['test/runner', '--rebaseline', '--browser=0'] + TESTS, cwd=root_dir)
 
   if not run(['git', 'status', '-uno', '--porcelain']):
-    print('no updates found')
-    return 1
+    print('test expectations are up-to-date')
+    return 0
 
   output = run(['git', 'status', '-uno', '--porcelain'])
   filenames = []
   for line in output.splitlines():
     status, filename = line.strip().split(' ', 1)
     filenames.append(filename)
 
-  commit_message = f'''
+  if check:
+    message = f'''Test expectations are out-of-date
+
+The following ({len(filenames)}) test expectation files were updated by
+running the tests with `--rebaseline`:
+
+```
+'''
+  else:
+    message = f'''
 Automatic rebaseline of codesize expectations. NFC
 
 This is an automatic change generated by tools/maint/rebaseline_tests.py.
@@ -100,16 +110,20 @@ def main(argv):
 '''
 
   for file in filenames:
-    commit_message += process_changed_file(file)
+    message += process_changed_file(file)
+
+  message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n'
 
-  commit_message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n'
+  message += '```\n'
 
-  commit_message += '```\n'
+  if check:
+
+    return 1
 
   if args.new_branch:
     run(['git', 'checkout', '-b', 'rebaseline_tests'])
   run(['git', 'add', '-u', '.'])
-  run(['git', 'commit', '-F', '-'], input=commit_message)
+  run(['git', 'commit', '-F', '-'], input=message)
 
   print(commit_message)
   return 0