llvm · ldionne · Oct 3, 2025 · Oct 3, 2025
diff --git a/.github/workflows/flake8_docs.yaml → .github/workflows/tox.yaml b/.github/workflows/flake8_docs.yaml → .github/workflows/tox.yaml
@@ -1,4 +1,4 @@
-name: Python Flake8, Docs, mypy
+name: Run Tox tests
 
 on:
   - push
@@ -24,3 +24,5 @@ jobs:
       run: tox -e docs
     - name: Tox mypy
       run: tox -e mypy
+    - name: Tox py3
+      run: tox -e py3
diff --git a/docs/developer_guide.rst b/docs/developer_guide.rst
@@ -24,20 +24,20 @@ all times, therefore you should run the regression tests as part of your
 development work-flow, just like you do when developing on other LLVM
 sub-projects.
 
-The LNT regression tests make use of lit and other tools like FileCheck. At
-the moment, probably the easiest way to get them installed is to compile LLVM
-and use the binaries that are generated there. Assuming you've build LLVM
-into $LLVMBUILD, and installed lnt in $LNTINSTALL you can run the regression
-tests using the following command::
+The LNT regression tests make use of lit and other tools like [filecheck](https://github.com/AntonLydike/filecheck).
+To run the tests, we recomment using ``tox`` in a virtual environment::
 
-     PATH=$LLVMBUILD/bin:$LNTINSTALL/bin:$PATH llvm-lit -sv ./tests
+    python3 -m venv .venv
+    source .venv/bin/activate
+    pip install tox
+    tox
 
-If you don't like temporary files being created in your LNT source directory,
-you can run the tests in a different directory too::
+You can also run individual unit tests with ``lit`` directly::
 
-     mkdir ../run_lnt_tests
-     cd ../run_lnt_tests
-     PATH=$LLVMBUILD/bin:$LNTINSTALL/bin:$PATH llvm-lit -sv ../lnt/tests
+    pip install lit
+    lit -sv ./tests
+
+However, that requires manually setting up the testing environment (``filecheck``, etc).
 
 For simple changes, adding a regression test and making sure all regression
 tests pass, is often a good enough testing approach. For some changes, the
@@ -73,4 +73,4 @@ default. You can enable them by passing additional flags to lit:
 
 Example::
 
-     PATH=$LLVMBUILD/bin:$LNTINSTALL/bin:$PATH llvm-lit -sv -Dpostgres=1 -Dmysql=1 -Dtidylib=1 ../lnt/tests
+    lit -sv -Dpostgres=1 -Dmysql=1 -Dtidylib=1 ./tests
diff --git a/tests/Formats/json.py b/tests/Formats/json.py
@@ -1,4 +1,4 @@
-# RUN: lnt convert --to=json %S/Inputs/test.json | FileCheck %s
-# RUN: lnt convert --to=json < %S/Inputs/test.json | FileCheck %s
+# RUN: lnt convert --to=json %S/Inputs/test.json | filecheck %s
+# RUN: lnt convert --to=json < %S/Inputs/test.json | filecheck %s
 
 # CHECK: {"a": 1}
diff --git a/tests/Formats/plist.py b/tests/Formats/plist.py
@@ -1,4 +1,4 @@
-# RUN: lnt convert --to=json %S/Inputs/test.plist | FileCheck %s
-# RUN: lnt convert --to=json < %S/Inputs/test.plist | FileCheck %s
+# RUN: lnt convert --to=json %S/Inputs/test.plist | filecheck %s
+# RUN: lnt convert --to=json < %S/Inputs/test.plist | filecheck %s
 
 # CHECK: {"a": 1}
diff --git a/tests/lnttool/PostgresDB.shtest b/tests/lnttool/PostgresDB.shtest
@@ -22,7 +22,7 @@ lnt import "${TESTDIR}/instance" "${SHARED_INPUTS}/sample-b-small.plist" --show-
 #
 lnt updatedb "${TESTDIR}/instance" --testsuite nts --delete-run 1 \
 	--show-sql >& "${TESTDIR}/runrm.out"
-# RUN: FileCheck --check-prefix CHECK-RUNRM %s < "%t.install/runrm.out"
+# RUN: filecheck --check-prefix CHECK-RUNRM %s < "%t.install/runrm.out"
 
 # CHECK-RUNRM: DELETE FROM "NT_Sample" WHERE "NT_Sample"."ID" = %(ID)s
 # CHECK-RUNRM-NEXT: ({'ID': 1}, {'ID': 2})
@@ -38,7 +38,7 @@ lnt import "${TESTDIR}/instance" "${SHARED_INPUTS}/sample-a-small.plist" --show-
 lnt updatedb "${TESTDIR}/instance" --testsuite nts \
 	--delete-machine "LNT SAMPLE MACHINE" \
 	--show-sql >& "${TESTDIR}/machinerm.out"
-# RUN: FileCheck --check-prefix CHECK-MACHINERM %s < "%t.install/machinerm.out"
+# RUN: filecheck --check-prefix CHECK-MACHINERM %s < "%t.install/machinerm.out"
 
 # CHECK-MACHINERM: DELETE FROM "NT_Sample" WHERE "NT_Sample"."ID" = %(ID)s
 # CHECK-MACHINERM-NEXT: ({'ID': 3}, {'ID': 4}, {'ID': 5})

diff --git a/tests/lnttool/Profile.py b/tests/lnttool/Profile.py
@@ -1,13 +1,13 @@
-# RUN: lnt profile getVersion %S/Inputs/test.lntprof | FileCheck --check-prefix=CHECK-GETVERSION %s
+# RUN: lnt profile getVersion %S/Inputs/test.lntprof | filecheck --check-prefix=CHECK-GETVERSION %s
 # CHECK-GETVERSION: 1
 
-# RUN: lnt profile getTopLevelCounters %S/Inputs/test.lntprof | FileCheck --check-prefix=CHECK-GETTLC %s
+# RUN: lnt profile getTopLevelCounters %S/Inputs/test.lntprof | filecheck --check-prefix=CHECK-GETTLC %s
 # CHECK-GETTLC: {"cycles": 12345.0, "branch-misses": 200.0}
 
-# RUN: lnt profile getFunctions --sortkeys %S/Inputs/test.lntprof | FileCheck --check-prefix=CHECK-GETFUNCTIONS %s
+# RUN: lnt profile getFunctions --sortkeys %S/Inputs/test.lntprof | filecheck --check-prefix=CHECK-GETFUNCTIONS %s
 # CHECK-GETFUNCTIONS: {"fn1": {"counters": {"branch-misses": 10.0, "cycles": 45.0}, "length": 2}}
 
-# RUN: lnt profile getCodeForFunction %S/Inputs/test.lntprof fn1 | FileCheck --check-prefix=CHECK-GETFN1 %s
+# RUN: lnt profile getCodeForFunction %S/Inputs/test.lntprof fn1 | filecheck --check-prefix=CHECK-GETFN1 %s
 # CHECK-GETFN1: [{}, 1048576, "add r0, r0, r0"], [{"cycles": 100.0}, 1048580, "sub r1, r0, r0"]]
 
 # RUN: mkdir -p %t

diff --git a/tests/lnttool/UpdateDB.py b/tests/lnttool/UpdateDB.py
@@ -9,7 +9,7 @@
 #
 # RUN: lnt updatedb %t.install --testsuite nts \
 # RUN:     --delete-run 1 --show-sql >& %t.out
-# RUN: FileCheck --check-prefix CHECK-RUNRM %s < %t.out
+# RUN: filecheck --check-prefix CHECK-RUNRM %s < %t.out
 
 # CHECK-RUNRM: DELETE FROM "NT_Sample" WHERE "NT_Sample"."ID" = ?
 # CHECK-RUNRM-NEXT: ((1,), (2,))
@@ -25,7 +25,7 @@
 # RUN:     --show-sample-count
 # RUN: lnt updatedb %t.install --testsuite nts \
 # RUN:     --delete-machine "LNT SAMPLE MACHINE" --show-sql >& %t.out
-# RUN: FileCheck --check-prefix CHECK-MACHINERM %s < %t.out
+# RUN: filecheck --check-prefix CHECK-MACHINERM %s < %t.out
 
 # CHECK-MACHINERM: DELETE FROM "NT_Sample" WHERE "NT_Sample"."ID" = ?
 # CHECK-MACHINERM-NEXT: ((1,), (2,))

diff --git a/tests/lnttool/admin.shtest b/tests/lnttool/admin.shtest
@@ -12,7 +12,7 @@ cd "$DIR"
 rm -rf lntadmin.yaml
 lnt admin create-config > create_config.stdout
 mv lntadmin.yaml create_config.txt
-# RUN: FileCheck %s --check-prefix=CREATE_CONFIG < %t.tmp/create_config.txt
+# RUN: filecheck %s --check-prefix=CREATE_CONFIG < %t.tmp/create_config.txt
 # CREATE_CONFIG: lnt_url: "http://localhost:8000"
 # CREATE_CONFIG-NEXT: database: default
 # CREATE_CONFIG-NEXT: testsuite: nts
@@ -28,11 +28,11 @@ auth_token: test_token
 __EOF__
 
 lnt admin post-run "${SHARED_INPUTS}/sample-a-small.plist" > post_run.stdout
-# RUN: FileCheck %s --check-prefix=POST_RN < %t.tmp/post_run.stdout
+# RUN: filecheck %s --check-prefix=POST_RN < %t.tmp/post_run.stdout
 # POST_RN: http://localhost:9092/api/db_default/v4/nts/runs/3
 
 lnt admin machine-info 1 > machine_info.stdout
-# RUN: FileCheck %s --check-prefix=MACHINE_INFO < %t.tmp/machine_info.stdout
+# RUN: filecheck %s --check-prefix=MACHINE_INFO < %t.tmp/machine_info.stdout
 # MACHINE_INFO: name: localhost__clang_DEV__x86_64
 # MACHINE_INFO: id: 1
 # MACHINE_INFO: hardware: x86_64
@@ -42,7 +42,7 @@ lnt admin machine-info 1 > machine_info.stdout
 
 rm -rf run_3.json
 lnt admin get-run 3 > get_run.stdout
-# RUN: FileCheck %s --check-prefix=GET_RN < %t.tmp/run_3.json
+# RUN: filecheck %s --check-prefix=GET_RN < %t.tmp/run_3.json
 # GET_RN: {
 # GET_RN:   "generated_by":
 # GET_RN:   "machine": {
@@ -60,18 +60,18 @@ lnt admin get-run 3 > get_run.stdout
 # GET_RN: }
 
 lnt admin list-machines > list_machines.stdout
-# RUN: FileCheck %s --check-prefix=LIST_MACHINES < %t.tmp/list_machines.stdout
+# RUN: filecheck %s --check-prefix=LIST_MACHINES < %t.tmp/list_machines.stdout
 # LIST_MACHINES: localhost__clang_DEV__x86_64:1
 # LIST_MACHINES-NEXT: LNT SAMPLE MACHINE:2
 
 lnt admin list-runs 1 > list_runs.stdout
-# RUN: FileCheck %s --check-prefix=LIST_RUNS < %t.tmp/list_runs.stdout
+# RUN: filecheck %s --check-prefix=LIST_RUNS < %t.tmp/list_runs.stdout
 # LIST_RUNS: llvm_project_revision=154331 1
 # LIST_RUNS: llvm_project_revision=152289 2
 
 rm -rf machine_1.json
 lnt admin get-machine 1 > get_machine.stdout
-# RUN: FileCheck %s --check-prefix=GET_MACHINE_JSON < %t.tmp/machine_1.json
+# RUN: filecheck %s --check-prefix=GET_MACHINE_JSON < %t.tmp/machine_1.json
 # GET_MACHINE_JSON: {
 # GET_MACHINE_JSON:   "generated_by":
 # GET_MACHINE_JSON:   "machine": {
@@ -88,20 +88,20 @@ lnt admin get-machine 1 > get_machine.stdout
 # GET_MACHINE_JSON: }
 
 lnt admin rm-machine 1 > rm_machine.stdout
-# RUN: FileCheck %s --check-prefix=RM_MACHINE < %t.tmp/rm_machine.stdout
+# RUN: filecheck %s --check-prefix=RM_MACHINE < %t.tmp/rm_machine.stdout
 # RM_MACHINE: Deleting runs 1 2 (2/2)
 # RM_MACHINE: Deleted machine localhost__clang_DEV__x86_64:1
 
 lnt admin list-machines > list_machines2.stdout
-# RUN: FileCheck %s --check-prefix=LIST_MACHINES2 < %t.tmp/list_machines2.stdout
+# RUN: filecheck %s --check-prefix=LIST_MACHINES2 < %t.tmp/list_machines2.stdout
 # LIST_MACHINES2-NOT: localhost__clang_DEV__x86_64:1
 # LIST_MACHINES2: LNT SAMPLE MACHINE:2
 
 lnt admin rename-machine 2 hal9000
 # No output
 
 lnt admin list-machines > list_machines3.stdout
-# RUN: FileCheck %s --check-prefix=LIST_MACHINES3 < %t.tmp/list_machines3.stdout
+# RUN: filecheck %s --check-prefix=LIST_MACHINES3 < %t.tmp/list_machines3.stdout
 # LIST_MACHINES3: hal9000:2
 
 # Just post again so we have a machine to merge

diff --git a/tests/lnttool/checkformat.shtest b/tests/lnttool/checkformat.shtest
@@ -1,5 +1,5 @@
 # Check an old plist format
-# RUN: lnt checkformat %{shared_inputs}/sample-a-small.plist 2>&1 | FileCheck %s --check-prefix=CHECK0
+# RUN: lnt checkformat %{shared_inputs}/sample-a-small.plist 2>&1 | filecheck %s --check-prefix=CHECK0
 #
 # CHECK0: Import succeeded.
 # CHECK0: Imported Data
@@ -14,7 +14,7 @@
 #
 #
 # Check an old json format
-# RUN: lnt checkformat %{shared_inputs}/sample-report.json 2>&1 | FileCheck %s --check-prefix=CHECK1
+# RUN: lnt checkformat %{shared_inputs}/sample-report.json 2>&1 | filecheck %s --check-prefix=CHECK1
 #
 # CHECK1: Import succeeded.
 # CHECK1: Imported Data
@@ -29,7 +29,7 @@
 #
 #
 # Check new-style json format
-# RUN: lnt checkformat %{src_root}/docs/report-example.json 2>&1 | FileCheck %s --check-prefix=CHECK2
+# RUN: lnt checkformat %{src_root}/docs/report-example.json 2>&1 | filecheck %s --check-prefix=CHECK2
 #
 # CHECK2: Import succeeded.
 # CHECK2: Imported Data
@@ -43,14 +43,14 @@
 # CHECK2: PASS : 10
 #
 #
-# RUN: lnt checkformat %S/Inputs/minimal.json 2>&1 | FileCheck %s --check-prefix=MINIMAL
+# RUN: lnt checkformat %S/Inputs/minimal.json 2>&1 | filecheck %s --check-prefix=MINIMAL
 # MINIMAL: Import succeeded.
 # MINIMAL: Added Machines: 1
 # MINIMAL: Added Runs    : 1
 #
 #
 # Check invalid format
-# RUN: lnt checkformat %S/Inputs/invalid_submission0.json 2>&1 | FileCheck %s --check-prefix=CHECKFAIL0
+# RUN: lnt checkformat %S/Inputs/invalid_submission0.json 2>&1 | filecheck %s --check-prefix=CHECKFAIL0
 #
 # CHECKFAIL0: Import Failed:
 # CHECKFAIL0: could not parse input format
diff --git a/tests/lnttool/email_tools.py b/tests/lnttool/email_tools.py
@@ -9,7 +9,7 @@
 
 # RUN: lnt send-run-comparison --dry-run --to [email protected] \
 # RUN: --from [email protected]  \
-# RUN: --host localhost %t.instance 1 2 | FileCheck %s --check-prefix CHECK0
+# RUN: --host localhost %t.instance 1 2 | filecheck %s --check-prefix CHECK0
 #
 # CHECK0: From: [email protected]
 # CHECK0: To: [email protected]
@@ -37,7 +37,7 @@
 
 # RUN: lnt send-daily-report --dry-run --from [email protected] \
 # RUN: --host localhost --testsuite nts --filter-machine-regex=machine.? \
-# RUN: %t.instance [email protected] | FileCheck %s --check-prefix CHECK1
+# RUN: %t.instance [email protected] | filecheck %s --check-prefix CHECK1
 #
 # CHECK1: From: [email protected]
 # CHECK1: To: [email protected]

diff --git a/tests/lnttool/showtests.shtest b/tests/lnttool/showtests.shtest
@@ -1,4 +1,4 @@
-# RUN: lnt showtests | FileCheck %s
+# RUN: lnt showtests | filecheck %s
 # CHECK: Available tests:
 # CHECK-NEXT: compile    - Single file compile-time performance testing
 # CHECK-NEXT: nt         - LLVM test-suite compile and execution tests

diff --git a/tests/lnttool/submit.shtest b/tests/lnttool/submit.shtest
@@ -14,7 +14,7 @@ SHARED_INPUTS="$3"
 SRC_ROOT="$4"
 
 lnt submit "http://localhost:9091/db_default/submitRun" "${SHARED_INPUTS}/sample-report.json" -v > "${OUTPUT_DIR}/submit_verbose.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-VERBOSE < %t.tmp/submit_verbose.txt
+# RUN: filecheck %s --check-prefix=CHECK-VERBOSE < %t.tmp/submit_verbose.txt
 #
 # CHECK-VERBOSE: Import succeeded.
 # CHECK-VERBOSE: --- Tested: 10 tests --
@@ -33,21 +33,21 @@ lnt submit "http://localhost:9091/db_default/submitRun" "${SHARED_INPUTS}/sample
 
 # Make sure the old --commit=1 style argument is still accepted.
 lnt submit "http://localhost:9091/db_default/submitRun" --commit=1 "${SHARED_INPUTS}/sample-report1.json" > "${OUTPUT_DIR}/submit0.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-SUBMIT0 < %t.tmp/submit0.txt
+# RUN: filecheck %s --check-prefix=CHECK-SUBMIT0 < %t.tmp/submit0.txt
 # CHECK-SUBMIT0: http://localhost:9091/db_default/v4/nts/4
 
 # Submit sample-report1.json again and check it keeps the same URL
 lnt submit "http://localhost:9091/db_default/submitRun" --commit=1 "${SHARED_INPUTS}/sample-report1.json" > "${OUTPUT_DIR}/submit0-resubmit.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-RESUBMIT0 < %t.tmp/submit0-resubmit.txt
+# RUN: filecheck %s --check-prefix=CHECK-RESUBMIT0 < %t.tmp/submit0-resubmit.txt
 # CHECK-RESUBMIT0: http://localhost:9091/db_default/v4/nts/4
 
 lnt submit "http://localhost:9091/db_default/submitRun" --commit 1 "${SHARED_INPUTS}/sample-report2.json" > "${OUTPUT_DIR}/submit1.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-SUBMIT1 < %t.tmp/submit1.txt
+# RUN: filecheck %s --check-prefix=CHECK-SUBMIT1 < %t.tmp/submit1.txt
 # CHECK-SUBMIT1: http://localhost:9091/db_default/v4/nts/5
 
 
 lnt submit "http://localhost:9091/db_default/v4/compile/submitRun" "${INPUTS}/compile_submission.json" -v > "${OUTPUT_DIR}/submit_compile.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-COMPILE0 < %t.tmp/submit_compile.txt
+# RUN: filecheck %s --check-prefix=CHECK-COMPILE0 < %t.tmp/submit_compile.txt
 #
 # CHECK-COMPILE0: --- Tested: 10 tests --
 #
@@ -63,7 +63,7 @@ lnt submit "http://localhost:9091/db_default/v4/compile/submitRun" "${INPUTS}/co
 # CHECK-COMPILE0: Results available at: http://localhost:9091/db_default/v4/compile/5
 
 lnt submit "http://localhost:9091/db_default/submitRun" "${SRC_ROOT}/docs/report-example.json" -v > "${OUTPUT_DIR}/submit_newformat.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-NEWFORMAT < %t.tmp/submit_newformat.txt
+# RUN: filecheck %s --check-prefix=CHECK-NEWFORMAT < %t.tmp/submit_newformat.txt
 #
 # CHECK-NEWFORMAT: Import succeeded.
 # CHECK-NEWFORMAT: --- Tested: 10 tests --
@@ -83,7 +83,7 @@ lnt submit "http://localhost:9091/db_default/submitRun" "${SRC_ROOT}/docs/report
 # test-suite based on the Info.Run.tag field instead of the URL. The result
 # should be the same as using the "correct" URL.
 lnt submit "http://localhost:9091/db_default/submitRun" "${INPUTS}/compile_submission1.json" -v > "${OUTPUT_DIR}/submit_compile1.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-COMPILE1 < %t.tmp/submit_compile1.txt
+# RUN: filecheck %s --check-prefix=CHECK-COMPILE1 < %t.tmp/submit_compile1.txt
 #
 # CHECK-COMPILE1: Import succeeded.
 #
@@ -93,7 +93,7 @@ lnt submit "http://localhost:9091/db_default/submitRun" "${INPUTS}/compile_submi
 # CHECK-COMPILE1: Results available at: http://localhost:9091/db_default/v4/compile/6
 
 # Check some error handling/reporting
-# RUN: FileCheck %s --check-prefix=CHECK-ERRORS < %t.tmp/submit_errors.txt
+# RUN: filecheck %s --check-prefix=CHECK-ERRORS < %t.tmp/submit_errors.txt
 rm -f "${OUTPUT_DIR}/submit_errors.txt"
 
 echo "=== compile_submission.json badsuite" >> "${OUTPUT_DIR}/submit_errors.txt"
@@ -132,7 +132,7 @@ not lnt submit "http://localhost:9091/db_default/v4/compile/submitRun" "${INPUTS
 
 # Adding extra fields to the machine in a submission is fine.
 lnt submit "http://localhost:9091/db_default/v4/compile/submitRun" "${INPUTS}/compile_submission_machine_diff_fine.json" -v > "${OUTPUT_DIR}/submit_compile_machine_diff_fine.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-MACHINEDIFF < %t.tmp/submit_compile_machine_diff_fine.txt
+# RUN: filecheck %s --check-prefix=CHECK-MACHINEDIFF < %t.tmp/submit_compile_machine_diff_fine.txt
 #
 # CHECK-MACHINEDIFF: Imported Data
 # CHECK-MACHINEDIFF: -------------
@@ -147,7 +147,7 @@ lnt submit "http://localhost:9091/db_default/v4/compile/submitRun" "${INPUTS}/co
 
 # Test updating existing machine
 lnt submit "http://localhost:9091/db_default/v4/compile/submitRun" "${INPUTS}/compile_submission_machine_diff_reject.json" --select-machine=update -v > "${OUTPUT_DIR}/submit_compile_machine_update.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-UPDATEMACHINE < %t.tmp/submit_compile_machine_update.txt
+# RUN: filecheck %s --check-prefix=CHECK-UPDATEMACHINE < %t.tmp/submit_compile_machine_update.txt
 #
 # CHECK-UPDATEMACHINE: Imported Data
 # CHECK-UPDATEMACHINE: -------------
@@ -162,7 +162,7 @@ lnt submit "http://localhost:9091/db_default/v4/compile/submitRun" "${INPUTS}/co
 
 # Test creation of new machines on information mismatch in split mode.
 lnt submit "http://localhost:9091/db_default/v4/compile/submitRun" "${INPUTS}/compile_submission_machine_diff_split.json" --select-machine=split -v > "${OUTPUT_DIR}/submit_compile_machine_split.txt"
-# RUN: FileCheck %s --check-prefix=CHECK-SPLITMACHINE < %t.tmp/submit_compile_machine_split.txt
+# RUN: filecheck %s --check-prefix=CHECK-SPLITMACHINE < %t.tmp/submit_compile_machine_split.txt
 #
 # We should have added a new machine:
 # CHECK-SPLITMACHINE: Imported Data

diff --git a/tests/lnttool/submit_badurl.shtest b/tests/lnttool/submit_badurl.shtest
@@ -1,2 +1,2 @@
-# RUN: not lnt submit "http://doesnotexist.invalid/db_default/submitRun" %{shared_inputs}/sample-report.json 2>&1 | FileCheck %s
+# RUN: not lnt submit "http://doesnotexist.invalid/db_default/submitRun" %{shared_inputs}/sample-report.json 2>&1 | filecheck %s
 # CHECK: error: could not resolve 'http://doesnotexist.invalid/db_default/submitRun':
diff --git a/tests/runtest/exclude_stat.py b/tests/runtest/exclude_stat.py
@@ -7,8 +7,8 @@
 # RUN:   --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
 # RUN:   --exclude-stat-from-submission compile \
 # RUN:   --no-timestamp > %t.log 2> %t.err
-# RUN: FileCheck --check-prefix CHECK-STDOUT < %t.log %s
-# RUN: FileCheck --check-prefix CHECK-REPORT < %t.SANDBOX/build/report.json %s
+# RUN: filecheck --check-prefix CHECK-STDOUT < %t.log %s
+# RUN: filecheck --check-prefix CHECK-REPORT < %t.SANDBOX/build/report.json %s
 # CHECK-STDOUT: Import succeeded.
 # CHECK-REPORT:     "Name": "nts.{{[^.]+}}.exec"
 # CHECK-REPORT-NOT: "Name": "nts.{{[^.]+}}.compile"
diff --git a/tests/runtest/multisample.py b/tests/runtest/multisample.py
@@ -6,8 +6,8 @@
 # RUN:   --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
 # RUN:   --no-timestamp --multisample 5 > %t.log 2> %t.err
 #
-# RUN: FileCheck --check-prefix CHECK-STDOUT < %t.log %s
-# RUN: FileCheck --check-prefix CHECK-STDERR < %t.err %s
+# RUN: filecheck --check-prefix CHECK-STDOUT < %t.log %s
+# RUN: filecheck --check-prefix CHECK-STDERR < %t.err %s
 #
 # CHECK-STDOUT: Import succeeded.
 # CHECK-STDOUT: Added Machines: 1

diff --git a/tests/runtest/nt-darwin.py b/tests/runtest/nt-darwin.py
@@ -6,7 +6,7 @@
 # RUN:   --test-suite %S/Inputs/test-suite \
 # RUN:   --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
 # RUN:   --no-timestamp --use-isolation > %t.log 2> %t.err
-# RUN: FileCheck --check-prefix CHECK-SANDBOX < %t.err %s
+# RUN: filecheck --check-prefix CHECK-SANDBOX < %t.err %s
 #
 # CHECK-SANDBOX: creating sandbox profile