Merge branch 'validate-type-def-features' into type-uniquer

Author: tlively

.github/workflows/ci.yml

@@ -39,6 +39,7 @@ jobs:
         sudo ./llvm.sh ${LLVM_VERSION}
         sudo apt-get install clang-format clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION}
     - run: flake8
+    - run: ruff check
     - run: ./scripts/clang-format-diff.sh
     - name: clang-tidy
       run: |

.ruff.toml

@@ -0,0 +1,42 @@
+target-version = "py310"
+
+exclude = [
+  'third_party',
+  'test/lit/lit.cfg.py',
+  'test/spec/testsuite',
+]
+
+[lint]
+select = [
+  "ARG",
+  "ASYNC",
+  "B",
+  "C4",
+  "C90",
+  "COM",
+  "E",
+  "F",
+  "I",
+  "PERF",
+  "PIE",
+  "PL",
+  "UP",
+  "W",
+  "YTT",
+]
+
+ignore = [
+  "C901", # https://docs.astral.sh/ruff/rules/complex-structure/
+  "B006", # https://docs.astral.sh/ruff/rules/mutable-argument-default/
+  "B011", # https://docs.astral.sh/ruff/rules/assert-false/
+  "B023", # https://docs.astral.sh/ruff/rules/function-uses-loop-variable/
+  "E501", # https://docs.astral.sh/ruff/rules/line-too-long/
+  "PERF401", # https://docs.astral.sh/ruff/rules/manual-list-comprehension/
+  "PLR0912", # https://docs.astral.sh/ruff/rules/too-many-branches/
+  "PLR0913", # https://docs.astral.sh/ruff/rules/too-many-arguments/
+  "PLR0915", # https://docs.astral.sh/ruff/rules/too-many-statements/
+  "PLR2004", # https://docs.astral.sh/ruff/rules/magic-value-comparison/
+  "PLW0603", # https://docs.astral.sh/ruff/rules/global-statement/
+  "PLW1510", # https://docs.astral.sh/ruff/rules/subprocess-run-without-check/
+  "PLW2901", # https://docs.astral.sh/ruff/rules/redefined-loop-name/
+]

CHANGELOG.md

@@ -14,6 +14,7 @@ full changeset diff at the end of each section.
 
 Current Trunk
 -------------
+- The c api now has separate functions for `CallRef` and `ReturnCallRef` matching the semantics of `Call` and `ReturnCall`.
 
 v125
 ----

check.py

@@ -15,24 +15,16 @@
 # limitations under the License.
 
 import glob
+import io
 import os
 import subprocess
 import sys
 import unittest
 from collections import OrderedDict
-from concurrent.futures import ThreadPoolExecutor
+from multiprocessing.pool import ThreadPool
 from pathlib import Path
-import queue
-import io
-import threading
-from functools import partial
 
-from scripts.test import binaryenjs
-from scripts.test import lld
-from scripts.test import shared
-from scripts.test import support
-from scripts.test import wasm2js
-from scripts.test import wasm_opt
+from scripts.test import binaryenjs, lld, shared, support, wasm2js, wasm_opt
 
 
 def get_changelog_version():
@@ -41,7 +33,7 @@ def get_changelog_version():
     lines = [line for line in lines if len(line.split()) == 1]
     lines = [line for line in lines if line.startswith('v')]
     version = lines[0][1:]
-    print("Parsed CHANGELOG.md version: %s" % version)
+    print(f"Parsed CHANGELOG.md version: {version}")
     return int(version)
 
 
@@ -55,16 +47,16 @@ def run_version_tests():
                    not any(f.endswith(s) for s in not_executable_suffix) and
                    any(os.path.basename(f).startswith(s) for s in executable_prefix)]
     executables = sorted(executables)
-    assert len(executables)
+    assert executables
 
     changelog_version = get_changelog_version()
     for e in executables:
-        print('.. %s --version' % e)
+        print(f'.. {e} --version')
         proc = subprocess.run([e, '--version'], capture_output=True, text=True)
-        assert len(proc.stderr) == 0, 'Expected no stderr, got:\n%s' % proc.stderr
+        assert len(proc.stderr) == 0, f'Expected no stderr, got:\n{proc.stderr}'
         out = proc.stdout
-        assert os.path.basename(e).replace('.exe', '') in out, 'Expected version to contain program name, got:\n%s' % out
-        assert len(out.strip().splitlines()) == 1, 'Expected only version info, got:\n%s' % out
+        assert os.path.basename(e).replace('.exe', '') in out, f'Expected version to contain program name, got:\n{out}'
+        assert len(out.strip().splitlines()) == 1, f'Expected only version info, got:\n{out}'
         parts = out.split()
         assert parts[1] == 'version'
         version = int(parts[2])
@@ -158,7 +150,8 @@ def run_wasm_reduce_tests():
         print('..', os.path.basename(t))
         # convert to wasm
         support.run_command(shared.WASM_AS + [t, '-o', 'a.wasm', '-all'])
-        support.run_command(shared.WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec -all ' % shared.WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm', '--timeout=4'])
+        cmd = shared.WASM_OPT[0]
+        support.run_command(shared.WASM_REDUCE + ['a.wasm', f'--command={cmd} b.wasm --fuzz-exec -all ', '-t', 'b.wasm', '-w', 'c.wasm', '--timeout=4'])
         expected = t + '.txt'
         support.run_command(shared.WASM_DIS + ['c.wasm', '-o', 'a.wat'])
         with open('a.wat') as seen:
@@ -171,14 +164,15 @@ def run_wasm_reduce_tests():
         # TODO: re-enable multivalue once it is better optimized
         support.run_command(shared.WASM_OPT + [os.path.join(shared.options.binaryen_test, 'lit/basic/signext.wast'), '-ttf', '-Os', '-o', 'a.wasm', '--detect-features', '--disable-multivalue'])
         before = os.stat('a.wasm').st_size
-        support.run_command(shared.WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec --detect-features' % shared.WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm'])
+        cmd = shared.WASM_OPT[0]
+        support.run_command(shared.WASM_REDUCE + ['a.wasm', f'--command={cmd} b.wasm --fuzz-exec --detect-features', '-t', 'b.wasm', '-w', 'c.wasm'])
         after = os.stat('c.wasm').st_size
         # This number is a custom threshold to check if we have shrunk the
         # output sufficiently
         assert after < 0.85 * before, [before, after]
 
 
-def run_spec_test(wast, stdout=None, stderr=None):
+def run_spec_test(wast, stdout=None):
     cmd = shared.WASM_SHELL + [wast]
     output = support.run_command(cmd, stdout=stdout, stderr=subprocess.PIPE)
     # filter out binaryen interpreter logging that the spec suite
@@ -187,7 +181,7 @@ def run_spec_test(wast, stdout=None, stderr=None):
     return '\n'.join(filtered) + '\n'
 
 
-def run_opt_test(wast, stdout=None, stderr=None):
+def run_opt_test(wast, stdout=None):
     # check optimization validation
     cmd = shared.WASM_OPT + [wast, '-O', '-all', '-q']
     support.run_command(cmd, stdout=stdout)
@@ -203,7 +197,7 @@ def check_expected(actual, expected, stdout=None):
             shared.fail(actual, expected)
 
 
-def run_one_spec_test(wast: Path, stdout=None, stderr=None):
+def run_one_spec_test(wast: Path, stdout=None):
     test_name = wast.name
 
     # /path/to/binaryen/test/spec/foo.wast -> test-spec-foo
@@ -218,7 +212,7 @@ def run_one_spec_test(wast: Path, stdout=None, stderr=None):
 
     # some spec tests should fail (actual process failure, not just assert_invalid)
     try:
-        actual = run_spec_test(str(wast), stdout=stdout, stderr=stderr)
+        actual = run_spec_test(str(wast), stdout=stdout)
     except Exception as e:
         if ('wasm-validator error' in str(e) or 'error: ' in str(e)) and '.fail.' in test_name:
             print('<< test failed as expected >>', file=stdout)
@@ -240,67 +234,45 @@ def run_one_spec_test(wast: Path, stdout=None, stderr=None):
             print(f'        testing split module {i}', file=stdout)
             split_name = base_name + f'_split{i}.wast'
             support.write_wast(split_name, module)
-            run_opt_test(split_name, stdout=stdout, stderr=stderr)    # also that our optimizer doesn't break on it
+            run_opt_test(split_name, stdout=stdout)    # also that our optimizer doesn't break on it
 
-            result_wast_file = shared.binary_format_check(split_name, verify_final_result=False, base_name=base_name, stdout=stdout, stderr=stderr)
+            result_wast_file = shared.binary_format_check(split_name, verify_final_result=False, base_name=base_name, stdout=stdout)
             with open(result_wast_file) as f:
                 result_wast = f.read()
                 # add the asserts, and verify that the test still passes
                 transformed_spec_file.write(result_wast + '\n' + '\n'.join(asserts))
 
     # compare all the outputs to the expected output
-    actual = run_spec_test(transformed_path, stdout=stdout, stderr=stderr)
+    actual = run_spec_test(transformed_path, stdout=stdout)
     check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', test_name + '.log'), stdout=stdout)
 
 
-def run_spec_test_with_wrapped_stdout(output_queue, wast: Path):
+def run_spec_test_with_wrapped_stdout(wast: Path):
     out = io.StringIO()
     try:
-        ret = run_one_spec_test(wast, stdout=out, stderr=out)
+        run_one_spec_test(wast, stdout=out)
     except Exception as e:
+        # Serialize exceptions into the output string buffer
+        # so they can be reported on the main thread.
         print(e, file=out)
         raise
-    finally:
-        # If a test fails, it's important to keep its output
-        output_queue.put(out.getvalue())
-    return ret
+    return out.getvalue()
 
 
 def run_spec_tests():
     print('\n[ checking wasm-shell spec testcases... ]\n')
 
-    output_queue = queue.Queue()
-
-    stop_printer = object()
-
-    def printer():
-        while True:
-            string = output_queue.get()
-            if string is stop_printer:
-                break
-
-            print(string, end="")
-
-    printing_thread = threading.Thread(target=printer)
-    printing_thread.start()
-
     worker_count = os.cpu_count()
     print("Running with", worker_count, "workers")
-    executor = ThreadPoolExecutor(max_workers=worker_count)
-    try:
-        results = executor.map(partial(run_spec_test_with_wrapped_stdout, output_queue), map(Path, shared.options.spec_tests))
-        for _ in results:
-            # Iterating joins the threads. No return value here.
-            pass
-    except KeyboardInterrupt:
-        # Hard exit to avoid threads continuing to run after Ctrl-C.
-        # There's no concern of deadlocking during shutdown here.
-        os._exit(1)
-    finally:
-        executor.shutdown(cancel_futures=True)
-
-        output_queue.put(stop_printer)
-        printing_thread.join()
+    test_paths = [Path(x) for x in shared.options.spec_tests]
+    with ThreadPool(processes=worker_count) as pool:
+        try:
+            for result in pool.imap_unordered(run_spec_test_with_wrapped_stdout, test_paths):
+                print(result, end="")
+        except KeyboardInterrupt:
+            # Hard exit to avoid threads continuing to run after Ctrl-C.
+            # There's no concern of deadlocking during shutdown here.
+            os._exit(1)
 
 
 def run_validator_tests():
@@ -437,7 +409,7 @@ def main():
 
     for r in shared.requested:
         if r not in all_suites:
-            print('invalid test suite: %s (see --list-suites)\n' % r)
+            print(f'invalid test suite: {r} (see --list-suites)\n')
             return 1
 
     if not shared.requested:

requirements-dev.txt

@@ -4,5 +4,6 @@
 # Install with `pip3 install -r requirements-dev.txt`
 
 flake8==7.3.0
+ruff==0.14.1
 filecheck==0.0.22
 lit==0.11.0.post1

scripts/auto_update_tests.py

@@ -19,12 +19,7 @@
 import sys
 from collections import OrderedDict
 
-from test import binaryenjs
-from test import lld
-from test import shared
-from test import support
-from test import wasm2js
-from test import wasm_opt
+from test import binaryenjs, lld, shared, support, wasm2js, wasm_opt
 
 
 def update_example_tests():
@@ -119,7 +114,8 @@ def update_reduce_tests():
         print('..', os.path.basename(t))
         # convert to wasm
         support.run_command(shared.WASM_AS + [t, '-o', 'a.wasm', '-all'])
-        print(support.run_command(shared.WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec -all' % shared.WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm']))
+        cmd = shared.WASM_OPT[0]
+        print(support.run_command(shared.WASM_REDUCE + ['a.wasm', f'--command={cmd} b.wasm --fuzz-exec -all', '-t', 'b.wasm', '-w', 'c.wasm']))
         expected = t + '.txt'
         support.run_command(shared.WASM_DIS + ['c.wasm', '-o', expected])
 

scripts/bundle_clusterfuzz.py

@@ -155,7 +155,7 @@
     for i, test in enumerate(all_tests):
         if not fuzzing.is_fuzzable(test):
             continue
-        for wast, asserts in support.split_wast(test):
+        for wast, _asserts in support.split_wast(test):
             if not wast:
                 continue
             support.write_wast(temp_wasm, wast)

scripts/clusterfuzz/embed_wasms.py

@@ -54,7 +54,7 @@
 wasm_index = 0
 
 
-def replace_wasm(text):
+def replace_wasm(_text):
     global wasm_index
     wasm_file = in_wasms[wasm_index]
     wasm_index += 1

scripts/clusterfuzz/run.py

@@ -23,14 +23,13 @@
 bundle_clusterfuzz.py.
 '''
 
-import os
 import getopt
 import math
+import os
 import random
 import subprocess
 import sys
 
-
 # The V8 flags we put in the "fuzzer flags" files, which tell ClusterFuzz how to
 # run V8. By default we apply all staging flags.
 FUZZER_FLAGS = '--wasm-staging --experimental-wasm-custom-descriptors'
@@ -141,7 +140,7 @@ def get_wasm_contents(name, output_dir, extra_args=[]):
 
     # wasm-opt may fail to run in rare cases (when the fuzzer emits code it
     # detects as invalid). Just try again in such a case.
-    for attempt in range(0, 100):
+    for attempt in range(100):
         # Generate random data.
         random_size = system_random.randint(1, MAX_RANDOM_SIZE)
         with open(input_data_file_path, 'wb') as file:
@@ -186,7 +185,7 @@ def get_wasm_contents(name, output_dir, extra_args=[]):
     global temp_files
     temp_files += [
         wasm_file_path,
-        input_data_file_path
+        input_data_file_path,
     ]
 
     # Convert to a string, and wrap into a typed array.
@@ -261,7 +260,7 @@ def get_js_file_contents(i, output_dir):
             'build(secondBinary, true)',
         ]
 
-    for i in range(num):
+    for _ in range(num):
         choice = system_random.choice(extra_js_operations)
         if choice == 'CALL_EXPORTS':
             # The random seed can be any unsigned 32-bit number.

scripts/foreach.py

@@ -15,8 +15,8 @@
 # limitations under the License.
 
 import os
-import sys
 import subprocess
+import sys
 
 from test import support
 
@@ -33,7 +33,7 @@ def main():
     cmd = sys.argv[3:]
     returncode = 0
     all_modules = open(infile).read()
-    for i, (module, asserts) in enumerate(support.split_wast(infile)):
+    for i, (module, _asserts) in enumerate(support.split_wast(infile)):
         tempname = tempfile + '.' + str(i)
         with open(tempname, 'w') as temp:
             print(module, file=temp)