[fuzzing] Give each testcase handler a list of feature flags it requires. (#2225)

That way we can still test new flags on modes that do support them (e.g. FuzzExec runs on everything)

Author: kripken

scripts/fuzz_opt.py

@@ -28,11 +28,15 @@
 
 NANS = True
 
+# feature options that are always passed to the tools.
 # exceptions: https://github.com/WebAssembly/binaryen/issues/2195
 # simd: known issues with d8
 # atomics, bulk memory: doesn't work in wasm2js
 # truncsat: https://github.com/WebAssembly/binaryen/issues/2198
-FEATURE_OPTS = ['--all-features', '--disable-exception-handling', '--disable-simd', '--disable-threads', '--disable-bulk-memory', '--disable-nontrapping-float-to-int']
+CONSTANT_FEATURE_OPTS = ['--all-features']
+
+# possible feature options that are sometimes passed to the tools.
+POSSIBLE_FEATURE_OPTS = ['--disable-exception-handling', '--disable-simd', '--disable-threads', '--disable-bulk-memory', '--disable-nontrapping-float-to-int']
 
 FUZZ_OPTS = []
 
@@ -75,6 +79,21 @@ def randomize_pass_debug():
     del os.environ['BINARYEN_PASS_DEBUG']
 
 
+def randomize_feature_opts():
+  global FEATURE_OPTS
+  FEATURE_OPTS = CONSTANT_FEATURE_OPTS[:]
+  # half the time apply all the possible opts. this lets all test runners work at max
+  # capacity at least half the time, as otherwise if they need almost all the opts, the
+  # chance of getting them is exponentially small.
+  if random.random() < 0.5:
+    FEATURE_OPTS += POSSIBLE_FEATURE_OPTS
+  else:
+    for possible in POSSIBLE_FEATURE_OPTS:
+      if random.random() < 0.5:
+        FEATURE_OPTS.append(possible)
+  print('feature opts:', ' '.join(FEATURE_OPTS))
+
+
 # Test outputs we want to ignore are marked this way.
 IGNORE = '[binaryen-fuzzer-ignore]'
 
@@ -145,7 +164,7 @@ def run_bynterp(wasm, args):
 
 
 def run_d8(wasm):
-  return run_vm(['d8', in_binaryen('scripts', 'fuzz_shell.js'), '--', wasm])
+  return run_vm(['d8'] + V8_OPTS + [in_binaryen('scripts', 'fuzz_shell.js'), '--', wasm])
 
 
 # Each test case handler receives two wasm files, one before and one after some changes
@@ -159,6 +178,9 @@ def handle_pair(self, before_wasm, after_wasm, opts):
     self.handle(before_wasm)
     self.handle(after_wasm)
 
+  def can_run_on_feature_opts(self, feature_opts):
+    return True
+
 
 # Run VMs and compare results
 class CompareVMs(TestCaseHandler):
@@ -199,6 +221,9 @@ def compare_vs(self, before, after):
       if NANS:
         break
 
+  def can_run_on_feature_opts(self, feature_opts):
+    return all([x in feature_opts for x in ['--disable-simd']])
+
 
 # Fuzz the interpreter with --fuzz-exec. This tests everything in a single command (no
 # two separate binaries) so it's easy to reproduce.
@@ -244,6 +269,9 @@ def run(self, wasm):
       out = IGNORE
     return out
 
+  def can_run_on_feature_opts(self, feature_opts):
+    return all([x in feature_opts for x in ['--disable-exception-handling', '--disable-simd', '--disable-threads', '--disable-bulk-memory', '--disable-nontrapping-float-to-int']])
+
 
 class Asyncify(TestCaseHandler):
   def handle_pair(self, before_wasm, after_wasm, opts):
@@ -286,6 +314,9 @@ def do_asyncify(wasm):
     compare(before, before_asyncify, 'Asyncify (before/before_asyncify)')
     compare(before, after_asyncify, 'Asyncify (before/after_asyncify)')
 
+  def can_run_on_feature_opts(self, feature_opts):
+    return all([x in feature_opts for x in ['--disable-exception-handling', '--disable-simd']])
+
 
 # The global list of all test case handlers
 testcase_handlers = [
@@ -300,6 +331,7 @@ def do_asyncify(wasm):
 # Do one test, given an input file for -ttf and some optimizations to run
 def test_one(random_input, opts):
   randomize_pass_debug()
+  randomize_feature_opts()
 
   bytes = 0
 
@@ -319,7 +351,9 @@ def test_one(random_input, opts):
   shutil.copyfile('a.js', 'b.js')
 
   for testcase_handler in testcase_handlers:
-    testcase_handler.handle_pair(before_wasm='a.wasm', after_wasm='b.wasm', opts=opts + FUZZ_OPTS + FEATURE_OPTS)
+    print('running testcase handler:', testcase_handler.__class__.__name__)
+    if testcase_handler.can_run_on_feature_opts(FEATURE_OPTS):
+      testcase_handler.handle_pair(before_wasm='a.wasm', after_wasm='b.wasm', opts=opts + FUZZ_OPTS + FEATURE_OPTS)
 
   return bytes