Finish dataflow removal (#3986)

Most of it was removed last year. There's little chance we will use it
again.

Author: jonathanmetzman

src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py

@@ -65,11 +65,10 @@ class LibFuzzerOptions(engine.FuzzOptions):
   """LibFuzzer engine options."""
 
   def __init__(self, corpus_dir, arguments, strategies, fuzz_corpus_dirs,
-               extra_env, use_dataflow_tracing, is_mutations_run):
+               extra_env, is_mutations_run):
     super().__init__(corpus_dir, arguments, strategies)
     self.fuzz_corpus_dirs = fuzz_corpus_dirs
     self.extra_env = extra_env
-    self.use_dataflow_tracing = use_dataflow_tracing
     self.is_mutations_run = is_mutations_run
     self.merge_back_new_testcases = True
 
@@ -144,8 +143,7 @@ def prepare(self, corpus_dir, target_path, build_dir):
     subset_size = engine_common.random_choice(
         engine_common.CORPUS_SUBSET_NUM_TESTCASES)
 
-    if (not strategy_info.use_dataflow_tracing and
-        strategy_pool.do_strategy(strategy.CORPUS_SUBSET_STRATEGY) and
+    if (strategy_pool.do_strategy(strategy.CORPUS_SUBSET_STRATEGY) and
         shell.get_directory_file_count(corpus_dir) > subset_size):
       # Copy |subset_size| testcases into 'subset' directory.
       corpus_subset_dir = self._create_temp_corpus_dir('subset')
@@ -177,10 +175,10 @@ def prepare(self, corpus_dir, target_path, build_dir):
 
     strategies = stats.process_strategies(
         strategy_info.fuzzing_strategies, name_modifier=lambda x: x)
-    return LibFuzzerOptions(
-        corpus_dir, arguments.list(), strategies,
-        strategy_info.additional_corpus_dirs, strategy_info.extra_env,
-        strategy_info.use_dataflow_tracing, strategy_info.is_mutations_run)
+    return LibFuzzerOptions(corpus_dir, arguments.list(), strategies,
+                            strategy_info.additional_corpus_dirs,
+                            strategy_info.extra_env,
+                            strategy_info.is_mutations_run)
 
   def _create_empty_testcase_file(self, reproducers_dir):
     """Create an empty testcase file in temporary directory."""
@@ -271,7 +269,7 @@ def fuzz(self, target_path, options, reproducers_dir, max_time):
       A FuzzResult object.
     """
     profiler.start_if_needed('libfuzzer_fuzz')
-    libfuzzer.set_sanitizer_options(target_path, fuzz_options=options)
+    libfuzzer.set_sanitizer_options(target_path)
     runner = libfuzzer.get_runner(target_path)
 
     # Directory to place new units.

src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py

@@ -53,7 +53,6 @@
     'arguments',
     'additional_corpus_dirs',
     'extra_env',
-    'use_dataflow_tracing',
     'is_mutations_run',
 ])
 
@@ -63,9 +62,6 @@
 CRASH_TESTCASE_REGEX = (r'.*Test unit written to\s*'
                         r'(.*(crash|oom|timeout|leak)-.*)')
 
-# Currently matches oss-fuzz/infra/base-images/base-runner/collect_dft#L34.
-DATAFLOW_TRACE_DIR_SUFFIX = '_dft'
-
 # List of all strategies that affect LD_PRELOAD.
 MUTATOR_STRATEGIES = [
     strategy.PEACH_GRAMMAR_MUTATION_STRATEGY.name,
@@ -1154,7 +1150,6 @@ def get_runner(fuzzer_path, temp_dir=None, use_minijail=None):
     temp_dir = fuzzer_utils.get_temp_dir()
 
   build_dir = environment.get_value('BUILD_DIR')
-  dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR')
   is_android = environment.is_android()
   is_fuchsia = environment.platform() == 'FUCHSIA'
 
@@ -1176,11 +1171,6 @@ def get_runner(fuzzer_path, temp_dir=None, use_minijail=None):
     minijail_chroot.add_binding(
         minijail.ChrootBinding(build_dir, build_dir, writeable=False))
 
-    if dataflow_build_dir:
-      minijail_chroot.add_binding(
-          minijail.ChrootBinding(
-              dataflow_build_dir, dataflow_build_dir, writeable=False))
-
     # Also bind the build dir to /out to make it easier to hardcode references
     # to data files.
     minijail_chroot.add_binding(
@@ -1313,18 +1303,14 @@ def parse_log_stats(log_lines):
   return log_stats
 
 
-def set_sanitizer_options(fuzzer_path, fuzz_options=None):
-  """Sets sanitizer options based on .options file overrides, FuzzOptions (if
-  provided), and what this script requires."""
+def set_sanitizer_options(fuzzer_path):
+  """Sets sanitizer options based on .options file overrides and what this
+  script requires."""
   engine_common.process_sanitizer_options_overrides(fuzzer_path)
   sanitizer_options_var = environment.get_current_memory_tool_var()
   sanitizer_options = environment.get_memory_tool_options(
       sanitizer_options_var, {})
   sanitizer_options['exitcode'] = constants.TARGET_ERROR_EXITCODE
-  if fuzz_options and fuzz_options.use_dataflow_tracing:
-    # Focus function feature does not work without symbolization.
-    sanitizer_options['symbolize'] = 1
-    environment.update_symbolizer_options(sanitizer_options)
   environment.set_memory_tool_options(sanitizer_options_var, sanitizer_options)
 
 
@@ -1424,7 +1410,6 @@ def pick_strategies(strategy_pool,
                     existing_arguments,
                     grammar=None):
   """Pick strategies."""
-  build_directory = environment.get_value('BUILD_DIR')
   fuzzing_strategies = []
   arguments = fuzzer_options.FuzzerArguments({})
   additional_corpus_dirs = []
@@ -1435,24 +1420,6 @@ def pick_strategies(strategy_pool,
   is_mutations_run = (not environment.is_ephemeral() and
                       candidate_generator != engine_common.Generator.NONE)
 
-  # Depends on the presense of DFSan instrumented build.
-  dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR')
-  use_dataflow_tracing = (
-      dataflow_build_dir and
-      strategy_pool.do_strategy(strategy.DATAFLOW_TRACING_STRATEGY))
-  if use_dataflow_tracing:
-    dataflow_binary_path = os.path.join(
-        dataflow_build_dir, os.path.relpath(fuzzer_path, build_directory))
-    dataflow_trace_dir = dataflow_binary_path + DATAFLOW_TRACE_DIR_SUFFIX
-    if os.path.exists(dataflow_trace_dir):
-      arguments[constants.DATA_FLOW_TRACE_FLAGNAME] = str(dataflow_trace_dir)
-      arguments[constants.FOCUS_FUNCTION_FLAGNAME] = 'auto'
-      fuzzing_strategies.append(strategy.DATAFLOW_TRACING_STRATEGY.name)
-    else:
-      logs.log_warn(
-          'Dataflow trace is not found in dataflow build, skipping strategy.')
-      use_dataflow_tracing = False
-
   # Generate new testcase mutations using radamsa, etc.
   if is_mutations_run:
     new_testcase_mutations_directory = create_corpus_directory('mutations')
@@ -1476,8 +1443,7 @@ def pick_strategies(strategy_pool,
     arguments[constants.VALUE_PROFILE_FLAGNAME] = 1
     fuzzing_strategies.append(strategy.VALUE_PROFILE_STRATEGY.name)
 
-  if not use_dataflow_tracing and should_set_fork_flag(existing_arguments,
-                                                       strategy_pool):
+  if should_set_fork_flag(existing_arguments, strategy_pool):
     max_fuzz_threads = environment.get_value('MAX_FUZZ_THREADS', 1)
     num_fuzz_processes = max(1, utils.cpu_count() // max_fuzz_threads)
     arguments[constants.FORK_FLAGNAME] = num_fuzz_processes
@@ -1496,7 +1462,7 @@ def pick_strategies(strategy_pool,
     fuzzing_strategies.append(strategy.USE_EXTRA_SANITIZERS_STRATEGY.name)
 
   return StrategyInfo(fuzzing_strategies, arguments, additional_corpus_dirs,
-                      extra_env, use_dataflow_tracing, is_mutations_run)
+                      extra_env, is_mutations_run)
 
 
 def should_set_fork_flag(existing_arguments, strategy_pool):

src/clusterfuzz/_internal/build_management/build_manager.py

@@ -494,9 +494,6 @@ def _unpack_build(self,
       return False
 
     if unpack_everything:
-      # Set a random fuzz target now that the build has been unpacked, if we
-      # didn't set one earlier. For an auxiliary build, fuzz target is already
-      # specified during main build unpacking.
       self._pick_fuzz_target(
           self._get_fuzz_targets_from_dir(build_dir), target_weights)
     else:

src/clusterfuzz/_internal/cron/project_setup.py

@@ -902,20 +902,6 @@ def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name,
 
         create_pubsub_topics_for_queue_id(job.platform)
 
-      if (template.engine == 'libfuzzer' and
-          template.architecture == 'x86_64' and
-          'dataflow' in info.get('fuzzing_engines', DEFAULT_ENGINES)):
-        # Dataflow binaries are built with dataflow sanitizer, but can be used
-        # as an auxiliary build with libFuzzer builds (e.g. with ASan or UBSan).
-        dataflow_build_bucket_path = self._get_build_bucket_path(
-            project_name=project,
-            info=info,
-            engine='dataflow',
-            memory_tool='dataflow',
-            architecture=template.architecture)
-        job.environment_string += (
-            f'DATAFLOW_BUILD_BUCKET_PATH = {dataflow_build_bucket_path}\n')
-
       if self._additional_vars:
         additional_vars = {}
         additional_vars.update(self._additional_vars.get('all', {}))
@@ -1088,7 +1074,6 @@ def main():
             'honggfuzz': bucket_config.get('honggfuzz'),
             'googlefuzztest': bucket_config.get('googlefuzztest'),
             'none': bucket_config.get('no_engine'),
-            'dataflow': bucket_config.get('dataflow'),
             'centipede': bucket_config.get('centipede'),
         },
         fuzzer_entities=fuzzer_entities,

src/clusterfuzz/_internal/fuzzing/strategy.py

@@ -29,8 +29,6 @@
 # Supported fuzzing strategies.
 CORPUS_MUTATION_RADAMSA_STRATEGY = Strategy(
     name='corpus_mutations_radamsa', probability=0.15, manually_enable=False)
-DATAFLOW_TRACING_STRATEGY = Strategy(
-    name='dataflow_tracing', probability=0.50, manually_enable=True)
 CORPUS_SUBSET_STRATEGY = Strategy(
     name='corpus_subset', probability=0.50, manually_enable=True)
 FORK_STRATEGY = Strategy(name='fork', probability=0.50, manually_enable=False)
@@ -51,7 +49,6 @@
     VALUE_PROFILE_STRATEGY,
     FORK_STRATEGY,
     CORPUS_SUBSET_STRATEGY,
-    DATAFLOW_TRACING_STRATEGY,
     PEACH_GRAMMAR_MUTATION_STRATEGY,
     USE_EXTRA_SANITIZERS_STRATEGY,
 ]
@@ -81,7 +78,6 @@
 
 LIBFUZZER_STRATEGIES_WITH_BOOLEAN_VALUE = [
     CORPUS_MUTATION_RADAMSA_STRATEGY,
-    DATAFLOW_TRACING_STRATEGY,
     RANDOM_MAX_LENGTH_STRATEGY,
     VALUE_PROFILE_STRATEGY,
     USE_EXTRA_SANITIZERS_STRATEGY,

src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py

@@ -193,10 +193,6 @@
     'mode': 'NULLABLE',
     'name': 'strategy_corpus_subset',
     'type': 'INTEGER'
-}, {
-    'mode': 'NULLABLE',
-    'name': 'strategy_dataflow_tracing',
-    'type': 'INTEGER'
 }, {
     'mode': 'NULLABLE',
     'name': 'number_of_executed_units',

src/clusterfuzz/_internal/tests/appengine/handlers/cron/project_setup_test.py

@@ -205,7 +205,6 @@ def setUp(self):
             'build_buckets': {
                 'afl': 'clusterfuzz-builds-afl',
                 'centipede': 'clusterfuzz-builds-centipede',
-                'dataflow': 'clusterfuzz-builds-dataflow',
                 'honggfuzz': 'clusterfuzz-builds-honggfuzz',
                 'libfuzzer': 'clusterfuzz-builds',
                 'libfuzzer_i386': 'clusterfuzz-builds-i386',
@@ -288,8 +287,8 @@ def test_execute(self):
         }),
         ('lib6', {
             'homepage': 'http://example6.com',
-            'sanitizers': ['address', 'dataflow', 'memory', 'undefined'],
-            'fuzzing_engines': ['libfuzzer', 'afl', 'dataflow'],
+            'sanitizers': ['address', 'memory', 'undefined'],
+            'fuzzing_engines': ['libfuzzer', 'afl'],
             'auto_ccs': '[email protected]',
             'vendor_ccs': ['[email protected]', '[email protected]'],
         }),
@@ -509,9 +508,7 @@ def test_execute(self):
         'QUARANTINE_BUCKET = lib6-quarantine.clusterfuzz-external.appspot.com\n'
         'BACKUP_BUCKET = lib6-backup.clusterfuzz-external.appspot.com\n'
         'AUTOMATIC_LABELS = Proj-lib6,Engine-libfuzzer\n'
-        'FILE_GITHUB_ISSUE = False\n'
-        'DATAFLOW_BUILD_BUCKET_PATH = '
-        'gs://clusterfuzz-builds-dataflow/lib6/lib6-dataflow-([0-9]+).zip\n')
+        'FILE_GITHUB_ISSUE = False\n')
 
     job = data_types.Job.query(
         data_types.Job.name == 'libfuzzer_asan_lib7').get()
@@ -1793,7 +1790,6 @@ def setUp(self):
                 'experimental_sanitizers': ['memory'],
                 'build_buckets': {
                     'afl': 'clusterfuzz-builds-afl',
-                    'dataflow': 'clusterfuzz-builds-dataflow',
                     'honggfuzz': 'clusterfuzz-builds-honggfuzz',
                     'googlefuzztest': 'clusterfuzz-builds-googlefuzztest',
                     'libfuzzer': 'clusterfuzz-builds',
@@ -1829,7 +1825,6 @@ def setUp(self):
                 'build_type': 'FUZZ_TARGET_BUILD_BUCKET_PATH',
                 'build_buckets': {
                     'afl': 'clusterfuzz-builds-afl-dbg',
-                    'dataflow': 'clusterfuzz-builds-dataflow-dbg',
                     'honggfuzz': 'clusterfuzz-builds-honggfuzz-dbg',
                     'googlefuzztest': 'clusterfuzz-builds-googlefuzztest-dbg',
                     'libfuzzer': 'clusterfuzz-builds-dbg',

src/clusterfuzz/_internal/tests/core/bot/fuzzers/libFuzzer/engine_test.py

@@ -91,7 +91,6 @@ def setUp(self):
         }),
         additional_corpus_dirs=['/new_corpus_dir'],
         extra_env={'extra_env': '1'},
-        use_dataflow_tracing=False,
         is_mutations_run=True)
 
   def test_prepare(self):
@@ -111,7 +110,6 @@ def test_prepare(self):
     self.assertCountEqual(['/new_corpus_dir', '/corpus_dir'],
                           options.fuzz_corpus_dirs)
     self.assertDictEqual({'extra_env': '1'}, options.extra_env)
-    self.assertFalse(options.use_dataflow_tracing)
     self.assertTrue(options.is_mutations_run)
 
     self.mock.unpack_seed_corpus_if_needed.assert_called_with(
@@ -157,7 +155,6 @@ def test_no_mutations(self):
         strategies=[],
         fuzz_corpus_dirs=[],
         extra_env={},
-        use_dataflow_tracing=False,
         is_mutations_run=False)
     self.assertEqual(1800.0,
                      engine_impl.fuzz_additional_processing_timeout(options))
@@ -171,7 +168,6 @@ def test_mutations(self):
         strategies=[],
         fuzz_corpus_dirs=[],
         extra_env={},
-        use_dataflow_tracing=False,
         is_mutations_run=True)
     self.assertEqual(2400.0,
                      engine_impl.fuzz_additional_processing_timeout(options))
@@ -249,7 +245,7 @@ def test_fuzz(self):
         '-dict=blah.dict',
         '-max_len=9001',
         '-use_value_profile=1',
-    ], [], ['/corpus'], {}, False, False)
+    ], [], ['/corpus'], {}, False)
 
     with open(os.path.join(TEST_DIR, 'crash.txt'), encoding='utf-8') as f:
       fuzz_output = f.read()
@@ -383,7 +379,6 @@ def mock_merge(*args, **kwargs):  # pylint: disable=unused-argument
         'startup_crash_count': 0,
         'strategy_corpus_mutations_radamsa': 0,
         'strategy_corpus_subset': 0,
-        'strategy_dataflow_tracing': 0,
         'strategy_extra_sanitizers': 0,
         'strategy_fork': 0,
         'strategy_peach_grammar_mutation': '',

src/clusterfuzz/_internal/tests/core/bot/fuzzers/libFuzzer/libfuzzer_stats_test.py

@@ -93,7 +93,6 @@ def test_parse_log(self):
         'slow_unit_count': 0,
         'slow_units_count': 0,
         'startup_crash_count': 0,
-        'strategy_dataflow_tracing': 0,
         'strategy_extra_sanitizers': 0,
         'strategy_corpus_mutations_radamsa': 1,
         'strategy_corpus_subset': 50,