Set a proper timeout for corpus minimization after a fuzzing session (google#4719)

This change also adds the collected new_units_added stat to the
centipede schema.

Author: alhijazi

src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py

@@ -188,6 +188,19 @@ def _get_arguments(self, fuzzer_path):
 
     return arguments
 
+  def fuzz_additional_processing_timeout(self, options):
+    """Return the maximum additional timeout in seconds for additional
+    operations in fuzz() (e.g. merging back new items).
+
+    Args:
+      options: A FuzzOptions object.
+
+    Returns:
+      An int representing the number of seconds required.
+    """
+    del options
+    return engine_common.get_merge_timeout(engine_common.DEFAULT_MERGE_TIMEOUT)
+
   # pylint: disable=unused-argument
   def prepare(self, corpus_dir, target_path, build_dir):
     """Prepares for a fuzzing session, by generating options.
@@ -302,39 +315,14 @@ def fuzz(self, target_path, options, reproducers_dir, max_time):  # pylint: disa
 
     old_corpus_len = shell.get_directory_file_count(options.corpus_dir)
     logs.info(f'Corpus length before fuzzing: {old_corpus_len}')
-
     fuzz_result = runner.run_and_wait(
         additional_args=options.arguments, timeout=timeout)
     log_lines = fuzz_result.output.splitlines()
     fuzz_result.output = Engine.trim_logs(fuzz_result.output)
 
-    workdir = options.workdir
-
-    try:
-      time_for_minimize = timeout - fuzz_result.time_executed
-
-      self.minimize_corpus(
-          target_path=target_path,
-          arguments=[],
-          # New units, in addition to the main corpus units,
-          # are placed in new_corpus_dir. Minimize and merge back
-          # to the main corpus_dir.
-          input_dirs=[options.new_corpus_dir],
-          output_dir=options.corpus_dir,
-          reproducers_dir=reproducers_dir,
-          max_time=time_for_minimize,
-          # Use the same workdir that was used for fuzzing.
-          # This allows us to skip rerunning the fuzzing inputs.
-          workdir=workdir)
-    except:
-      # TODO(alhijazi): Convert to a warning if this becomes a problem
-      # caused by user code rather than by ClusterFuzz or Centipede.
-      logs.error('Corpus minimization failed.')
-      # If we fail to minimize, fall back to moving the new units
-      # from the new corpus_dir to the main corpus_dir.
-      engine_common.move_mergeable_units(options.new_corpus_dir,
-                                         options.corpus_dir)
+    logs.info('Fuzzing run completed.', fuzzing_logs=log_lines)
 
+    workdir = options.workdir
     reproducer_path = _get_reproducer_path(fuzz_result.output, reproducers_dir)
     crashes = []
     if reproducer_path:
@@ -364,6 +352,30 @@ def fuzz(self, target_path, options, reproducers_dir, max_time):  # pylint: disa
     stats['average_exec_per_sec'] = num_execs_avg / fuzz_time_secs_avg
     stats.update(_parse_centipede_logs(log_lines))
 
+    try:
+      self.minimize_corpus(
+          target_path=target_path,
+          arguments=[],
+          # New units, in addition to the main corpus units,
+          # are placed in new_corpus_dir. Minimize and merge back
+          # to the main corpus_dir.
+          input_dirs=[options.new_corpus_dir],
+          output_dir=options.corpus_dir,
+          reproducers_dir=reproducers_dir,
+          max_time=engine_common.get_merge_timeout(
+              engine_common.DEFAULT_MERGE_TIMEOUT),
+          # Use the same workdir that was used for fuzzing.
+          # This allows us to skip rerunning the fuzzing inputs.
+          workdir=workdir)
+    except:
+      # TODO(alhijazi): Convert to a warning if this becomes a problem
+      # caused by user code rather than by ClusterFuzz or Centipede.
+      logs.error('Corpus minimization failed.')
+      # If we fail to minimize, fall back to moving the new units
+      # from the new corpus_dir to the main corpus_dir.
+      engine_common.move_mergeable_units(options.new_corpus_dir,
+                                         options.corpus_dir)
+
     new_corpus_len = shell.get_directory_file_count(options.corpus_dir)
     logs.info(f'Corpus length after fuzzing: {new_corpus_len}')
     new_units_added = new_corpus_len - old_corpus_len

src/clusterfuzz/_internal/bot/fuzzers/engine_common.py

@@ -74,6 +74,9 @@
 
 HEXDIGITS_SET = set(string.hexdigits)
 
+# Allow 30 minutes to merge the testcases back into the corpus.
+DEFAULT_MERGE_TIMEOUT = 30 * 60
+
 
 class Generator:
   """Generators we can use."""

src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py

@@ -235,7 +235,7 @@ def _merge_new_units(self, target_path, corpus_dir, new_corpus_dir,
           output_corpus_dir=merge_corpus,
           reproducers_dir=None,
           max_time=engine_common.get_merge_timeout(
-              libfuzzer.DEFAULT_MERGE_TIMEOUT))
+              engine_common.DEFAULT_MERGE_TIMEOUT))
 
       engine_common.move_mergeable_units(merge_corpus, corpus_dir)
       new_corpus_len = shell.get_directory_file_count(corpus_dir)

src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py

@@ -39,9 +39,6 @@
 # Maximum length of a random chosen length for `-max_len`.
 MAX_VALUE_FOR_MAX_LENGTH = 10000
 
-# Allow 30 minutes to merge the testcases back into the corpus.
-DEFAULT_MERGE_TIMEOUT = 30 * 60
-
 StrategyInfo = collections.namedtuple('StrategiesInfo', [
     'fuzzing_strategies',
     'arguments',
@@ -1296,7 +1293,7 @@ def get_fuzz_timeout(is_mutations_run, total_timeout=None):
   """Get the fuzz timeout."""
   fuzz_timeout = (
       engine_common.get_hard_timeout(total_timeout=total_timeout) -
-      engine_common.get_merge_timeout(DEFAULT_MERGE_TIMEOUT))
+      engine_common.get_merge_timeout(engine_common.DEFAULT_MERGE_TIMEOUT))
 
   if is_mutations_run:
     fuzz_timeout -= engine_common.get_new_testcase_mutations_timeout()

src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py

@@ -855,6 +855,10 @@
     'mode': 'NULLABLE',
     'name': 'average_exec_per_sec',
     'type': 'FLOAT'
+}, {
+    'mode': 'NULLABLE',
+    'name': 'new_units_added',
+    'type': 'INTEGER'
 }] + _COMMON_COLUMNS
 
 _SCHEMA = {