Merge branch 'master' into fluentd_removal

Author: vitaliset

src/clusterfuzz/_internal/bot/fuzzers/afl/launcher.py

@@ -43,7 +43,6 @@
 from clusterfuzz._internal.bot.fuzzers.afl.fuzzer import write_dummy_file
 from clusterfuzz._internal.fuzzing import strategy
 from clusterfuzz._internal.metrics import logs
-from clusterfuzz._internal.metrics import profiler
 from clusterfuzz._internal.platforms import android
 from clusterfuzz._internal.system import environment
 from clusterfuzz._internal.system import new_process
@@ -1661,7 +1660,6 @@ def main(argv):
   # same python process.
   logs.configure('run_fuzzer')
   _verify_system_config()
-  profiler.start_if_needed('afl_launcher')
 
   build_directory = environment.get_value('BUILD_DIR')
   fuzzer_path = engine_common.find_fuzzer_path(build_directory, target_name)

src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py

@@ -147,10 +147,6 @@ def _parse_centipede_logs(log_lines: List[str]) -> Dict[str, int]:
 class Engine(engine.Engine):
   """Centipede engine implementation."""
 
-  def __init__(self):
-    super().__init__()
-    self.workdir = self._create_temp_dir('workdir')
-
   @property
   def name(self):
     return 'centipede'
@@ -202,7 +198,8 @@ def prepare(self, corpus_dir, target_path, build_dir):
     # 1. Centipede-readable corpus file;
     # 2. Centipede-readable feature file;
     # 3. Crash reproducing inputs.
-    arguments[constants.WORKDIR_FLAGNAME] = str(self.workdir)
+    workdir = self._create_temp_dir('workdir')
+    arguments[constants.WORKDIR_FLAGNAME] = str(workdir)
 
     # Directory corpus_dir saves the corpus files required by ClusterFuzz.
     arguments[constants.CORPUS_DIR_FLAGNAME] = corpus_dir
@@ -301,7 +298,12 @@ def fuzz(self, target_path, options, reproducers_dir, max_time):  # pylint: disa
               int(fuzz_result.time_executed)))
 
     stats_filename = f'fuzzing-stats-{os.path.basename(target_path)}.000000.csv'
-    stats_file = os.path.join(self.workdir, stats_filename)
+    args = fuzzer_options.FuzzerArguments.from_list(options.arguments)
+    assert args is not None
+    assert constants.WORKDIR_FLAGNAME in args
+
+    workdir = args[constants.WORKDIR_FLAGNAME]
+    stats_file = os.path.join(workdir, stats_filename)
     stats = _parse_centipede_stats(stats_file)
     if not stats:
       stats = {}
@@ -505,9 +507,10 @@ def minimize_testcase(self, target_path, arguments, input_path, output_path,
       TimeoutError: If the testcase minimization exceeds max_time.
     """
     runner = _get_runner(target_path)
+    workdir = self._create_temp_dir('workdir')
     args = [
         f'--binary={target_path}',
-        f'--workdir={self.workdir}',
+        f'--workdir={workdir}',
         f'--minimize_crash={input_path}',
         f'--num_runs={constants.NUM_RUNS_PER_MINIMIZATION}',
         '--seed=1',
@@ -517,7 +520,7 @@ def minimize_testcase(self, target_path, arguments, input_path, output_path,
       logs.warning(
           'Testcase minimization timed out.', fuzzer_output=result.output)
       raise TimeoutError('Minimization timed out.')
-    minimum_testcase = self._get_smallest_crasher(self.workdir)
+    minimum_testcase = self._get_smallest_crasher(workdir)
     if minimum_testcase:
       shutil.copyfile(minimum_testcase, output_path)
     else:

src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py

@@ -29,7 +29,6 @@
 from clusterfuzz._internal.bot.fuzzers.libFuzzer import stats
 from clusterfuzz._internal.fuzzing import strategy
 from clusterfuzz._internal.metrics import logs
-from clusterfuzz._internal.metrics import profiler
 from clusterfuzz._internal.system import environment
 from clusterfuzz._internal.system import shell
 from clusterfuzz.fuzz import engine
@@ -280,7 +279,6 @@ def fuzz(self, target_path, options, reproducers_dir, max_time):
     Returns:
       A FuzzResult object.
     """
-    profiler.start_if_needed('libfuzzer_fuzz')
     libfuzzer.set_sanitizer_options(target_path)
     runner = libfuzzer.get_runner(target_path)
 

src/clusterfuzz/_internal/bot/tasks/commands.py

@@ -320,10 +320,13 @@ def process_command_impl(task_name, task_argument, job_name, high_end,
     # A misconfiguration led to this point. Clean up the job if necessary.
     # TODO(ochang): Remove the first part of this check once we migrate off the
     # old untrusted worker architecture.
+    # If the job base quque is '-android', which is the default Android queue
+    # ignore the mismatch since with subqueues, it is expected
     if (not environment.get_value('DEBUG_TASK') and
         not environment.is_tworker() and
         not environment.is_trusted_host(ensure_connected=False) and
-        job_base_queue_suffix != bot_base_queue_suffix):
+        job_base_queue_suffix != bot_base_queue_suffix and
+        job_base_queue_suffix != '-android'):
       # This happens rarely, store this as a hard exception.
       logs.error('Wrong platform for job %s: job queue [%s], bot queue [%s].' %
                  (job_name, job_base_queue_suffix, bot_base_queue_suffix))

src/clusterfuzz/_internal/metrics/profiler.py

@@ -13,14 +13,16 @@
 # limitations under the License.
 """Profiling functions."""
 from clusterfuzz._internal.base import utils
+from clusterfuzz._internal.config import local_config
 from clusterfuzz._internal.metrics import logs
 from clusterfuzz._internal.system import environment
 
 
 def start_if_needed(service):
-  """Start Google Cloud Profiler if |USE_PYTHON_PROFILER| environment variable
-  is set."""
-  if not environment.get_value('USE_PYTHON_PROFILER'):
+  """Start Google Cloud Profiler if profiling key in project config
+  is enabled."""
+  config = local_config.ProjectConfig()
+  if not config.get('profiling.enabled', False):
     return True
 
   project_id = utils.get_application_id()

src/local/butler/package.py

@@ -27,9 +27,9 @@
 MIN_SUPPORTED_NODEJS_VERSION = 4
 
 
-def _clear_zip(target_zip_path):
+def _clear_zip(target_zip_path, clear_manifest=True):
   """Remove zip and manifest file."""
-  if os.path.exists(constants.PACKAGE_TARGET_MANIFEST_PATH):
+  if clear_manifest and os.path.exists(constants.PACKAGE_TARGET_MANIFEST_PATH):
     os.remove(constants.PACKAGE_TARGET_MANIFEST_PATH)
 
   if os.path.exists(target_zip_path):
@@ -144,7 +144,7 @@ def package(revision,
       add_target_zip_name = utils.get_platform_deployment_filename(
           platform_name, release=add_release)
       add_target_zip_path = os.path.join(target_zip_dir, add_target_zip_name)
-      _clear_zip(add_target_zip_path)
+      _clear_zip(add_target_zip_path, clear_manifest=False)
       shutil.copy2(target_zip_path, add_target_zip_path)
       print('\n%s is ready.' % add_target_zip_path)
       targets_zip_paths.append(add_target_zip_path)