Merge branch 'master' into test_remove_fluentd

vitaliset · web-flow · commit 7a27d778ad4c · 2025-01-21T15:15:40.000-03:00
diff --git a/.github/workflows/staleness.yml b/.github/workflows/staleness.yml
@@ -0,0 +1,36 @@
+name: "Close stale issues and PRs"
+
+permissions: read-all
+
+on:
+  schedule:
+    - cron: "0 * * * *"
+
+jobs:
+  stale:
+    permissions:
+      issues: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
+        with:
+          days-before-stale: 60
+          days-before-close: 14
+          operations-per-run: 100
+          remove-stale-when-updated: true
+          exempt-issue-labels: "good first issue,backlog"
+          exempt-all-assignees: true
+          ignore-updates: false
+          stale-issue-label: stale
+          stale-issue-message: |
+            This issue has not had any activity for 60 days and will be automatically closed in two weeks
+          stale-pr-label: stale
+          stale-pr-message: |
+            This pull request has not had any activity for 60 days and will be automatically closed in two weeks
+          close-issue-label: "autoclosed"
+          close-issue-message: |
+            Automatically closing stale issue
+          close-pr-label: "autoclosed"
+          close-pr-message: |
+            Automatically closing stale pull request
diff --git a/src/appengine/handlers/upload_testcase.py b/src/appengine/handlers/upload_testcase.py
@@ -392,7 +392,9 @@ def do_post(self):
     trusted_agreement_signed = request.get(
         'trustedAgreement') == TRUSTED_AGREEMENT_TEXT.strip()
 
-    if (not trusted_agreement_signed and
+    # Chrome is the only ClusterFuzz deployment where there are trusted bots running utasks.
+    # This check also fails on oss-fuzz because of the way it abuses platform.
+    if (not trusted_agreement_signed and utils.is_chromium() and        
         task_utils.is_remotely_executing_utasks() and
         ((platform_id and platform_id != 'Linux') or
          job.platform.lower() != 'linux')):
diff --git a/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py b/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py
@@ -313,6 +313,11 @@ def fuzz(self, target_path, options, reproducers_dir, max_time):  # pylint: disa
         'actual_duration': actual_duration,
         'fuzzing_time_percent': fuzzing_time_percent,
     })
+    fuzz_time_secs_avg = stats.get('FuzzTimeSec_Avg', 1.0)
+    if fuzz_time_secs_avg == 0.0:
+      fuzz_time_secs_avg = 1.0
+    num_execs_avg = stats.get('NumExecs_Avg', 0.0)
+    stats['average_exec_per_sec'] = num_execs_avg / fuzz_time_secs_avg
     stats.update(_parse_centipede_logs(log_lines))
     return engine.FuzzResult(fuzz_result.output, fuzz_result.command, crashes,
                              stats, fuzz_result.time_executed)
diff --git a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/fuzzer.py b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/fuzzer.py
@@ -42,7 +42,7 @@ def get_arguments(fuzzer_path) -> options.FuzzerArguments:
     rss_limit_mb = arguments.get('rss_limit_mb', constructor=int)
     timeout = arguments.get('timeout', constructor=int)
 
-  if timeout is None or timeout > constants.DEFAULT_TIMEOUT_LIMIT:
+  if timeout is None:
     arguments[constants.TIMEOUT_FLAGNAME] = constants.DEFAULT_TIMEOUT_LIMIT
 
   if not rss_limit_mb and (utils.is_chromium() or
diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/minimize_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/minimize_task.py
@@ -1370,9 +1370,9 @@ def do_js_minimization(test_function, get_temp_file, data, deadline, threads,
 
 
 def _run_libfuzzer_testcase(fuzz_target,
-                            testcase,
-                            testcase_file_path,
-                            crash_retries=1):
+                            testcase: data_types.Testcase,
+                            testcase_file_path: str,
+                            crash_retries: int = 1) -> CrashResult:
   """Run libFuzzer testcase, and return the CrashResult."""
   # Cleanup any existing application instances and temp directories.
   process_handler.cleanup_stale_processes()
@@ -1429,8 +1429,16 @@ def _run_libfuzzer_tool(
     expected_crash_state: str,
     minimize_task_input: uworker_msg_pb2.MinimizeTaskInput,  # pylint: disable=no-member
     fuzz_target: Optional[data_types.FuzzTarget],
-    set_dedup_flags: bool = False):
-  """Run libFuzzer tool to either minimize or cleanse."""
+    set_dedup_flags: bool = False
+) -> tuple[str, CrashResult, str] | tuple[None, None, None]:
+  """Run libFuzzer tool to either minimize or cleanse.
+
+  Returns (None, None, None) in case of failure.
+  Otherwise sets `testcase.minimized_keys` and returns:
+
+    (testcase_file_path, crash_result, minimized_keys)
+
+  """
   memory_tool_options_var = environment.get_current_memory_tool_var()
   saved_memory_tool_options = environment.get_value(memory_tool_options_var)
 
@@ -1565,8 +1573,6 @@ def do_libfuzzer_minimization(
   """Use libFuzzer's built-in minimizer where appropriate."""
   timeout = environment.get_value('LIBFUZZER_MINIMIZATION_TIMEOUT', 600)
   rounds = environment.get_value('LIBFUZZER_MINIMIZATION_ROUNDS', 5)
-  current_testcase_path = testcase_file_path
-  last_crash_result = None
 
   # Get initial crash state.
   initial_crash_result = _run_libfuzzer_testcase(
@@ -1637,7 +1643,10 @@ def do_libfuzzer_minimization(
   if env:
     testcase.set_metadata('env', env, False)
 
-  minimized_keys = None
+  current_testcase_path = testcase_file_path
+  last_crash_result = None
+  last_minimized_keys = None
+
   # We attempt minimization multiple times in case one round results in an
   # incorrect state, or runs into another issue such as a slow unit.
   for round_number in range(1, rounds + 1):
@@ -1653,6 +1662,7 @@ def do_libfuzzer_minimization(
         set_dedup_flags=True)
     if output_file_path:
       last_crash_result = crash_result
+      last_minimized_keys = minimized_keys
       current_testcase_path = output_file_path
 
   if not last_crash_result:
@@ -1663,8 +1673,8 @@ def do_libfuzzer_minimization(
     minimize_task_output = uworker_msg_pb2.MinimizeTaskOutput(  # pylint: disable=no-member
         last_crash_result_dict=crash_result_dict,
         memory_tool_options=memory_tool_options)
-    if minimized_keys:
-      minimize_task_output.minimized_keys = str(minimized_keys)
+    if last_minimized_keys:
+      minimize_task_output.minimized_keys = str(last_minimized_keys)
     return uworker_msg_pb2.Output(  # pylint: disable=no-member
         error_type=uworker_msg_pb2.ErrorType.LIBFUZZER_MINIMIZATION_FAILED,  # pylint: disable=no-member
         minimize_task_output=minimize_task_output)
@@ -1673,7 +1683,7 @@ def do_libfuzzer_minimization(
 
   if utils.is_oss_fuzz():
     # Scrub the testcase of non-essential data.
-    cleansed_testcase_path, minimized_keys = do_libfuzzer_cleanse(
+    cleansed_testcase_path, last_minimized_keys = do_libfuzzer_cleanse(
         fuzz_target, testcase, current_testcase_path,
         expected_state.crash_state, minimize_task_input)
     if cleansed_testcase_path:
@@ -1690,8 +1700,8 @@ def do_libfuzzer_minimization(
   minimize_task_output = uworker_msg_pb2.MinimizeTaskOutput(  # pylint: disable=no-member
       last_crash_result_dict=last_crash_result_dict,
       memory_tool_options=memory_tool_options)
-  if minimized_keys:
-    minimize_task_output.minimized_keys = str(minimized_keys)
+  if last_minimized_keys:
+    minimize_task_output.minimized_keys = str(last_minimized_keys)
   return uworker_msg_pb2.Output(minimize_task_output=minimize_task_output)  # pylint: disable=no-member
 
 
diff --git a/src/clusterfuzz/_internal/bot/testcase_manager.py b/src/clusterfuzz/_internal/bot/testcase_manager.py
@@ -673,10 +673,10 @@ def _get_crash_state(self, round_number: int,
     return state
 
   def reproduce_with_retries(self,
-                             retries,
-                             expected_state=None,
-                             expected_security_flag=None,
-                             flaky_stacktrace=False):
+                             retries: int,
+                             expected_state: CrashInfo | None = None,
+                             expected_security_flag: bool | None = None,
+                             flaky_stacktrace: bool = False) -> CrashResult:
     """Try reproducing a crash with retries."""
     self._pre_run_cleanup()
     crash_result = None
@@ -775,7 +775,7 @@ def test_for_crash_with_retries(fuzz_target,
                                 http_flag=False,
                                 use_gestures=True,
                                 compare_crash=True,
-                                crash_retries=None):
+                                crash_retries=None) -> CrashResult:
   """Test for a crash and return crash parameters like crash type, crash state,
   crash stacktrace, etc."""
   logs.info('Testing for crash.')
diff --git a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py
@@ -1146,7 +1146,7 @@ def get_attachment(self, resource_name):
 
   def find_issues(self, keywords=None, only_open=None):
     """Finds issues."""
-    return self.find_issues_with_filters(keywords, only_open)
+    return self.find_issues_with_filters(keywords=keywords, only_open=only_open)
 
   def find_issues_with_filters(self,
                                keywords=None,
diff --git a/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py b/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py
@@ -851,6 +851,10 @@
     'mode': 'NULLABLE',
     'name': 'fuzzing_time_percent',
     'type': 'FLOAT'
+}, {
+    'mode': 'NULLABLE',
+    'name': 'average_exec_per_sec',
+    'type': 'FLOAT'
 }] + _COMMON_COLUMNS
 
 _SCHEMA = {
diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py
@@ -29,7 +29,6 @@
 from local.butler import common
 from local.butler import constants
 from local.butler import package
-from local.butler import py_unittest
 from src.clusterfuzz._internal.base import utils
 from src.clusterfuzz._internal.config import local_config
 from src.clusterfuzz._internal.system import environment
@@ -458,14 +457,6 @@ def _is_safe_deploy_day():
   return day_now_in_ny not in {4, 5, 6}  # The days of the week are 0-indexed.
 
 
-def _enforce_tests_pass():
-  config = local_config.Config()
-  if not config.get('project.enforce_tests_before_deploy', False):
-    return
-  py_unittest.run_tests(target='core', parallel=True)
-  py_unittest.run_tests(target='appengine', parallel=True)
-
-
 def _enforce_safe_day_to_deploy():
   """Checks that is not an unsafe day (Friday, Saturday, or Sunday) to
   deploy for chrome ClusterFuzz."""
@@ -530,7 +521,6 @@ def execute(args):
     print('gsutil not found in PATH.')
     sys.exit(1)
 
-  _enforce_tests_pass()
   _enforce_safe_day_to_deploy()
 
   # Build templates before deployment.
diff --git a/src/local/butler/package.py b/src/local/butler/package.py
@@ -21,7 +21,6 @@
 from local.butler import appengine
 from local.butler import common
 from local.butler import constants
-from local.butler import py_unittest
 from src.clusterfuzz._internal.base import utils
 
 MIN_SUPPORTED_NODEJS_VERSION = 4
@@ -84,8 +83,6 @@ def package(revision,
     print('You do not have nodejs, or your nodejs is not at least version 4.')
     sys.exit(1)
 
-  py_unittest.execute(args={})
-
   common.install_dependencies(platform_name=platform_name)
 
   # This needs to be done before packaging step to let src/appengine/config be
diff --git a/src/local/butler/py_unittest.py b/src/local/butler/py_unittest.py
@@ -126,27 +126,25 @@ def run_one_test_parallel(args):
     raise
 
 
-def run_tests_single_core(pattern, unsuppress_output, test_directory,
-                          top_level_dir):
+def run_tests_single_core(args, test_directory, top_level_dir):
   """Run tests (single CPU)."""
   suites = unittest.loader.TestLoader().discover(
-      test_directory, pattern=pattern, top_level_dir=top_level_dir)
+      test_directory, pattern=args.pattern, top_level_dir=top_level_dir)
 
   # TODO(mbarbella): Re-implement code coverage after migrating to Python 3.
   # Verbosity=2 since we want to see real-time test execution with test name
   # and result.
   result = TrackedTestRunner(
-      verbosity=2, buffer=not unsuppress_output).run(suites)
+      verbosity=2, buffer=not args.unsuppress_output).run(suites)
 
   if result.errors or result.failures:
     sys.exit(1)
 
 
-def run_tests_parallel(pattern, unsuppress_output, test_directory,
-                       top_level_dir):
+def run_tests_parallel(args, test_directory, top_level_dir):
   """Run tests (multiple CPUs)."""
   suites = unittest.loader.TestLoader().discover(
-      test_directory, pattern=pattern, top_level_dir=top_level_dir)
+      test_directory, pattern=args.pattern, top_level_dir=top_level_dir)
 
   test_classes = []  # pylint: disable=protected-access
   for suite in suites:
@@ -183,7 +181,7 @@ def run_tests_parallel(pattern, unsuppress_output, test_directory,
   tests_per_cpu = max(1, len(test_modules) // cpu_count)
   for i in range(0, len(test_modules), tests_per_cpu):
     group = test_modules[i:i + tests_per_cpu]
-    test_args.append((group, not unsuppress_output))
+    test_args.append((group, not args.unsuppress_output))
 
   results = pool.map_async(run_one_test_parallel, test_args)
 
@@ -217,12 +215,7 @@ def run_tests_parallel(pattern, unsuppress_output, test_directory,
     sys.exit(1)
 
 
-def run_tests(target=None,
-              config_dir=None,
-              verbose=None,
-              pattern=None,
-              unsuppress_output=None,
-              parallel=None):
+def execute(args):
   """Run Python unit tests. For unittests involved appengine, sys.path needs
   certain modification."""
   os.environ['PY_UNITTESTS'] = 'True'
@@ -242,7 +235,7 @@ def run_tests(target=None,
   os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test')
 
   top_level_dir = os.path.join('src', 'clusterfuzz', '_internal')
-  if target == 'appengine':
+  if args.target == 'appengine':
     # Build template files.
     appengine.build_templates()
 
@@ -255,12 +248,12 @@ def run_tests(target=None,
         sys.path[i] = os.path.abspath(
             os.path.join('src', 'appengine', 'third_party'))
 
-  elif target == 'core':
+  elif args.target == 'core':
     test_directory = CORE_TEST_DIRECTORY
   else:
     # Config module tests.
-    os.environ['CONFIG_DIR_OVERRIDE'] = config_dir
-    test_directory = os.path.join(config_dir, 'modules')
+    os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir
+    test_directory = os.path.join(args.config_dir, 'modules')
     top_level_dir = None
 
     # Modules may use libs from our App Engine directory.
@@ -276,29 +269,17 @@ def run_tests(target=None,
   # Needed for NDB to work with cloud datastore emulator.
   os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true'
 
-  if verbose:
+  if args.verbose:
     # Force logging to console for this process and child processes.
     os.environ['LOG_TO_CONSOLE'] = 'True'
   else:
     # Disable logging.
     logging.disable(logging.CRITICAL)
 
-  if pattern is None:
-    pattern = '*_test.py'
+  if args.pattern is None:
+    args.pattern = '*_test.py'
 
-  if parallel:
-    run_tests_parallel(pattern, unsuppress_output, test_directory,
-                       top_level_dir)
+  if args.parallel:
+    run_tests_parallel(args, test_directory, top_level_dir)
   else:
-    run_tests_single_core(pattern, unsuppress_output, test_directory,
-                          top_level_dir)
-
-
-def execute(args):
-  run_tests(
-      target=args.target,
-      config_dir=args.config_dir,
-      verbose=args.verbose,
-      pattern=args.pattern,
-      unsuppress_output=args.unsuppress_output,
-      parallel=args.parallel)
+    run_tests_single_core(args, test_directory, top_level_dir)
diff --git a/src/local/butler/scripts/setup.py b/src/local/butler/scripts/setup.py
@@ -300,12 +300,12 @@ def __init__(self):
 logs: "Link to fuzzing logs"
 corpus_backup: "Backup copy of the minimized corpus generated based on code coverage"'''
 
-    self.stats_columns = """sum(t.number_of_executed_units) as tests_executed,
+    self.stats_columns = """sum(t.NumExecs_Max) as tests_executed,
 custom(j.new_crashes) as new_crashes,
 _EDGE_COV as edge_coverage,
 _COV_REPORT as cov_report,
 _CORPUS_SIZE as corpus_size,
-avg(t.FuzzTimeSec_Avg / t.NumExecs_Avg) as avg_exec_per_sec,
+avg(t.average_exec_per_sec) as avg_exec_per_sec,
 avg(t.fuzzing_time_percent) as fuzzing_time_percent,
 avg(t.crash_count*100) as regular_crash_percent,
 avg(t.oom_count*100) as oom_percent,
