Force unit tests to run before deployments (#4575)

Followup for the analyze/triage chrome incident

Author: vitorguidi

src/local/butler/deploy.py

@@ -29,6 +29,7 @@
 from local.butler import common
 from local.butler import constants
 from local.butler import package
+from local.butler import py_unittest
 from src.clusterfuzz._internal.base import utils
 from src.clusterfuzz._internal.config import local_config
 from src.clusterfuzz._internal.system import environment
@@ -457,6 +458,14 @@ def _is_safe_deploy_day():
   return day_now_in_ny not in {4, 5, 6}  # The days of the week are 0-indexed.
 
 
+def _enforce_tests_pass():
+  config = local_config.Config()
+  if not config.get('project.enforce_tests_before_deploy', False):
+    return
+  py_unittest.run_tests(target='core', parallel=True)
+  py_unittest.run_tests(target='appengine', parallel=True)
+
+
 def _enforce_safe_day_to_deploy():
   """Checks that is not an unsafe day (Friday, Saturday, or Sunday) to
   deploy for chrome ClusterFuzz."""
@@ -521,6 +530,7 @@ def execute(args):
     print('gsutil not found in PATH.')
     sys.exit(1)
 
+  _enforce_tests_pass()
   _enforce_safe_day_to_deploy()
 
   # Build templates before deployment.

src/local/butler/package.py

@@ -21,6 +21,7 @@
 from local.butler import appengine
 from local.butler import common
 from local.butler import constants
+from local.butler import py_unittest
 from src.clusterfuzz._internal.base import utils
 
 MIN_SUPPORTED_NODEJS_VERSION = 4
@@ -83,6 +84,8 @@ def package(revision,
     print('You do not have nodejs, or your nodejs is not at least version 4.')
     sys.exit(1)
 
+  py_unittest.execute(args={})
+
   common.install_dependencies(platform_name=platform_name)
 
   # This needs to be done before packaging step to let src/appengine/config be

src/local/butler/py_unittest.py

@@ -45,7 +45,7 @@ def __init__(self, *args, **kwargs):
     self.slow_tests = []
 
   def startTest(self, test):
-    self._start_time = time.time()
+    self._start_time = time.time()  # pylint: disable=attribute-defined-outside-init
     super().startTest(test)
 
   def addSuccess(self, test):
@@ -126,25 +126,27 @@ def run_one_test_parallel(args):
     raise
 
 
-def run_tests_single_core(args, test_directory, top_level_dir):
+def run_tests_single_core(pattern, unsuppress_output, test_directory,
+                          top_level_dir):
   """Run tests (single CPU)."""
   suites = unittest.loader.TestLoader().discover(
-      test_directory, pattern=args.pattern, top_level_dir=top_level_dir)
+      test_directory, pattern=pattern, top_level_dir=top_level_dir)
 
   # TODO(mbarbella): Re-implement code coverage after migrating to Python 3.
   # Verbosity=2 since we want to see real-time test execution with test name
   # and result.
   result = TrackedTestRunner(
-      verbosity=2, buffer=not args.unsuppress_output).run(suites)
+      verbosity=2, buffer=not unsuppress_output).run(suites)
 
   if result.errors or result.failures:
     sys.exit(1)
 
 
-def run_tests_parallel(args, test_directory, top_level_dir):
+def run_tests_parallel(pattern, unsuppress_output, test_directory,
+                       top_level_dir):
   """Run tests (multiple CPUs)."""
   suites = unittest.loader.TestLoader().discover(
-      test_directory, pattern=args.pattern, top_level_dir=top_level_dir)
+      test_directory, pattern=pattern, top_level_dir=top_level_dir)
 
   test_classes = []  # pylint: disable=protected-access
   for suite in suites:
@@ -181,7 +183,7 @@ def run_tests_parallel(args, test_directory, top_level_dir):
   tests_per_cpu = max(1, len(test_modules) // cpu_count)
   for i in range(0, len(test_modules), tests_per_cpu):
     group = test_modules[i:i + tests_per_cpu]
-    test_args.append((group, not args.unsuppress_output))
+    test_args.append((group, not unsuppress_output))
 
   results = pool.map_async(run_one_test_parallel, test_args)
 
@@ -215,7 +217,12 @@ def run_tests_parallel(args, test_directory, top_level_dir):
     sys.exit(1)
 
 
-def execute(args):
+def run_tests(target=None,
+              config_dir=None,
+              verbose=None,
+              pattern=None,
+              unsuppress_output=None,
+              parallel=None):
   """Run Python unit tests. For unittests involved appengine, sys.path needs
   certain modification."""
   os.environ['PY_UNITTESTS'] = 'True'
@@ -235,7 +242,7 @@ def execute(args):
   os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test')
 
   top_level_dir = os.path.join('src', 'clusterfuzz', '_internal')
-  if args.target == 'appengine':
+  if target == 'appengine':
     # Build template files.
     appengine.build_templates()
 
@@ -248,12 +255,12 @@ def execute(args):
         sys.path[i] = os.path.abspath(
             os.path.join('src', 'appengine', 'third_party'))
 
-  elif args.target == 'core':
+  elif target == 'core':
     test_directory = CORE_TEST_DIRECTORY
   else:
     # Config module tests.
-    os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir
-    test_directory = os.path.join(args.config_dir, 'modules')
+    os.environ['CONFIG_DIR_OVERRIDE'] = config_dir
+    test_directory = os.path.join(config_dir, 'modules')
     top_level_dir = None
 
     # Modules may use libs from our App Engine directory.
@@ -269,17 +276,29 @@ def execute(args):
   # Needed for NDB to work with cloud datastore emulator.
   os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true'
 
-  if args.verbose:
+  if verbose:
     # Force logging to console for this process and child processes.
     os.environ['LOG_TO_CONSOLE'] = 'True'
   else:
     # Disable logging.
     logging.disable(logging.CRITICAL)
 
-  if args.pattern is None:
-    args.pattern = '*_test.py'
+  if pattern is None:
+    pattern = '*_test.py'
 
-  if args.parallel:
-    run_tests_parallel(args, test_directory, top_level_dir)
+  if parallel:
+    run_tests_parallel(pattern, unsuppress_output, test_directory,
+                       top_level_dir)
   else:
-    run_tests_single_core(args, test_directory, top_level_dir)
+    run_tests_single_core(pattern, unsuppress_output, test_directory,
+                          top_level_dir)
+
+
+def execute(args):
+  run_tests(
+      target=args.target,
+      config_dir=args.config_dir,
+      verbose=args.verbose,
+      pattern=args.pattern,
+      unsuppress_output=args.unsuppress_output,
+      parallel=args.parallel)