Allow deployment to subset of bots (#3637)

Author: jonathanmetzman

src/clusterfuzz/_internal/bot/tasks/update_task.py

@@ -75,7 +75,9 @@ def _deployment_file_url(filename):
   if not deployment_bucket:
     return None
 
-  return 'gs://{bucket}/{name}'.format(bucket=deployment_bucket, name=filename)
+  if environment.get_value('USE_TEST_DEPLOYMENT'):
+    return f'gs://{deployment_bucket}/test-deployment/{filename}'
+  return f'gs://{deployment_bucket}/{filename}'
 
 
 def get_source_url():

src/local/butler/deploy.py

@@ -91,7 +91,8 @@ def _deploy_app_prod(project,
                      deployment_bucket,
                      yaml_paths,
                      package_zip_paths,
-                     deploy_appengine=True):
+                     deploy_appengine=True,
+                     test_deployment=False):
   """Deploy app in production."""
   if deploy_appengine:
     services = _get_services(yaml_paths)
@@ -109,9 +110,13 @@ def _deploy_app_prod(project,
 
   if package_zip_paths:
     for package_zip_path in package_zip_paths:
-      _deploy_zip(deployment_bucket, package_zip_path)
+      _deploy_zip(
+          deployment_bucket, package_zip_path, test_deployment=test_deployment)
 
-    _deploy_manifest(deployment_bucket, constants.PACKAGE_TARGET_MANIFEST_PATH)
+    _deploy_manifest(
+        deployment_bucket,
+        constants.PACKAGE_TARGET_MANIFEST_PATH,
+        test_deployment=test_deployment)
 
 
 def _deploy_app_staging(project, yaml_paths):
@@ -212,22 +217,31 @@ def find_file_exceeding_limit(path, limit):
   return None
 
 
-def _deploy_zip(bucket_name, zip_path):
+def _deploy_zip(bucket_name, zip_path, test_deployment=False):
   """Deploy zip to GCS."""
-  common.execute('gsutil cp %s gs://%s/%s' % (zip_path, bucket_name,
-                                              os.path.basename(zip_path)))
+  if test_deployment:
+    common.execute(f'gsutil cp {zip_path} gs://{bucket_name}/test-deployment/'
+                   f'{os.path.basename(zip_path)}')
+  else:
+    common.execute('gsutil cp %s gs://%s/%s' % (zip_path, bucket_name,
+                                                os.path.basename(zip_path)))
 
 
-def _deploy_manifest(bucket_name, manifest_path):
+def _deploy_manifest(bucket_name, manifest_path, test_deployment=False):
   """Deploy source manifest to GCS."""
   if sys.version_info.major == 3:
     manifest_suffix = '.3'
   else:
     manifest_suffix = ''
 
-  common.execute('gsutil cp %s '
-                 'gs://%s/clusterfuzz-source.manifest%s' %
-                 (manifest_path, bucket_name, manifest_suffix))
+  if test_deployment:
+    common.execute(f'gsutil cp {manifest_path} '
+                   f'gs://{bucket_name}/test-deployment/'
+                   f'clusterfuzz-source.manifest{manifest_suffix}')
+  else:
+    common.execute(f'gsutil cp {manifest_path} '
+                   f'gs://{bucket_name}/'
+                   f'clusterfuzz-source.manifest{manifest_suffix}')
 
 
 def _update_deployment_manager(project, name, config_path):
@@ -385,7 +399,8 @@ def _prod_deployment_helper(config_dir,
                             package_zip_paths,
                             deploy_appengine=True,
                             deploy_k8s=True,
-                            python3=True):
+                            python3=True,
+                            test_deployment=False):
   """Helper for production deployment."""
   config = local_config.Config()
   deployment_bucket = config.get('project.deployment.bucket')
@@ -415,11 +430,12 @@ def _prod_deployment_helper(config_dir,
       deployment_bucket,
       yaml_paths,
       package_zip_paths,
-      deploy_appengine=deploy_appengine)
+      deploy_appengine=deploy_appengine,
+      test_deployment=test_deployment)
 
   if deploy_appengine:
-    common.execute('python butler.py run setup --config-dir {config_dir} '
-                   '--non-dry-run'.format(config_dir=config_dir))
+    common.execute(
+        f'python butler.py run setup --config-dir {config_dir} --non-dry-run')
 
   if deploy_k8s:
     _deploy_terraform(config_dir)
@@ -515,6 +531,12 @@ def execute(args):
   deploy_zips = 'zips' in args.targets
   deploy_appengine = 'appengine' in args.targets
   deploy_k8s = 'k8s' in args.targets
+  test_deployment = 'test_deployment' in args.targets
+
+  if test_deployment:
+    deploy_appengine = False
+    deploy_k8s = False
+    deploy_zips = True
 
   is_python3 = sys.version_info.major == 3
   package_zip_paths = []
@@ -546,7 +568,8 @@ def execute(args):
         package_zip_paths,
         deploy_appengine,
         deploy_k8s,
-        python3=is_python3)
+        python3=is_python3,
+        test_deployment=test_deployment)
 
   with open(constants.PACKAGE_TARGET_MANIFEST_PATH) as f:
     print('Source updated to %s' % f.read())