Merge branch 'master' into fastfast

Author: jonathanmetzman

src/clusterfuzz/_internal/cron/schedule_corpus_pruning.py

@@ -17,12 +17,13 @@
 from clusterfuzz._internal.base import utils
 from clusterfuzz._internal.datastore import data_types
 from clusterfuzz._internal.datastore import fuzz_target_utils
+from clusterfuzz._internal.datastore import ndb_utils
 from clusterfuzz._internal.metrics import logs
 
 
 def get_tasks_to_schedule():
   """Return (task_target, job_name, queue_name) arguments to schedule a task."""
-  for job in data_types.Job.query():
+  for job in ndb_utils.get_all_from_query(data_types.Job.query()):
     if not utils.string_is_true(job.get_environment().get('CORPUS_PRUNE')):
       continue
 

src/clusterfuzz/_internal/cron/schedule_fuzz.py

@@ -201,7 +201,7 @@ def schedule_fuzz_tasks() -> bool:
   project = batch_config.get('project')
   available_cpus = get_available_cpus(project, region)
   # TODO(metzman): Remove this as we move from experimental code to production.
-  available_cpus = min(available_cpus, 1000)
+  available_cpus = min(available_cpus, 2500)
   fuzz_tasks = get_fuzz_tasks(available_cpus)
   if not fuzz_tasks:
     logs.error('No fuzz tasks found to schedule.')

src/clusterfuzz/_internal/google_cloud_utils/storage.py

@@ -1352,6 +1352,11 @@ def _sign_urls_for_existing_file(
   return (download_url, delete_url)
 
 
+def _mappable_sign_urls_for_existing_file(url_and_include_delete_urls):
+  url, include_delete_urls = url_and_include_delete_urls
+  return _sign_urls_for_existing_file(url, include_delete_urls)
+
+
 def sign_urls_for_existing_files(urls,
                                  include_delete_urls) -> List[Tuple[str, str]]:
   logs.info('Signing URLs for existing files.')