Do safe preprocess optimizations. (#4444)

1. Don't unnecessarily list features about blobs other than names
2. Memoize leak blacklist stuff.
3. Properly skip cleanup.

Author: jonathanmetzman

src/clusterfuzz/_internal/bot/tasks/commands.py

@@ -80,6 +80,8 @@ class AlreadyRunningError(Error):
 def cleanup_task_state():
   """Cleans state before and after a task is executed."""
   # Cleanup stale processes.
+  if not environment.is_tworker():
+    return
   process_handler.cleanup_stale_processes()
 
   # Clear build urls, temp and testcase directories.
@@ -468,7 +470,6 @@ def process_command_impl(task_name, task_argument, job_name, high_end,
     return run_command(task_name, task_argument, job_name, uworker_env)
   finally:
     # Final clean up.
-    if not environment.is_tworker():
-      cleanup_task_state()
+    cleanup_task_state()
     if 'CF_TASK_ID' in os.environ:
       del os.environ['CF_TASK_ID']

src/clusterfuzz/_internal/fuzzing/leak_blacklist.py

@@ -17,6 +17,7 @@
 import re
 
 from clusterfuzz._internal.base import errors
+from clusterfuzz._internal.base import memoize
 from clusterfuzz._internal.datastore import data_handler
 from clusterfuzz._internal.datastore import data_types
 from clusterfuzz._internal.datastore import ndb_utils
@@ -65,7 +66,9 @@ def cleanup_global_blacklist():
   ndb_utils.delete_multi(blacklists_to_delete)
 
 
+@memoize.wrap(memoize.Memcache(60 * 10))
 def get_global_blacklisted_functions():
+  """Gets global blacklisted functions."""
   # Copy global blacklist into local blacklist.
   global_blacklists = data_types.Blacklist.query(
       data_types.Blacklist.tool_name == LSAN_TOOL_NAME)

src/clusterfuzz/_internal/google_cloud_utils/storage.py

@@ -125,7 +125,7 @@ def get_bucket(self, name):
     """Get a bucket."""
     raise NotImplementedError
 
-  def list_blobs(self, remote_path, recursive=True):
+  def list_blobs(self, remote_path, recursive=True, names_only=False):
     """List the blobs under the remote path."""
     raise NotImplementedError
 
@@ -228,7 +228,7 @@ def get_bucket(self, name):
 
       raise
 
-  def list_blobs(self, remote_path, recursive=True):
+  def list_blobs(self, remote_path, recursive=True, names_only=False):
     """List the blobs under the remote path."""
     bucket_name, path = get_bucket_name_and_path(remote_path)
 
@@ -244,7 +244,13 @@ def list_blobs(self, remote_path, recursive=True):
     else:
       delimiter = '/'
 
-    iterator = bucket.list_blobs(prefix=path, delimiter=delimiter)
+    if names_only:
+      fields = 'items(name),nextPageToken'
+    else:
+      fields = None
+
+    iterator = bucket.list_blobs(
+        prefix=path, delimiter=delimiter, fields=fields)
     for blob in iterator:
       properties['bucket'] = bucket_name
       properties['name'] = blob.name
@@ -567,8 +573,9 @@ def _list_files_nonrecursive(self, fs_path):
     for filename in os.listdir(fs_path):
       yield os.path.join(fs_path, filename)
 
-  def list_blobs(self, remote_path, recursive=True):
+  def list_blobs(self, remote_path, recursive=True, names_only=False):
     """List the blobs under the remote path."""
+    del names_only
     bucket, _ = get_bucket_name_and_path(remote_path)
     fs_path = self.convert_path(remote_path)
 
@@ -1070,7 +1077,8 @@ def get_blobs(cloud_storage_path, recursive=True):
     exception_types=_TRANSIENT_ERRORS)
 def list_blobs(cloud_storage_path, recursive=True):
   """Return blob names under the given cloud storage path."""
-  for blob in _provider().list_blobs(cloud_storage_path, recursive=recursive):
+  for blob in _provider().list_blobs(
+      cloud_storage_path, recursive=recursive, names_only=True):
     yield blob['name']