Use data bundle bucket name from the database. (#4661)

This allows data bundles to be moved, and to be stored in different
buckets than `${BUNDLE_NAME}-corpus.${BUCKET_DOMAIN}`.

Bug: https://crbug.com/387828386

Author: letitz

src/appengine/handlers/corpora.py

@@ -67,7 +67,6 @@ def post(self):
 
     user_email = helpers.get_user_email()
     bucket_name = data_handler.get_data_bundle_bucket_name(name)
-    bucket_url = data_handler.get_data_bundle_bucket_url(name)
 
     if not data_handler.create_data_bundle_bucket_and_iams(name, [user_email]):
       raise helpers.EarlyExitError('Failed to create bucket %s.' % bucket_name,
@@ -82,12 +81,13 @@ def post(self):
     data_bundle.bucket_name = bucket_name
     data_bundle.put()
 
+    bucket_url = data_bundle.bucket_url()
     template_values = {
         'title':
             'Success',
         'message': (
             'Upload data to the corpus using: '
-            'gsutil -d -m rsync -r <local_corpus_directory> %s' % bucket_url),
+            f'gsutil -d -m rsync -r <local_corpus_directory> {bucket_url}'),
     }
     return self.render('message.html', template_values)
 

src/clusterfuzz/_internal/bot/tasks/setup.py

@@ -741,8 +741,7 @@ def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
 
   # Check when the bucket url had last updates. If no new updates, no need to
   # update directory.
-  bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
-  last_updated_time = storage.last_updated(bucket_url)
+  last_updated_time = storage.last_updated(data_bundle.bucket_url())
   if last_updated_time and last_sync_time > last_updated_time:
     logs.info(
         'Data bundle %s has no new content from last sync.' % data_bundle.name)

src/clusterfuzz/_internal/datastore/data_handler.py

@@ -1029,11 +1029,6 @@ def get_data_bundle_bucket_name(data_bundle_name):
   return '%s-corpus.%s' % (data_bundle_name, domain)
 
 
-def get_data_bundle_bucket_url(data_bundle_name):
-  """Return data bundle bucket url on GCS."""
-  return 'gs://%s' % get_data_bundle_bucket_name(data_bundle_name)
-
-
 def get_value_from_fuzzer_environment_string(fuzzer_name,
                                              variable_pattern,
                                              default=None):

src/clusterfuzz/_internal/datastore/data_types.py

@@ -791,6 +791,10 @@ class DataBundle(Model):
   # where the libFuzzer binary will run (untrusted).
   sync_to_worker = ndb.BooleanProperty(default=False)
 
+  def bucket_url(self) -> str:
+    """Returns the GCS URL of the bucket storing this data bundle's contents."""
+    return f'gs://{self.bucket_name}'
+
 
 class Config(Model):
   """Configuration."""

src/clusterfuzz/_internal/fuzzing/corpus_manager.py

@@ -23,7 +23,6 @@
 from clusterfuzz._internal.base import utils
 from clusterfuzz._internal.bot.tasks import task_types
 from clusterfuzz._internal.bot.tasks.utasks import uworker_io
-from clusterfuzz._internal.datastore import data_handler
 from clusterfuzz._internal.google_cloud_utils import storage
 from clusterfuzz._internal.metrics import logs
 from clusterfuzz._internal.protos import uworker_msg_pb2
@@ -607,8 +606,7 @@ def get_proto_data_bundle_corpus(
   workers to download the data bundle files using the fastest means available to
   them."""
   data_bundle_corpus = uworker_msg_pb2.DataBundleCorpus()  # pylint: disable=no-member
-  data_bundle_corpus.gcs_url = data_handler.get_data_bundle_bucket_url(
-      data_bundle.name)
+  data_bundle_corpus.gcs_url = data_bundle.bucket_url()
   data_bundle_corpus.data_bundle.CopyFrom(
       uworker_io.entity_to_protobuf(data_bundle))
   if task_types.task_main_runs_on_uworker():

src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/corpus_pruning_task_test.py

@@ -454,8 +454,9 @@ def setUp(self):
     self.quarantine_corpus.rsync_from_disk(
         os.path.join(TEST_DIR, 'quarantine'), delete=True)
 
-    self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET
-    data_types.DataBundle(name='bundle', sync_to_worker=True).put()
+    data_types.DataBundle(
+        name='bundle', bundle_name=TEST_GLOBAL_BUCKET,
+        sync_to_worker=True).put()
 
     self.fuzzer = data_types.Fuzzer(
         revision=1,