Sign WEB_TESTS_URL in fuzzing preprocess and use HttpZipFile for faster download

- Add `web_tests_url` field to `FuzzTaskInput` proto.
- Sign `WEB_TESTS_URL` in `fuzz_task.py` preprocess and pass it to `utask_main`.
- In `utask_main`, call `update_tests_if_needed` with the signed URL.
- Update `update_tests_if_needed` in `update_task.py` to accept `tests_url`.
- Use `archive.HttpZipFile` for streaming extraction of the signed URL if compatible, avoiding large file download to disk.
- Recompile `uworker_msg.proto`.

Author: google-labs-jules[bot]

src/clusterfuzz/_internal/bot/tasks/update_task.py

@@ -288,14 +288,15 @@ def update_source_code():
             f'(release = {utils.get_clusterfuzz_release()}).')
 
 
-def update_tests_if_needed():
+def update_tests_if_needed(tests_url=None):
   """Updates layout tests every day."""
   data_directory = environment.get_value('FUZZ_DATA')
   error_occured = False
   expected_task_duration = 60 * 60  # 1 hour.
   retry_limit = environment.get_value('FAIL_RETRIES')
   temp_archive = os.path.join(data_directory, 'temp.zip')
-  tests_url = environment.get_value('WEB_TESTS_URL')
+  if not tests_url:
+    tests_url = environment.get_value('WEB_TESTS_URL')
 
   # Check if we have a valid tests url.
   if not tests_url:
@@ -323,10 +324,20 @@ def update_tests_if_needed():
   for _ in range(retry_limit):
     try:
       shell.remove_directory(data_directory, recreate=True)
-      storage.copy_file_from(tests_url, temp_archive)
-      with archive.open(temp_archive) as reader:
-        reader.extract_all(data_directory, trusted=True)
-      shell.remove_file(temp_archive)
+      if tests_url.startswith('http') and archive.HttpZipFile.is_uri_compatible(
+          tests_url):
+        with archive.ZipArchiveReader(archive.HttpZipFile(tests_url)) as reader:
+          reader.extract_all(data_directory, trusted=True)
+      else:
+        if tests_url.startswith('http'):
+          storage.download_signed_url_to_file(tests_url, temp_archive)
+        else:
+          storage.copy_file_from(tests_url, temp_archive)
+
+        with archive.open(temp_archive) as reader:
+          reader.extract_all(data_directory, trusted=True)
+        shell.remove_file(temp_archive)
+
       error_occured = False
       break
     except:

src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py

@@ -38,6 +38,7 @@
 from clusterfuzz._internal.bot.tasks import setup
 from clusterfuzz._internal.bot.tasks import task_creation
 from clusterfuzz._internal.bot.tasks import trials
+from clusterfuzz._internal.bot.tasks import update_task
 from clusterfuzz._internal.bot.tasks.utasks import fuzz_task_knobs
 from clusterfuzz._internal.bot.tasks.utasks import uworker_handle_errors
 from clusterfuzz._internal.bot.tasks.utasks import uworker_io
@@ -2097,6 +2098,10 @@ def _get_fuzz_target(uworker_input):
 
 def utask_main(uworker_input):
   """Runs the given fuzzer for one round."""
+  if uworker_input.fuzz_task_input.HasField('web_tests_url'):
+    update_task.update_tests_if_needed(
+        uworker_input.fuzz_task_input.web_tests_url)
+
   # Sets fuzzing logs context before running the fuzzer.
   fuzz_target = _get_fuzz_target(uworker_input)
   with logs.fuzzer_log_context(uworker_input.fuzzer_name,
@@ -2191,6 +2196,11 @@ def _utask_preprocess(fuzzer_name, job_type, uworker_env):
             use_backup=True).serialize())
 
   fuzz_task_input.trials.extend(trials.preprocess_get_db_trials())
+  web_tests_url = environment.get_value('WEB_TESTS_URL')
+  if web_tests_url:
+    fuzz_task_input.web_tests_url = storage.get_signed_download_url(
+        web_tests_url)
+
   for _ in range(MAX_CRASHES_UPLOADED):
     url = fuzz_task_input.crash_upload_urls.add()
     url.key = blobs.generate_new_blob_name()

src/clusterfuzz/_internal/protos/uworker_msg.proto

@@ -53,6 +53,7 @@ message FuzzTaskInput {
   repeated string global_blacklisted_functions = 6;
   repeated BlobUploadUrl crash_upload_urls = 7;
   repeated google.protobuf.Any trials = 8;
+  optional string web_tests_url = 9;
 }
 
 message DataBundleCorpus {