Optimize WEB_TESTS_URL download and extraction

- Add `web_tests_url` to `FuzzTaskInput` proto and recompile.
- Sign `WEB_TESTS_URL` in `fuzz_task` preprocess.
- In `utask_main`, if blackbox fuzzing, pass signed URL to `update_tests_if_needed`.
- In `update_tests_if_needed`, handle signed URL (downloading to file if http).
- Use `unzip -q` for extraction if available for speed, falling back to `archive` module.
- Refactor defensive checks in `fuzz_task`.

Author: google-labs-jules[bot]

src/clusterfuzz/_internal/bot/tasks/update_task.py

@@ -16,6 +16,7 @@
 import datetime
 import os
 import platform
+import shutil
 import sys
 import time
 
@@ -324,20 +325,18 @@ def update_tests_if_needed(tests_url=None):
   for _ in range(retry_limit):
     try:
       shell.remove_directory(data_directory, recreate=True)
-      if tests_url.startswith('http') and archive.HttpZipFile.is_uri_compatible(
-          tests_url):
-        with archive.ZipArchiveReader(archive.HttpZipFile(tests_url)) as reader:
-          reader.extract_all(data_directory, trusted=True)
+      if tests_url.startswith('http'):
+        storage.download_signed_url_to_file(tests_url, temp_archive)
       else:
-        if tests_url.startswith('http'):
-          storage.download_signed_url_to_file(tests_url, temp_archive)
-        else:
-          storage.copy_file_from(tests_url, temp_archive)
+        storage.copy_file_from(tests_url, temp_archive)
 
+      if shutil.which('unzip'):
+        shell.execute_command(
+            'unzip -q -o %s -d %s' % (temp_archive, data_directory))
+      else:
         with archive.open(temp_archive) as reader:
           reader.extract_all(data_directory, trusted=True)
-        shell.remove_file(temp_archive)
-
+      shell.remove_file(temp_archive)
       error_occured = False
       break
     except:

src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py

@@ -2098,7 +2098,7 @@ def _get_fuzz_target(uworker_input):
 
 def utask_main(uworker_input):
   """Runs the given fuzzer for one round."""
-  if uworker_input.fuzz_task_input.HasField('web_tests_url'):
+  if not engine.get(uworker_input.fuzzer_name):
     update_task.update_tests_if_needed(
         uworker_input.fuzz_task_input.web_tests_url)