Remove uploads in Redis code

I changed the code in API to store uploads into GCS directly instead of redis in codecov/codecov-api#960.

Since then, the code to download from redis and write to GCS in worker became obsolete, so lets remove it.

Author: Swatinem

services/archive.py

@@ -86,8 +86,7 @@ def write_file(
     ) -> None:
         """
         Writes a generic file to the archive -- it's typically recommended to
-        not use this in lieu of the convenience methods write_raw_upload and
-        write_chunks
+        not use this in lieu of the convenience method `write_chunks`
         """
         self.storage.write_file(
             self.root,
@@ -97,29 +96,6 @@ def write_file(
             is_already_gzipped=is_already_gzipped,
         )
 
-    def write_raw_upload(
-        self, commit_sha, report_id, data, *, is_already_gzipped=False
-    ) -> str:
-        """
-        Convenience write method, writes a raw upload to a destination.
-        Returns the path it writes.
-        """
-        # create a custom report path for a raw upload.
-        # write the file.
-        path = "/".join(
-            (
-                "v4/raw",
-                self.get_now().strftime("%Y-%m-%d"),
-                self.storage_hash,
-                commit_sha,
-                "%s.txt" % report_id,
-            )
-        )
-
-        self.write_file(path, data, is_already_gzipped=is_already_gzipped)
-
-        return path
-
     def write_computed_comparison(self, comparison, data) -> str:
         path = MinioEndpoints.computed_comparison.get_path(
             version="v4", repo_hash=self.storage_hash, comparison_id=comparison.id

services/redis.py

@@ -1,6 +1,4 @@
 import logging
-import zlib
-from typing import Optional
 
 from redis import Redis
 from shared.config import get_config
@@ -24,15 +22,3 @@ def get_redis_connection() -> Redis:
 
 def _get_redis_instance_from_url(url) -> Redis:
     return Redis.from_url(url)
-
-
-def download_archive_from_redis(
-    redis_connection: Redis, redis_key: str
-) -> Optional[str]:
-    raw_uploaded_report = redis_connection.get(redis_key)
-    gzipped = redis_key.endswith("/gzip")
-    if gzipped:
-        raw_uploaded_report = zlib.decompress(raw_uploaded_report, zlib.MAX_WBITS | 16)
-    if raw_uploaded_report is not None:
-        return raw_uploaded_report.decode()
-    return None

tasks/tests/integration/test_upload_e2e.py

@@ -33,7 +33,8 @@ def write_raw_upload(
     upload_json: dict | None = None,
 ):
     report_id = uuid4().hex
-    written_path = archive_service.write_raw_upload(commitid, report_id, contents)
+    written_path = f"upload/{report_id}.txt"
+    archive_service.write_file(written_path, contents)
 
     upload_json = upload_json or {}
     upload_json.update({"reportid": report_id, "url": written_path})

tasks/upload.py

@@ -32,11 +32,10 @@
 from helpers.github_installation import get_installation_name_for_owner_for_task
 from helpers.save_commit_error import save_commit_error
 from rollouts import NEW_TA_TASKS
-from services.archive import ArchiveService
 from services.bundle_analysis.report import BundleAnalysisReportService
 from services.processing.state import ProcessingState
 from services.processing.types import UploadArguments
-from services.redis import download_archive_from_redis, get_redis_connection
+from services.redis import get_redis_connection
 from services.report import (
     BaseReportService,
     NotReadyToBuildReportYetError,
@@ -164,38 +163,14 @@ def arguments_list(self):
             for arg in arguments:
                 yield orjson.loads(arg)
 
-    def normalize_arguments(self, commit: Commit, arguments: UploadArguments):
-        """
-        Normalizes and validates the argument list from the user.
 
-        Does things like:
-
-            - replacing a redis-stored value with a storage one (by doing an upload)
-            - Removing unnecessary sensitive information for the arguments
-        """
-        commit_sha = commit.commitid
-        reportid = arguments.get("reportid")
-        if redis_key := arguments.pop("redis_key", None):
-            archive_service = ArchiveService(commit.repository)
-            content = download_archive_from_redis(self.redis_connection, redis_key)
-            written_path = archive_service.write_raw_upload(
-                commit_sha, reportid, content
-            )
-            log.info(
-                "Writing report content from redis to storage",
-                extra=dict(path=written_path),
-            )
-            arguments["url"] = written_path
-        arguments.pop("token", None)
-
-        flags: list | str | None = arguments.get("flags")
-        if not flags:
-            flags = []
-        elif isinstance(flags, str):
-            flags = [flag.strip() for flag in flags.split(",")]
-        arguments["flags"] = flags
-
-        return arguments
+def normalize_flags(arguments: UploadArguments):
+    flags: list | str | None = arguments.get("flags")
+    if not flags:
+        flags = []
+    elif isinstance(flags, str):
+        flags = [flag.strip() for flag in flags.split(",")]
+    arguments["flags"] = flags
 
 
 def _should_debounce_processing(upload_context: UploadContext) -> Optional[float]:
@@ -605,7 +580,9 @@ def _possibly_create_uploads_to_insert(
         upload_flag_map: dict[Upload, list | str | None] = {}
 
         for arguments in upload_context.arguments_list():
-            arguments = upload_context.normalize_arguments(commit, arguments)
+            arguments.pop("token", None)
+            normalize_flags(arguments)
+
             if "upload_id" not in arguments:
                 upload = report_service.create_report_upload(arguments, commit_report)
                 arguments["upload_id"] = upload.id_