Store v2 reports in GCS instead of Redis

Worker has some dedicated code to move an uploaded report from Redis over to GCS.
Instead of having that code, we can just store the report into GCS in the first place.

Additionally, this PR also cleans up the `services.archive`, replacing it fully with the `shared.api_archive.archive`.

It also forwards all the task arguments directly to the `Upload` task, in addition to storing it in Redis, to be forward compatible to some more planned upload processing improvements.

Author: Swatinem

graphql_api/dataloader/bundle_analysis.py

@@ -1,5 +1,6 @@
 from typing import Union
 
+from shared.api_archive.archive import ArchiveService
 from shared.bundle_analysis import (
     BundleAnalysisReportLoader,
     MissingBaseReportError,
@@ -10,7 +11,6 @@
 from core.models import Commit
 from graphql_api.types.comparison.comparison import MissingBaseReport, MissingHeadReport
 from reports.models import CommitReport
-from services.archive import ArchiveService
 from services.bundle_analysis import BundleAnalysisComparison, BundleAnalysisReport
 
 

graphql_api/tests/test_bundle_analysis_measurements.py

@@ -1,6 +1,7 @@
 from unittest.mock import patch
 
 from django.test import TransactionTestCase
+from shared.api_archive.archive import ArchiveService
 from shared.bundle_analysis import StoragePaths
 from shared.bundle_analysis.storage import get_bucket_name
 from shared.django_apps.core.tests.factories import (
@@ -11,10 +12,7 @@
 from shared.storage.memory import MemoryStorageService
 
 from reports.models import CommitReport
-from reports.tests.factories import (
-    CommitReportFactory,
-)
-from services.archive import ArchiveService
+from reports.tests.factories import CommitReportFactory
 from timeseries.tests.factories import MeasurementFactory
 
 from .helper import GraphQLTestHelper

graphql_api/tests/test_commit.py

@@ -6,6 +6,7 @@
 
 import yaml
 from django.test import TransactionTestCase
+from shared.api_archive.archive import ArchiveService
 from shared.bundle_analysis import StoragePaths
 from shared.bundle_analysis.storage import get_bucket_name
 from shared.django_apps.core.tests.factories import (
@@ -30,7 +31,6 @@
     UploadFactory,
     UploadFlagMembershipFactory,
 )
-from services.archive import ArchiveService
 from services.comparison import MissingComparisonReport
 from services.components import Component
 from services.profiling import CriticalFile

graphql_api/tests/test_pull.py

@@ -4,6 +4,7 @@
 
 from django.test import TransactionTestCase
 from freezegun import freeze_time
+from shared.api_archive.archive import ArchiveService
 from shared.bundle_analysis import StoragePaths
 from shared.bundle_analysis.storage import get_bucket_name
 from shared.django_apps.core.tests.factories import (
@@ -18,7 +19,6 @@
 from core.models import Commit
 from reports.models import CommitReport
 from reports.tests.factories import CommitReportFactory, ReportLevelTotalsFactory
-from services.archive import ArchiveService
 
 from .helper import GraphQLTestHelper, paginate_connection
 

profiling/serializers.py

@@ -1,7 +1,7 @@
 from rest_framework import serializers
+from shared.api_archive.archive import ArchiveService
 
 from profiling.models import ProfilingCommit, ProfilingUpload
-from services.archive import ArchiveService
 
 
 class CreatableProfilingCommitRelatedField(serializers.SlugRelatedField):

profiling/tests/test_views.py

@@ -2,20 +2,20 @@
 
 from django.urls import reverse
 from rest_framework.test import APIClient
+from shared.api_archive.archive import ArchiveService
 from shared.django_apps.core.tests.factories import (
     RepositoryFactory,
     RepositoryTokenFactory,
 )
 
 from profiling.models import ProfilingCommit, ProfilingUpload
-from services.archive import ArchiveService
 from services.task import TaskService
 
 
 def test_simple_profiling_apicall(db, mocker):
     mocked_call = mocker.patch.object(TaskService, "normalize_profiling_upload")
     mocker.patch(
-        "services.archive.StorageService.create_presigned_put",
+        "shared.api_archive.archive.StorageService.create_presigned_put",
         return_value="banana.txt",
     )
     repo = RepositoryFactory.create(active=True)

profiling/views.py

@@ -3,11 +3,11 @@
 
 from rest_framework.generics import CreateAPIView
 from rest_framework.permissions import BasePermission
+from shared.api_archive.archive import ArchiveService, MinioEndpoints
 
 from codecov_auth.authentication.repo_auth import RepositoryTokenAuthentication
 from profiling.models import ProfilingCommit
 from profiling.serializers import ProfilingCommitSerializer, ProfilingUploadSerializer
-from services.archive import ArchiveService, MinioEndpoints
 from services.task import TaskService
 
 log = logging.getLogger(__name__)

requirements.in

@@ -20,7 +20,7 @@ factory-boy
 fakeredis
 freezegun
 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
-https://github.com/codecov/shared/archive/ebafd805394f22b4ab9e1f6e9a864ab28396dfe4.tar.gz#egg=shared
+https://github.com/codecov/shared/archive/ce68890c789aa1bc187438e71db13de2d0110d1c.tar.gz#egg=shared
 google-cloud-pubsub
 gunicorn>=22.0.0
 https://github.com/photocrowd/django-cursor-pagination/archive/f560902696b0c8509e4d95c10ba0d62700181d84.tar.gz

requirements.txt

@@ -419,7 +419,7 @@ sentry-sdk[celery]==2.13.0
     #   shared
 setproctitle==1.1.10
     # via -r requirements.in
-shared @ https://github.com/codecov/shared/archive/ebafd805394f22b4ab9e1f6e9a864ab28396dfe4.tar.gz
+shared @ https://github.com/codecov/shared/archive/ce68890c789aa1bc187438e71db13de2d0110d1c.tar.gz
     # via -r requirements.in
 simplejson==3.17.2
     # via -r requirements.in