feat(release-files): Store multiple release archives (#26529)

Instead of merging multiple archive uploads for one release, store all
uploaded archives as-is and create an artifact index to keep track of
the location of each individual file.

In order to hide the archives and index file from the user, use the new
`public_objects` manager when querying user-facing release files.

Author: jjbayer

src/sentry/api/endpoints/debug_files.py

@@ -429,13 +429,13 @@ def expose_release(release, count):
 
         def serialize_results(results):
             file_counts = (
-                Release.objects.filter(id__in=[r["id"] for r in results])
-                .annotate(count=Count("releasefile"))
-                .values("count", "id")
+                ReleaseFile.public_objects.filter(release_id__in=[r["id"] for r in results])
+                .values("release_id")
+                .annotate(count=Count("id"))
             )
-            file_count_map = {r["id"]: r["count"] for r in file_counts}
+            file_count_map = {r["release_id"]: r["count"] for r in file_counts}
             return serialize(
-                [expose_release(r, file_count_map[r["id"]]) for r in results], request.user
+                [expose_release(r, file_count_map.get(r["id"], 0)) for r in results], request.user
             )
 
         return self.paginate(

src/sentry/api/endpoints/organization_release_file_details.py

@@ -52,7 +52,7 @@ def get(self, request, organization, version, file_id):
             raise ResourceDoesNotExist
 
         try:
-            releasefile = ReleaseFile.objects.get(release=release, id=file_id)
+            releasefile = ReleaseFile.public_objects.get(release=release, id=file_id)
         except ReleaseFile.DoesNotExist:
             raise ResourceDoesNotExist
 
@@ -88,7 +88,7 @@ def put(self, request, organization, version, file_id):
             raise ResourceDoesNotExist
 
         try:
-            releasefile = ReleaseFile.objects.get(release=release, id=file_id)
+            releasefile = ReleaseFile.public_objects.get(release=release, id=file_id)
         except ReleaseFile.DoesNotExist:
             raise ResourceDoesNotExist
 
@@ -127,7 +127,7 @@ def delete(self, request, organization, version, file_id):
             raise ResourceDoesNotExist
 
         try:
-            releasefile = ReleaseFile.objects.get(release=release, id=file_id)
+            releasefile = ReleaseFile.public_objects.get(release=release, id=file_id)
         except ReleaseFile.DoesNotExist:
             raise ResourceDoesNotExist
 

src/sentry/api/endpoints/organization_release_files.py

@@ -57,7 +57,9 @@ def get(self, request, organization, version):
             raise ResourceDoesNotExist
 
         file_list = (
-            ReleaseFile.objects.filter(release=release).select_related("file").order_by("name")
+            ReleaseFile.public_objects.filter(release=release)
+            .select_related("file")
+            .order_by("name")
         )
 
         return self.paginate(

src/sentry/api/endpoints/organization_release_meta.py

@@ -77,7 +77,7 @@ def get(self, request, organization, version):
             for pr in project_releases
         ]
 
-        release_file_count = ReleaseFile.objects.filter(release=release).count()
+        release_file_count = ReleaseFile.public_objects.filter(release=release).count()
 
         return Response(
             {

src/sentry/api/endpoints/project_release_file_details.py

@@ -56,7 +56,7 @@ def get(self, request, project, version, file_id):
             raise ResourceDoesNotExist
 
         try:
-            releasefile = ReleaseFile.objects.get(release=release, id=file_id)
+            releasefile = ReleaseFile.public_objects.get(release=release, id=file_id)
         except ReleaseFile.DoesNotExist:
             raise ResourceDoesNotExist
 
@@ -92,7 +92,7 @@ def put(self, request, project, version, file_id):
             raise ResourceDoesNotExist
 
         try:
-            releasefile = ReleaseFile.objects.get(release=release, id=file_id)
+            releasefile = ReleaseFile.public_objects.get(release=release, id=file_id)
         except ReleaseFile.DoesNotExist:
             raise ResourceDoesNotExist
 
@@ -132,7 +132,7 @@ def delete(self, request, project, version, file_id):
             raise ResourceDoesNotExist
 
         try:
-            releasefile = ReleaseFile.objects.get(release=release, id=file_id)
+            releasefile = ReleaseFile.public_objects.get(release=release, id=file_id)
         except ReleaseFile.DoesNotExist:
             raise ResourceDoesNotExist
 

src/sentry/api/endpoints/project_release_files.py

@@ -45,7 +45,9 @@ def get(self, request, project, version):
             raise ResourceDoesNotExist
 
         file_list = (
-            ReleaseFile.objects.filter(release=release).select_related("file").order_by("name")
+            ReleaseFile.public_objects.filter(release=release)
+            .select_related("file")
+            .order_by("name")
         )
 
         if query:

src/sentry/lang/javascript/processor.py

@@ -5,8 +5,8 @@
 
 from django.utils.encoding import force_bytes, force_text
 
-from sentry.models.releasefile import ReleaseArchive
-from sentry.tasks.assemble import RELEASE_ARCHIVE_FILENAME
+from sentry.models.releasefile import ARTIFACT_INDEX_FILENAME, ReleaseArchive, read_artifact_index
+from sentry.utils import json
 
 __all__ = ["JavaScriptStacktraceProcessor"]
 
@@ -392,17 +392,58 @@ def get_from_archive(url: str, archive: ReleaseArchive) -> Tuple[bytes, dict]:
     raise KeyError(f"Not found in archive: '{url}'")
 
 
+@metrics.wraps("sourcemaps.load_artifact_index")
+def get_artifact_index(release, dist):
+    dist_name = dist and dist.name or None
+
+    ident = ReleaseFile.get_ident(ARTIFACT_INDEX_FILENAME, dist_name)
+    cache_key = f"artifact-index:v1:{release.id}:{ident}"
+    result = cache.get(cache_key)
+    if result == -1:
+        index = None
+    elif result:
+        index = json.loads(result)
+    else:
+        index = read_artifact_index(release, dist)
+        cache_value = -1 if index is None else json.dumps(index)
+        # Only cache for a short time to keep the manifest up-to-date
+        cache.set(cache_key, cache_value, timeout=60)
+
+    return index
+
+
+def get_index_entry(release, dist, url) -> Optional[dict]:
+    index = get_artifact_index(release, dist)
+    if index:
+        for candidate in ReleaseFile.normalize(url):
+            entry = index.get("files", {}).get(candidate)
+            if entry:
+                return entry
+
+    return None
+
+
 @metrics.wraps("sourcemaps.fetch_release_archive")
-def fetch_release_archive(release, dist) -> Optional[IO]:
+def fetch_release_archive_for_url(release, dist, url) -> Optional[IO]:
     """Fetch release archive and cache if possible.
 
+    Multiple archives might have been uploaded, so we need the URL
+    to get the correct archive from the artifact index.
+
     If return value is not empty, the caller is responsible for closing the stream.
     """
-    dist_name = dist and dist.name or None
-    releasefile_ident = ReleaseFile.get_ident(RELEASE_ARCHIVE_FILENAME, dist_name)
-    cache_key = get_release_file_cache_key(
-        release_id=release.id, releasefile_ident=releasefile_ident
-    )
+    info = get_index_entry(release, dist, url)
+    if info is None:
+        # Cannot write negative cache entry here because ID of release archive
+        # is not yet known
+        return None
+
+    archive_ident = info["archive_ident"]
+
+    # TODO(jjbayer): Could already extract filename from info and return
+    # it later
+
+    cache_key = get_release_file_cache_key(release_id=release.id, releasefile_ident=archive_ident)
 
     result = cache.get(cache_key)
 
@@ -412,11 +453,13 @@ def fetch_release_archive(release, dist) -> Optional[IO]:
         return BytesIO(result)
     else:
         qs = ReleaseFile.objects.filter(
-            release=release, dist=dist, ident=releasefile_ident
+            release=release, dist=dist, ident=archive_ident
         ).select_related("file")
         try:
             releasefile = qs[0]
         except IndexError:
+            # This should not happen when there is an archive_ident in the manifest
+            logger.error("sourcemaps.missing_archive", exc_info=sys.exc_info())
             # Cache as nonexistent:
             cache.set(cache_key, -1, 60)
             return None
@@ -460,11 +503,10 @@ def fetch_release_artifact(url, release, dist):
         return result_from_cache(url, result)
 
     start = time.monotonic()
-
-    release_file = fetch_release_archive(release, dist)
-    if release_file is not None:
+    archive_file = fetch_release_archive_for_url(release, dist, url)
+    if archive_file is not None:
         try:
-            archive = ReleaseArchive(release_file)
+            archive = ReleaseArchive(archive_file)
         except BaseException as exc:
             logger.error("Failed to initialize archive for release %s", release.id, exc_info=exc)
             # TODO(jjbayer): cache error and return here
@@ -473,8 +515,10 @@ def fetch_release_artifact(url, release, dist):
                 try:
                     fp, headers = get_from_archive(url, archive)
                 except KeyError:
-                    logger.debug(
-                        "Release artifact %r not found in archive (release_id=%s)", url, release.id
+                    # The manifest mapped the url to an archive, but the file
+                    # is not there.
+                    logger.error(
+                        "Release artifact %r not found in archive %s", url, archive_file.id
                     )
                     cache.set(cache_key, -1, 60)
                     metrics.timing(

src/sentry/models/file.py

@@ -405,7 +405,6 @@ def putfile(self, fileobj, blob_size=DEFAULT_BLOB_SIZE, commit=True, logger=noop
 
             blob_fileobj = ContentFile(contents)
             blob = FileBlob.from_file(blob_fileobj, logger=logger)
-
             results.append(FileBlobIndex.objects.create(file=self, blob=blob, offset=offset))
             offset += blob.size
         self.size = offset