rebuild-index

Author: joshuarli

.github/workflows/backfill_metadata.yml renamed to .github/workflows/rebuild-index.yml

@@ -1,10 +1,10 @@
-name: backfill metadata
+name: rebuild index
 
 on:
   workflow_dispatch:
 
 jobs:
-  backfill-metadata:
+  rebuild-index:
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -18,4 +18,4 @@ jobs:
       with:
         workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool
         service_account: [email protected]
-    - run: python3 -uS bin/backfill-core-metadata --pypi-url https://pypi.devinfra.sentry.io
+    - run: python3 -uS bin/rebuild-index --pypi-url https://pypi.devinfra.sentry.io

bin/backfill-core-metadata

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os.path
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib.parse
+import urllib.request
+from collections.abc import Sequence
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pypi-url", required=True)
+    args = parser.parse_args(argv)
+
+    url = urllib.parse.urljoin(args.pypi_url, "packages.json")
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        packages_json = f"{tmpdir}/packages.json"
+        with urllib.request.urlopen(url) as resp, open(packages_json, "wb") as f:
+            shutil.copyfileobj(resp, f)
+
+        index = f"{tmpdir}/index"
+        os.makedirs(index)
+
+        subprocess.check_call(
+            (
+                sys.executable,
+                "-mdumb_pypi.main",
+                f"--package-list-json={packages_json}",
+                f"--output-dir={index}",
+                f'--packages-url={urllib.parse.urljoin(args.pypi_url, "wheels")}',
+                "--title=sentry pypi",
+                "--logo=https://avatars.githubusercontent.com/u/1396951?s=24",
+                "--logo-width=36",
+            )
+        )
+
+        subprocess.check_call(
+            (
+                "gcloud",
+                "storage",
+                "cp",
+                # clobber, no caching
+                "--cache-control",
+                "no-store",
+                "-r",
+                *(os.path.join(index, name) for name in os.listdir(index)),
+                "gs://pypi.devinfra.sentry.io",
+            )
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())