Fix GCP bucket upload (#258)

nikhilwoodruff · web-flow · commit 493adff3007b · 2025-05-20T11:37:26.000+01:00
* Google Cloud Bucket upload fails Fixes #257 * No-change * Add perms * Use GCP creds * Move GCP auth code to main action * Try WIF again * No-change * Re-add test script * No-change * No-change * Add google auth step * Move GCP code back to prod config * Versioning * Format * Remove unnecessary permissions
diff --git a/changelog_entry.yaml b/changelog_entry.yaml
@@ -0,0 +1,4 @@
+- bump: patch
+  changes:
+    fixed:
+    - GCP uploads use permissions correctly
diff --git a/policyengine_us_data/storage/upload_completed_datasets.py b/policyengine_us_data/storage/upload_completed_datasets.py
@@ -6,10 +6,14 @@
 from policyengine_us_data.storage import STORAGE_FOLDER
 from policyengine_us_data.utils.huggingface import upload
 from google.cloud import storage
+import google.auth
 
 
 def upload_datasets():
-    storage_client = storage.Client()
+    credentials, project_id = google.auth.default()
+    storage_client = storage.Client(
+        credentials=credentials, project=project_id
+    )
     bucket = storage_client.bucket("policyengine-us-data")
 
     datasets_to_upload = [
diff --git a/pyproject.toml b/pyproject.toml
@@ -25,6 +25,7 @@ dependencies = [
     "microimpute",
     "pip-system-certs",
     "google-cloud-storage",
+    "google-auth",
 ]
 
 [project.optional-dependencies]

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ dependencies = [`
`25`	`25`	`"microimpute",`
`26`	`26`	`"pip-system-certs",`
`27`	`27`	`"google-cloud-storage",`
	`28`	`+ "google-auth",`
`28`	`29`	`]`
`29`	`30`
`30`	`31`	`[project.optional-dependencies]`