ci/cd troubleshooting

Nathan Parker · Nathan Parker · commit eda5b0699a8f · 2025-12-02T19:35:28.000-08:00
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -12,6 +12,10 @@ env:
   PYTHON_VERSION: '3.11'
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  GCP_PROJECT_ID: nathan-playground-368310
+  GCP_REGION: us-west1
+  AR_REGISTRY: us-west1-docker.pkg.dev
+  AR_REPOSITORY: cloud-run-source-deploy
 
 jobs:
   # Linting and code quality checks
@@ -105,18 +109,37 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Log in to Container Registry
+      - name: Log in to GitHub Container Registry
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Authenticate to Google Cloud
+        if: github.event_name != 'pull_request'
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SA_KEY_PRODUCTION }}
+
+      - name: Set up Cloud SDK
+        if: github.event_name != 'pull_request'
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Configure Docker for Artifact Registry
+        if: github.event_name != 'pull_request'
+        env:
+          GCP_REGION: ${{ env.GCP_REGION }}
+        run: |
+          gcloud auth configure-docker ${GCP_REGION}-docker.pkg.dev --quiet
+
       - name: Extract metadata
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+            ${{ env.AR_REGISTRY }}/${{ env.GCP_PROJECT_ID }}/${{ env.AR_REPOSITORY }}/supply-graph-ai
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -137,6 +160,17 @@ jobs:
           cache-to: type=gha,mode=max
           platforms: linux/amd64,linux/arm64
 
+      - name: Verify Artifact Registry push
+        if: github.event_name != 'pull_request'
+        env:
+          GCP_REGION: ${{ env.GCP_REGION }}
+          GCP_PROJECT_ID: ${{ env.GCP_PROJECT_ID }}
+          AR_REGISTRY: ${{ env.AR_REGISTRY }}
+          AR_REPOSITORY: ${{ env.AR_REPOSITORY }}
+        run: |
+          echo "Verifying images were pushed to Artifact Registry..."
+          gcloud artifacts docker images list ${AR_REGISTRY}/${GCP_PROJECT_ID}/${AR_REPOSITORY}/supply-graph-ai --limit=10 || echo "No images found yet"
+
   # Security scanning
   security:
     name: Security Scan
@@ -207,17 +241,71 @@ jobs:
 
       - name: Deploy to Cloud Run
         id: deploy
+        env:
+          PROJECT_ID: ${{ env.GCP_PROJECT_ID }}
+          SA_EMAIL: supply-graph-ai@${{ env.GCP_PROJECT_ID }}.iam.gserviceaccount.com
+          AR_REGISTRY: ${{ env.AR_REGISTRY }}
+          AR_REPOSITORY: ${{ env.AR_REPOSITORY }}
         run: |
           # Determine image tag
+          # For main branch, the build step creates a "latest" tag
           if [ "${{ github.event_name }}" == "release" ]; then
             IMAGE_TAG="${{ github.event.release.tag_name }}"
+          elif [ "${{ github.ref }}" == "refs/heads/main" ]; then
+            IMAGE_TAG="latest"
           else
             IMAGE_TAG="main"
           fi
           
+          # Use Artifact Registry image (Cloud Run doesn't support GHCR)
+          AR_IMAGE="${AR_REGISTRY}/${PROJECT_ID}/${AR_REPOSITORY}/supply-graph-ai:${IMAGE_TAG}"
+          
+          echo "=== Deployment Configuration ==="
+          echo "IMAGE_TAG: ${IMAGE_TAG}"
+          echo "AR_REGISTRY: ${AR_REGISTRY}"
+          echo "PROJECT_ID: ${PROJECT_ID}"
+          echo "AR_REPOSITORY: ${AR_REPOSITORY}"
+          echo "AR_IMAGE: ${AR_IMAGE}"
+          echo "================================"
+          
+          echo "Verifying image exists..."
+          gcloud artifacts docker images describe ${AR_IMAGE} || {
+            echo "ERROR: Image ${AR_IMAGE} not found in Artifact Registry"
+            echo "Trying with 'latest' tag instead..."
+            AR_IMAGE_LATEST="${AR_REGISTRY}/${PROJECT_ID}/${AR_REPOSITORY}/supply-graph-ai:latest"
+            gcloud artifacts docker images describe ${AR_IMAGE_LATEST} && {
+              echo "Found image with 'latest' tag, using that instead"
+              AR_IMAGE=${AR_IMAGE_LATEST}
+            } || {
+              echo "Available images in repository:"
+              gcloud artifacts docker images list ${AR_REGISTRY}/${PROJECT_ID}/${AR_REPOSITORY}/supply-graph-ai || true
+              exit 1
+            }
+          }
+          
+          echo "Deploying with image: ${AR_IMAGE}"
+          echo "Full deployment command will use: --image ${AR_IMAGE}"
+          
+          # Check current service configuration (if it exists)
+          echo "Checking current service configuration..."
+          CURRENT_IMAGE=$(gcloud run services describe supply-graph-ai \
+            --region="${{ env.GCP_REGION }}" \
+            --format="value(spec.template.spec.containers[0].image)" 2>/dev/null || echo "")
+          if [ -n "${CURRENT_IMAGE}" ]; then
+            echo "Current service image: ${CURRENT_IMAGE}"
+          else
+            echo "Service does not exist yet"
+          fi
+          
+          # Deploy with explicit image override (image flag must be provided)
+          # Using explicit variable to ensure it's set correctly
+          IMAGE_TO_DEPLOY="${AR_IMAGE}"
+          echo "Final image to deploy: ${IMAGE_TO_DEPLOY}"
+          
           gcloud run deploy supply-graph-ai \
-            --image ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG} \
-            --region us-central1 \
+            --image "${IMAGE_TO_DEPLOY}" \
+            --service-account "${SA_EMAIL}" \
+            --region "${{ env.GCP_REGION }}" \
             --platform managed \
             --no-allow-unauthenticated \
             --set-env-vars ENVIRONMENT=production \
diff --git a/docs/development/ci-cd-deployment-permissions.md b/docs/development/ci-cd-deployment-permissions.md
@@ -0,0 +1,117 @@
+# CI/CD Deployment Permissions
+
+## Overview
+
+For CI/CD deployment to Cloud Run, the service account used in GitHub Actions needs specific IAM permissions to deploy services.
+
+## Required Permissions
+
+The service account needs the **Cloud Run Admin** role to deploy services, and permission to act as itself:
+
+```bash
+export PROJECT_ID="nathan-playground-368310"
+export SA_EMAIL="supply-graph-ai@${PROJECT_ID}.iam.gserviceaccount.com"
+
+# Grant Cloud Run Admin role (includes all necessary permissions)
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+    --member="serviceAccount:${SA_EMAIL}" \
+    --role="roles/run.admin"
+
+# Grant permission to act as the service account itself (required for deployment)
+gcloud iam service-accounts add-iam-policy-binding ${SA_EMAIL} \
+    --member="serviceAccount:${SA_EMAIL}" \
+    --role="roles/iam.serviceAccountUser"
+
+# Grant permission to act as the Compute Engine default service account
+# (required even when specifying a different service account)
+# Note: The Compute Engine default SA uses PROJECT_NUMBER, not PROJECT_ID
+PROJECT_NUMBER=$(gcloud projects describe ${PROJECT_ID} --format="value(projectNumber)")
+export COMPUTE_SA="${PROJECT_NUMBER}-compute@developer.gserviceaccount.com"
+gcloud iam service-accounts add-iam-policy-binding ${COMPUTE_SA} \
+    --member="serviceAccount:${SA_EMAIL}" \
+    --role="roles/iam.serviceAccountUser"
+```
+
+## What This Role Includes
+
+The `roles/run.admin` role includes all necessary permissions for Cloud Run deployment:
+- `run.services.get` - Get service information
+- `run.services.create` - Create new services
+- `run.services.update` - Update existing services
+- `run.services.delete` - Delete services (if needed)
+- `run.revisions.create` - Create new revisions
+- `run.revisions.get` - Get revision information
+- `run.revisions.update` - Update revisions
+- `iam.serviceAccounts.actAs` - Act as the runtime service account
+- And other Cloud Run management permissions
+
+## Alternative: Minimum Required Permissions
+
+If you prefer to grant only the minimum required permissions instead of the full admin role:
+
+```bash
+# Grant individual permissions (more granular control)
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+    --member="serviceAccount:${SA_EMAIL}" \
+    --role="roles/run.developer"
+
+# Also need to act as the service account
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+    --member="serviceAccount:${SA_EMAIL}" \
+    --role="roles/iam.serviceAccountUser"
+```
+
+However, `roles/run.admin` is recommended for CI/CD deployments as it provides all necessary permissions in one role.
+
+## Verify Permissions
+
+After granting permissions, verify the service account has the required role:
+
+```bash
+gcloud projects get-iam-policy $PROJECT_ID \
+    --flatten="bindings[].members" \
+    --filter="bindings.members:serviceAccount:${SA_EMAIL}" \
+    --format="table(bindings.role)"
+```
+
+You should see `roles/run.admin` (or the individual permissions) in the output.
+
+## Artifact Registry Permissions
+
+The service account also needs permission to push images to Artifact Registry:
+
+```bash
+# Grant Artifact Registry Writer role (for pushing images)
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+    --member="serviceAccount:${SA_EMAIL}" \
+    --role="roles/artifactregistry.writer"
+```
+
+## Summary of All Required Permissions
+
+Your service account needs:
+1. `roles/run.admin` - For Cloud Run deployment operations
+2. `roles/iam.serviceAccountUser` on itself - To act as itself during deployment
+3. `roles/iam.serviceAccountUser` on Compute Engine default SA - Required by Cloud Run
+4. `roles/artifactregistry.writer` - To push images to Artifact Registry
+
+## Important Notes
+
+1. **Separate Service Accounts**: Consider using a separate service account for CI/CD deployment (e.g., `ci-cd-deployer@...`) rather than the runtime service account. This follows the principle of least privilege.
+
+2. **Service Account User Role**: If deploying with a different service account than the runtime service account, you may also need `roles/iam.serviceAccountUser` to allow the deployment service account to act as the runtime service account.
+
+3. **Region-Specific**: Cloud Run permissions are typically project-wide, but ensure the service account has access to the specific region where you're deploying.
+
+## Troubleshooting
+
+If you still get permission errors after granting `roles/run.admin`:
+
+1. **Wait a few minutes** - IAM policy changes can take a few minutes to propagate
+2. **Verify the service account email** - Ensure you're using the correct service account
+3. **Check project ID** - Ensure you're granting permissions in the correct project
+4. **Verify API is enabled** - Ensure Cloud Run API is enabled:
+   ```bash
+   gcloud services enable run.googleapis.com
+   ```
+
