feat: Install the right DB version when updating the DB contents (#1426)

jcpitre · web-flow · commit 69ccfabc1e5f · 2025-10-31T10:47:59.000-04:00
diff --git a/.github/actions/resolve-api-meta/action.yml b/.github/actions/resolve-api-meta/action.yml
@@ -0,0 +1,80 @@
+name: Resolve API commit/version
+description: Resolve deployed API commit SHA and version from the metadata endpoint
+inputs:
+  api_base_url:
+    description: Base URL host for the API (e.g. api-dev.mobilitydatabase.org)
+    required: false
+    default: api.mobilitydatabase.org
+  api_refresh_token:
+    description: API refresh token
+    required: false
+outputs:
+  COMMIT_SHA:
+    description: Resolved commit SHA
+    value: ${{ steps.resolve.outputs.COMMIT_SHA }}
+  API_VERSION:
+    description: Resolved API version
+    value: ${{ steps.resolve.outputs.API_VERSION }}
+  RESOLVED:
+    description: Whether a commit SHA was resolved (true/false)
+    value: ${{ steps.resolve.outputs.RESOLVED }}
+runs:
+  using: composite
+  steps:
+    - id: resolve
+      name: Resolve via API and expose outputs
+      shell: bash
+      env:
+        API_BASE_URL: ${{ inputs.api_base_url }}
+        API_REFRESH_TOKEN: ${{ inputs.api_refresh_token }}
+      run: |
+        # Do not exit on failure; this action should never abort the caller workflow.
+        set -u
+        COMMIT_SHA=""
+        API_VERSION=""
+        RESOLVED="false"
+
+        if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then
+          echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..."
+
+          # Exchange refresh token -> access token (handle failures gracefully)
+          REPLY_JSON=$(curl --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \
+            --header 'Content-Type: application/json' \
+            --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }" ) || {
+              echo "Warning: token exchange failed; will fallback to 'main'" >&2
+              REPLY_JSON=""
+          }
+
+          if [[ -n "${REPLY_JSON}" ]]; then
+            ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token 2>/dev/null || echo "")
+            if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then
+              echo "Warning: Could not obtain access token from reply; will fallback to 'main'" >&2
+            else
+              META_JSON=$(curl --silent --show-error \
+                -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                -H 'accept: application/json' \
+                "https://${API_BASE_URL}/v1/metadata" ) || {
+                  echo "Warning: metadata request failed; will fallback to 'main'" >&2
+                  META_JSON=""
+              }
+
+              if [[ -n "${META_JSON}" ]]; then
+                COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash 2>/dev/null || echo "")
+                API_VERSION=$(echo "${META_JSON}" | jq -r .version 2>/dev/null || echo "")
+                if [[ -n "${COMMIT_SHA}" && "${COMMIT_SHA}" != "null" ]]; then
+                  RESOLVED="true"
+                  echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})"
+                else
+                  echo "Warning: commit_hash missing in metadata; will fallback to 'main'" >&2
+                fi
+              fi
+            fi
+          fi
+        else
+          echo "No API refresh token provided; skipping API metadata resolution and falling back to 'main'."
+        fi
+
+        # Expose outputs (empty COMMIT_SHA and RESOLVED=false indicate fallback to 'main')
+        echo "COMMIT_SHA=${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
+        echo "API_VERSION=${API_VERSION}" >> "$GITHUB_OUTPUT"
+        echo "RESOLVED=${RESOLVED}" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml
@@ -10,6 +10,17 @@ on:
   repository_dispatch: # Update on mobility-database-catalog repo dispatch
     types: [ catalog-sources-updated, gbfs-systems-updated ]
   workflow_dispatch:
+    inputs:
+      DRY_RUN:
+        description: Dry run. Skip applying schema and content updates
+        required: false
+        default: false
+        type: boolean
+      INSTALL_LATEST:
+        description: Install the latest (main) API version when true; when false install the currently deployed version.
+        required: false
+        default: false
+        type: boolean
 jobs:
   update:
     uses: ./.github/workflows/db-update.yml
@@ -19,6 +30,14 @@ jobs:
       DB_NAME: ${{ vars.DEV_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }}
       DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
+      API_BASE_URL: api-dev.mobilitydatabase.org
+      # DRY_RUN is only if requested by the user in a workflow_dispatch
+      DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }}
+      # We want to use the currently installed version (not the latest) if we received a dispatch from
+      #   mobility_database_catalog.
+      # For a workflow_dispatch (manual trigger), we use the value set by the user.
+      INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }}
+
     secrets:
       DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }}
@@ -28,6 +47,7 @@ jobs:
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
       POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      API_TEST_REFRESH_TOKEN: ${{ secrets.DEV_API_TEST_REFRESH_TOKEN }}
   notify-slack-on-failure:
     needs: [ update ]
     if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')
diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml
@@ -1,10 +1,22 @@
 # Update the Mobility Database Schema
 name: Database Update - PROD
 on:
-  workflow_dispatch:
+  workflow_dispatch:  # Manual trigger
+    inputs:
+      DRY_RUN:
+        description: Dry run. Skip applying schema and content updates
+        required: false
+        default: false
+        type: boolean
+      INSTALL_LATEST:
+        description: Install the latest (main) API version when true; when false install the currently deployed version.
+        required: false
+        default: false
+        type: boolean
   workflow_call:
   repository_dispatch: # Update on mobility-database-catalog repo dispatch
     types: [ catalog-sources-updated, gbfs-systems-updated ]
+
 jobs:
   update:
     uses: ./.github/workflows/db-update.yml
@@ -14,6 +26,16 @@ jobs:
       DB_NAME: ${{ vars.PROD_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
       DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
+      API_BASE_URL: api.mobilitydatabase.org
+      # DRY_RUN is only if requested by the user in a workflow_dispatch
+      DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }}
+      # We want to use the currently installed version (not the latest) if we received a dispatch from
+      #   mobility_database_catalog.
+      # We want to use the latest version if the trigger was workflow_call, because currently it's used for
+      #   upgrading (e.g. release.yml)
+      # For a workflow_dispatch (manual trigger), we use the value set by the user.
+      INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_call' && true) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }}
+
     secrets:
       DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }}
@@ -23,6 +45,7 @@ jobs:
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
       POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      API_TEST_REFRESH_TOKEN: ${{ secrets.PROD_API_TEST_REFRESH_TOKEN }}
 
   notify-slack-on-failure:
     needs: [ update ]
diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml
@@ -2,6 +2,17 @@
 name: Database Update - QA
 on:
   workflow_dispatch:
+    inputs:
+      DRY_RUN:
+        description: Dry run. Skip applying schema and content updates
+        required: false
+        default: false
+        type: boolean
+      INSTALL_LATEST:
+        description: Install the latest (main) API version when true; when false install the currently deployed version.
+        required: false
+        default: false
+        type: boolean
   workflow_call:
   repository_dispatch: # Update on mobility-database-catalog repo dispatch
     types: [ catalog-sources-updated, gbfs-systems-updated ]
@@ -15,6 +26,16 @@ jobs:
       DB_NAME: ${{ vars.QA_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
       DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
+      API_BASE_URL: api-qa.mobilitydatabase.org
+      # DRY_RUN is only if requested by the user in a workflow_dispatch
+      DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }}
+      # We want to use the currently installed version (not the latest) if we received a dispatch from
+      #   mobility_database_catalog.
+      # We want to use the latest version if the trigger was workflow_call, because currently it's used for
+      #   upgrading (e.g. release.yml)
+      # For a workflow_dispatch (manual trigger), we use the value set by the user.
+      INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_call' && true) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }}
+
     secrets:
       DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }}
@@ -24,6 +45,7 @@ jobs:
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
       POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      API_TEST_REFRESH_TOKEN: ${{ secrets.QA_API_TEST_REFRESH_TOKEN }}
   notify-slack-on-failure:
     needs: [ update ]
     if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')
diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml
@@ -46,6 +46,9 @@ on:
       POSTGRE_SQL_INSTANCE_NAME:
         description: PostgreSQL Instance Name
         required: true        
+      API_TEST_REFRESH_TOKEN:
+        description: API refresh token used to resolve deployed API commit (used on repository_dispatch)
+        required: false
     inputs:
       PROJECT_ID:
         description: GCP Project ID
@@ -67,19 +70,61 @@ on:
         description: GCP region
         required: true
         type: string
+      API_BASE_URL:
+        description: Base URL host for the API used to resolve version/commit (e.g. api-dev.mobilitydatabase.org)
+        required: false
+        default: api.mobilitydatabase.org
+        type: string
+      INSTALL_LATEST:
+        description: Install the latest (main) API version when true; when false keep the currently deployed version.
+        required: false
+        default: false
+        type: boolean
+      DRY_RUN:
+        description: Skip applying schema and content updates
+        required: false
+        default: false
+        type: boolean
 
 env:
   python_version: '3.11'
   liquibase_version: '4.33.0'
 
 jobs:
+  resolve-api-meta:
+    name: 'Resolve API commit/version'
+    runs-on: ubuntu-latest
+    # Run this job for all triggers; the action itself will skip resolution when API_BASE_URL or token is not provided.
+    # Keeping it unconditional ensures CHECKOUT_REF is always set (defaults to 'main') for downstream jobs.
+    outputs:
+      # Use resolved commit when available; otherwise default to 'main'.
+      CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }}
+    steps:
+      - name: Checkout repo (for scripts and local action)
+        uses: actions/checkout@v4
+      - name: Resolve API commit/version
+        id: resolve
+        if: ${{ inputs.INSTALL_LATEST == false }}
+        uses: ./.github/actions/resolve-api-meta
+        with:
+          api_base_url: ${{ inputs.API_BASE_URL }}
+          api_refresh_token: ${{ secrets.API_TEST_REFRESH_TOKEN }}
+
   db-schema-update:
     name: 'Database Schema Update'
     permissions: write-all
     runs-on: ubuntu-latest
+    needs: [resolve-api-meta]
+    # Run the schema update when the resolved checkout target is 'main' (install latest/main).
+    # This covers both explicit INSTALL_LATEST runs and cases where resolution failed and CHECKOUT_REF fell back to 'main'.
+    if: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF == 'main' }}
     steps:
-    - name: Checkout code
+    - name: Checkout repo
       uses: actions/checkout@v4
+      with:
+        # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main')
+        ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }}
+        fetch-depth: 0
 
     - name: Authenticate to Google Cloud QA/PROD
       uses: google-github-actions/auth@v2
@@ -126,6 +171,7 @@ jobs:
         liquibase --version
     
     - name: Run Liquibase
+      if: ${{ !inputs.DRY_RUN }}
       working-directory: ${{ github.workspace }}/liquibase
       run: |
         export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml"
@@ -139,11 +185,15 @@ jobs:
     name: 'Database Content Update'
     permissions: write-all
     runs-on: ubuntu-latest
-    needs: db-schema-update
-    if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }}
+    needs: [resolve-api-meta, db-schema-update]
+    if: ${{ always() }}
     steps:
-    - name: Checkout code
+    - name: Checkout repo
       uses: actions/checkout@v4
+      with:
+        # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main')
+        ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }}
+        fetch-depth: 0
 
     - name: Setup python
       uses: actions/setup-python@v5
@@ -212,11 +262,11 @@ jobs:
       run: echo "PATH=$(realpath sources.csv)" >> $GITHUB_OUTPUT
 
     - name: GTFS - Update Database Content
-      if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }}
+      if: ${{ !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }}
       run: scripts/populate-db.sh ${{ steps.getpath.outputs.PATH }} > populate.log
 
     - name: GTFS - Upload log file for verification
-      if: ${{ always() && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }}
+      if: ${{ always() && !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }}
       uses: actions/upload-artifact@v4
       with:
         name: populate-${{ inputs.ENVIRONMENT }}.log
@@ -232,11 +282,11 @@ jobs:
       run: echo "PATH=$(realpath systems.csv)" >> $GITHUB_OUTPUT
 
     - name: GBFS - Update Database Content
-      if: ${{ env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual' }}
+      if: ${{ !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }}
       run: scripts/populate-db.sh ${{ steps.getsyspath.outputs.PATH }} gbfs >> populate-gbfs.log
 
     - name: GBFS - Upload log file for verification
-      if: ${{ always() && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }}
+      if: ${{ always() && !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }}
       uses: actions/upload-artifact@v4
       with:
         name: populate-gbfs-${{ inputs.ENVIRONMENT }}.log
@@ -245,7 +295,7 @@ jobs:
 
   update-gcp-secret:
     name: Update GCP Secrets
-    if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }}
+    if: ${{ contains('repository_dispatch,workflow_dispatch', github.event_name) && !inputs.DRY_RUN }}
     runs-on: ubuntu-latest
     steps:
       - name: Authenticate to Google Cloud
@@ -283,6 +333,3 @@ jobs:
             echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..."
             echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID
           fi
-    
-      
-
