First draft

Author: jcpitre

.github/workflows/db-update-dev.yml

@@ -19,6 +19,7 @@ jobs:
       DB_NAME: ${{ vars.DEV_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }}
       DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
+      API_BASE_URL: api-dev.mobilitydatabase.org
     secrets:
       DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }}
@@ -28,6 +29,7 @@ jobs:
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
       POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      API_TEST_REFRESH_TOKEN: ${{ secrets.DEV_API_TEST_REFRESH_TOKEN }}
   notify-slack-on-failure:
     needs: [ update ]
     if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')

.github/workflows/db-update-prod.yml

@@ -14,6 +14,7 @@ jobs:
       DB_NAME: ${{ vars.PROD_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
       DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
+      API_BASE_URL: api.mobilitydatabase.org
     secrets:
       DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }}
@@ -23,6 +24,7 @@ jobs:
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
       POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      API_TEST_REFRESH_TOKEN: ${{ secrets.PROD_API_TEST_REFRESH_TOKEN }}
 
   notify-slack-on-failure:
     needs: [ update ]

.github/workflows/db-update-qa.yml

@@ -15,6 +15,7 @@ jobs:
       DB_NAME: ${{ vars.QA_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
       DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
+      API_BASE_URL: api-qa.mobilitydatabase.org
     secrets:
       DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }}
@@ -24,6 +25,7 @@ jobs:
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
       POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      API_TEST_REFRESH_TOKEN: ${{ secrets.QA_API_TEST_REFRESH_TOKEN }}
   notify-slack-on-failure:
     needs: [ update ]
     if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')

.github/workflows/db-update.yml

@@ -46,6 +46,9 @@ on:
       POSTGRE_SQL_INSTANCE_NAME:
         description: PostgreSQL Instance Name
         required: true        
+      API_TEST_REFRESH_TOKEN:
+        description: API refresh token used to resolve deployed API commit (used on repository_dispatch)
+        required: false
     inputs:
       PROJECT_ID:
         description: GCP Project ID
@@ -67,18 +70,57 @@ on:
         description: GCP region
         required: true
         type: string
+      API_BASE_URL:
+        description: Base URL host for the API used to resolve version/commit (e.g. api-dev.mobilitydatabase.org)
+        required: false
+        default: api.mobilitydatabase.org
+        type: string
 
 env:
   python_version: '3.11'
   liquibase_version: '4.33.0'
 
 jobs:
+  resolve-api-meta:
+    name: 'Resolve API commit/version'
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'repository_dispatch' && inputs.API_BASE_URL != '' && secrets.API_TEST_REFRESH_TOKEN != '' }}
+    outputs:
+      COMMIT_SHA: ${{ steps.resolve.outputs.COMMIT_SHA }}
+      API_VERSION: ${{ steps.resolve.outputs.API_VERSION }}
+    steps:
+      - name: Checkout repo (for scripts)
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          sparse-checkout: |
+            scripts
+          sparse-checkout-cone: true
+      - name: Resolve API commit/version
+        id: resolve
+        env:
+          API_BASE_URL: ${{ inputs.API_BASE_URL }}
+          API_REFRESH_TOKEN: ${{ secrets.API_TEST_REFRESH_TOKEN }}
+          EVENT: ${{ github.event_name }}
+        run: |
+          bash scripts/resolve_api_meta.sh
+
   db-schema-update:
     name: 'Database Schema Update'
     permissions: write-all
     runs-on: ubuntu-latest
+    needs: [resolve-api-meta]
+    if: ${{ always() }}
     steps:
-    - name: Checkout code
+    - name: Checkout code at API commit
+      if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }}
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ needs.resolve-api-meta.outputs.COMMIT_SHA }}
+        fetch-depth: 0
+
+    - name: Checkout code (default)
+      if: ${{ needs.resolve-api-meta.result != 'success' || needs.resolve-api-meta.outputs.COMMIT_SHA == '' }}
       uses: actions/checkout@v4
 
     - name: Authenticate to Google Cloud QA/PROD
@@ -139,10 +181,18 @@ jobs:
     name: 'Database Content Update'
     permissions: write-all
     runs-on: ubuntu-latest
-    needs: db-schema-update
-    if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }}
+    needs: [resolve-api-meta, db-schema-update]
+    if: ${{ always() && (github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch') }}
     steps:
-    - name: Checkout code
+    - name: Checkout code at API commit
+      if: ${{ needs.resolve-api-meta.result == 'success' && needs.resolve-api-meta.outputs.COMMIT_SHA != '' }}
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ needs.resolve-api-meta.outputs.COMMIT_SHA }}
+        fetch-depth: 0
+
+    - name: Checkout code (default)
+      if: ${{ needs.resolve-api-meta.result != 'success' || needs.resolve-api-meta.outputs.COMMIT_SHA == '' }}
       uses: actions/checkout@v4
 
     - name: Setup python
@@ -159,6 +209,8 @@ jobs:
       uses: google-github-actions/setup-gcloud@v2
 
     - name: Update .env file
+      env:
+        API_VERSION: ${{ needs.resolve-api-meta.outputs.API_VERSION }}
       run: |
         echo "PGUSER=${{ secrets.DB_USER_NAME }}" > config/.env.local
         echo "POSTGRES_USER=${{ secrets.DB_USER_NAME }}" >> config/.env.local
@@ -168,6 +220,7 @@ jobs:
         echo "POSTGRES_PORT=5432" >> config/.env.local
         echo "POSTGRES_HOST=localhost" >> config/.env.local
         echo "ENV=${{ inputs.ENVIRONMENT }}" >> config/.env.local
+        if [[ -n "${API_VERSION}" ]]; then echo "API_VERSION=${API_VERSION}" >> config/.env.local; fi
         cat config/.env.local
 
     - name: Load secrets from 1Password
@@ -283,6 +336,3 @@ jobs:
             echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..."
             echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID
           fi
-    
-      
-

scripts/resolve_api_meta.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Resolve API metadata and set GitHub Action step outputs COMMIT_SHA and API_VERSION
+# Inputs (via env):
+#   EVENT            - GitHub event name (e.g., repository_dispatch)
+#   API_BASE_URL     - API host (e.g., api-dev.mobilitydatabase.org)
+#   API_REFRESH_TOKEN- Refresh token to obtain access token (only used on repository_dispatch)
+# Outputs:
+#   Prints COMMIT_SHA and API_VERSION to stdout as NAME=VALUE lines
+#   Also exports to $GITHUB_ENV if defined so the script is runnable by hand
+
+set -euo pipefail
+# QA
+#export API_REFRESH_TOKEN="AMf-vBzv1rT8AQ0uZrzNDD5wuMxSvxLVbXRmViTaOVuP8eh-uDdWLDpsDHYMrGNqq2sqn1ya_-i8YXZdWh9GnLoPSLbbWY99hmb-JUrJ_NXz4pJ5v2ysm3kCpjy02zN2uI5csAi1YmGSrlJoUQazNO4ntkVHkgdpsyMSBwgIGDczwX4qANzEjsRjtRCQlCbm_MnWsLaBKrukn5qxFbyVszzOloG5piIivTL700I9cPslxlzirmYYrj3jsYIX00RKBF3pPvTpiiOGskRjeZi_UvI5spux0tkFuZJZGt-vNKWeLT9MSJp6S3Y5os4PQMjCJ-StCP4Qkqwja5EiDRdVhqJVCT7XYn_MAQBYGFUYvFOQu9S9siKlhXO9Mcc2NiH89eaJ1EUlkXRnzY9nahP82cmt8VOKHyu6GAT3-l4V9_9zJ7wjyQJ0wSf_5wZlLxFSScLaIADZdRya"
+# https://beta.mobilitydatabase.org/account
+#export API_REFRESH_TOKEN="AMf-vByLiUKvxTqH_4vkQTJ8aStgCvrNGDyMINJNpYVdRCQliF6q0FmZxy7Y29rgNce4HGnrr2La3lja5CsOPn8-Vx6RA0enAv5RVxnBC08G-c6ZAiwCMfvUT-vT49ZLeULmUIM6BlCQKGdXSxAOljg6QJNH0wXdvhHaLUGVGdhoeKW3pg692ZCCWTVCWOFSDhHhS-d-8ywSd7nsFwka0ZMBRAtOeY-lgvwsz7Wo21hCTaHyUYSoodKzWFjUUJ9W7nA33OB6lKBA9tZGap3pYlO-Vo47jl-M2GcZn77R-sgsgEbPXfHxc0NMS1ZbNXKeWmrcqPYcgcEoBFdFrnQcd1bD-b1O8_zTnDyjjRekLq8bGKLnDfRg-qZvUaNxtKPUqFnh-pmvO7gXMxwj7Tnoc5cMmQxHGZ2euuxaVRHNMoAxnEzqIzZ7nxIM4NX36eUZCuLLSTXvOnmsdMuhQ_7oI2wX-YKxRGHnrw"
+# https://mobility-feeds-dev.web.app/account
+export API_REFRESH_TOKEN="AMf-vBxJqJLukaDstkJl_Pi-GonY1suIN_3O5Fp7pHfYcD_XVZwSJczM_815UpEJFQN9ShzaE5KRqsyO4sRnFUBAA6KLQgcP-7Mx9yNzRBGtBi-e37X50CNwqUvsuUfOjFZNeKPfaO1ipuCA9LQLWf-5e29DmyxEgO1Fy9UguKm5KlDRrcIkUjTzSBHuSRu06j1_th4TR4l0X5OteNGsG6F6N2lrihj2Z5Idx6PoJ02_5fgLOhop8mlGd-ktNpQ3J46lAz7BX7_UgPkbugwIoD9YdYvnIlrSp5hl3ri7mmauN0rOV1HEMn-aynd7Zqkc0lmNWWL3BqQe-Ik0sOt23HZISK7Y2EITrXEtXxPN0-824BKUAoLN-_eutLk3NFRJW19jQc-RrZRP1LzXdWQETKhXPRBWCf33U1N0Yv2qzBKTzhvFjh3yULRVnZjUxh7CnZeYDKbxAWYj"
+
+COMMIT_SHA=""
+API_VERSION=""
+
+if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then
+  echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..."
+  REPLY_JSON=$(curl --fail --silent --show-error --location "https://${mobility-feeds-dev}/v1/tokens" \
+    --header 'Content-Type: application/json' \
+    --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }")
+  ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token)
+  if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then
+    echo "Error: Could not obtain access token from reply" >&2
+    exit 1
+  fi
+  META_JSON=$(curl --fail --silent --show-error \
+    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    -H 'accept: application/json' \
+    "https://${API_BASE_URL}/v1/metadata")
+  COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash)
+  API_VERSION=$(echo "${META_JSON}" | jq -r .version)
+  if [[ -z "${COMMIT_SHA}" || "${COMMIT_SHA}" == "null" ]]; then
+    echo "Error: Could not extract commit_hash from metadata" >&2
+    echo "Metadata reply: ${META_JSON}" >&2
+    exit 1
+  fi
+  echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})"
+else
+  echo "No token provided; skipping API metadata resolution."
+fi
+
+# Output values to stdout in a parse-friendly format
+echo "COMMIT_SHA=${COMMIT_SHA}"
+echo "API_VERSION=${API_VERSION}"
+
+# Optionally export to $GITHUB_ENV for subsequent steps when available
+if [[ -n "${GITHUB_ENV:-}" ]]; then
+  {
+    echo "COMMIT_SHA=${COMMIT_SHA}"
+    echo "API_VERSION=${API_VERSION}"
+  } >> "$GITHUB_ENV"
+fi