Skip to content

Commit a1e3b60

Browse files
jcpitrejcpitre
authored
feat: 260 refactor database update (#1436)
1 parent 66d16a1 commit a1e3b60

File tree

9 files changed

+654
-290
lines changed

9 files changed

+654
-290
lines changed

‎.github/actions/notify-slack/action.yml‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ inputs:
66
required: true
77
type: string
88
PRIORITY:
9-
description: 'Priority level of the alert (medium/high)'
9+
description: 'Priority level of the alert (medium/high/test)'
1010
required: true
1111
type: string
1212
STEP:
@@ -31,19 +31,29 @@ runs:
3131
id: alert_message
3232
shell: bash
3333
run: |
34-
if [[ "${{ inputs.PRIORITY }}" == "high" ]]; then
35-
if [[ -n "${{ inputs.STEP }}" ]]; then
36-
message="🚨 High Priority Alert: [${{ github.workflow }}] failed at step \"${{ inputs.STEP }}\". Immediate attention is required to avoid production impact."
37-
else
38-
message="🚨 High Priority Alert: [${{ github.workflow }}] failed. Immediate attention is required to avoid production impact."
39-
fi
40-
else
41-
if [[ -n "${{ inputs.STEP }}" ]]; then
42-
message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue at step \"${{ inputs.STEP }}\". This may affect ongoing integration processes."
43-
else
44-
message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue. This may affect ongoing integration processes."
45-
fi
46-
fi
34+
case "${{ inputs.PRIORITY }}" in
35+
high)
36+
if [[ -n "${{ inputs.STEP }}" ]]; then
37+
message="🚨 High Priority Alert: [${{ github.workflow }}] failed at step \"${{ inputs.STEP }}\". Immediate attention is required to avoid production impact."
38+
else
39+
message="🚨 High Priority Alert: [${{ github.workflow }}] failed. Immediate attention is required to avoid production impact."
40+
fi
41+
;;
42+
test)
43+
if [[ -n "${{ inputs.STEP }}" ]]; then
44+
message="🧪 Test Alert: [${{ github.workflow }}] reached step \"${{ inputs.STEP }}\". This is a test notification only."
45+
else
46+
message="🧪 Test Alert: [${{ github.workflow }}]. This is a test notification only."
47+
fi
48+
;;
49+
*)
50+
if [[ -n "${{ inputs.STEP }}" ]]; then
51+
message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue at step \"${{ inputs.STEP }}\". This may affect ongoing integration processes."
52+
else
53+
message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue. This may affect ongoing integration processes."
54+
fi
55+
;;
56+
esac
4757
4858
# Construct the JSON payload and save it to a file
4959
jq -n --arg message "$message" --arg run_url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" '{
@@ -74,5 +84,3 @@ runs:
7484
env:
7585
SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }}
7686
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
77-
78-

‎.github/actions/resolve-api-meta/action.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
name: Resolve API commit/version
2+
description: Resolve deployed API commit SHA and version from the metadata endpoint
3+
inputs:
4+
api_base_url:
5+
description: Base URL host for the API (e.g. api-dev.mobilitydatabase.org)
6+
required: false
7+
default: api.mobilitydatabase.org
8+
api_refresh_token:
9+
description: API refresh token
10+
required: false
11+
outputs:
12+
COMMIT_SHA:
13+
description: Resolved commit SHA
14+
value: ${{ steps.resolve.outputs.COMMIT_SHA }}
15+
API_VERSION:
16+
description: Resolved API version
17+
value: ${{ steps.resolve.outputs.API_VERSION }}
18+
RESOLVED:
19+
description: Whether a commit SHA was resolved (true/false)
20+
value: ${{ steps.resolve.outputs.RESOLVED }}
21+
runs:
22+
using: composite
23+
steps:
24+
- id: resolve
25+
name: Resolve via API and expose outputs
26+
shell: bash
27+
env:
28+
API_BASE_URL: ${{ inputs.api_base_url }}
29+
API_REFRESH_TOKEN: ${{ inputs.api_refresh_token }}
30+
run: |
31+
# Do not exit on failure; this action should never abort the caller workflow.
32+
set -u
33+
COMMIT_SHA=""
34+
API_VERSION=""
35+
RESOLVED="false"
36+
37+
if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then
38+
echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..."
39+
40+
# Exchange refresh token -> access token (handle failures gracefully)
41+
REPLY_JSON=$(curl --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \
42+
--header 'Content-Type: application/json' \
43+
--data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }" ) || {
44+
echo "Warning: token exchange failed; will fallback to 'main'" >&2
45+
REPLY_JSON=""
46+
}
47+
48+
if [[ -n "${REPLY_JSON}" ]]; then
49+
ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token 2>/dev/null || echo "")
50+
if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then
51+
echo "Warning: Could not obtain access token from reply; will fallback to 'main'" >&2
52+
else
53+
META_JSON=$(curl --silent --show-error \
54+
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
55+
-H 'accept: application/json' \
56+
"https://${API_BASE_URL}/v1/metadata" ) || {
57+
echo "Warning: metadata request failed; will fallback to 'main'" >&2
58+
META_JSON=""
59+
}
60+
61+
if [[ -n "${META_JSON}" ]]; then
62+
COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash 2>/dev/null || echo "")
63+
API_VERSION=$(echo "${META_JSON}" | jq -r .version 2>/dev/null || echo "")
64+
if [[ -n "${COMMIT_SHA}" && "${COMMIT_SHA}" != "null" ]]; then
65+
RESOLVED="true"
66+
echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})"
67+
else
68+
echo "Warning: commit_hash missing in metadata; will fallback to 'main'" >&2
69+
fi
70+
fi
71+
fi
72+
fi
73+
else
74+
echo "No API refresh token provided; skipping API metadata resolution and falling back to 'main'."
75+
fi
76+
77+
# Expose outputs (empty COMMIT_SHA and RESOLVED=false indicate fallback to 'main')
78+
echo "COMMIT_SHA=${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
79+
echo "API_VERSION=${API_VERSION}" >> "$GITHUB_OUTPUT"
80+
echo "RESOLVED=${RESOLVED}" >> "$GITHUB_OUTPUT"

‎.github/workflows/catalog-update.yml‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
# Called when the Mobility Catalog is updated.
2+
name: Mobility Catalog Update
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
DRY_RUN:
7+
description: Dry run. Skip applying schema and content updates
8+
required: false
9+
default: true
10+
type: boolean
11+
repository_dispatch: # Update on mobility-database-catalog repo dispatch
12+
types: [ catalog-sources-updated, gbfs-systems-updated ]
13+
14+
env:
15+
python_version: '3.11'
16+
liquibase_version: '4.33.0'
17+
18+
jobs:
19+
resolve-api-meta-qa:
20+
name: QA Resolve API commit/version
21+
runs-on: ubuntu-latest
22+
outputs:
23+
CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }}
24+
steps:
25+
- name: Checkout repo (for scripts and local action)
26+
uses: actions/checkout@v4
27+
28+
- name: Resolve API commit/version
29+
id: resolve
30+
uses: ./.github/actions/resolve-api-meta
31+
with:
32+
api_base_url: api-qa.mobilitydatabase.org
33+
api_refresh_token: ${{ secrets.QA_API_TEST_REFRESH_TOKEN }}
34+
35+
update-content-qa:
36+
name: QA Update DB content
37+
needs: [ resolve-api-meta-qa ]
38+
uses: ./.github/workflows/db-update-content.yml
39+
with:
40+
PROJECT_ID: ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
41+
REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
42+
DB_NAME: ${{ vars.QA_POSTGRE_SQL_DB_NAME }}
43+
ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
44+
DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
45+
DRY_RUN: ${{ github.event_name == 'repository_dispatch' || inputs.DRY_RUN }}
46+
CHECKOUT_REF: ${{ needs.resolve-api-meta-qa.outputs.CHECKOUT_REF }}
47+
secrets:
48+
DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }}
49+
DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }}
50+
DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
51+
GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
52+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
53+
OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
54+
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
55+
56+
resolve-api-meta-prod:
57+
name: PROD Resolve API commit/version
58+
runs-on: ubuntu-latest
59+
outputs:
60+
CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }}
61+
steps:
62+
- name: Checkout repo (for scripts and local action)
63+
uses: actions/checkout@v4
64+
65+
- name: Resolve API commit/version
66+
id: resolve
67+
uses: ./.github/actions/resolve-api-meta
68+
with:
69+
api_base_url: api.mobilitydatabase.org
70+
api_refresh_token: ${{ secrets.PROD_API_TEST_REFRESH_TOKEN }}
71+
72+
update-content-prod:
73+
name: PROD Update DB content
74+
needs: [ resolve-api-meta-prod ]
75+
uses: ./.github/workflows/db-update-content.yml
76+
with:
77+
PROJECT_ID: ${{ vars.PROD_MOBILITY_FEEDS_PROJECT_ID }}
78+
REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
79+
DB_NAME: ${{ vars.PROD_POSTGRE_SQL_DB_NAME }}
80+
ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
81+
DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
82+
DRY_RUN: ${{ github.event_name == 'repository_dispatch' || inputs.DRY_RUN }}
83+
CHECKOUT_REF: ${{ needs.resolve-api-meta-prod.outputs.CHECKOUT_REF }}
84+
secrets:
85+
DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }}
86+
DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }}
87+
DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
88+
GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
89+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
90+
OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
91+
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
92+
93+
resolve-api-meta-dev:
94+
name: DEV Resolve API commit/version
95+
runs-on: ubuntu-latest
96+
outputs:
97+
CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }}
98+
steps:
99+
- name: Checkout repo (for scripts and local action)
100+
uses: actions/checkout@v4
101+
102+
- name: Resolve API commit/version
103+
id: resolve
104+
uses: ./.github/actions/resolve-api-meta
105+
with:
106+
api_base_url: api-dev.mobilitydatabase.org
107+
api_refresh_token: ${{ secrets.DEV_API_TEST_REFRESH_TOKEN }}
108+
109+
update-content-dev:
110+
name: Dev Update DB content
111+
needs: [ resolve-api-meta-dev ]
112+
uses: ./.github/workflows/db-update-content.yml
113+
with:
114+
PROJECT_ID: ${{ vars.DEV_MOBILITY_FEEDS_PROJECT_ID }}
115+
REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
116+
DB_NAME: ${{ vars.DEV_POSTGRE_SQL_DB_NAME }}
117+
ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }}
118+
# dev uses the QA sql instance
119+
DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
120+
DRY_RUN: ${{ github.event_name == 'repository_dispatch' || inputs.DRY_RUN }}
121+
CHECKOUT_REF: ${{ needs.resolve-api-meta-dev.outputs.CHECKOUT_REF }}
122+
secrets:
123+
DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }}
124+
DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }}
125+
GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }}
126+
# dev uses the QA sql instance
127+
DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
128+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
129+
OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
130+
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
131+
132+
notify-slack-on-failure:
133+
# Run after all relevant jobs and notify if any failed
134+
needs: [ resolve-api-meta-qa, update-content-qa, resolve-api-meta-prod, update-content-prod, resolve-api-meta-dev, update-content-dev ]
135+
if: ${{ always() && contains(join(needs.*.result, ','), 'failure') }}
136+
runs-on: ubuntu-latest
137+
steps:
138+
- name: Checkout code
139+
uses: actions/checkout@v4
140+
- name: Notify Slack
141+
uses: ./.github/actions/notify-slack
142+
with:
143+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
144+
PRIORITY: "high"

0 commit comments

Comments
 (0)