Skip to content

Commit 9ded67f

Browse files
jcpitrejcpitre
committed
Adding catalog-update.yml
1 parent 8a418c4 commit 9ded67f

File tree

2 files changed

+109
-78
lines changed

2 files changed

+109
-78
lines changed

‎.github/workflows/db-update-content.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,3 +167,43 @@ jobs:
167167
name: populate-gbfs-${{ inputs.ENVIRONMENT }}.log
168168
path: populate-gbfs.log
169169

170+
update-gcp-secret:
171+
name: Update GCP Secrets
172+
if: ${{ !inputs.DRY_RUN }}
173+
runs-on: ubuntu-latest
174+
steps:
175+
- name: Authenticate to Google Cloud
176+
uses: google-github-actions/auth@v2
177+
with:
178+
credentials_json: ${{ secrets.GCP_MOBILITY_FEEDS_SA_KEY }}
179+
180+
- name: Google Cloud Setup
181+
uses: google-github-actions/setup-gcloud@v2
182+
183+
- name: Load secrets from 1Password
184+
id: onepw_secrets
185+
uses: 1password/[email protected]
186+
with:
187+
export-env: true # Export loaded secrets as environment variables
188+
env:
189+
# This alternate service account token gives access to a vault writable by some third
190+
# party people who can update the list of feeds requiring authorization and their tokens
191+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
192+
JSON_FEEDS_WITH_TOKENS: "op://lijd6lj7lyw7dajea6x3zgf53m/l6sr2cnpjj3cbw3t5amlu7vui4/credential"
193+
194+
- name: Create or Update Auth Secret
195+
env:
196+
PROJECT_ID: ${{ inputs.PROJECT_ID }}
197+
ENVIRONMENT: ${{ inputs.ENVIRONMENT }}
198+
SECRET_VALUE: ${{ env.JSON_FEEDS_WITH_TOKENS }}
199+
SECRET_NAME: FEEDS_CREDENTIALS
200+
run: |
201+
echo "Processing secret $SECRET_NAME in project $PROJECT_ID..."
202+
203+
if gcloud secrets describe $SECRET_NAME --project=$PROJECT_ID; then
204+
echo "Secret $SECRET_NAME already exists in project $PROJECT_ID, updating..."
205+
echo -n "$SECRET_VALUE" | gcloud secrets versions add $SECRET_NAME --data-file=- --project=$PROJECT_ID
206+
else
207+
echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..."
208+
echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID
209+
fi

‎.github/workflows/db-update.yml‎

Lines changed: 69 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,53 +3,43 @@ name: Database Update
33
on:
44
workflow_call:
55
inputs:
6-
DRY_RUN:
6+
DRY_RUN: #
77
description: Dry run. Skip applying schema and content updates
88
required: false
99
default: true
1010
type: boolean
11-
PROJECT_ID:
11+
PROJECT_ID: #
1212
description: GCP Project ID (forwarded to child workflows)
1313
required: false
1414
type: string
15-
REGION:
15+
REGION: #
1616
description: GCP region (forwarded to child workflows)
1717
required: false
1818
type: string
19-
DB_NAME:
19+
DB_NAME: #
2020
description: PostgreSQL Database Name (forwarded to child workflows)
2121
required: false
2222
type: string
23-
ENVIRONMENT:
24-
description: Environment label (forwarded to child workflows)
25-
required: false
26-
type: string
27-
DB_ENVIRONMENT:
23+
DB_ENVIRONMENT: #
2824
description: Environment where DB is deployed (forwarded to child workflows)
2925
required: false
3026
type: string
3127
secrets:
32-
DB_USER_PASSWORD:
28+
DB_USER_PASSWORD: #
3329
description: PostgreSQL User Password
3430
required: true
35-
DB_USER_NAME:
31+
DB_USER_NAME: #
3632
description: PostgreSQL User Name
3733
required: true
38-
POSTGRE_SQL_INSTANCE_NAME:
34+
POSTGRE_SQL_INSTANCE_NAME: #
3935
description: PostgreSQL Instance Name
4036
required: true
41-
DB_GCP_MOBILITY_FEEDS_SA_KEY:
37+
DB_GCP_MOBILITY_FEEDS_SA_KEY: #
4238
description: Service account key for DB environment
4339
required: true
44-
GCP_MOBILITY_FEEDS_SA_KEY:
45-
description: Service account key for GCP (general)
46-
required: true
47-
OP_SERVICE_ACCOUNT_TOKEN:
40+
OP_SERVICE_ACCOUNT_TOKEN: #
4841
description: 1Password Service Account Token
4942
required: true
50-
OP_FEEDS_SERVICE_ACCOUNT_TOKEN:
51-
description: 1Password token for feeds secret
52-
required: true
5343

5444
jobs:
5545
print-event-name:
@@ -67,67 +57,68 @@ jobs:
6757
echo "inputs.DB_ENVIRONMENT=${{ inputs.DB_ENVIRONMENT || 'unset' }}"
6858
6959
db-update-schema:
70-
name: Call DB schema update
71-
uses: ./.github/workflows/db-update-schema.yml
72-
with:
73-
PROJECT_ID: ${{ inputs.PROJECT_ID }}
74-
REGION: ${{ inputs.REGION }}
75-
DB_NAME: ${{ inputs.DB_NAME }}
76-
DB_ENVIRONMENT: ${{ inputs.DB_ENVIRONMENT }}
77-
DRY_RUN: ${{ inputs.DRY_RUN }}
78-
secrets: inherit
79-
80-
db-update-content:
81-
name: Call DB content update
82-
needs: [ db-update-schema ]
83-
uses: ./.github/workflows/db-update-content.yml
84-
with:
85-
PROJECT_ID: ${{ inputs.PROJECT_ID }}
86-
REGION: ${{ inputs.REGION }}
87-
DB_NAME: ${{ inputs.DB_NAME }}
88-
ENVIRONMENT: ${{ inputs.ENVIRONMENT }}
89-
DB_ENVIRONMENT: ${{ inputs.DB_ENVIRONMENT }}
90-
DRY_RUN: ${{ inputs.DRY_RUN }}
91-
CHECKOUT_REF: main
92-
secrets: inherit
93-
94-
update-gcp-secret:
95-
name: Update GCP Secrets
96-
if: ${{ !inputs.DRY_RUN }}
60+
db-schema-update:
61+
name: 'Database Schema Update'
62+
permissions: write-all
9763
runs-on: ubuntu-latest
9864
steps:
99-
- name: Authenticate to Google Cloud
100-
uses: google-github-actions/auth@v2
101-
with:
102-
credentials_json: ${{ secrets.GCP_MOBILITY_FEEDS_SA_KEY }}
65+
- name: Checkout repo
66+
uses: actions/checkout@v4
67+
with:
68+
ref: main
10369

104-
- name: Google Cloud Setup
105-
uses: google-github-actions/setup-gcloud@v2
70+
- name: Authenticate to Google Cloud QA/PROD
71+
uses: google-github-actions/auth@v2
72+
with:
73+
credentials_json: ${{ secrets.DB_GCP_MOBILITY_FEEDS_SA_KEY }}
10674

107-
- name: Load secrets from 1Password
108-
id: onepw_secrets
109-
uses: 1password/[email protected]
110-
with:
111-
export-env: true # Export loaded secrets as environment variables
112-
env:
113-
# This alternate service account token gives access to a vault writable by some third
114-
# party people who can update the list of feeds requiring authorization and their tokens
115-
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
116-
JSON_FEEDS_WITH_TOKENS: "op://lijd6lj7lyw7dajea6x3zgf53m/l6sr2cnpjj3cbw3t5amlu7vui4/credential"
75+
- name: Google Cloud Setup
76+
uses: google-github-actions/setup-gcloud@v2
11777

118-
- name: Create or Update Auth Secret
119-
env:
120-
PROJECT_ID: ${{ inputs.PROJECT_ID }}
121-
ENVIRONMENT: ${{ inputs.ENVIRONMENT }}
122-
SECRET_VALUE: ${{ env.JSON_FEEDS_WITH_TOKENS }}
123-
SECRET_NAME: FEEDS_CREDENTIALS
124-
run: |
125-
echo "Processing secret $SECRET_NAME in project $PROJECT_ID..."
78+
- name: Load secrets from 1Password
79+
uses: 1password/[email protected]
80+
with:
81+
export-env: true # Export loaded secrets as environment variables
82+
env:
83+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
84+
GCP_FEED_SSH_USER: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_SSH_USER/username"
85+
GCP_FEED_BASTION_NAME: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_BASTION_NAME/username"
86+
GCP_FEED_BASTION_SSH_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_BASTION_SSH_KEY/private key"
87+
88+
- name: Tunnel
89+
run: |
90+
mkdir -p ~/.ssh
91+
echo "${{ env.GCP_FEED_BASTION_SSH_KEY }}" > ~/.ssh/id_rsa
92+
chmod 600 ~/.ssh/id_rsa
93+
./scripts/tunnel-create.sh -project_id ${{ inputs.PROJECT_ID }} -zone ${{ inputs.REGION }}-a -instance ${{ env.GCP_FEED_BASTION_NAME }}-${{ inputs.DB_ENVIRONMENT}} -target_account ${{ env.GCP_FEED_SSH_USER }} -db_instance ${{ secrets.POSTGRE_SQL_INSTANCE_NAME }}
94+
sleep 10 # Wait for the tunnel to establish
95+
96+
- name: Test Database Connection Through Tunnel
97+
run: |
98+
sudo apt-get update && sudo apt-get install -y postgresql-client
99+
PGPASSWORD=${{ secrets.DB_USER_PASSWORD }} psql -h localhost -p 5432 -U ${{ secrets.DB_USER_NAME }} -d ${{ inputs.DB_NAME }} -c "SELECT version();"
100+
101+
- name: Install Liquibase
102+
env:
103+
LIQUIBASE_VERSION: ${{ env.liquibase_version }}
104+
run: |
105+
curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz
106+
rm -rf liquibase-dist
107+
mkdir liquibase-dist
108+
tar -xzf liquibase.tar.gz -C liquibase-dist
109+
sudo rm -rf /usr/local/liquibase
110+
sudo mv liquibase-dist /usr/local/liquibase
111+
sudo ln -sf /usr/local/liquibase/liquibase /usr/local/bin/liquibase
112+
liquibase --version
113+
114+
- name: Run Liquibase
115+
if: ${{ !inputs.DRY_RUN }}
116+
working-directory: ${{ github.workspace }}/liquibase
117+
run: |
118+
export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml"
119+
export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:5432/${{ inputs.DB_NAME }}
120+
export LIQUIBASE_COMMAND_USERNAME=${{ secrets.DB_USER_NAME }}
121+
export LIQUIBASE_COMMAND_PASSWORD=${{ secrets.DB_USER_PASSWORD }}
122+
export LIQUIBASE_LOG_LEVEL=FINE
123+
liquibase update
126124
127-
if gcloud secrets describe $SECRET_NAME --project=$PROJECT_ID; then
128-
echo "Secret $SECRET_NAME already exists in project $PROJECT_ID, updating..."
129-
echo -n "$SECRET_VALUE" | gcloud secrets versions add $SECRET_NAME --data-file=- --project=$PROJECT_ID
130-
else
131-
echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..."
132-
echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID
133-
fi

0 commit comments

Comments
 (0)