Skip to content

Commit ffa6f7e

Browse files
alistairewjalistairewj
committed
use composite action for demo download, split psql and mysql
1 parent bbb2bed commit ffa6f7e

File tree

3 files changed

+110
-89
lines changed

3 files changed

+110
-89
lines changed

‎.github/actions/download-demo/action.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: "Download MIMIC-IV Demo"
2+
description: "Downloads MIMIC-IV demo data from GCP"
3+
inputs:
4+
gcp-project-id:
5+
required: true
6+
description: "The Google Cloud Project ID used for billing usage"
7+
gcp-sa-key:
8+
required: true
9+
description: "The service account key used to authenticate with GCP"
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Authenticate with GCP
14+
uses: 'google-github-actions/auth@v0'
15+
with:
16+
project_id: ${{ inputs.gcp-project-id }}
17+
credentials_json: ${{ inputs.gcp-sa-key }}
18+
19+
- name: Set up Cloud SDK
20+
uses: 'google-github-actions/setup-gcloud@v0'
21+
22+
- name: Download demo data from GCP
23+
run: |
24+
echo "Downloading MIMIC-IV demo from GCP."
25+
gsutil -q -u $PROJECT_ID -m cp -r gs://mimic-iv-archive/v2.0/demo ./
26+
env:
27+
PROJECT_ID: ${{ inputs.gcp-project-id }}
28+
shell: bash

‎.github/workflows/mysql.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: mysql demo db build
2+
on: pull_request
3+
4+
jobs:
5+
mimic-iv-mysql:
6+
runs-on: ubuntu-22.04
7+
8+
steps:
9+
- name: Check out repository code
10+
uses: actions/checkout@v3
11+
12+
- name: Download demo data
13+
uses: ./.github/actions/download-demo
14+
with:
15+
gcp-project-id: ${{ secrets.GCP_PROJECT_ID }}
16+
gcp-sa-key: ${{ secrets.GCP_SA_KEY }}
17+
18+
- name: Extract demo data to local folder
19+
run: |
20+
mv demo/hosp/*.csv.gz ./
21+
mv demo/icu/*.csv.gz ./
22+
mv demo/ed/*.csv.gz ./
23+
gzip -d *.csv.gz
24+
25+
- name: Start MySQL service
26+
run: |
27+
sudo /etc/init.d/mysql start
28+
mysql -u root -proot -e "SET GLOBAL local_infile=1;"
29+
mysql -u root -proot -e "SET GLOBAL sql_notes=0;"
30+
mysql -u root -proot -e "create database mimic"
31+
32+
- name: Load icu/hosp demo data
33+
run: |
34+
echo "Loading data into mysql."
35+
mysql -u root -proot --local-infile=1 mimic < mimic-iv/buildmimic/mysql/load.sql
36+
mysql -u root -proot mimic < mimic-iv/buildmimic/mysql/validate_demo.sql > results
37+
38+
# if we find "FAILED", then we did not pass the build
39+
if grep -F -q "FAILED" results; then
40+
echo "Failed the following row counts:"
41+
head -n 1 results
42+
grep "FAILED" results
43+
exit 1
44+
else
45+
echo "Built and loaded demo data successfully."
46+
cat results
47+
fi
48+
49+
- name: Load ed demo data
50+
run: |
51+
echo "Loading data into mysql."
52+
mysql -u root -proot --local-infile=1 mimic < mimic-iv-ed/buildmimic/mysql/load.sql
53+
mysql -u root -proot mimic < mimic-iv-ed/buildmimic/mysql/validate_demo.sql > results
54+
55+
# if we find "FAILED", then we did not pass the build
56+
if grep -F -q "FAILED" results; then
57+
echo "Failed the following row counts:"
58+
head -n 1 results
59+
grep "FAILED" results
60+
exit 1
61+
else
62+
echo "Built and loaded demo data successfully."
63+
cat results
64+
fi

.github/workflows/build-db.yml renamed to .github/workflows/psql.yml

Lines changed: 18 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
name: Test DB build scripts
1+
name: psql demo db build
22
on: pull_request
33

44
jobs:
5-
build-mimic-iv-psql:
5+
mimic-iv-psql:
66
# Containers must run in Linux based operating systems
77
runs-on: ubuntu-latest
88
# Docker Hub image that `container-job` executes in
@@ -28,26 +28,16 @@ jobs:
2828
- name: Check out repository code
2929
uses: actions/checkout@v3
3030

31-
- name: Install psql command
32-
run: |
33-
apt-get update
34-
apt-get install --yes --no-install-recommends postgresql-client
35-
36-
- id: 'auth'
37-
uses: 'google-github-actions/auth@v0'
31+
- name: Download demo data
32+
uses: ./.github/actions/download-demo
3833
with:
39-
project_id: ${{ secrets.GCP_PROJECT_ID }}
40-
credentials_json: ${{ secrets.GCP_SA_KEY }}
34+
gcp-project-id: ${{ secrets.GCP_PROJECT_ID }}
35+
gcp-sa-key: ${{ secrets.GCP_SA_KEY }}
4136

42-
- name: 'Set up Cloud SDK'
43-
uses: 'google-github-actions/setup-gcloud@v0'
44-
45-
- name: Download demo data from GCP
37+
- name: Install postgresql-client
4638
run: |
47-
echo "Downloading MIMIC-IV demo from GCP."
48-
gsutil -q -u $PROJECT_ID -m cp -r gs://mimic-iv-archive/v2.0/demo ./
49-
env:
50-
PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
39+
apt-get update
40+
apt-get install --yes --no-install-recommends postgresql-client
5141
5242
- name: Load icu/hosp data into PostgreSQL
5343
run: |
@@ -72,10 +62,17 @@ jobs:
7262
# The hostname used to communicate with the PostgreSQL service container
7363
POSTGRES_HOST: postgres
7464
PGPASSWORD: postgres
75-
# The default PostgreSQL port
76-
POSTGRES_PORT: 5432
7765
BUILDCODE_PATH: mimic-iv/buildmimic/postgres
7866

67+
- name: Build mimic-iv concepts
68+
run: |
69+
psql -h $POSTGRES_HOST -U postgres -f postgres-functions.sql
70+
psql -h $POSTGRES_HOST -U postgres -f postgres-make-concepts.sql
71+
working-directory: ./mimic-iv/concepts_postgres
72+
env:
73+
POSTGRES_HOST: postgres
74+
PGPASSWORD: postgres
75+
7976
- name: Load ed data into PostgreSQL
8077
run: |
8178
echo "Loading data into psql."
@@ -102,71 +99,3 @@ jobs:
10299
# The default PostgreSQL port
103100
POSTGRES_PORT: 5432
104101
BUILDCODE_PATH: mimic-iv-ed/buildmimic/postgres
105-
106-
build-mimic-iv-mysql:
107-
runs-on: ubuntu-22.04
108-
109-
steps:
110-
- name: Check out repository code
111-
uses: actions/checkout@v3
112-
113-
- id: 'auth'
114-
uses: 'google-github-actions/auth@v0'
115-
with:
116-
project_id: ${{ secrets.GCP_PROJECT_ID }}
117-
credentials_json: ${{ secrets.GCP_SA_KEY }}
118-
119-
- name: 'Set up Cloud SDK'
120-
uses: 'google-github-actions/setup-gcloud@v0'
121-
122-
- name: Download data from GCP
123-
run: |
124-
echo "Downloading MIMIC-IV demo from GCP."
125-
gsutil -q -u $PROJECT_ID -m cp -r gs://mimic-iv-archive/v2.0/demo ./
126-
mv demo/hosp/*.csv.gz ./
127-
mv demo/icu/*.csv.gz ./
128-
mv demo/ed/*.csv.gz ./
129-
gzip -d *.csv.gz
130-
env:
131-
PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
132-
133-
- name: Start MySQL service
134-
run: |
135-
sudo /etc/init.d/mysql start
136-
mysql -u root -proot -e "SET GLOBAL local_infile=1;"
137-
mysql -u root -proot -e "SET GLOBAL sql_notes=0;"
138-
mysql -u root -proot -e "create database mimic"
139-
140-
- name: Load icu/hosp demo data
141-
run: |
142-
echo "Loading data into mysql."
143-
mysql -u root -proot --local-infile=1 mimic < mimic-iv/buildmimic/mysql/load.sql
144-
mysql -u root -proot mimic < mimic-iv/buildmimic/mysql/validate_demo.sql > results
145-
146-
# if we find "FAILED", then we did not pass the build
147-
if grep -F -q "FAILED" results; then
148-
echo "Failed the following row counts:"
149-
head -n 1 results
150-
grep "FAILED" results
151-
exit 1
152-
else
153-
echo "Built and loaded demo data successfully."
154-
cat results
155-
fi
156-
157-
- name: Load ed demo data
158-
run: |
159-
echo "Loading data into mysql."
160-
mysql -u root -proot --local-infile=1 mimic < mimic-iv-ed/buildmimic/mysql/load.sql
161-
mysql -u root -proot mimic < mimic-iv-ed/buildmimic/mysql/validate_demo.sql > results
162-
163-
# if we find "FAILED", then we did not pass the build
164-
if grep -F -q "FAILED" results; then
165-
echo "Failed the following row counts:"
166-
head -n 1 results
167-
grep "FAILED" results
168-
exit 1
169-
else
170-
echo "Built and loaded demo data successfully."
171-
cat results
172-
fi

0 commit comments

Comments
 (0)