Skip to content

Commit e1d2dae

Browse files
alistairewjalistairewj
authored
Merge pull request #1469 from MIT-LCP/mimiciv_v2.2_updates
MIMIC-IV v2.2 updates
2 parents ed1702c + 25570f6 commit e1d2dae

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2388
-200
lines changed

‎.github/actions/download-demo/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ runs:
2222
- name: Download demo data from GCP
2323
run: |
2424
echo "Downloading MIMIC-IV demo from GCP."
25-
gsutil -q -u $PROJECT_ID -m cp -r gs://mimic-iv-archive/v2.0/demo ./
25+
gsutil -q -u $PROJECT_ID -m cp -r gs://mimic-iv-archive/v2.2/demo ./
2626
env:
2727
PROJECT_ID: ${{ inputs.gcp-project-id }}
2828
shell: bash

‎.github/workflows/duckdb.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: duckdb demo build
2+
on:
3+
pull_request_review:
4+
types: [submitted]
5+
6+
jobs:
7+
mimic-iv-sqlite:
8+
# only run if PR is approved
9+
if: github.event.review.state == 'approved'
10+
runs-on: ubuntu-20.04
11+
12+
steps:
13+
- name: Check out repository code
14+
uses: actions/checkout@v3
15+
16+
- name: Install dependencies and download CLI
17+
run: |
18+
apt-get install unzip
19+
wget https://github.com/duckdb/duckdb/releases/download/v0.6.1/duckdb_cli-linux-amd64.zip
20+
unzip duckdb_cli-linux-amd64.zip
21+
22+
- name: Download demo data
23+
uses: ./.github/actions/download-demo
24+
with:
25+
gcp-project-id: ${{ secrets.GCP_PROJECT_ID }}
26+
gcp-sa-key: ${{ secrets.GCP_SA_KEY }}
27+
28+
- name: Load icu/hosp data into duckdb
29+
run: |
30+
echo "Running duckdb build."
31+
./${BUILDCODE_PATH}/import_duckdb.sh
32+
33+
echo `md5sum mimic4.db`
34+
35+
env:
36+
BUILDCODE_PATH: mimic-iv/buildmimic/duckdb

‎mimic-iv-ed/buildmimic/bigquery/schemas/ed/diagnosis.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
"type": "STRING",
2020
"mode": "REQUIRED"
2121
},
22+
{
23+
"name": "icd_version",
24+
"type": "INT64",
25+
"mode": "REQUIRED"
26+
},
2227
{
2328
"name": "icd_title",
2429
"type": "STRING",

‎mimic-iv-note/buildmimic/bigquery/schemas/discharge.json‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
[
2+
{
3+
"name": "note_id",
4+
"type": "STRING",
5+
"mode": "REQUIRED"
6+
},
7+
{
8+
"name": "subject_id",
9+
"type": "INT64",
10+
"mode": "REQUIRED"
11+
},
12+
{
13+
"name": "hadm_id",
14+
"type": "INT64",
15+
"mode": "NULLABLE"
16+
},
17+
{
18+
"name": "note_type",
19+
"type": "STRING",
20+
"mode": "REQUIRED"
21+
},
22+
{
23+
"name": "note_seq",
24+
"type": "INT64",
25+
"mode": "REQUIRED"
26+
},
27+
{
28+
"name": "charttime",
29+
"type": "DATETIME",
30+
"mode": "REQUIRED"
31+
},
32+
{
33+
"name": "storetime",
34+
"type": "DATETIME",
35+
"mode": "NULLABLE"
36+
},
37+
{
38+
"name": "text",
39+
"type": "STRING",
40+
"mode": "REQUIRED"
41+
}
42+
]

‎mimic-iv-note/buildmimic/bigquery/schemas/discharge_detail.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
[
2+
{
3+
"name": "note_id",
4+
"type": "STRING",
5+
"mode": "REQUIRED"
6+
},
7+
{
8+
"name": "subject_id",
9+
"type": "INT64",
10+
"mode": "REQUIRED"
11+
},
12+
{
13+
"name": "field_name",
14+
"type": "STRING",
15+
"mode": "REQUIRED"
16+
},
17+
{
18+
"name": "field_value",
19+
"type": "STRING",
20+
"mode": "REQUIRED"
21+
},
22+
{
23+
"name": "field_ordinal",
24+
"type": "INT64",
25+
"mode": "REQUIRED"
26+
}
27+
]

‎mimic-iv-note/buildmimic/bigquery/schemas/radiology.json‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
[
2+
{
3+
"name": "note_id",
4+
"type": "STRING",
5+
"mode": "REQUIRED"
6+
},
7+
{
8+
"name": "subject_id",
9+
"type": "INT64",
10+
"mode": "REQUIRED"
11+
},
12+
{
13+
"name": "hadm_id",
14+
"type": "INT64",
15+
"mode": "NULLABLE"
16+
},
17+
{
18+
"name": "note_type",
19+
"type": "STRING",
20+
"mode": "REQUIRED"
21+
},
22+
{
23+
"name": "note_seq",
24+
"type": "INT64",
25+
"mode": "REQUIRED"
26+
},
27+
{
28+
"name": "charttime",
29+
"type": "DATETIME",
30+
"mode": "REQUIRED"
31+
},
32+
{
33+
"name": "storetime",
34+
"type": "DATETIME",
35+
"mode": "NULLABLE"
36+
},
37+
{
38+
"name": "text",
39+
"type": "STRING",
40+
"mode": "REQUIRED"
41+
}
42+
]

‎mimic-iv-note/buildmimic/bigquery/schemas/radiology_detail.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
[
2+
{
3+
"name": "note_id",
4+
"type": "STRING",
5+
"mode": "REQUIRED"
6+
},
7+
{
8+
"name": "subject_id",
9+
"type": "INT64",
10+
"mode": "REQUIRED"
11+
},
12+
{
13+
"name": "field_name",
14+
"type": "STRING",
15+
"mode": "REQUIRED"
16+
},
17+
{
18+
"name": "field_value",
19+
"type": "STRING",
20+
"mode": "REQUIRED"
21+
},
22+
{
23+
"name": "field_ordinal",
24+
"type": "INT64",
25+
"mode": "REQUIRED"
26+
}
27+
]

‎mimic-iv-note/buildmimic/postgres/README.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Load MIMIC-IV-Note into a PostgreSQL database
2+
3+
The scripts in this folder create the schema for MIMIC-IV-Note and load the data into the appropriate tables for PostgreSQL v10+.
4+
5+
## Quickstart
6+
7+
```sh
8+
# clone repo
9+
git clone https://github.com/MIT-LCP/mimic-code.git
10+
cd mimic-code
11+
# download data
12+
wget -r -N -c -np --user <USERNAME> --ask-password https://physionet.org/files/mimiciv/2.2/
13+
mv physionet.org/files/mimiciv mimiciv && rmdir physionet.org/files && rm physionet.org/robots.txt && rmdir physionet.org
14+
# if mimiciv not exists
15+
# createdb mimiciv
16+
psql -d mimiciv -f mimic-iv-note/buildmimic/postgres/create.sql
17+
psql -d mimiciv -v ON_ERROR_STOP=1 -v mimic_data_dir=mimiciv/2.2 -f mimic-iv-note/buildmimic/postgres/load_gz.sql
18+
```

‎mimic-iv-note/buildmimic/postgres/create.sql‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
-------------------------------------------
2+
-- Create the tables and MIMIC-IV-Note schema --
3+
-------------------------------------------
4+
5+
----------------------
6+
-- Creating schemas --
7+
----------------------
8+
9+
DROP SCHEMA IF EXISTS mimiciv_note CASCADE;
10+
CREATE SCHEMA mimiciv_note;
11+
12+
---------------------
13+
-- Creating tables --
14+
---------------------
15+
16+
-- schema
17+
18+
DROP TABLE IF EXISTS mimiciv_note.discharge;
19+
CREATE TABLE mimiciv_note.discharge
20+
(
21+
note_id VARCHAR(25) NOT NULL,
22+
subject_id INTEGER NOT NULL,
23+
hadm_id INTEGER NOT NULL,
24+
note_type VARCHAR(2) NOT NULL,
25+
note_seq SMALLINT NOT NULL,
26+
charttime TIMESTAMP NOT NULL,
27+
storetime TIMESTAMP,
28+
text TEXT NOT NULL
29+
);
30+
31+
DROP TABLE IF EXISTS mimiciv_note.radiology;
32+
CREATE TABLE mimiciv_note.radiology
33+
(
34+
note_id VARCHAR(25) NOT NULL,
35+
subject_id INTEGER NOT NULL,
36+
hadm_id INTEGER,
37+
note_type VARCHAR(2) NOT NULL,
38+
note_seq SMALLINT NOT NULL,
39+
charttime TIMESTAMP NOT NULL,
40+
storetime TIMESTAMP,
41+
text TEXT NOT NULL
42+
);
43+
44+
DROP TABLE IF EXISTS mimiciv_note.discharge_detail;
45+
CREATE TABLE mimiciv_note.discharge_detail
46+
(
47+
note_id VARCHAR(25) NOT NULL,
48+
subject_id INTEGER NOT NULL,
49+
field_name VARCHAR(255) NOT NULL,
50+
field_value TEXT NOT NULL,
51+
field_ordinal INTEGER NOT NULL
52+
);
53+
54+
55+
DROP TABLE IF EXISTS mimiciv_note.radiology_detail;
56+
CREATE TABLE mimiciv_note.radiology_detail
57+
(
58+
note_id VARCHAR(25) NOT NULL,
59+
subject_id INTEGER NOT NULL,
60+
field_name VARCHAR(255) NOT NULL,
61+
field_value TEXT NOT NULL,
62+
field_ordinal INTEGER NOT NULL
63+
);

‎mimic-iv-note/buildmimic/postgres/load.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---------------------------------------------
2+
-- Load data into the MIMIC-IV-Note schema --
3+
---------------------------------------------
4+
5+
-- To run from a terminal:
6+
-- psql "dbname=<DBNAME> user=<USER>" -v mimic_data_dir=<PATH TO DATA DIR> -f load_gz.sql
7+
\cd :mimic_data_dir
8+
9+
-- making sure that all tables are emtpy and correct encoding is defined -utf8-
10+
SET CLIENT_ENCODING TO 'utf8';
11+
12+
\COPY mimiciv_note.discharge FROM 'discharge.csv' DELIMITER ',' CSV HEADER NULL '';
13+
\COPY mimiciv_note.radiology FROM 'radiology.csv' DELIMITER ',' CSV HEADER NULL '';
14+
\COPY mimiciv_note.discharge_detail FROM 'discharge_detail.csv' DELIMITER ',' CSV HEADER NULL '';
15+
\COPY mimiciv_note.radiology_detail FROM 'radiology_detail.csv' DELIMITER ',' CSV HEADER NULL '';

0 commit comments

Comments
 (0)