Merge pull request #1363 from schu/schu/fix-mimic-iv-concepts-postgres-setup

mimic-iv/concepts: fix `postgres-make-concepts` and minor updates

Author: alistairewj

mimic-iv/concepts/README.md

@@ -3,7 +3,7 @@
 This folder contains scripts to generate useful abstractions of raw MIMIC-IV data ("concepts").
 The scripts are written using the **BigQuery Standard SQL Dialect**. Concepts are categorized into folders if possible, otherwise they remain in the top-level directory. The [postgres](/mimic-iv/concepts/postgres) subfolder contains automatically generated PostgreSQL versions of these scripts; [see below for how these were generated](#postgresql-concepts). Concepts are categorized into folders if possible, otherwise they remain in the top-level directory.
 
-The concepts are organized into individual SQL scripts, with each script generating a table. The BigQuery `mimic_derived` dataset under `physionet-data` contains the concepts pregenerated. Access to this dataset is available to MIMIC-IV approved users: see the [cloud instructions](https://mimic.mit.edu/docs/gettingstarted/cloud/) on how to access MIMIC-IV on BigQuery (which includes the derived concepts).
+The concepts are organized into individual SQL scripts, with each script generating a table. The BigQuery `mimiciv_derived` dataset under `physionet-data` contains the concepts pregenerated. Access to this dataset is available to MIMIC-IV approved users: see the [cloud instructions](https://mimic.mit.edu/docs/gettingstarted/cloud/) on how to access MIMIC-IV on BigQuery (which includes the derived concepts).
 
 * [List of the concept folders and their content](#concept-index)
 * [Generating the concept tables on BigQuery](#generating-the-concepts-on-bigquery)

mimic-iv/concepts/convert_bigquery_to_postgres.sh

@@ -4,12 +4,10 @@
 # String replacements are necessary for some queries.
 export REGEX_SCHEMA='s/`physionet-data.(mimiciv_hosp|mimiciv_icu|mimiciv_derived).([A-Za-z0-9_-]+)`/\1.\2/g'
 # Note that these queries are very senstive to changes, e.g. adding whitespaces after comma can already change the behavior.
-export REGEX_DATETIME_DIFF="s/DATETIME_DIFF\(([^,]+), ?([^,]+), ?(DAY|MINUTE|SECOND|HOUR|YEAR)\)/DATETIME_DIFF(\1, \2, '\3')/g"
+export REGEX_DATETIME_DIFF="s/DATETIME_DIFF\(([^,]+), ?(.*), ?(DAY|MINUTE|SECOND|HOUR|YEAR)\)/DATETIME_DIFF(\1, \2, '\3')/g"
 export REGEX_DATETIME_TRUNC="s/DATETIME_TRUNC\(([^,]+), ?(DAY|MINUTE|SECOND|HOUR|YEAR)\)/DATE_TRUNC('\2', \1)/g"
 # Add necessary quotes to INTERVAL, e.g. "INTERVAL 5 hour" to "INTERVAL '5' hour"
 export REGEX_INTERVAL="s/interval ([[:digit:]]+) (hour|day|month|year)/INTERVAL '\1' \2/gI"
-# Add numeric cast to ROUND(), e.g. "ROUND(1.234, 2)" to "ROUND( CAST(1.234 as numeric), 2)".
-export PERL_REGEX_ROUND='s/ROUND\(((.|\n)*?)\, /ROUND\( CAST\( \1 as numeric\)\,/g'
 # Specific queries for some problems that arose with some files.
 export REGEX_INT="s/CAST\(hr AS INT64\)/CAST\(hr AS bigint\)/g"
 export REGEX_ARRAY="s/GENERATE_ARRAY\(-24, CEIL\(DATETIME\_DIFF\(it\.outtime_hr, it\.intime_hr, HOUR\)\)\)/ARRAY\(SELECT \* FROM generate\_series\(-24, CEIL\(DATETIME\_DIFF\(it\.outtime_hr, it\.intime_hr, HOUR\)\)\)\)/g"
@@ -51,7 +49,7 @@ do
 
   # for two scripts, add a perl replace to cast rounded values as numeric
   if [[ "${tbl}" == "icustay_times" ]] || [[ "${tbl}" == "urine_output" ]]; then
-    cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" | sed -r -e "${REGEX_SECONDS}" | perl -0777 -pe "${PERL_REGEX_ROUND}" >> "postgres/${d}/${tbl}.sql"
+    cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" | sed -r -e "${REGEX_SECONDS}" >> "postgres/${d}/${tbl}.sql"
   else
     cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" | sed -r -e "${REGEX_SECONDS}" >> "postgres/${d}/${tbl}.sql"
   fi
@@ -66,10 +64,19 @@ echo " done!"
 # (2) output to the postgres subfolder
 # (3) add a line to the postgres-make-concepts.sql script to generate this table
 
-# order of the folders is important for a few tables here:
-# * scores (sofa et al) depend on labs, icustay_hourly
-# * sepsis depends on score (sofa.sql in particular)
-# * organfailure depends on measurement and firstday
+# we control the order by skipping tables listed in the below var
+DIR_AND_TABLES_TO_SKIP='demographics.icustay_times demographics.weight_durations measurement.urine_output organfailure.kdigo_uo organfailure.kdigo_stages firstday.first_day_sofa sepsis.sepsis3 medication.vasoactive_agent medication.norepinephrine_equivalent_dose'
+
+# create an array to store tables for which the order of generation matters
+# i.e. these tables cannot be generated in alphabetical order, as done in the later loop
+TABLES_TO_SKIP=()
+for dir_and_table in $DIR_AND_TABLES_TO_SKIP;
+do
+  tbl=`echo ${dir_and_table} | cut -d. -f2`
+  TABLES_TO_SKIP+=($tbl)
+done
+
+echo $TABLES_TO_SKIP
 # the order *only* matters during the conversion step because our loop is
 # inserting table build commands into the postgres-make-concepts.sql file
 for d in demographics measurement comorbidity medication treatment firstday organfailure score sepsis;
@@ -93,19 +100,24 @@ do
             echo -n " ${tbl} .."
             echo "-- THIS SCRIPT IS AUTOMATICALLY GENERATED. DO NOT EDIT IT DIRECTLY." > "postgres/${d}/${tbl}.sql"
             echo "DROP TABLE IF EXISTS ${tbl}; CREATE TABLE ${tbl} AS " >> "postgres/${d}/${tbl}.sql"
-            cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" | perl -0777 -pe "${PERL_REGEX_ROUND}" >> "postgres/${d}/${fn}"
+            cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" >> "postgres/${d}/${fn}"
 
-            echo "\i ${d}/${fn}" >> postgres/postgres-make-concepts.sql
+            if [[ ! " ${TABLES_TO_SKIP[*]} " =~ " ${tbl} " ]]; then
+                # this table is *not* in our skip array
+                # therefore, we print it out to the make concepts script
+                echo "\i ${d}/${fn}" >> postgres/postgres-make-concepts.sql
+            fi
         fi
     done
     echo " done!"
 done
 
 # finally generate first_day_sofa which depends on concepts in firstday folder
 echo "" >> postgres/postgres-make-concepts.sql
-echo "-- final tables dependent on previous concepts" >> postgres/postgres-make-concepts.sql
+echo "-- final tables which were dependent on one or more prior tables" >> postgres/postgres-make-concepts.sql
 
-for dir_and_table in firstday.first_day_sofa sepsis.sepsis3
+echo -n "final:"
+for dir_and_table in $DIR_AND_TABLES_TO_SKIP
 do
   d=`echo ${dir_and_table} | cut -d. -f1`
   tbl=`echo ${dir_and_table} | cut -d. -f2`

mimic-iv/concepts/demographics/icustay_detail.sql

@@ -16,7 +16,7 @@ SELECT ie.subject_id, ie.hadm_id, ie.stay_id
 
 -- icu level factors
 , ie.intime as icu_intime, ie.outtime as icu_outtime
-, ROUND(DATETIME_DIFF(ie.outtime, ie.intime, HOUR)/24.0, 2) as los_icu
+, ROUND(CAST(DATETIME_DIFF(ie.outtime, ie.intime, HOUR)/24.0 AS NUMERIC), 2) as los_icu
 , DENSE_RANK() OVER (PARTITION BY ie.hadm_id ORDER BY ie.intime) AS icustay_seq
 
 -- first ICU stay *for the current hospitalization*

mimic-iv/concepts/firstday/first_day_height.sql

@@ -21,7 +21,7 @@ WITH ce AS
 SELECT
     ie.subject_id
     , ie.stay_id
-    , ROUND(AVG(height), 2) AS height
+    , ROUND(CAST(AVG(height) AS NUMERIC), 2) AS height
 FROM `physionet-data.mimiciv_icu.icustays` ie
 LEFT JOIN `physionet-data.mimiciv_derived.height` ht
     ON ie.stay_id = ht.stay_id

mimic-iv/concepts/make_concepts.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# This script generates the concepts in the BigQuery table mimic_derived.
+# This script generates the concepts in the BigQuery table mimiciv_derived.
 export TARGET_DATASET=mimiciv_derived
 
 # specify bigquery query command options

mimic-iv/concepts/measurement/blood_differential.sql

@@ -94,31 +94,31 @@ subject_id, hadm_id, charttime, specimen_id
 
 , wbc
 -- impute absolute count if percentage & WBC is available
-, ROUND(CASE
+, ROUND(CAST(CASE
     WHEN basophils_abs IS NULL AND basophils IS NOT NULL AND impute_abs = 1
         THEN basophils * wbc / 100
     ELSE basophils_abs
-END, 4) AS basophils_abs
-, ROUND(CASE
+END AS NUMERIC), 4) AS basophils_abs
+, ROUND(CAST(CASE
     WHEN eosinophils_abs IS NULL AND eosinophils IS NOT NULL AND impute_abs = 1
         THEN eosinophils * wbc / 100
     ELSE eosinophils_abs
-END, 4) AS eosinophils_abs
-, ROUND(CASE
+END AS NUMERIC), 4) AS eosinophils_abs
+, ROUND(CAST(CASE
     WHEN lymphocytes_abs IS NULL AND lymphocytes IS NOT NULL AND impute_abs = 1
         THEN lymphocytes * wbc / 100
     ELSE lymphocytes_abs
-END, 4) AS lymphocytes_abs
-, ROUND(CASE
+END AS NUMERIC), 4) AS lymphocytes_abs
+, ROUND(CAST(CASE
     WHEN monocytes_abs IS NULL AND monocytes IS NOT NULL AND impute_abs = 1
         THEN monocytes * wbc / 100
     ELSE monocytes_abs
-END, 4) AS monocytes_abs
-, ROUND(CASE
+END AS NUMERIC), 4) AS monocytes_abs
+, ROUND(CAST(CASE
     WHEN neutrophils_abs IS NULL AND neutrophils IS NOT NULL AND impute_abs = 1
         THEN neutrophils * wbc / 100
     ELSE neutrophils_abs
-END, 4) AS neutrophils_abs
+END AS NUMERIC), 4) AS neutrophils_abs
 
 , basophils
 , eosinophils

mimic-iv/concepts/measurement/height.sql

@@ -4,7 +4,7 @@ WITH ht_in AS
   SELECT 
     c.subject_id, c.stay_id, c.charttime
     -- Ensure that all heights are in centimeters
-    , ROUND(c.valuenum * 2.54, 2) AS height
+    , ROUND(CAST(c.valuenum * 2.54 AS NUMERIC), 2) AS height
     , c.valuenum as height_orig
   FROM `physionet-data.mimiciv_icu.chartevents` c
   WHERE c.valuenum IS NOT NULL
@@ -16,7 +16,7 @@ WITH ht_in AS
   SELECT 
     c.subject_id, c.stay_id, c.charttime
     -- Ensure that all heights are in centimeters
-    , ROUND(c.valuenum, 2) AS height
+    , ROUND(CAST(c.valuenum AS NUMERIC), 2) AS height
   FROM `physionet-data.mimiciv_icu.chartevents` c
   WHERE c.valuenum IS NOT NULL
   -- Height cm

mimic-iv/concepts/measurement/oxygen_delivery.sql

@@ -61,7 +61,7 @@ with ce_stg1 as
         , itemid
         , value AS o2_device
     , ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY value) as rn
-    FROM mimic_icu.chartevents
+    FROM mimiciv_icu.chartevents
     WHERE itemid = 226732 -- oxygen delivery device(s)
 )
 , stg AS
@@ -95,4 +95,4 @@ SELECT
     , MAX(CASE WHEN rn = 4 THEN o2_device ELSE NULL END) AS o2_delivery_device_4
 FROM stg
 GROUP BY subject_id, charttime
-;
+;

mimic-iv/concepts/measurement/urine_output_rate.sql

@@ -81,9 +81,9 @@ select
 , CASE WHEN uo_tm_12hr >= 12 THEN ROUND(CAST((ur.urineoutput_12hr/wd.weight/uo_tm_12hr) AS NUMERIC), 4) END AS uo_mlkghr_12hr
 , CASE WHEN uo_tm_24hr >= 24 THEN ROUND(CAST((ur.urineoutput_24hr/wd.weight/uo_tm_24hr) AS NUMERIC), 4) END AS uo_mlkghr_24hr
 -- time of earliest UO measurement that was used to calculate the rate
-, ROUND(uo_tm_6hr, 2) AS uo_tm_6hr
-, ROUND(uo_tm_12hr, 2) AS uo_tm_12hr
-, ROUND(uo_tm_24hr, 2) AS uo_tm_24hr
+, ROUND(CAST(uo_tm_6hr AS NUMERIC), 2) AS uo_tm_6hr
+, ROUND(CAST(uo_tm_12hr AS NUMERIC), 2) AS uo_tm_12hr
+, ROUND(CAST(uo_tm_24hr AS NUMERIC), 2) AS uo_tm_24hr
 from ur_stg ur
 LEFT JOIN `physionet-data.mimiciv_derived.weight_durations` wd
     ON ur.stay_id = wd.stay_id

mimic-iv/concepts/measurement/vitalsign.sql

@@ -12,10 +12,10 @@ select
   , AVG(case when itemid = 220180 and valuenum > 0 and valuenum < 300 then valuenum else null end) as dbp_ni
   , AVG(case when itemid = 220181 and valuenum > 0 and valuenum < 300 then valuenum else null end) as mbp_ni
   , AVG(case when itemid in (220210,224690) and valuenum > 0 and valuenum < 70 then valuenum else null end) as resp_rate
-  , ROUND(
+  , ROUND(CAST(
       AVG(case when itemid in (223761) and valuenum > 70 and valuenum < 120 then (valuenum-32)/1.8 -- converted to degC in valuenum call
               when itemid in (223762) and valuenum > 10 and valuenum < 50  then valuenum else null end)
-    , 2) as temperature
+    AS NUMERIC), 2) as temperature
   , MAX(CASE WHEN itemid = 224642 THEN value ELSE NULL END) AS temperature_site
   , AVG(case when itemid in (220277) and valuenum > 0 and valuenum <= 100 then valuenum else null end) as spo2
   , AVG(case when itemid in (225664,220621,226537) and valuenum > 0 then valuenum else null end) as glucose