Merge pull request #1336 from MIT-LCP/mimiciv_v2_update

Update derived concepts for mimic-iv

Author: alistairewj

mimic-iv/concepts/comorbidity/charlson.sql

@@ -19,7 +19,7 @@ WITH diag AS
         hadm_id
         , CASE WHEN icd_version = 9 THEN icd_code ELSE NULL END AS icd9_code
         , CASE WHEN icd_version = 10 THEN icd_code ELSE NULL END AS icd10_code
-    FROM `physionet-data.mimic_hosp.diagnoses_icd` diag
+    FROM `physionet-data.mimiciv_hosp.diagnoses_icd` diag
 )
 , com AS
 (
@@ -253,7 +253,7 @@ WITH diag AS
             SUBSTR(icd10_code, 1, 3) IN ('B20','B21','B22','B24')
             THEN 1 
             ELSE 0 END) AS aids
-    FROM `physionet-data.mimic_core.admissions` ad
+    FROM `physionet-data.mimiciv_hosp.admissions` ad
     LEFT JOIN diag
     ON ad.hadm_id = diag.hadm_id
     GROUP BY ad.hadm_id
@@ -268,7 +268,7 @@ WITH diag AS
     WHEN age <= 60 THEN 2
     WHEN age <= 70 THEN 3
     ELSE 4 END AS age_score
-    FROM `physionet-data.mimic_derived.age`
+    FROM `physionet-data.mimiciv_derived.age`
 )
 SELECT 
     ad.subject_id
@@ -303,7 +303,7 @@ SELECT
     + 2*paraplegia + 2*renal_disease 
     + 6*aids
     AS charlson_comorbidity_index
-FROM `physionet-data.mimic_core.admissions` ad
+FROM `physionet-data.mimiciv_hosp.admissions` ad
 LEFT JOIN com
 ON ad.hadm_id = com.hadm_id
 LEFT JOIN ag

mimic-iv/concepts/convert_bigquery_to_postgres.sh

@@ -2,12 +2,12 @@
 # This shell script converts BigQuery .sql files into PostgreSQL .sql files.
 
 # String replacements are necessary for some queries.
-export REGEX_SCHEMA='s/`physionet-data.(mimic_core|mimic_icu|mimic_derived|mimic_hosp).(.+?)`/\1.\2/g'
+export REGEX_SCHEMA='s/`physionet-data.(mimiciv_hosp|mimiciv_icu|mimiciv_derived).([A-Za-z0-9_-]+)`/\1.\2/g'
 # Note that these queries are very senstive to changes, e.g. adding whitespaces after comma can already change the behavior.
-export REGEX_DATETIME_DIFF="s/DATETIME_DIFF\((.+?),\s?(.+?),\s?(DAY|MINUTE|SECOND|HOUR|YEAR)\)/DATETIME_DIFF(\1,\2,'\3')/g"
-export REGEX_DATETIME_TRUNC="s/DATETIME_TRUNC\((.+?),\s?(DAY|MINUTE|SECOND|HOUR|YEAR)\)/DATE_TRUNC('\2', \1)/g"
+export REGEX_DATETIME_DIFF="s/DATETIME_DIFF\(([^,]+), ?([^,]+), ?(DAY|MINUTE|SECOND|HOUR|YEAR)\)/DATETIME_DIFF(\1, \2, '\3')/g"
+export REGEX_DATETIME_TRUNC="s/DATETIME_TRUNC\(([^,]+), ?(DAY|MINUTE|SECOND|HOUR|YEAR)\)/DATE_TRUNC('\2', \1)/g"
 # Add necessary quotes to INTERVAL, e.g. "INTERVAL 5 hour" to "INTERVAL '5' hour"
-export REGEX_INTERVAL="s/interval\s([[:digit:]]+)\s(hour|day|month|year)/INTERVAL '\1' \2/gI"
+export REGEX_INTERVAL="s/interval ([[:digit:]]+) (hour|day|month|year)/INTERVAL '\1' \2/gI"
 # Add numeric cast to ROUND(), e.g. "ROUND(1.234, 2)" to "ROUND( CAST(1.234 as numeric), 2)".
 export PERL_REGEX_ROUND='s/ROUND\(((.|\n)*?)\, /ROUND\( CAST\( \1 as numeric\)\,/g'
 # Specific queries for some problems that arose with some files.
@@ -83,7 +83,7 @@ do
         # only run SQL queries
         if [[ "${fn: -4}" == ".sql" ]]; then
             # table name is file name minus extension
-            tbl="${fn::-4}"
+            tbl="${fn%????}"
 
             # skip first_day_sofa as it depends on other firstday queries, we'll generate it later
             # we also skipped tables generated in the "Dependencies" loop above.

mimic-iv/concepts/demographics/age.sql

@@ -22,7 +22,7 @@ SELECT
 	, pa.anchor_age
 	, pa.anchor_year
 	, DATETIME_DIFF(ad.admittime, DATETIME(pa.anchor_year, 1, 1, 0, 0, 0), YEAR) + pa.anchor_age AS age
-FROM `physionet-data.mimic_core.admissions` ad
-INNER JOIN `physionet-data.mimic_core.patients` pa
+FROM `physionet-data.mimiciv_hosp.admissions` ad
+INNER JOIN `physionet-data.mimiciv_hosp.patients` pa
 ON ad.subject_id = pa.subject_id
 ;

mimic-iv/concepts/demographics/icustay_detail.sql

@@ -7,7 +7,7 @@ SELECT ie.subject_id, ie.hadm_id, ie.stay_id
 , adm.admittime, adm.dischtime
 , DATETIME_DIFF(adm.dischtime, adm.admittime, DAY) as los_hospital
 , DATETIME_DIFF(adm.admittime, DATETIME(pat.anchor_year, 1, 1, 0, 0, 0), YEAR) + pat.anchor_age as admission_age
-, adm.ethnicity
+, adm.race
 , adm.hospital_expire_flag
 , DENSE_RANK() OVER (PARTITION BY adm.subject_id ORDER BY adm.admittime) AS hospstay_seq
 , CASE
@@ -24,8 +24,8 @@ SELECT ie.subject_id, ie.hadm_id, ie.stay_id
     WHEN DENSE_RANK() OVER (PARTITION BY ie.hadm_id ORDER BY ie.intime) = 1 THEN True
     ELSE False END AS first_icu_stay
 
-FROM `physionet-data.mimic_icu.icustays` ie
-INNER JOIN `physionet-data.mimic_core.admissions` adm
+FROM `physionet-data.mimiciv_icu.icustays` ie
+INNER JOIN `physionet-data.mimiciv_hosp.admissions` adm
     ON ie.hadm_id = adm.hadm_id
-INNER JOIN `physionet-data.mimic_core.patients` pat
+INNER JOIN `physionet-data.mimiciv_hosp.patients` pat
     ON ie.subject_id = pat.subject_id

mimic-iv/concepts/demographics/icustay_hourly.sql

@@ -27,7 +27,7 @@ select
   --  we allow 24 hours before ICU admission (to grab labs before admit)
   , GENERATE_ARRAY(-24, CEIL(DATETIME_DIFF(it.outtime_hr, it.intime_hr, HOUR))) as hrs
 
-  from `physionet-data.mimic_derived.icustay_times` it
+  from `physionet-data.mimiciv_derived.icustay_times` it
 )
 SELECT stay_id
 , CAST(hr AS INT64) as hr

mimic-iv/concepts/demographics/icustay_times.sql

@@ -6,7 +6,7 @@ WITH t1 AS
     select ce.stay_id
     , min(charttime) as intime_hr
     , max(charttime) as outtime_hr
-    FROM `physionet-data.mimic_icu.chartevents` ce
+    FROM `physionet-data.mimiciv_icu.chartevents` ce
     -- only look at heart rate
     where ce.itemid = 220045
     group by ce.stay_id
@@ -16,6 +16,6 @@ select
   ie.subject_id, ie.hadm_id, ie.stay_id
   , t1.intime_hr
   , t1.outtime_hr
-FROM `physionet-data.mimic_icu.icustays` ie
+FROM `physionet-data.mimiciv_icu.icustays` ie
 left join t1
   on ie.stay_id = t1.stay_id;

mimic-iv/concepts/demographics/weight_durations.sql

@@ -9,7 +9,7 @@ WITH wt_stg as
           else 'daily' end as weight_type
       -- TODO: eliminate obvious outliers if there is a reasonable weight
       , c.valuenum as weight
-    FROM `physionet-data.mimic_icu.chartevents` c
+    FROM `physionet-data.mimiciv_icu.chartevents` c
     WHERE c.valuenum IS NOT NULL
       AND c.itemid in
       (
@@ -42,7 +42,7 @@ WITH wt_stg as
       else wt_stg1.charttime end as starttime
     , wt_stg1.weight
   from wt_stg1
-  INNER JOIN `physionet-data.mimic_icu.icustays` ie
+  INNER JOIN `physionet-data.mimiciv_icu.icustays` ie
     on ie.stay_id = wt_stg1.stay_id
 )
 , wt_stg3 as
@@ -87,7 +87,7 @@ WITH wt_stg as
     , wt.starttime as endtime
     , wt.weight
     , wt.weight_type
-  from `physionet-data.mimic_icu.icustays` ie
+  from `physionet-data.mimiciv_icu.icustays` ie
   inner join
   -- the below subquery returns one row for each unique stay_id
   -- the row contains: the first starttime and the corresponding weight

mimic-iv/concepts/firstday/first_day_bg.sql

@@ -23,8 +23,8 @@ select
     , MIN(glucose) AS glucose_min, MAX(glucose) AS glucose_max
     , MIN(potassium) AS potassium_min, MAX(potassium) AS potassium_max
     , MIN(sodium) AS sodium_min, MAX(sodium) AS sodium_max
-FROM `physionet-data.mimic_icu.icustays` ie
-LEFT JOIN `physionet-data.mimic_derived.bg` bg
+FROM `physionet-data.mimiciv_icu.icustays` ie
+LEFT JOIN `physionet-data.mimiciv_derived.bg` bg
     ON ie.subject_id = bg.subject_id
     AND bg.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR)
     AND bg.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY)

mimic-iv/concepts/firstday/first_day_bg_art.sql

@@ -23,8 +23,8 @@ select
     , MIN(glucose) AS glucose_min, MAX(glucose) AS glucose_max
     , MIN(potassium) AS potassium_min, MAX(potassium) AS potassium_max
     , MIN(sodium) AS sodium_min, MAX(sodium) AS sodium_max
-FROM `physionet-data.mimic_icu.icustays` ie
-LEFT JOIN `physionet-data.mimic_derived.bg` bg
+FROM `physionet-data.mimiciv_icu.icustays` ie
+LEFT JOIN `physionet-data.mimiciv_derived.bg` bg
     ON ie.subject_id = bg.subject_id
     AND bg.specimen = 'ART.'
     AND bg.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR)

mimic-iv/concepts/firstday/first_day_gcs.sql

@@ -21,7 +21,7 @@ WITH gcs_final AS
             PARTITION BY gcs.stay_id
             ORDER BY gcs.GCS
         ) as gcs_seq
-    FROM `physionet-data.mimic_derived.gcs` gcs
+    FROM `physionet-data.mimiciv_derived.gcs` gcs
 )
 SELECT
     ie.subject_id
@@ -33,7 +33,7 @@ SELECT
     , gcs_verbal
     , gcs_eyes
     , gcs_unable
-FROM `physionet-data.mimic_icu.icustays` ie
+FROM `physionet-data.mimiciv_icu.icustays` ie
 LEFT JOIN gcs_final gs
     ON ie.stay_id = gs.stay_id
     AND gs.gcs_seq = 1