add CAST(... AS NUMERIC) for ROUND() calls, remove perl regex

Author: alistairewj

mimic-iv/concepts/convert_bigquery_to_postgres.sh

@@ -8,8 +8,6 @@ export REGEX_DATETIME_DIFF="s/DATETIME_DIFF\(([^,]+), ?(.*), ?(DAY|MINUTE|SECOND
 export REGEX_DATETIME_TRUNC="s/DATETIME_TRUNC\(([^,]+), ?(DAY|MINUTE|SECOND|HOUR|YEAR)\)/DATE_TRUNC('\2', \1)/g"
 # Add necessary quotes to INTERVAL, e.g. "INTERVAL 5 hour" to "INTERVAL '5' hour"
 export REGEX_INTERVAL="s/interval ([[:digit:]]+) (hour|day|month|year)/INTERVAL '\1' \2/gI"
-# Add numeric cast to ROUND(), e.g. "ROUND(1.234, 2)" to "ROUND( CAST(1.234 as numeric), 2)".
-export PERL_REGEX_ROUND='s/ROUND\(((.|\n)*?)(\, [24]\))/ROUND\( CAST\( \1 as numeric\)\3/g'
 # Specific queries for some problems that arose with some files.
 export REGEX_INT="s/CAST\(hr AS INT64\)/CAST\(hr AS bigint\)/g"
 export REGEX_ARRAY="s/GENERATE_ARRAY\(-24, CEIL\(DATETIME\_DIFF\(it\.outtime_hr, it\.intime_hr, HOUR\)\)\)/ARRAY\(SELECT \* FROM generate\_series\(-24, CEIL\(DATETIME\_DIFF\(it\.outtime_hr, it\.intime_hr, HOUR\)\)\)\)/g"
@@ -51,7 +49,7 @@ do
 
   # for two scripts, add a perl replace to cast rounded values as numeric
   if [[ "${tbl}" == "icustay_times" ]] || [[ "${tbl}" == "urine_output" ]]; then
-    cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" | sed -r -e "${REGEX_SECONDS}" | perl -0777 -pe "${PERL_REGEX_ROUND}" >> "postgres/${d}/${tbl}.sql"
+    cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" | sed -r -e "${REGEX_SECONDS}" >> "postgres/${d}/${tbl}.sql"
   else
     cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" | sed -r -e "${REGEX_SECONDS}" >> "postgres/${d}/${tbl}.sql"
   fi
@@ -93,7 +91,7 @@ do
             echo -n " ${tbl} .."
             echo "-- THIS SCRIPT IS AUTOMATICALLY GENERATED. DO NOT EDIT IT DIRECTLY." > "postgres/${d}/${tbl}.sql"
             echo "DROP TABLE IF EXISTS ${tbl}; CREATE TABLE ${tbl} AS " >> "postgres/${d}/${tbl}.sql"
-            cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" | perl -0777 -pe "${PERL_REGEX_ROUND}" >> "postgres/${d}/${fn}"
+            cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" >> "postgres/${d}/${fn}"
 
             echo "\i ${d}/${fn}" >> postgres/postgres-make-concepts.sql
         fi

mimic-iv/concepts/demographics/icustay_detail.sql

@@ -16,7 +16,7 @@ SELECT ie.subject_id, ie.hadm_id, ie.stay_id
 
 -- icu level factors
 , ie.intime as icu_intime, ie.outtime as icu_outtime
-, ROUND(DATETIME_DIFF(ie.outtime, ie.intime, HOUR)/24.0, 2) as los_icu
+, ROUND(CAST(DATETIME_DIFF(ie.outtime, ie.intime, HOUR)/24.0 AS NUMERIC), 2) as los_icu
 , DENSE_RANK() OVER (PARTITION BY ie.hadm_id ORDER BY ie.intime) AS icustay_seq
 
 -- first ICU stay *for the current hospitalization*

mimic-iv/concepts/firstday/first_day_height.sql

@@ -21,7 +21,7 @@ WITH ce AS
 SELECT
     ie.subject_id
     , ie.stay_id
-    , ROUND(AVG(height), 2) AS height
+    , ROUND(CAST(AVG(height) AS NUMERIC), 2) AS height
 FROM `physionet-data.mimiciv_icu.icustays` ie
 LEFT JOIN `physionet-data.mimiciv_derived.height` ht
     ON ie.stay_id = ht.stay_id

mimic-iv/concepts/measurement/blood_differential.sql

@@ -94,31 +94,31 @@ subject_id, hadm_id, charttime, specimen_id
 
 , wbc
 -- impute absolute count if percentage & WBC is available
-, ROUND(CASE
+, ROUND(CAST(CASE
     WHEN basophils_abs IS NULL AND basophils IS NOT NULL AND impute_abs = 1
         THEN basophils * wbc / 100
     ELSE basophils_abs
-END, 4) AS basophils_abs
-, ROUND(CASE
+END AS NUMERIC), 4) AS basophils_abs
+, ROUND(CAST(CASE
     WHEN eosinophils_abs IS NULL AND eosinophils IS NOT NULL AND impute_abs = 1
         THEN eosinophils * wbc / 100
     ELSE eosinophils_abs
-END, 4) AS eosinophils_abs
-, ROUND(CASE
+END AS NUMERIC), 4) AS eosinophils_abs
+, ROUND(CAST(CASE
     WHEN lymphocytes_abs IS NULL AND lymphocytes IS NOT NULL AND impute_abs = 1
         THEN lymphocytes * wbc / 100
     ELSE lymphocytes_abs
-END, 4) AS lymphocytes_abs
-, ROUND(CASE
+END AS NUMERIC), 4) AS lymphocytes_abs
+, ROUND(CAST(CASE
     WHEN monocytes_abs IS NULL AND monocytes IS NOT NULL AND impute_abs = 1
         THEN monocytes * wbc / 100
     ELSE monocytes_abs
-END, 4) AS monocytes_abs
-, ROUND(CASE
+END AS NUMERIC), 4) AS monocytes_abs
+, ROUND(CAST(CASE
     WHEN neutrophils_abs IS NULL AND neutrophils IS NOT NULL AND impute_abs = 1
         THEN neutrophils * wbc / 100
     ELSE neutrophils_abs
-END, 4) AS neutrophils_abs
+END AS NUMERIC), 4) AS neutrophils_abs
 
 , basophils
 , eosinophils

mimic-iv/concepts/measurement/height.sql

@@ -4,7 +4,7 @@ WITH ht_in AS
   SELECT 
     c.subject_id, c.stay_id, c.charttime
     -- Ensure that all heights are in centimeters
-    , ROUND(c.valuenum * 2.54, 2) AS height
+    , ROUND(CAST(c.valuenum * 2.54 AS NUMERIC), 2) AS height
     , c.valuenum as height_orig
   FROM `physionet-data.mimiciv_icu.chartevents` c
   WHERE c.valuenum IS NOT NULL
@@ -16,7 +16,7 @@ WITH ht_in AS
   SELECT 
     c.subject_id, c.stay_id, c.charttime
     -- Ensure that all heights are in centimeters
-    , ROUND(c.valuenum, 2) AS height
+    , ROUND(CAST(c.valuenum AS NUMERIC), 2) AS height
   FROM `physionet-data.mimiciv_icu.chartevents` c
   WHERE c.valuenum IS NOT NULL
   -- Height cm

mimic-iv/concepts/measurement/urine_output_rate.sql

@@ -81,9 +81,9 @@ select
 , CASE WHEN uo_tm_12hr >= 12 THEN ROUND(CAST((ur.urineoutput_12hr/wd.weight/uo_tm_12hr) AS NUMERIC), 4) END AS uo_mlkghr_12hr
 , CASE WHEN uo_tm_24hr >= 24 THEN ROUND(CAST((ur.urineoutput_24hr/wd.weight/uo_tm_24hr) AS NUMERIC), 4) END AS uo_mlkghr_24hr
 -- time of earliest UO measurement that was used to calculate the rate
-, ROUND(uo_tm_6hr, 2) AS uo_tm_6hr
-, ROUND(uo_tm_12hr, 2) AS uo_tm_12hr
-, ROUND(uo_tm_24hr, 2) AS uo_tm_24hr
+, ROUND(CAST(uo_tm_6hr AS NUMERIC), 2) AS uo_tm_6hr
+, ROUND(CAST(uo_tm_12hr AS NUMERIC), 2) AS uo_tm_12hr
+, ROUND(CAST(uo_tm_24hr AS NUMERIC), 2) AS uo_tm_24hr
 from ur_stg ur
 LEFT JOIN `physionet-data.mimiciv_derived.weight_durations` wd
     ON ur.stay_id = wd.stay_id

mimic-iv/concepts/medication/norepinephrine_equivalent_dose.sql

@@ -3,15 +3,16 @@
 -- by Goradia et al. 2020.
 SELECT stay_id, starttime, endtime
 -- calculate the dose
-, ROUND(COALESCE(norepinephrine, 0)
+, ROUND(CAST(
+    COALESCE(norepinephrine, 0)
   + COALESCE(epinephrine, 0)
   + COALESCE(phenylephrine/10, 0)
   + COALESCE(dopamine/100, 0)
   -- + metaraminol/8 -- metaraminol not used in BIDMC
   + COALESCE(vasopressin*2.5, 0)
   -- angotensin_ii*10 -- angitensin ii rarely used, currently not incorporated
   -- (it could be included due to norepinephrine sparing effects)
-  , 4) AS norepinephrine_equivalent_dose
+  AS NUMERIC), 4) AS norepinephrine_equivalent_dose
   -- angotensin_ii*10 -- angitensin ii rarely used, currently not incorporated
   -- (it could be included due to norepinephrine sparing effects)
 FROM `physionet-data.mimiciv_derived.vasoactive_agent`