move table check code into an array, add tables to it

Author: alistairewj

mimic-iv/concepts/convert_bigquery_to_postgres.sh

@@ -64,10 +64,19 @@ echo " done!"
 # (2) output to the postgres subfolder
 # (3) add a line to the postgres-make-concepts.sql script to generate this table
 
-# order of the folders is important for a few tables here:
-# * scores (sofa et al) depend on labs, icustay_hourly
-# * sepsis depends on score (sofa.sql in particular)
-# * organfailure depends on measurement and firstday
+# we control the order by skipping tables listed in the below var
+DIR_AND_TABLES_TO_SKIP='demographics.icustay_times demographics.weight_durations measurement.urine_output organfailure.kdigo_uo organfailure.kdigo_stages firstday.first_day_sofa sepsis.sepsis3 medication.vasoactive_agent medication.norepinephrine_equivalent_dose'
+
+# create an array to store tables for which the order of generation matters
+# i.e. these tables cannot be generated in alphabetical order, as done in the later loop
+TABLES_TO_SKIP=()
+for dir_and_table in $DIR_AND_TABLES_TO_SKIP;
+do
+  tbl=`echo ${dir_and_table} | cut -d. -f2`
+  TABLES_TO_SKIP+=($tbl)
+done
+
+echo $TABLES_TO_SKIP
 # the order *only* matters during the conversion step because our loop is
 # inserting table build commands into the postgres-make-concepts.sql file
 for d in demographics measurement comorbidity medication treatment firstday organfailure score sepsis;
@@ -93,17 +102,22 @@ do
             echo "DROP TABLE IF EXISTS ${tbl}; CREATE TABLE ${tbl} AS " >> "postgres/${d}/${tbl}.sql"
             cat "${d}/${tbl}.sql" | sed -r -e "${REGEX_ARRAY}" | sed -r -e "${REGEX_HOUR_INTERVAL}" | sed -r -e "${REGEX_INT}" | sed -r -e "${REGEX_DATETIME_DIFF}" | sed -r -e "${REGEX_DATETIME_TRUNC}" | sed -r -e "${REGEX_SCHEMA}" | sed -r -e "${REGEX_INTERVAL}" >> "postgres/${d}/${fn}"
 
-            echo "\i ${d}/${fn}" >> postgres/postgres-make-concepts.sql
+            if [[ ! " ${TABLES_TO_SKIP[*]} " =~ " ${tbl} " ]]; then
+                # this table is *not* in our skip array
+                # therefore, we print it out to the make concepts script
+                echo "\i ${d}/${fn}" >> postgres/postgres-make-concepts.sql
+            fi
         fi
     done
     echo " done!"
 done
 
 # finally generate first_day_sofa which depends on concepts in firstday folder
 echo "" >> postgres/postgres-make-concepts.sql
-echo "-- final tables dependent on previous concepts" >> postgres/postgres-make-concepts.sql
+echo "-- final tables which were dependent on one or more prior tables" >> postgres/postgres-make-concepts.sql
 
-for dir_and_table in firstday.first_day_sofa sepsis.sepsis3
+echo -n "final:"
+for dir_and_table in $DIR_AND_TABLES_TO_SKIP
 do
   d=`echo ${dir_and_table} | cut -d. -f1`
   tbl=`echo ${dir_and_table} | cut -d. -f2`