[build] improve dataloading and not dropping tables during build

Author: SPTKL

pluto_build/01_dataloading.sh

@@ -56,77 +56,7 @@ import dof_condo
 wait
 
 ## Load local CSV files
- psql $BUILD_ENGINE -c "
- DROP TABLE IF EXISTS pluto_input_research;
- CREATE TABLE pluto_input_research (
-    bbl text,
-    field text,
-    old_value text,
-    new_value text,
-    Type text,
-    reason text,
-    version text
- );
-
-DROP TABLE IF EXISTS pluto_input_landuse_bldgclass;
-CREATE TABLE pluto_input_landuse_bldgclass (
-    bldgclass text,
-    landuse text,
-    landusevalue text
- );
-
-DROP TABLE IF EXISTS pluto_input_condolot_descriptiveattributes;
-CREATE TABLE pluto_input_condolot_descriptiveattributes (
-    CondNO text,
-    Boro text,
-    PARID text,
-    BC text,
-    TC text,
-    LandSize text,
-    Story text,
-    YearBuilt text
-);
-
-DROP TABLE IF EXISTS pluto_input_condo_bldgclass;
-CREATE TABLE pluto_input_condo_bldgclass (
-    code character varying,
-    description character varying,
-    type character varying,
-    dcpcreated character varying,
-    logic character varying
-);
-
-DROP TABLE IF EXISTS pluto_input_bsmtcode;
-CREATE TABLE pluto_input_bsmtcode (
-    bsmnt_type character varying,
-    bsmntgradient character varying,
-    bsmtcode character varying,
-    bsmnt_typevalue character varying,
-    bsmntgradientvalue character varying,
-    bsmtcodevalue character varying
-);
-
-DROP TABLE IF EXISTS dcp_zoning_maxfar;
-CREATE TABLE dcp_zoning_maxfar (
-    zonedist character varying,
-    contextual character varying,
-    zoningdistricttype character varying,
-    residfar character varying,
-    facilfar character varying,
-    commfar character varying,
-    mnffar character varying,
-    column1 character varying,
-    verified character varying,
-    extra character varying
-)
-"
-
-imports_csv pluto_input_research &
-imports_csv pluto_input_landuse_bldgclass &
-imports_csv pluto_input_condolot_descriptiveattributes &
-imports_csv pluto_input_condo_bldgclass &
-imports_csv pluto_input_bsmtcode &
-imports_csv dcp_zoning_maxfar
+psql $BUILD_ENGINE -f sql/_create.sql
 
 # Create data version table 
- psql $BUILD_ENGINE -f sql/source_data_versions.sql
+psql $BUILD_ENGINE -f sql/source_data_versions.sql

pluto_build/02_build.sh

@@ -4,7 +4,6 @@ source bin/config.sh
 echo "Starting to build PLUTO ..."
 psql $BUILD_ENGINE -f sql/preprocessing.sql
 psql $BUILD_ENGINE -f sql/pts_clean.sql
-psql $BUILD_ENGINE -c "DROP TABLE pluto_pts;"
 psql $BUILD_ENGINE -f sql/create_rpad_geo.sql
 
 echo 'Reporting records that did not get geocoded...'
@@ -36,7 +35,6 @@ psql $BUILD_ENGINE -f sql/condono.sql
 echo 'Adding on CAMA data attributes'
 psql $BUILD_ENGINE -f sql/landuse.sql
 psql $BUILD_ENGINE -f sql/create_cama_primebbl.sql
-psql $BUILD_ENGINE -c "DROP TABLE pluto_input_cama_dof;"
 
 psql $BUILD_ENGINE -f sql/cama_bsmttype.sql
 psql $BUILD_ENGINE -f sql/cama_lottype.sql
@@ -46,7 +44,6 @@ psql $BUILD_ENGINE -f sql/cama_bldgarea_2.sql
 psql $BUILD_ENGINE -f sql/cama_bldgarea_3.sql
 psql $BUILD_ENGINE -f sql/cama_bldgarea_4.sql
 psql $BUILD_ENGINE -f sql/cama_easements.sql
-psql $BUILD_ENGINE -c "DROP TABLE pluto_input_geocodes;"
 
 echo 'Adding on data attributes from other sources'
 psql $BUILD_ENGINE -f sql/lpc.sql
@@ -83,7 +80,6 @@ psql $BUILD_ENGINE -f sql/zoning_parks.sql
 psql $BUILD_ENGINE -f sql/zoning_correctdups.sql
 psql $BUILD_ENGINE -f sql/zoning_correctgaps.sql
 psql $BUILD_ENGINE -f sql/zoning_splitzone.sql
-psql $BUILD_ENGINE -c "DROP TABLE dof_dtm;"
 psql $BUILD_ENGINE -c "VACUUM ANALYZE pluto;"
 
 echo 'Filling in FAR values'
@@ -117,9 +113,5 @@ psql $BUILD_ENGINE -v ON_ERROR_STOP=1 -f sql/plutomapid_1.sql
 psql $BUILD_ENGINE -v ON_ERROR_STOP=1 -f sql/plutomapid_2.sql
 psql $BUILD_ENGINE -f sql/shorelineclip.sql
 
-# echo 'Backfilling'
-# psql $BUILD_ENGINE -v ON_ERROR_STOP=1 -f sql/backfill.sql
-# psql $BUILD_ENGINE -c "VACUUM ANALYZE pluto;"
-
 echo 'Done'
 exit 0

pluto_build/sql/_create.sql

@@ -0,0 +1,70 @@
+DROP TABLE IF EXISTS pluto_input_research;
+CREATE TABLE pluto_input_research (
+    bbl text,
+    field text,
+    old_value text,
+    new_value text,
+    type text,
+    reason text,
+    version text
+);
+\COPY pluto_input_research FROM 'data/pluto_input_research.csv' DELIMITER ',' CSV HEADER;
+
+DROP TABLE IF EXISTS pluto_input_landuse_bldgclass;
+CREATE TABLE pluto_input_landuse_bldgclass (
+    bldgclass text,
+    landuse text,
+    landusevalue text
+);
+\COPY pluto_input_landuse_bldgclass FROM 'data/pluto_input_landuse_bldgclass.csv' DELIMITER ',' CSV HEADER;
+
+
+DROP TABLE IF EXISTS pluto_input_condolot_descriptiveattributes;
+CREATE TABLE pluto_input_condolot_descriptiveattributes (
+    CondNO text,
+    Boro text,
+    PARID text,
+    BC text,
+    TC text,
+    LandSize text,
+    Story text,
+    YearBuilt text
+);
+\COPY pluto_input_condolot_descriptiveattributes FROM 'data/pluto_input_condolot_descriptiveattributes.csv' DELIMITER ',' CSV HEADER;
+
+DROP TABLE IF EXISTS pluto_input_condo_bldgclass;
+CREATE TABLE pluto_input_condo_bldgclass (
+    code character varying,
+    description character varying,
+    type character varying,
+    dcpcreated character varying,
+    logic character varying
+);
+\COPY pluto_input_condo_bldgclass FROM 'data/pluto_input_condo_bldgclass.csv' DELIMITER ',' CSV HEADER;
+
+DROP TABLE IF EXISTS pluto_input_bsmtcode;
+CREATE TABLE pluto_input_bsmtcode (
+    bsmnt_type character varying,
+    bsmntgradient character varying,
+    bsmtcode character varying,
+    bsmnt_typevalue character varying,
+    bsmntgradientvalue character varying,
+    bsmtcodevalue character varying
+);
+\COPY pluto_input_bsmtcode FROM 'data/pluto_input_bsmtcode.csv' DELIMITER ',' CSV HEADER;
+
+
+DROP TABLE IF EXISTS dcp_zoning_maxfar;
+CREATE TABLE dcp_zoning_maxfar (
+    zonedist character varying,
+    contextual character varying,
+    zoningdistricttype character varying,
+    residfar character varying,
+    facilfar character varying,
+    commfar character varying,
+    mnffar character varying,
+    column1 character varying,
+    verified character varying,
+    extra character varying
+);
+\COPY dcp_zoning_maxfar FROM 'data/dcp_zoning_maxfar.csv' DELIMITER ',' CSV HEADER;