Add QAQC tables for Condo Checks

pluto_build/01_dataloading.sh

@@ -55,7 +55,8 @@ import_public lpc_historic_districts &
 import_public lpc_landmarks &
 import_public dcp_colp &
 import_public dof_dtm &
-import_public dof_condo
+import_public dof_condo &
+import_public dcp_housing 
 
 wait
 

pluto_build/01_dataloading_minor.sh

@@ -58,6 +58,7 @@ import_public dpr_greenthumb $DPR_GREENTHUMB_VERSION &
 import_public dsny_frequencies $DSNY_FREQUENCIES_VERSION &
 import_public lpc_historic_districts $LPC_HISTORIC_DISTRICTS_VERSION &
 import_public lpc_landmarks $LPC_LANDMARKS_VESRSION
+import_public dcp_housing $DCP_HOUSING
 
 # import pluto corrections
 import_public pluto_corrections $PLUTO_CORRECTIONS_VERSION &

pluto_build/05_qaqc.sh

@@ -44,7 +44,7 @@ function QAQC {
 }
 
 # QAQC MISMATCH ANALYSIS
-for file in sql/qaqc_aggregate.sql sql/qaqc_mismatch.sql sql/qaqc_null.sql sql/qaqc_outlier.sql
+for file in sql/qaqc_aggregate.sql sql/qaqc_mismatch.sql sql/qaqc_null.sql sql/qaqc_outlier.sql sql/qaqc_pts_condo.sql sql/qaqc_housing_units.sql
 do
   for mapped in true false
   do

pluto_build/06_export.sh

@@ -58,7 +58,7 @@ mkdir -p output/dof &&
 
 mkdir -p output/qaqc && 
   (cd output/qaqc
-    for table in qaqc_aggregate qaqc_expected qaqc_mismatch qaqc_null qaqc_outlier
+    for table in qaqc_aggregate qaqc_expected qaqc_mismatch qaqc_null qaqc_outlier qaqc_pts_condo qaqc_housing_units
     do
       psql $BUILD_ENGINE -c "\COPY ( 
           SELECT * FROM ${table}

pluto_build/sql/qaqc_housing_units.sql

@@ -0,0 +1,45 @@
+-- Create qaqc table All PLUTO records where there is a match in Housing Database 
+-- and the PLUTO residential units value does not match the housing database certificates 
+-- of occupancy value. Have flag indicating if bbl has residential unit correction in manual corrections table.
+-- select PLUTO records that have a match in the HousingDB subset where unitsres does not equal units co
+
+
+-- Data Dictionary 
+-- bbl - billing BBL
+-- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). 
+-- units_res - The number of residential units as reported by dcp_housing database
+-- units_co - The number of units listed on the DOB issued Certificate of Occupancy 
+-- new_value 
+-- old_value 
+
+
+CREATE TABLE IF NOT EXISTS qaqc_housing_units(
+  bbl text,
+  job_number text,
+  unitsres text,
+  units_co text,
+  new_value text,
+  old_value text
+);
+
+INSERT INTO qaqc_housing_units
+WITH base as(
+SELECT DISTINCT round(a.bbl::numeric,0)::text as bbl,b.job_number,a.unitsres,round(b.units_co::numeric,0)::text as units_co
+FROM pluto a, dcp_housing b
+WHERE b.bbl||b.date_lastupdt IN (
+-- get the most recent DOB record for a BBL based on date of last update field
+	SELECT bbl||max(date_lastupdt::date) maxDate
+        FROM dcp_housing
+      GROUP BY bbl)
+AND round(a.bbl::numeric,0)::text=b.bbl 
+AND a.unitsres<>round(b.units_co::numeric,0)::text),
+-- select only corrections to unitsres field
+corrections_subset as (
+SELECT * 
+FROM pluto_corrections 
+WHERE field='unitsres')
+-- combine PLUTO, DOB, and corrections data for final output
+SELECT a.*, c.new_value, c.old_value
+FROM base a
+LEFT JOIN corrections_subset c
+ON a.bbl=c.bbl;

pluto_build/sql/qaqc_pts_condo.sql

@@ -0,0 +1,74 @@
+-- Record PTS records that meet the criteria of
+-- The units and coop_apts values are greater than one for the billing bbl AND
+-- Two or more unit bbl records have units and coop_apts values greater than 1
+
+-- Data Dictionary 
+
+-- primebbl - Billing BBL  
+-- bbl - Unit level BBl
+-- units - The number of units listed by Department of Finance for the property (compare to units_co in dcp_housing)
+-- coop_apts - The number of coop apartments listed for the property 
+-- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). 
+-- units_co - The number of units listed on the DOB issued Certificate of Occupancy 
+-- date_lastupdt - The date of the last update to the DOB record for the job filing.
+-- new_value - The new number of units as reported by the pluto_corrections file
+-- old_value - The previous number of units as reported by the pluto_correction file
+
+
+CREATE TABLE IF NOT EXISTS qaqc_pts_condo(
+	primebbl text,
+	bbl text,
+	units numeric,
+	coop_apts numeric,
+	job_number text,
+	units_co text,
+	date_lastupdt varchar,
+	new_value text,
+	old_value text
+);
+
+INSERT INTO qaqc_pts_condo
+WITH pts_subset as (
+SELECT primebbl, bbl, units, coop_apts
+FROM pluto_rpad_geo
+	WHERE primebbl IN (
+		SELECT primebbl FROM (
+			SELECT primebbl, COUNT(*)
+			FROM pluto_rpad_geo
+			WHERE tl NOT LIKE '75%'
+			AND RIGHT(primebbl,4) LIKE '75%'
+			AND units::integer > 1
+			AND coop_apts::integer > 1
+			GROUP BY primebbl, units, coop_apts) as badbases 
+		WHERE count>1)
+	AND primebbl IN (
+		SELECT primebbl FROM (
+			SELECT primebbl	
+			FROM pluto_rpad_geo
+			WHERE tl LIKE '75%'
+			AND units::integer > 1
+			AND coop_apts::integer > 1) as badbillings)),
+-- get the most recent DOB record for a BBL based on date of last update field
+dob_subset as (
+SELECT * FROM dcp_housing b
+WHERE b.bbl||b.date_lastupdt IN ( 
+	SELECT bbl||max(date_lastupdt::date) maxDate
+        FROM dcp_housing
+      GROUP BY bbl)),
+-- select only corrections to unitsres field
+corrections_subset as (
+SELECT * 
+FROM pluto_corrections 
+WHERE field='unitsres'),
+-- Join PTS and DOB subsets, preserving all PTS records
+pts_dob as ( 
+SELECT a.*, b.job_number, round(b.units_co::numeric,0)::text as units_co, b.date_lastupdt
+FROM pts_subset a
+LEFT JOIN dob_subset b
+ON a.bbl=b.bbl
+AND a.coop_apts::text<>round(b.units_co::numeric,0)::text)
+-- Join on corrections to produce final output
+SELECT a.*, c.new_value, c.old_value
+FROM pts_dob a
+LEFT JOIN corrections_subset c
+ON a.bbl=c.bbl;

pluto_build/sql/source_data_versions.sql

@@ -70,4 +70,6 @@ CREATE TABLE source_data_versions as(
         (SELECT 'fema_pfirms2015_100yr' as schema_name, v from fema_pfirms2015_100yr limit 1)
     union
         (SELECT 'doitt_buildingcentroids' as schema_name, v from pluto_input_numbldgs limit 1)
+    union
+        (SELECT 'dcp_housing' as schema_name, v from dcp_housing limit 1)
 ) a order by schema_name);

pluto_build/version.env

@@ -16,5 +16,6 @@ DPR_GREENTHUMB_VERSION=20221101
 DSNY_FREQUENCIES_VERSION=20221105
 LPC_HISTORIC_DISTRICTS_VERSION=20220526
 LPC_LANDMARKS_VESRSION=20220613
+DCP_HOUSING=22Q2
 
 PLUTO_CORRECTIONS_VERSION=21v4