slight tweaks with devdb cos (#2159)

* add dob_cofos ingest templates

* refactor _co.sql

Author: fvankrieken

ingest_templates/dob_bis_cofos.yml

@@ -0,0 +1,35 @@
+id: dob_bis_cofos
+acl: public-read
+
+attributes:
+  name: DOB BIS Certificate Of Occupancy
+  description: >-
+    A Certificate of Occupancy (CO) states a building's legal use and/or type of
+    permitted occupancy. New buildings must have a CO, and existing buildings must have
+    a current or amended CO when there is a change in use, egress or type of occupancy.
+    No one may legally occupy a building until the Department has issued a Certificate
+    of Occupancy or Temporary Certificate of Occupancy. The Department issues a final
+    Certificate of Occupancy when the completed work matches the submitted plans for new
+    buildings or major alterations. It issues a Letter of Completion for minor
+    alterations to properties. These documents confirm the work complies with all
+    applicable laws, all paperwork has been completed, all fees owed to the Department
+    have been paid, all relevant violations have been resolved and all necessary
+    approvals have been received from other City Agencies.
+
+    This dataset contains all Certificates of Occupancy issued from 7/12/2012 to March
+    2021. PDFs of the actual COs are viewable in www.nyc.gov/bis.
+
+    For COs issued since March 2021, see DOB NOW: Certificate of Occupancy dataset.
+  url: https://data.cityofnewyork.us/Housing-Development/DOB-Certificate-Of-Occupancy/bs8b-p36w/about_data
+
+ingestion:
+  source:
+    type: socrata
+    org: nyc
+    uid: bs8b-p36w
+    format: csv
+  file_format:
+    type: csv
+  processing_steps:
+  - name: clean_column_names
+    args: { "lower": True }

ingest_templates/dob_now_cofos.yml

@@ -0,0 +1,38 @@
+id: dob_now_cofos
+acl: public-read
+
+attributes:
+  name: DOB NOW Certificate Of Occupancy
+  description: >-
+    This data set includes certificates of occupancy issued through the New York City
+    Department of Buildings' DOB NOW: Certificate of Occupancy module. This module was
+    released in March of 2021, anbd from that point onward this data set should be
+    utilized instead of the "DOB Certificate of Occupancy" data set. The data is
+    collected because the Department of Buildings tracks Certificates of Occupancies
+    issued. This data include items such as job filing name, job filing label, BIN,
+    Address, and Certificate of Occupancy status, sequence, label, and issuance date.
+
+    "A Certificate of Occupancy (CO) states a legal use and/or type of permitted
+    occupancy of a building. New buildings must have a CO, and existing buildings must
+    have a current or amended CO when there is a change in use, egress or type of
+    occupancy. No one may legally occupy a building until the Department has issued a CO
+    or Temporary Certificate of Occupancy (TCO).
+
+    A CO confirms that the completed work complies with all applicable laws, all
+    paperwork has been completed, all fees owed to the Department have been paid, all
+    relevant violations have been resolved, and all necessary approvals have been
+    received from other City Agencies. The Department issues a final CO when the
+    completed work matches the submitted plans for new buildings or major alterations."
+  url: https://data.cityofnewyork.us/Housing-Development/DOB-NOW-Certificate-of-Occupancy/pkdm-hqz6/about_data
+
+ingestion:
+  source:
+    type: socrata
+    org: nyc
+    uid: pkdm-hqz6
+    format: csv
+  file_format:
+    type: csv
+  processing_steps:
+  - name: clean_column_names
+    args: { "lower": True }

products/developments/sql/_co.sql

@@ -28,62 +28,30 @@ IN PREVIOUS VERSION:
     co_.sql
 */
 DROP TABLE IF EXISTS co_devdb;
-WITH
-order_certtype AS (
-    SELECT
-        *,
-        CASE
-            WHEN certificatetype = 'T- TCO' THEN 2
-            WHEN certificatetype = 'C- CO' THEN 1
-        END AS certorder
+CREATE TABLE co_devdb AS
+WITH latest_co AS (
+    SELECT DISTINCT ON (jobnum)
+        jobnum,
+        numofdwellingunits::numeric,
+        effectivedate::date,
+        certificatetype
     FROM dob_cofos
+    WHERE jobnum IN (SELECT job_number FROM init_devdb)
+    ORDER BY
+        jobnum ASC, effectivedate::date DESC, certificatetype ASC
 ),
-order_co AS (
-    SELECT
-        jobnum AS job_number,
-        effectivedate::date AS effectivedate,
-        numofdwellingunits::numeric AS units,
-        certificatetype AS certtype,
-        row_number() OVER (
-            PARTITION BY jobnum
-            ORDER BY effectivedate::date DESC, certorder ASC
-        ) AS latest,
-        row_number() OVER (
-            PARTITION BY jobnum
-            ORDER BY effectivedate::date ASC
-        ) AS earliest
-    FROM order_certtype
-    WHERE jobnum IN (
-        SELECT DISTINCT job_number
-        FROM init_devdb
-    )
-),
-draft_co AS (
+earliest_co AS (
     SELECT
-        a.*,
-        b._date_complete
-    FROM (
-        SELECT
-            job_number,
-            effectivedate AS co_latest_effectivedate,
-            units AS co_latest_units,
-            certtype AS co_latest_certtype
-        FROM order_co
-        WHERE latest = 1
-    ) AS a
-    LEFT JOIN (
-        SELECT
-            job_number,
-            effectivedate AS _date_complete
-        FROM order_co
-        WHERE earliest = 1
-    ) AS b ON a.job_number = b.job_number
+        jobnum,
+        min(effectivedate::date) AS _date_complete
+    FROM dob_cofos
+    GROUP BY jobnum
 )
 SELECT
-    job_number,
-    _date_complete,
-    co_latest_effectivedate,
-    co_latest_units,
-    co_latest_certtype
-INTO co_devdb
-FROM draft_co;
+    latest_co.jobnum AS job_number,
+    earliest_co._date_complete,
+    latest_co.effectivedate AS co_latest_effectivedate,
+    latest_co.numofdwellingunits AS co_latest_units,
+    latest_co.certificatetype AS co_latest_certtype
+FROM latest_co
+INNER JOIN earliest_co ON latest_co.jobnum = earliest_co.jobnum;