deduplicate nysdec permit records

Author: fvankrieken

products/green_fast_track/models/_sources.yml

@@ -151,7 +151,6 @@ sources:
     columns:
     - name: dec_id
       tests:
-      - unique
       - not_null
     - name: facility_name
       tests:
@@ -164,7 +163,6 @@ sources:
     columns:
     - name: dec_id
       tests:
-      - unique
       - not_null
     - name: facility_name
       tests:

products/green_fast_track/models/staging/stg__nysdec_state_facility_permits.sql

@@ -1,15 +1,19 @@
 WITH source AS (
-    SELECT *
-    FROM {{ source('recipe_sources', 'nysdec_state_facility_permits') }}
+    SELECT * FROM {{ source('recipe_sources', 'nysdec_state_facility_permits') }}
+),
+deduplicated AS (
+    SELECT DISTINCT
+        dec_id,
+        facility_name,
+        geom
+    FROM source
 ),
-
 final AS (
     SELECT
         'state_facility_permits' AS variable_type,
         dec_id AS variable_id,
         facility_name,
         ST_TRANSFORM(geom::geometry, 2263) AS permit_geom
-    FROM source
+    FROM deduplicated
 )
-
 SELECT * FROM final

products/green_fast_track/models/staging/stg__nysdec_title_v_facility_permits.sql

@@ -1,14 +1,20 @@
 WITH source AS (
-    SELECT *
-    FROM {{ source('recipe_sources', 'nysdec_title_v_facility_permits') }}
+    SELECT * FROM {{ source('recipe_sources', 'nysdec_title_v_facility_permits') }}
+),
+deduplicated AS (
+    SELECT DISTINCT
+        dec_id,
+        facility_name,
+        year,
+        siccode,
+        geom
+    FROM source
 ),
-
 final AS (
     SELECT
         'title_v_permits' AS variable_type,
         dec_id AS variable_id,
         st_transform(geom::geometry, 2263) AS permit_geom
-    FROM source
+    FROM deduplicated
 )
-
 SELECT * FROM final