migrate nysdec permits to pull from source

.github/workflows/ceqr_dep_monthly.yml

@@ -39,6 +39,9 @@ jobs:
         python3 -m pip install mdpdf
         ./ceqr run recipe dep_cats_permits
 
+    - name: library archive
+      run: dcpy lifecycle scripts ingest_or_library_archive dep_cats_permits --latest --version "$(date '+%Y-%m-%d')" --push-to-s3
+
     - name: Create issue on success
       if: success()
       uses: JasonEtco/create-an-issue@v2

.github/workflows/ingest_arcgis_feature_server.yml

@@ -34,7 +34,9 @@ jobs:
         nysdec_priority_shorelines
         nysdec_priority_estuaries
         nysdec_priority_streams
+        nysdec_state_facility_permits
         nysdec_tidal_wetlands
+        nysdec_title_v_facility_permits
         nysparks_historicplaces
         nysparks_parks
         nysshpo_historic_buildings_points

dcpy/library/templates/dep_cats_permits.yml

@@ -7,7 +7,7 @@ dataset:
     #   uid: f4rp-2kvy
     #   format: csv
     url:
-      path: https://nyc3.digitaloceanspaces.com/edm-publishing/ceqr-app-data/dep_cats_permits/{{ version }}/dep_cats_permits.csv
+      path: https://nyc3.digitaloceanspaces.com/edm-publishing/datasets/dep_cats_permits/{{ version }}/dep_cats_permits.csv
     options:
       - "AUTODETECT_TYPE=NO"
       - "EMPTY_STRING_AS_NULL=YES"

ingest_templates/nysdec_state_facility_permits.yml

@@ -0,0 +1,21 @@
+id: nysdec_state_facility_permits
+acl: public-read
+
+attributes:
+  name: NYSDEC State Facility Permits
+  description: >-
+    State Facility Permits are issued to facilities considered to be mid-sized. They
+    have lower potential emissions, fewer permit requirements than Title V facilities,
+    and they meet the criteria of Subpart 201-5.
+  url: https://www.arcgis.com/home/item.html?id=a29c5e334de14f63884ebe4855345f41
+
+ingestion:
+  source:
+    type: arcgis_feature_server
+    server: nys_clearinghouse
+    dataset: Air_State_Facility_Permits_ASF
+  file_format:
+    type: geojson
+  processing_steps:
+  - name: clean_column_names
+    args: { "replace": { " ": "_" }, "lower": True }

ingest_templates/nysdec_title_v_facility_permits.yml

@@ -0,0 +1,21 @@
+id: nysdec_title_v_facility_permits
+acl: public-read
+
+attributes:
+  name: NYSDEC Title V Facility Permits
+  description: >-
+    Title V Permits are issued to facilities that are major sources of air emissions.
+    These facilities are usually considered to be the largest in the state and meet the
+    criteria of Subpart 201-6.
+  url: https://www.arcgis.com/home/item.html?id=4f1f53ba97934e1cafaf68a11f7c489f
+
+ingestion:
+  source:
+    type: arcgis_feature_server
+    server: nys_clearinghouse
+    dataset: Air_Title_V_Permits_ATV
+  file_format:
+    type: geojson
+  processing_steps:
+  - name: clean_column_names
+    args: { "replace": { " ": "_" }, "lower": True }

products/ceqr/ceqr_app/recipes/nysdec_state_facility_permits/runner.sh

@@ -9,12 +9,7 @@ VERSION=$DATE
     cd $BASEDIR
     mkdir -p output
 
-    docker run --rm\
-        -e EDM_DATA=$EDM_DATA\
-        -v $(pwd)/../:/recipes\
-        -e NAME=$NAME\
-        -w /recipes/$NAME\
-        nycplanning/docker-geosupport:latest python3 build.py
+    python3 build.py
     (
         cd output
 
@@ -37,4 +32,4 @@ VERSION=$DATE
     Upload $NAME $VERSION
     Upload $NAME latest
     rm -rf output
-)
+)

products/ceqr/ceqr_app/recipes/nysdec_title_v_facility_permits/runner.sh

@@ -9,11 +9,7 @@ VERSION=$DATE
     cd $BASEDIR
     mkdir -p output
 
-    docker run --rm\
-        -v $(pwd)/../:/recipes\
-        -e NAME=$NAME\
-        -w /recipes/$NAME\
-        nycplanning/docker-geosupport:latest python3 build.py | 
+    python3 build.py | 
     psql $EDM_DATA --set ON_ERROR_STOP=1 -v NAME=$NAME -v VERSION=$VERSION -f create.sql
 
     (
@@ -39,4 +35,4 @@ VERSION=$DATE
     Upload $NAME latest
     rm -rf output
 
-)
+)

products/green_fast_track/models/_sources.yml

@@ -149,9 +149,8 @@ sources:
 
   - name: nysdec_title_v_facility_permits
     columns:
-    - name: permit_id
+    - name: dec_id
       tests:
-      - unique
       - not_null
     - name: facility_name
       tests:
@@ -162,9 +161,8 @@ sources:
 
   - name: nysdec_state_facility_permits
     columns:
-    - name: permit_id
+    - name: dec_id
       tests:
-      - unique
       - not_null
     - name: facility_name
       tests:

products/green_fast_track/models/staging/stg__nysdec_state_facility_permits.sql

@@ -1,15 +1,19 @@
 WITH source AS (
-    SELECT *
-    FROM {{ source('recipe_sources', 'nysdec_state_facility_permits') }}
+    SELECT * FROM {{ source('recipe_sources', 'nysdec_state_facility_permits') }}
+),
+deduplicated AS (
+    SELECT DISTINCT
+        dec_id,
+        facility_name,
+        geom
+    FROM source
 ),
-
 final AS (
     SELECT
         'state_facility_permits' AS variable_type,
-        permit_id AS variable_id,
+        dec_id AS variable_id,
         facility_name,
         ST_TRANSFORM(geom::geometry, 2263) AS permit_geom
-    FROM source
+    FROM deduplicated
 )
-
 SELECT * FROM final

products/green_fast_track/models/staging/stg__nysdec_title_v_facility_permits.sql

@@ -1,14 +1,20 @@
 WITH source AS (
-    SELECT *
-    FROM {{ source('recipe_sources', 'nysdec_title_v_facility_permits') }}
+    SELECT * FROM {{ source('recipe_sources', 'nysdec_title_v_facility_permits') }}
+),
+deduplicated AS (
+    SELECT DISTINCT
+        dec_id,
+        facility_name,
+        year,
+        siccode,
+        geom
+    FROM source
 ),
-
 final AS (
     SELECT
         'title_v_permits' AS variable_type,
-        permit_id AS variable_id,
+        dec_id AS variable_id,
         st_transform(geom::geometry, 2263) AS permit_geom
-    FROM source
+    FROM deduplicated
 )
-
 SELECT * FROM final