WIP: docker image testing

.github/workflows/single-runner.yml

@@ -27,7 +27,7 @@ jobs:
       AWS_S3_BUCKET: edm-recipes
     steps:
       - uses: actions/checkout@v2.3.4
-      - uses: NYCPlanning/action-library-archive@v1.1
+      - uses: NYCPlanning/action-library-archive@v1.2
         name: Archive ${{ github.event.inputs.dataset }}
         with:
           name: ${{ github.event.inputs.dataset }}

.github/workflows/test-staging-docker.yml

@@ -0,0 +1,46 @@
+name: Staging Docker Image Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "The version of the dataset (i.e. 22v2, 21C) if needed (optional)"
+        required: false
+
+
+
+jobs:
+  dataloading:
+    runs-on: ubuntu-latest
+    env:
+      AWS_S3_ENDPOINT: ${{ secrets.DO_S3_ENDPOINT }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.DO_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DO_SECRET_ACCESS_KEY }}
+      AWS_S3_BUCKET: edm-recipes
+    strategy:
+      matrix:
+        dataset:
+          - dcp_atomicpolygons
+          - dcp_boroboundaries
+          - dcp_boroboundaries_wi
+          - dcp_cdboundaries
+          - dcp_cdboundaries_wi
+          - dcp_ct2010
+          - dcp_ct2010_wi
+          - dcp_ct2020
+          - dcp_ct2020_wi
+          - dcp_cb2010
+          - dcp_cb2010_wi
+          - dcp_cb2020
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - uses: NYCPlanning/action-library-archive@v1.2
+        name: Testing ${{ github.event.inputs.dataset }}
+        with:
+          test: true
+          name: ${{ github.event.inputs.dataset }}
+          latest: false
+          version: ${{ github.event.inputs.version }}
+          s3: true
+          compress: true
+          output_format: pgdump shapefile csv

library/archive.py

@@ -31,6 +31,7 @@ def __call__(
         clean: bool = False,
         latest: bool = False,
         name: str = None,
+        test: bool = False
         *args,
         **kwargs,
     ):
@@ -96,6 +97,9 @@ def __call__(
             if push:
                 key = _file.replace(base_path + "/", "")
                 self.s3.put(_file, key, acl, metadata={"version": version})
+            if push and test:
+                key = _file.replace(base_path + "/datasets", "datasets/test")
+                self.s3.put(_file, key, acl, metadata={"version": version})
             if push and latest:
                 # Upload file to a latest directory, where version metadata is version
                 # This allows us to get the version associated with each file in latest

library/ingest.py

@@ -54,16 +54,22 @@ def wrapper(self, *args, **kwargs) -> list:
                 self, *args, **kwargs
             )
             # initiate source and destination datasets
-            folder_path = f"{self.base_path}/datasets/{name}/{version}"
-
+            try:
+                subfolder = destination["subfolder"]
+                folder_path = f"{self.base_path}/datasets/{subfolder}/{name}/{version}"
+            except KeyError:
+                folder_path = f"{self.base_path}/datasets/{name}/{version}"
+            print(f"what is the folder_path: {folder_path}")
             if output_suffix:
                 destination_path = f"{folder_path}/{name}.{output_suffix}"
                 output_files.append(destination_path)
             else:
                 destination_path = None
+            print(f"what is the destination_path: {destination_path}")
 
             # Default dstDS is destination_path if no dstDS is specificed
             dstDS = destination_path if not dstDS else dstDS
+            print(dstDS)
             srcDS = generic_source(
                 path=source["url"]["gdalpath"],
                 options=source["options"],

library/templates/test_docker.yml

@@ -0,0 +1,33 @@
+dataset:
+  name: &name test_docker
+  version: "{{ version }}"
+  acl: public-read
+
+  source:
+    url: 
+      path: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyct2020_{{ version }}.zip
+      subpath: nyct2020_{{ version }}/nyct2020.shp
+    geometry:
+      SRS: EPSG:2263
+      type: MULTIPOLYGON
+    options:
+      - "AUTODETECT_TYPE=NO"
+      - "EMPTY_STRING_AS_NULL=YES"
+
+  destination:
+    name: *name
+    subfolder: test_docker_
+    geometry:
+      SRS: EPSG:4326
+      type: MULTIPOLYGON
+    options:
+      - "OVERWRITE=YES"
+      - "PRECISION=NO"
+    fields: []
+    sql: null
+
+  info:
+    description: |
+      ### The Census Tracts for the 2020 US Census (Clipped to shoreline).
+    url: https://www1.nyc.gov/site/planning/data-maps/open-data/districts-download-metadata.page
+    dependents: []