fix: update promotion workflow to output promotion status (#299)

Author: botanical

.github/workflows/pr.yml

@@ -29,8 +29,9 @@ jobs:
     runs-on: ubuntu-latest
     environment: staging
     outputs:
-      publishedCollections: ${{ steps.publish-collections.outputs.success_collections }}
-      commentId: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      published_collections: ${{ steps.publish-collections.outputs.success_collections }}
+      comment_id: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      has_new_files_to_promote: ${{ steps.changed-files.outputs.added_files_count != 0 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -84,12 +85,19 @@ jobs:
           done
           echo "added_files_count: ${ADDED_FILES_COUNT}"
 
+      - name: Exit early if no new files
+        if: ${{ steps.changed-files.outputs.added_files_count == 0 }}
+        run: |
+          echo "🕵️ No new files found. Exiting workflow early."
+          exit 0
+
       # Makes request to /dataset/publish endpoint
       # Outputs only files that were successfully published
       # Used by other steps
       # If none of the requests are successful, workflow fails
       # Updates the PR comment with status of collection publication
       - name: Publish all newly added collections to staging
+        if: ${{ steps.changed-files.outputs.added_files_count != 0 }}
         id: publish-collections
         env:
           ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
@@ -178,7 +186,23 @@ jobs:
           ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
+            # If the workflow fails at any point, the PR comment will be updated
+
+      - name: Update PR comment if no new files
+        if: ${{ steps.changed-files.outputs.added_files_count == 0 }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+          WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
+
+          🕵️ No new files found. No files will be published to staging or promoted to production."
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
   create-mdx-files-and-open-pr:
+    if: ${{ needs.publish-new-datasets.outputs.has_new_files_to_promote == 'true' }}
     runs-on: ubuntu-latest
     environment: staging
     needs: publish-new-datasets
@@ -188,12 +212,12 @@ jobs:
 
       - name: Use output from publish-new-datasets
         run: |
-          echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.publishedCollections }}"
+          echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.published_collections }}"
 
       # Creates a slim dataset mdx file for each collection based on the dataset config json
       - name: Create dataset mdx for given collections
         env:
-          PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.publishedCollections }}
+          PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.published_collections }}
         run: |
           echo $PUBLISHED_COLLECTION_FILES
           collection_ids=""
@@ -212,7 +236,7 @@ jobs:
       - name: Prepare temp directory for artifacts
         run: |
             mkdir -p /tmp/upload-artifacts
-            for file in $(echo "${{ needs.publish-new-datasets.outputs.publishedCollections }}" | tr ',' ' '); do
+            for file in $(echo "${{ needs.publish-new-datasets.outputs.published_collections }}" | tr ',' ' '); do
               if [ -f "$file" ]; then
                 cp "$file" /tmp/upload-artifacts/
               else
@@ -341,7 +365,7 @@ jobs:
       - name: Create veda-config PR with changes
         id: create-pr
         env:
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
           cd veda-config

.github/workflows/promote.yml

@@ -69,6 +69,14 @@ jobs:
         run: |
           pip install -r ./scripts/requirements.txt
           for file in downloaded-files/*.json; do
-            python3 ./scripts/promote_collection.py "$file" "production"
+            collection_id=$(jq -r '.collection' "$file")
+            response=$(python3 ./scripts/promote_collection.py "$file" "production")
             echo "Processed file: $file"
+            status_code=$(echo "$response" | jq -r '.statusCode')
+            echo "Status Code: $status_code"
+             if [ $status_code -eq 200 ] || [ $status_code -eq 201 ]; then
+                echo "$collection_id successfully promoted to Production ✅"
+              else
+                echo "$collection_id failed to promote to Production ❌"
+              fi
           done

ingestion-data/staging/dataset-config/test-changed-files.json

@@ -1,90 +0,0 @@
-{
-    "license": "CC0-1.0",
-    "stac_version": "1.0.0",
-    "links": [],
-    "spatial_extent": {
-      "xmin": -90.85,
-      "ymin": 29.03,
-      "xmax": -90.06,
-      "ymax": 29.65
-    },
-    "temporal_extent": {
-      "startdate": "2021-08-23T00:00:00.000Z",
-      "enddate": "2021-09-09T23:59:59.000Z"
-    },
-    "discovery_items": [
-      {
-        "upload": false,
-        "cogify": false,
-        "dry_run": false,
-        "filename_regex": "(.*)NDWI_Difference_(.*).tif$",
-        "use_multithreading": false,
-        "discovery": "s3",
-        "prefix": "planet-indices-v2/",
-        "bucket": "veda-data-store-staging"
-      }
-    ],
-    "sample_files": [
-      "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif"
-    ],
-    "data_type": "cog",
-    "stac_extensions": [
-      "https://stac-extensions.github.io/render/v1.0.0/schema.json",
-      "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json"
-    ],
-    "item_assets": {
-      "cog_default": {
-        "type": "image/tiff; application=geotiff; profile=cloud-optimized",
-        "roles": [
-          "data",
-          "layer"
-        ],
-        "title": "Default COG Layer",
-        "description": "Cloud optimized default layer to display on map"
-      }
-    },
-    "providers": [
-      {
-        "name": "NASA VEDA",
-        "roles": [
-          "host"
-        ],
-        "url": "https://www.earthdata.nasa.gov/dashboard/"
-      }
-    ],
-    "assets": {
-      "thumbnail": {
-        "title": "Thumbnail",
-        "type": "image/jpeg",
-        "roles": [
-          "thumbnail"
-        ],
-        "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg",
-        "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)"
-      }
-    },
-    "collection": "ida-ndwi-difference-TEST-SM2A-STAGING",
-    "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SM2A-STAGING",
-    "dashboard:time_density": "day",
-    "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.",
-    "renders": {
-      "dashboard": {
-        "resampling": "nearest",
-        "bidx": [
-          1
-        ],
-        "colormap_name": "rdbu",
-        "assets": [
-          "cog_default"
-        ],
-        "rescale": [
-          [
-            -1,
-            1
-          ]
-        ],
-        "title": "VEDA Dashboard Render Parameters"
-      }
-    },
-    "dashboard:is_periodic": true
-  }

ingestion-data/staging/dataset-config/test-sm2a-staging.json

@@ -63,8 +63,8 @@
         "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)"
       }
     },
-    "collection": "ida-ndwi-difference-TEST-2",
-    "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-2",
+    "collection": "ida-ndwi-difference-TEST-SM2A-PRODUCTION",
+    "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SM2A-PRODUCTION",
     "dashboard:time_density": "day",
     "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.",
     "renders": {

…dataset-config/test-changed-files-2.json …on-data/staging/dataset-config/test.jsoningestion-data/staging/dataset-config/test-changed-files-2.json renamed to ingestion-data/staging/dataset-config/test.json ingestion-data/staging/dataset-config/test-changed-files-2.json renamed to ingestion-data/staging/dataset-config/test.json

@@ -69,6 +69,7 @@ def publish_to_staging(payload):
     http_conn.close()
 
     print(json.dumps({"statusCode": response.status}))
+    print(response_data.decode())
     return {"statusCode": response.status, "body": response_data.decode()}
 
 
@@ -91,7 +92,7 @@ def promote_to_production(payload):
         "Authorization": "Basic " + api_token,
     }
 
-    payload["transfer"] = True
+    payload["conf"]["transfer"] = True
     body = {
         **payload,
         "dag_run_id": f"{promotion_dag}-{uuid.uuid4()}",
@@ -105,6 +106,8 @@ def promote_to_production(payload):
     response_data = response.read()
     http_conn.close()
 
+    print(json.dumps({"statusCode": response.status}))
+    print(response_data.decode())
     return {"statusCode": response.status, "body": response_data.decode()}