Skip to content

Commit 9876efc

Browse files
authored
fix: update promotion workflow to output promotion status (#299)
1 parent 431c8e4 commit 9876efc

File tree

6 files changed

+45
-190
lines changed

6 files changed

+45
-190
lines changed

.github/workflows/pr.yml

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ jobs:
2929
runs-on: ubuntu-latest
3030
environment: staging
3131
outputs:
32-
publishedCollections: ${{ steps.publish-collections.outputs.success_collections }}
33-
commentId: ${{ steps.init-comment.outputs.COMMENT_ID }}
32+
published_collections: ${{ steps.publish-collections.outputs.success_collections }}
33+
comment_id: ${{ steps.init-comment.outputs.COMMENT_ID }}
34+
has_new_files_to_promote: ${{ steps.changed-files.outputs.added_files_count != 0 }}
3435
steps:
3536
- uses: actions/checkout@v4
3637

@@ -84,12 +85,19 @@ jobs:
8485
done
8586
echo "added_files_count: ${ADDED_FILES_COUNT}"
8687
88+
- name: Exit early if no new files
89+
if: ${{ steps.changed-files.outputs.added_files_count == 0 }}
90+
run: |
91+
echo "🕵️ No new files found. Exiting workflow early."
92+
exit 0
93+
8794
# Makes request to /dataset/publish endpoint
8895
# Outputs only files that were successfully published
8996
# Used by other steps
9097
# If none of the requests are successful, workflow fails
9198
# Updates the PR comment with status of collection publication
9299
- name: Publish all newly added collections to staging
100+
if: ${{ steps.changed-files.outputs.added_files_count != 0 }}
93101
id: publish-collections
94102
env:
95103
ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
@@ -178,7 +186,23 @@ jobs:
178186
** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
179187
gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
180188
189+
# If the workflow fails at any point, the PR comment will be updated
190+
191+
- name: Update PR comment if no new files
192+
if: ${{ steps.changed-files.outputs.added_files_count == 0 }}
193+
env:
194+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
195+
COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
196+
run: |
197+
WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
198+
CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
199+
UPDATED_BODY="$CURRENT_BODY
200+
201+
🕵️ No new files found. No files will be published to staging or promoted to production."
202+
gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
203+
181204
create-mdx-files-and-open-pr:
205+
if: ${{ needs.publish-new-datasets.outputs.has_new_files_to_promote == 'true' }}
182206
runs-on: ubuntu-latest
183207
environment: staging
184208
needs: publish-new-datasets
@@ -188,12 +212,12 @@ jobs:
188212

189213
- name: Use output from publish-new-datasets
190214
run: |
191-
echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.publishedCollections }}"
215+
echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.published_collections }}"
192216
193217
# Creates a slim dataset mdx file for each collection based on the dataset config json
194218
- name: Create dataset mdx for given collections
195219
env:
196-
PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.publishedCollections }}
220+
PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.published_collections }}
197221
run: |
198222
echo $PUBLISHED_COLLECTION_FILES
199223
collection_ids=""
@@ -212,7 +236,7 @@ jobs:
212236
- name: Prepare temp directory for artifacts
213237
run: |
214238
mkdir -p /tmp/upload-artifacts
215-
for file in $(echo "${{ needs.publish-new-datasets.outputs.publishedCollections }}" | tr ',' ' '); do
239+
for file in $(echo "${{ needs.publish-new-datasets.outputs.published_collections }}" | tr ',' ' '); do
216240
if [ -f "$file" ]; then
217241
cp "$file" /tmp/upload-artifacts/
218242
else
@@ -341,7 +365,7 @@ jobs:
341365
- name: Create veda-config PR with changes
342366
id: create-pr
343367
env:
344-
COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
368+
COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }}
345369
PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
346370
run: |
347371
cd veda-config

.github/workflows/promote.yml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,14 @@ jobs:
6969
run: |
7070
pip install -r ./scripts/requirements.txt
7171
for file in downloaded-files/*.json; do
72-
python3 ./scripts/promote_collection.py "$file" "production"
72+
collection_id=$(jq -r '.collection' "$file")
73+
response=$(python3 ./scripts/promote_collection.py "$file" "production")
7374
echo "Processed file: $file"
75+
status_code=$(echo "$response" | jq -r '.statusCode')
76+
echo "Status Code: $status_code"
77+
if [ $status_code -eq 200 ] || [ $status_code -eq 201 ]; then
78+
echo "$collection_id successfully promoted to Production ✅"
79+
else
80+
echo "$collection_id failed to promote to Production ❌"
81+
fi
7482
done

ingestion-data/staging/dataset-config/test-changed-files.json

Lines changed: 0 additions & 90 deletions
This file was deleted.
Lines changed: 0 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -1,90 +0,0 @@
1-
{
2-
"license": "CC0-1.0",
3-
"stac_version": "1.0.0",
4-
"links": [],
5-
"spatial_extent": {
6-
"xmin": -90.85,
7-
"ymin": 29.03,
8-
"xmax": -90.06,
9-
"ymax": 29.65
10-
},
11-
"temporal_extent": {
12-
"startdate": "2021-08-23T00:00:00.000Z",
13-
"enddate": "2021-09-09T23:59:59.000Z"
14-
},
15-
"discovery_items": [
16-
{
17-
"upload": false,
18-
"cogify": false,
19-
"dry_run": false,
20-
"filename_regex": "(.*)NDWI_Difference_(.*).tif$",
21-
"use_multithreading": false,
22-
"discovery": "s3",
23-
"prefix": "planet-indices-v2/",
24-
"bucket": "veda-data-store-staging"
25-
}
26-
],
27-
"sample_files": [
28-
"s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif"
29-
],
30-
"data_type": "cog",
31-
"stac_extensions": [
32-
"https://stac-extensions.github.io/render/v1.0.0/schema.json",
33-
"https://stac-extensions.github.io/item-assets/v1.0.0/schema.json"
34-
],
35-
"item_assets": {
36-
"cog_default": {
37-
"type": "image/tiff; application=geotiff; profile=cloud-optimized",
38-
"roles": [
39-
"data",
40-
"layer"
41-
],
42-
"title": "Default COG Layer",
43-
"description": "Cloud optimized default layer to display on map"
44-
}
45-
},
46-
"providers": [
47-
{
48-
"name": "NASA VEDA",
49-
"roles": [
50-
"host"
51-
],
52-
"url": "https://www.earthdata.nasa.gov/dashboard/"
53-
}
54-
],
55-
"assets": {
56-
"thumbnail": {
57-
"title": "Thumbnail",
58-
"type": "image/jpeg",
59-
"roles": [
60-
"thumbnail"
61-
],
62-
"href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg",
63-
"description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)"
64-
}
65-
},
66-
"collection": "ida-ndwi-difference-TEST-SM2A-STAGING",
67-
"title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SM2A-STAGING",
68-
"dashboard:time_density": "day",
69-
"description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.",
70-
"renders": {
71-
"dashboard": {
72-
"resampling": "nearest",
73-
"bidx": [
74-
1
75-
],
76-
"colormap_name": "rdbu",
77-
"assets": [
78-
"cog_default"
79-
],
80-
"rescale": [
81-
[
82-
-1,
83-
1
84-
]
85-
],
86-
"title": "VEDA Dashboard Render Parameters"
87-
}
88-
},
89-
"dashboard:is_periodic": true
90-
}

ingestion-data/staging/dataset-config/test-changed-files-2.json renamed to ingestion-data/staging/dataset-config/test.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@
6363
"description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)"
6464
}
6565
},
66-
"collection": "ida-ndwi-difference-TEST-2",
67-
"title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-2",
66+
"collection": "ida-ndwi-difference-TEST-SM2A-PRODUCTION",
67+
"title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SM2A-PRODUCTION",
6868
"dashboard:time_density": "day",
6969
"description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.",
7070
"renders": {

scripts/promote_collection.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ def publish_to_staging(payload):
6969
http_conn.close()
7070

7171
print(json.dumps({"statusCode": response.status}))
72+
print(response_data.decode())
7273
return {"statusCode": response.status, "body": response_data.decode()}
7374

7475

@@ -91,7 +92,7 @@ def promote_to_production(payload):
9192
"Authorization": "Basic " + api_token,
9293
}
9394

94-
payload["transfer"] = True
95+
payload["conf"]["transfer"] = True
9596
body = {
9697
**payload,
9798
"dag_run_id": f"{promotion_dag}-{uuid.uuid4()}",
@@ -105,6 +106,8 @@ def promote_to_production(payload):
105106
response_data = response.read()
106107
http_conn.close()
107108

109+
print(json.dumps({"statusCode": response.status}))
110+
print(response_data.decode())
108111
return {"statusCode": response.status, "body": response_data.decode()}
109112

110113

0 commit comments

Comments
 (0)