Skip to content

Commit 9871cac

Browse files
harshavemula-uaharshavemula-ua
authored
Add manual workflow trigger with custom version tags and execution file fixes (#54)
* Add force build inputs for manual workflow triggers Allow manual workflow dispatch to specify custom version tags for forcingprocessor-deps and forcingprocessor, bypassing version change detection. This enables rebuilding containers without modifying versions.yml. * Fix permission issue in docker push script Use dedicated log file (docker_push_log.txt) for push operations to avoid permission conflicts with docker_build_log.txt created by other commands. Add fallback to /tmp if log file is not writable. * Remove docker_build_log.txt from datastream tee to avoid permission conflicts * Disable S3 check to avoid unnecessary retries * Set n_retries_allowed to 0 to prevent retry loop when ii_check_s3 is false
1 parent 890c11c commit 9871cac

File tree

3 files changed

+60
-13
lines changed

3 files changed

+60
-13
lines changed

‎.github/executions/fp_push_execution_arm.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,19 @@
66
"runuser -l ec2-user -c 'ARCH=aarch64 TAG=latest-arm64 docker compose -f /home/ec2-user/forcingprocessor/docker/docker-compose.yml build forcingprocessor-deps' >> /home/ec2-user/forcingprocessor/docker_build_log.txt 2>&1",
77
"runuser -l ec2-user -c 'TAG=latest-arm64 docker compose -f /home/ec2-user/forcingprocessor/docker/docker-compose.yml build forcingprocessor' >> /home/ec2-user/forcingprocessor/docker_build_log.txt 2>&1",
88
"runuser -l ec2-user -c 'docker images' >> /home/ec2-user/forcingprocessor/docker_build_log.txt 2>&1",
9-
"runuser -l ec2-user -c 'export FP_TAG=latest-arm64 && /home/ec2-user/forcingprocessor/datastreamcli/scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d /home/ec2-user/forcingprocessor/datastreamcli/data/datastream_test -g https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg -R /home/ec2-user/forcingprocessor/datastreamcli/configs/ngen/realization_sloth_nom_cfe_pet.json --s3_bucket ciroh-community-ngen-datastream --s3_prefix forcingprocessor/test --forcing_source NWM_RETRO_V3 2>&1 | tee -a /home/ec2-user/forcingprocessor/datastream_test_output.txt /home/ec2-user/forcingprocessor/docker_build_log.txt'",
9+
"runuser -l ec2-user -c 'export FP_TAG=latest-arm64 && /home/ec2-user/forcingprocessor/datastreamcli/scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d /home/ec2-user/forcingprocessor/datastreamcli/data/datastream_test -g https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg -R /home/ec2-user/forcingprocessor/datastreamcli/configs/ngen/realization_sloth_nom_cfe_pet.json --s3_bucket ciroh-community-ngen-datastream --s3_prefix forcingprocessor/test --forcing_source NWM_RETRO_V3 2>&1 | tee -a /home/ec2-user/forcingprocessor/datastream_test_output.txt'",
1010
"runuser -l ec2-user -c 'chmod +x /home/ec2-user/forcingprocessor/docker/docker_loginNpush.sh'",
1111
"runuser -l ec2-user -c '/home/ec2-user/forcingprocessor/docker/docker_loginNpush.sh ${DEPS_TAG} ${FP_TAG} \"${BUILD_ARGS}\"' >> /home/ec2-user/forcingprocessor/docker_login_log.txt 2>&1",
12-
"runuser -l ec2-user -c 'aws s3 cp /home/ec2-user/forcingprocessor/docker_login_log.txt s3://ciroh-community-ngen-datastream/forcingprocessor/test/docker_login_log.txt'",
12+
"runuser -l ec2-user -c 'aws s3 cp /home/ec2-user/forcingprocessor/docker_push_log.txt s3://ciroh-community-ngen-datastream/forcingprocessor/test/docker_push_log.txt'",
1313
"runuser -l ec2-user -c 'aws s3 cp /home/ec2-user/forcingprocessor/docker_build_log.txt s3://ciroh-community-ngen-datastream/forcingprocessor/test/docker_build_log.txt'",
1414
"runuser -l ec2-user -c 'aws s3 cp /home/ec2-user/forcingprocessor/datastream_test_output.txt s3://ciroh-community-ngen-datastream/forcingprocessor/test/datastream_test_output.txt'"
1515
],
1616
"run_options":{
1717
"ii_terminate_instance" : true,
1818
"ii_delete_volume" : true,
19-
"ii_check_s3" : true,
19+
"ii_check_s3" : false,
2020
"timeout_s" : 3600,
21-
"n_retries_allowed" : 2
21+
"n_retries_allowed" : 0
2222
},
2323
"instance_parameters" :
2424
{

‎.github/workflows/build_push_fp.yaml‎

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,16 @@ on:
77
required: false
88
default: ''
99
type: string
10+
force_build_deps:
11+
description: 'Force build forcingprocessor-deps (provide tag version, e.g., 1.0.0)'
12+
required: false
13+
default: ''
14+
type: string
15+
force_build_fp:
16+
description: 'Force build forcingprocessor (provide tag version, e.g., 1.0.0)'
17+
required: false
18+
default: ''
19+
type: string
1020
push:
1121
branches:
1222
- main
@@ -46,11 +56,42 @@ jobs:
4656
shell: bash
4757
run: |
4858
set -euo pipefail
49-
59+
60+
# Check for manual trigger with forced builds
61+
FORCE_DEPS="${{ inputs.force_build_deps }}"
62+
FORCE_FP="${{ inputs.force_build_fp }}"
63+
64+
if [ -n "$FORCE_DEPS" ] || [ -n "$FORCE_FP" ]; then
65+
echo "Manual trigger with forced build detected"
66+
67+
if [ -n "$FORCE_DEPS" ]; then
68+
echo "Force building forcingprocessor-deps with tag: $FORCE_DEPS"
69+
echo "build_deps=true" >> "$GITHUB_OUTPUT"
70+
echo "deps_version=$FORCE_DEPS" >> "$GITHUB_OUTPUT"
71+
else
72+
echo "build_deps=false" >> "$GITHUB_OUTPUT"
73+
CURRENT_DEPS=$(yq -r e '."forcingprocessor-deps"' versions.yml)
74+
echo "deps_version=$CURRENT_DEPS" >> "$GITHUB_OUTPUT"
75+
fi
76+
77+
if [ -n "$FORCE_FP" ]; then
78+
echo "Force building forcingprocessor with tag: $FORCE_FP"
79+
echo "build_fp=true" >> "$GITHUB_OUTPUT"
80+
echo "fp_version=$FORCE_FP" >> "$GITHUB_OUTPUT"
81+
else
82+
echo "build_fp=false" >> "$GITHUB_OUTPUT"
83+
CURRENT_FP=$(yq -r e '.forcingprocessor' versions.yml)
84+
echo "fp_version=$CURRENT_FP" >> "$GITHUB_OUTPUT"
85+
fi
86+
87+
exit 0
88+
fi
89+
90+
# Normal version change detection for push/PR events
5091
# Current versions (raw)
5192
CURRENT_DEPS=$(yq -r e '."forcingprocessor-deps"' versions.yml)
5293
CURRENT_FP=$(yq -r e '.forcingprocessor' versions.yml)
53-
94+
5495
# Ensure previous commit and file exist
5596
if git rev-parse HEAD~1 >/dev/null 2>&1 && git cat-file -e HEAD~1:versions.yml 2>/dev/null; then
5697
git show HEAD~1:versions.yml > previous_versions.yml
@@ -60,7 +101,7 @@ jobs:
60101
61102
PREVIOUS_DEPS=$(yq -r e '."forcingprocessor-deps"' previous_versions.yml)
62103
PREVIOUS_FP=$(yq -r e '.forcingprocessor' previous_versions.yml)
63-
104+
64105
# Check what changed and set outputs
65106
if [ "$CURRENT_DEPS" != "$PREVIOUS_DEPS" ]; then
66107
echo "forcingprocessor-deps changed: $PREVIOUS_DEPS -> $CURRENT_DEPS"
@@ -69,7 +110,7 @@ jobs:
69110
echo "build_deps=false" >> "$GITHUB_OUTPUT"
70111
fi
71112
echo "deps_version=$CURRENT_DEPS" >> "$GITHUB_OUTPUT"
72-
113+
73114
if [ "$CURRENT_FP" != "$PREVIOUS_FP" ]; then
74115
echo "forcingprocessor changed: $PREVIOUS_FP -> $CURRENT_FP"
75116
echo "build_fp=true" >> "$GITHUB_OUTPUT"
@@ -141,14 +182,14 @@ jobs:
141182
retention-days: 10
142183

143184
- name: Login to Docker Hub
144-
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
185+
if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch' && (inputs.force_build_deps != '' || inputs.force_build_fp != ''))
145186
uses: docker/login-action@v3
146187
with:
147188
username: ${{ secrets.DOCKERHUB_USERNAME }}
148189
password: ${{ secrets.DOCKERHUB_TOKEN }}
149190

150191
- name: Push docker containers
151-
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
192+
if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch' && (inputs.force_build_deps != '' || inputs.force_build_fp != ''))
152193
run: |
153194
if [ "${{ needs.detect-changes.outputs.build_deps }}" == "true" ]; then
154195
VERSION_TAG="${{ needs.detect-changes.outputs.deps_version }}"
@@ -356,7 +397,7 @@ jobs:
356397
create-manifest:
357398
name: Create and Push Manifest
358399
needs: [detect-changes, build-test-push-docker-arm]
359-
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && (needs.detect-changes.outputs.build_deps == 'true' || needs.detect-changes.outputs.build_fp == 'true')
400+
if: ((github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch') && (needs.detect-changes.outputs.build_deps == 'true' || needs.detect-changes.outputs.build_fp == 'true')
360401
runs-on: ubuntu-latest
361402
steps:
362403
- name: Log in to Docker Hub

‎docker/docker_loginNpush.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
#!/bin/bash
22
set -euo pipefail
33

4-
LOG_FILE="/home/ec2-user/forcingprocessor/docker_build_log.txt"
5-
mkdir -p "$(dirname "$LOG_FILE")" 2>/dev/null || LOG_FILE="/tmp/docker_build_log.txt"
4+
# Use a dedicated log file for push operations to avoid permission conflicts
5+
LOG_FILE="/home/ec2-user/forcingprocessor/docker_push_log.txt"
6+
mkdir -p "$(dirname "$LOG_FILE")" 2>/dev/null || true
7+
8+
# Ensure log file is writable, fallback to /tmp if not
9+
if ! touch "$LOG_FILE" 2>/dev/null; then
10+
LOG_FILE="/tmp/docker_push_log.txt"
11+
fi
612

713
log() { echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"; }
814

0 commit comments

Comments
 (0)