Skip to content

Commit 3702cd4

Browse files
harshavemula-uaharshavemula-ua
authored
Merge pull request #246 from CIROH-UA/test_containers
Docker Compose implementation and Test ARM builds
2 parents 475dda8 + 88d2208 commit 3702cd4

22 files changed

+414
-182
lines changed

‎.github/workflows/build_test_docker_arm.yaml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: Build, Test Datastream Docker Containers on ARM
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
branch_name:
6+
description: 'Branch name to build from (leave empty to use current branch)'
7+
required: false
8+
default: ''
9+
type: string
10+
push:
11+
paths:
12+
- 'docker/**'
13+
- 'forcingprocessor/**'
14+
- 'scripts/**'
15+
- 'python_tools/**'
16+
- 'research_datastream/terraform/test/test_execution_gp_arm_docker_buildNtester.json'
17+
- '!docs/**'
18+
- '!scripts/README.md'
19+
- '!docker/README.md'
20+
- '!forcingprocessor/README.md'
21+
- '!python_tools/README.md'
22+
23+
permissions:
24+
contents: read
25+
26+
jobs:
27+
build-test-docker-arm:
28+
runs-on: ubuntu-latest
29+
steps:
30+
- name: Checkout code
31+
uses: actions/checkout@v2
32+
- name: Setup Terraform
33+
uses: hashicorp/setup-terraform@v1
34+
- name: Configure AWS
35+
run: |
36+
aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
37+
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
38+
aws configure set region us-east-1
39+
40+
- name: Prepare execution config
41+
run: |
42+
if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ -n "${{ inputs.branch_name }}" ]; then
43+
BRANCH_NAME="${{ inputs.branch_name }}"
44+
else
45+
BRANCH_NAME="${{ github.ref_name }}"
46+
fi
47+
48+
cd research_datastream/terraform/test
49+
sed -i "s/\${BRANCH_NAME}/$BRANCH_NAME/g" test_execution_gp_arm_docker_buildNtester.json
50+
51+
if [ "${{ github.event_name }}" != "workflow_dispatch" ] && ! git diff --name-only HEAD~1 HEAD | grep -q '^docker/Dockerfile\.datastream$'; then
52+
sed -i "/docker-compose\.yml build datastream/d" test_execution_gp_arm_docker_buildNtester.json
53+
fi
54+
55+
if [ "${{ github.event_name }}" != "workflow_dispatch" ] && ! git diff --name-only HEAD~1 HEAD | grep -q '^docker/Dockerfile\.datastream-deps$'; then
56+
sed -i "/docker-compose\.yml build datastream-deps/d" test_execution_gp_arm_docker_buildNtester.json
57+
fi
58+
59+
if [ "${{ github.event_name }}" != "workflow_dispatch" ] && ! git diff --name-only HEAD~1 HEAD | grep -q '^docker/Dockerfile\.forcingprocessor$'; then
60+
sed -i "/docker-compose\.yml build forcingprocessor/d" test_execution_gp_arm_docker_buildNtester.json
61+
fi
62+
63+
cat test_execution_gp_arm_docker_buildNtester.json
64+
65+
- name: Build AWS Infra
66+
run: |
67+
cd research_datastream/terraform
68+
terraform init
69+
terraform validate
70+
../scripts/import_resources.sh ./test/variables_gitactions_arm.tfvars
71+
terraform apply -var-file=./test/variables_gitactions_arm.tfvars -auto-approve
72+
sleep 60
73+
74+
- name: Set permissions
75+
run : |
76+
cd research_datastream/terraform
77+
aws iam attach-role-policy --role-name datastream_ec2_role_github_actions_arm --policy-arn arn:aws:iam::aws:policy/SecretsManagerReadWrite
78+
aws secretsmanager put-resource-policy --secret-id docker_awiciroh_creds --resource-policy file://test/secret-policy.json --block-public-policy --region us-east-1
79+
if ! aws ec2 describe-key-pairs --key-names "actions_key_arm" --query 'KeyPairs[0].KeyName' --output text 2>/dev/null; then aws ec2 create-key-pair --key-name "actions_key_arm" --query 'KeyName' --output text && echo "Key pair 'actions_key_arm' created in AWS"; else echo "Key pair 'actions_key_arm' already exists"; fi
80+
sleep 60
81+
82+
- name: Build and Test arm docker containers with AWS infra
83+
run : |
84+
cd research_datastream/terraform
85+
execution_arn=$(aws stepfunctions start-execution --state-machine-arn $(cat ./sm_ARN.txt) --name docker_builder_$(env TZ=US/Eastern date +'%Y%m%d%H%M%S') --input "file://test/test_execution_gp_arm_docker_buildNtester.json" --region us-east-1 --query 'executionArn' --output text); echo "Execution ARN: $execution_arn"; status="RUNNING"; while [ "$status" != "SUCCEEDED" ]; do status=$(aws stepfunctions describe-execution --execution-arn "$execution_arn" --region us-east-1 --query 'status' --output text); echo "Current status: $status"; if [ "$status" == "FAILED" ]; then echo "State machine execution failed!"; exit 1; fi; sleep 5; done; echo "State machine execution succeeded!"
86+
87+
- name: Tear down infra
88+
if: always()
89+
run : |
90+
cd research_datastream/terraform
91+
terraform destroy -var-file=./test/variables_gitactions_arm.tfvars -auto-approve
92+
sleep 60

‎.github/workflows/build_test_docker_compose_x86.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: Build and Test Datastream Docker compose on x86
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
paths:
7+
- 'docker/**'
8+
- 'forcingprocessor/**'
9+
- 'scripts/**'
10+
- 'python_tools/**'
11+
- '!docs/**'
12+
- '!scripts/README.md'
13+
- '!docker/README.md'
14+
- '!forcingprocessor/README.md'
15+
- '!python_tools/README.md'
16+
17+
permissions:
18+
contents: read
19+
20+
jobs:
21+
build-test-docker-x86:
22+
runs-on: ubuntu-latest
23+
steps:
24+
- name: Checkout code
25+
uses: actions/checkout@v2
26+
27+
- name: Set up QEMU
28+
uses: docker/setup-qemu-action@v3
29+
30+
- name: Set up Docker Buildx
31+
uses: docker/setup-buildx-action@v3
32+
33+
- name: Login to Docker Hub
34+
uses: docker/login-action@v3
35+
with:
36+
username: ${{secrets.DOCKERHUB_USERNAME}}
37+
password: ${{secrets.DOCKERHUB_TOKEN}}
38+
39+
- name: Configure AWS
40+
run: |
41+
aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
42+
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
43+
aws configure set region us-east-1
44+
45+
- name: Build docker containers
46+
run: |
47+
# export ARCH=$(uname -m)
48+
# docker compose -f docker/docker-compose.yml build datastream-deps
49+
docker compose -f docker/docker-compose.yml build datastream
50+
docker compose -f docker/docker-compose.yml build forcingprocessor
51+
52+
- name: Test docker containers
53+
run : |
54+
curl -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg
55+
./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json

‎.github/workflows/build_test_docker_x86.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,11 @@ jobs:
5151
5252
- name: Build docker containers
5353
run : |
54-
./scripts/docker_builds.sh -f -d
55-
54+
# export ARCH=$(uname -m)
55+
# docker compose -f docker/docker-compose.yml build datastream-deps
56+
docker compose -f docker/docker-compose.yml build datastream
57+
docker compose -f docker/docker-compose.yml build forcingprocessor
5658
- name: Test docker containers
5759
run : |
5860
curl -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg
59-
./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
61+
./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json

‎.github/workflows/build_test_ngiab_ami.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,10 @@ jobs:
4646
4747
- name: Build docker containers
4848
run : |
49-
./scripts/docker_builds.sh -e -f -d
49+
export ARCH=$(uname -m)
50+
docker compose -f docker/docker-compose.yml build datastream-deps
51+
docker compose -f docker/docker-compose.yml build datastream
52+
docker compose -f docker/docker-compose.yml build forcingprocessor
5053
docker images
5154
5255

.github/workflows/build_test_push_docker_X86_arm_version_aware renamed to .github/workflows/build_test_push_docker_X86_arm_version_aware.yaml

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
name: Build, Test, and Push Datastream Docker Containers on X86 and ARM (Version Aware)
22

33
on:
4-
workflow_dispatch:
4+
workflow_dispatch:
5+
inputs:
6+
branch_name:
7+
description: 'Branch name to build from (leave empty to use current branch)'
8+
required: false
9+
default: ''
10+
type: string
511
push:
6-
branches:
7-
- main
12+
# branches:
13+
# - main
814
paths:
915
- 'versions.yml'
1016

@@ -79,7 +85,7 @@ jobs:
7985
fi
8086
echo "ds_version=$CURRENT_DS" >> "$GITHUB_OUTPUT"
8187

82-
build-test-docker-x86:
88+
build-test-push-docker-x86:
8389
needs: [detect-changes]
8490
runs-on: ubuntu-latest
8591
steps:
@@ -104,14 +110,15 @@ jobs:
104110
username: ${{ secrets.DOCKERHUB_USERNAME }}
105111
password: ${{ secrets.DOCKERHUB_TOKEN }}
106112

107-
- name: Install packages for datastream
108-
run: |
109-
curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset && sudo apt-get update && sudo apt-get install git pip pigz -y
113+
# - name: Install packages for datastream
114+
# run: |
115+
# curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset && sudo apt-get update && sudo apt-get install git pip pigz -y
110116

111117
- name: Build docker containers
112118
run: |
113-
chmod +x ./scripts/docker_builds.sh
114-
./scripts/docker_builds.sh -e -f -d -t latest-x86
119+
ARCH=$(uname -m) TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream-deps
120+
TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream
121+
TAG=latest-x86 docker compose -f docker/docker-compose.yml build forcingprocessor
115122
116123
- name: Prepare and test docker containers
117124
run: |
@@ -140,7 +147,7 @@ jobs:
140147
fi
141148
142149
build-test-push-docker-arm:
143-
needs: [detect-changes, build-test-docker-x86]
150+
needs: [detect-changes, build-test-push-docker-x86]
144151
runs-on: ubuntu-latest
145152
steps:
146153
- name: Checkout code
@@ -155,6 +162,23 @@ jobs:
155162
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
156163
aws configure set region us-east-1
157164
165+
- name: Prepare execution config with branch
166+
run: |
167+
# Determine the branch name
168+
if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ -n "${{ inputs.branch_name }}" ]; then
169+
BRANCH_NAME="${{ inputs.branch_name }}"
170+
echo "Using manual input branch: $BRANCH_NAME"
171+
else
172+
BRANCH_NAME="${{ github.ref_name }}"
173+
echo "Using current branch: $BRANCH_NAME"
174+
fi
175+
176+
# Replace ${BRANCH_NAME} with actual branch name in the file
177+
cd research_datastream/terraform/test
178+
sed -i "s/\${BRANCH_NAME}/$BRANCH_NAME/g" execution_gp_arm_docker_buildNtester.json
179+
cat execution_gp_arm_docker_buildNtester.json
180+
181+
158182
- name: Build AWS Infra
159183
run: |
160184
cd research_datastream/terraform
@@ -203,14 +227,15 @@ jobs:
203227
BUILD_FLAGS=$(echo "$BUILD_FLAGS" | sed 's/^ *//')
204228
205229
cd research_datastream/terraform
206-
sed -e "s/\${TAG}/$TAG/g" -e "s/\${BUILD_ARGS}/$BUILD_FLAGS/g" \
207-
test/execution_gp_arm_docker_buildNtester.json > test/execution_temp.json
230+
231+
sed -i -e "s/\${TAG}/$TAG/g" -e "s/\${BUILD_ARGS}/$BUILD_FLAGS/g" \
232+
test/execution_gp_arm_docker_buildNtester.json
208233
209234
echo "Generated execution file:"
210-
cat test/execution_temp.json
235+
cat test/execution_gp_arm_docker_buildNtester.json
211236
212237
echo "Building with TAG: $TAG and FLAGS: $BUILD_FLAGS"
213-
execution_arn=$(aws stepfunctions start-execution --state-machine-arn $(cat ./sm_ARN.txt) --name docker_builder_$(env TZ=US/Eastern date +'%Y%m%d%H%M%S') --input "file://test/execution_temp.json" --region us-east-1 --query 'executionArn' --output text); echo "Execution ARN: $execution_arn"; status="RUNNING"; while [ "$status" != "SUCCEEDED" ]; do status=$(aws stepfunctions describe-execution --execution-arn "$execution_arn" --region us-east-1 --query 'status' --output text); echo "Current status: $status"; if [ "$status" == "FAILED" ]; then echo "State machine execution failed!"; exit 1; fi; sleep 5; done; echo "State machine execution succeeded!"
238+
execution_arn=$(aws stepfunctions start-execution --state-machine-arn $(cat ./sm_ARN.txt) --name docker_builder_$(env TZ=US/Eastern date +'%Y%m%d%H%M%S') --input "file://test/execution_gp_arm_docker_buildNtester.json" --region us-east-1 --query 'executionArn' --output text); echo "Execution ARN: $execution_arn"; status="RUNNING"; while [ "$status" != "SUCCEEDED" ]; do status=$(aws stepfunctions describe-execution --execution-arn "$execution_arn" --region us-east-1 --query 'status' --output text); echo "Current status: $status"; if [ "$status" == "FAILED" ]; then echo "State machine execution failed!"; exit 1; fi; sleep 5; done; echo "State machine execution succeeded!"
214239
215240
216241
- name: Tear down infra

‎.github/workflows/build_test_push_docker_arm.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
name: Build, Test, and Push Datastream Docker Containers on ARM
22
on:
33
workflow_dispatch:
4+
inputs:
5+
branch_name:
6+
description: 'Branch name to build from (leave empty to use current branch)'
7+
required: false
8+
default: ''
9+
type: string
410
push:
511
branches:
612
- main
@@ -33,6 +39,23 @@ jobs:
3339
aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
3440
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
3541
aws configure set region us-east-1
42+
43+
- name: Prepare execution config with branch
44+
run: |
45+
# Determine the branch name
46+
if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ -n "${{ inputs.branch_name }}" ]; then
47+
BRANCH_NAME="${{ inputs.branch_name }}"
48+
echo "Using manual input branch: $BRANCH_NAME"
49+
else
50+
BRANCH_NAME="${{ github.ref_name }}"
51+
echo "Using current branch: $BRANCH_NAME"
52+
fi
53+
54+
# Replace ${BRANCH_NAME} with actual branch name in the file
55+
cd research_datastream/terraform/test
56+
sed -i "s/\${BRANCH_NAME}/$BRANCH_NAME/g" execution_gp_arm_docker_buildNtester.json
57+
cat execution_gp_arm_docker_buildNtester.json
58+
3659
- name: Build AWS Infra
3760
run: |
3861
cd research_datastream/terraform

‎.github/workflows/build_test_push_docker_x86.yaml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,9 @@ jobs:
4444
4545
- name: Build docker containers
4646
run : |
47-
./scripts/docker_builds.sh -e -f -d -t latest-x86
47+
TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream-deps
48+
TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream
49+
TAG=latest-x86 docker compose -f docker/docker-compose.yml build forcingprocessor
4850
4951
- name: Test docker containers
5052
run : |
@@ -59,4 +61,6 @@ jobs:
5961

6062
- name: Push docker containers
6163
run: |
62-
./scripts/docker_builds.sh -p -t latest-x86
64+
TAG=latest-x86 docker compose -f docker/docker-compose.yml push datastream-deps
65+
TAG=latest-x86 docker compose -f docker/docker-compose.yml push datastream
66+
TAG=latest-x86 docker compose -f docker/docker-compose.yml push forcingprocessor

‎.github/workflows/test_datastream_ngiab.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,9 @@ jobs:
5858
5959
- name: Build docker containers
6060
run : |
61-
./scripts/docker_builds.sh -f -d
62-
61+
docker compose -f docker/docker-compose.yml build datastream
62+
docker compose -f docker/docker-compose.yml build forcingprocessor
63+
6364
- name: Base test and NWM_RETRO_V3
6465
run: |
6566
sudo rm -rf $(pwd)/data/datastream_test

‎.github/workflows/test_datastream_options.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,9 @@ jobs:
6060
echo $AWS_SECRET_ACCESS_KEY
6161
6262
- name: Build docker containers
63-
run : |
64-
./scripts/docker_builds.sh -f -d
63+
run : |
64+
docker compose -f docker/docker-compose.yml build datastream
65+
docker compose -f docker/docker-compose.yml build forcingprocessor
6566
6667
- name: Get geopackage
6768
run: |

‎.github/workflows/test_hfsubset_v2_1.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@ jobs:
3131
uses: docker/setup-buildx-action@v3
3232

3333
- name: Build docker containers
34-
run : |
35-
./scripts/docker_builds.sh -f -d
34+
run : |
35+
docker compose -f docker/docker-compose.yml build datastream
36+
docker compose -f docker/docker-compose.yml build forcingprocessor
3637
3738
- name: Install packages for datastream
3839
run: |

0 commit comments

Comments
 (0)