Skip to content

Commit 76d949a

Browse files
sethAmazonsethAmazon
authored
Add retry to EC2 CI tests (#827)
1 parent b0bf174 commit 76d949a

File tree

1 file changed

+54
-46
lines changed

1 file changed

+54
-46
lines changed

‎.github/workflows/CI.yml‎

Lines changed: 54 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -645,13 +645,15 @@ jobs:
645645

646646
- name: Run testing suite on ec2
647647
if: steps.e2etest-ec2-1.outputs.cache-hit != 'true'
648-
run: |
649-
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
650-
cd testing-framework/terraform/ec2 && terraform init && terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}"
651-
- name: Destroy resources
652-
if: ${{ always() && steps.e2etest-ec2-1.outputs.cache-hit != 'true' }}
653-
run: |
654-
cd testing-framework/terraform/ec2 && terraform destroy -auto-approve
648+
uses: nick-invision/retry@v2
649+
with:
650+
max_attempts: 3
651+
timeout_minutes: 120
652+
command: |
653+
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
654+
cd testing-framework/terraform/ec2
655+
terraform init
656+
if terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi
655657
656658
e2etest-ec2-2:
657659
runs-on: ubuntu-latest
@@ -707,13 +709,15 @@ jobs:
707709

708710
- name: Run testing suite on ec2
709711
if: steps.e2etest-ec2-2.outputs.cache-hit != 'true'
710-
run: |
711-
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
712-
cd testing-framework/terraform/ec2 && terraform init && terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}"
713-
- name: Destroy resources
714-
if: ${{ always() && steps.e2etest-ec2-2.outputs.cache-hit != 'true'}}
715-
run: |
716-
cd testing-framework/terraform/ec2 && terraform destroy -auto-approve
712+
uses: nick-invision/retry@v2
713+
with:
714+
max_attempts: 3
715+
timeout_minutes: 120
716+
command: |
717+
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
718+
cd testing-framework/terraform/ec2
719+
terraform init
720+
if terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi
717721
718722
e2etest-ec2-3:
719723
runs-on: ubuntu-latest
@@ -769,14 +773,15 @@ jobs:
769773

770774
- name: Run testing suite on ec2
771775
if: steps.e2etest-ec2-3.outputs.cache-hit != 'true'
772-
run: |
773-
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
774-
cd testing-framework/terraform/ec2 && terraform init && terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}"
775-
776-
- name: Destroy resources
777-
if: ${{ always() && steps.e2etest-ec2-3.outputs.cache-hit != 'true' }}
778-
run: |
779-
cd testing-framework/terraform/ec2 && terraform destroy -auto-approve
776+
uses: nick-invision/retry@v2
777+
with:
778+
max_attempts: 3
779+
timeout_minutes: 120
780+
command: |
781+
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
782+
cd testing-framework/terraform/ec2
783+
terraform init
784+
if terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi
780785
781786
e2etest-ecs:
782787
runs-on: ubuntu-latest
@@ -825,14 +830,15 @@ jobs:
825830

826831
- name: Run testing suite on ecs
827832
if: steps.e2etest-ecs.outputs.cache-hit != 'true'
828-
run: |
829-
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
830-
cd testing-framework/terraform/ecs && terraform init && terraform apply -auto-approve -lock=false $opts -var="ecs_launch_type=${{ matrix.launch_type }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}"
831-
832-
- name: Destroy resources
833-
if: ${{ always() && steps.e2etest-ecs.outputs.cache-hit != 'true' }}
834-
run: |
835-
cd testing-framework/terraform/ecs && terraform destroy -auto-approve
833+
uses: nick-invision/retry@v2
834+
with:
835+
max_attempts: 3
836+
timeout_minutes: 120
837+
command: |
838+
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
839+
cd testing-framework/terraform/ecs
840+
terraform init
841+
if terraform apply -auto-approve -lock=false $opts -var="ecs_launch_type=${{ matrix.launch_type }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi
836842
837843
e2etest-eks:
838844
runs-on: ubuntu-latest
@@ -881,14 +887,15 @@ jobs:
881887

882888
- name: Run testing suite on eks
883889
if: steps.e2etest-eks.outputs.cache-hit != 'true'
884-
run: |
885-
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
886-
cd testing-framework/terraform/eks && terraform init && terraform apply -auto-approve -lock=false $opts -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}"
887-
888-
- name: Destroy resources
889-
if: ${{ always() && steps.e2etest-eks.outputs.cache-hit != 'true' }}
890-
run: |
891-
cd testing-framework/terraform/eks && terraform destroy -auto-approve
890+
uses: nick-invision/retry@v2
891+
with:
892+
max_attempts: 3
893+
timeout_minutes: 120
894+
command: |
895+
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
896+
cd testing-framework/terraform/eks
897+
terraform init
898+
if terraform apply -auto-approve -lock=false $opts -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi
892899
893900
e2etest-eks-fargate:
894901
runs-on: ubuntu-latest
@@ -937,14 +944,15 @@ jobs:
937944

938945
- name: Run testing suite on eks
939946
if: steps.e2etest-eks-fargate.outputs.cache-hit != 'true'
940-
run: |
941-
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
942-
cd testing-framework/terraform/eks && terraform init && terraform apply -auto-approve -lock=false $opts -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}"
943-
944-
- name: Destroy resources
945-
if: ${{ always() && steps.e2etest-eks-fargate.outputs.cache-hit != 'true' }}
946-
run: |
947-
cd testing-framework/terraform/eks && terraform destroy -auto-approve -var-file="../testcases/${{ matrix.testcase }}/parameters.tfvars"
947+
uses: nick-invision/retry@v2
948+
with:
949+
max_attempts: 3
950+
timeout_minutes: 120
951+
command: |
952+
if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
953+
cd testing-framework/terraform/eks
954+
terraform init
955+
if terraform apply -auto-approve -lock=false $opts -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve -var-file="../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else terraform destroy -auto-approve -var-file="../testcases/${{ matrix.testcase }}/parameters.tfvars" && exit 1 ; fi
948956
949957
e2etest-eks-adot-operator:
950958
runs-on: ubuntu-latest

0 commit comments

Comments
 (0)