Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/assets/php/createImageBuilderDataYaml.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ function tabEverything($content, $tab = 7)
$scriptBuild = tabEverything(file_get_contents($argv[1]));
$scriptValidate = tabEverything(file_get_contents($argv[2]));

$timeout = getenv('IMAGE_BUILDER_STEP_TIMEOUT_SECONDS') ?: '1200';

$yamlContent = <<<EOD
name: Install Server Software
description: Installs server software for EC2 instances
Expand All @@ -18,7 +20,7 @@ function tabEverything($content, $tab = 7)
steps:
- name: CustomBuildEc2Image
action: ExecuteBash
timeoutSeconds: 1200
timeoutSeconds: $timeout
onFailure: Abort
maxAttempts: 1
inputs:
Expand Down
83 changes: 72 additions & 11 deletions .github/workflows/aws.yml
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,12 @@ on:
required: false
type: number
default: 30

imageBuilderStepTimeoutSeconds:
description: 'Timeout for each image builder step'
required: false
type: number
default: 1200
deployALB:
description: 'Deploy an Application Load Balancer (true or false)'
required: false
Expand Down Expand Up @@ -306,6 +312,60 @@ on:
required: false
type: string

vpcStackPropagationSeconds:
description: 'Time to wait for VPC stack exports to propagate'
required: false
type: number
default: 240

dnsPropagationTimeoutSeconds:
description: 'Maximum wait for DNS propagation when validating certificates'
required: false
type: number
default: 300

dnsPropagationCheckIntervalSeconds:
description: 'Interval between DNS propagation checks'
required: false
type: number
default: 10

albStackPauseSeconds:
description: 'Pause before creating the ALB stack'
required: false
type: number
default: 3

eksClusterCheckIntervalSeconds:
description: 'Interval between EKS cluster status checks'
required: false
type: number
default: 30

imageBuilderDistributionSleepSeconds:
description: 'Pause after creating or updating image builder distribution configuration'
required: false
type: number
default: 240

instanceRefreshInitialSleepSeconds:
description: 'Initial wait before polling instance refresh status'
required: false
type: number
default: 240

instanceRefreshCheckIntervalSeconds:
description: 'Interval between instance refresh status checks'
required: false
type: number
default: 60

instanceWarmupSeconds:
description: 'InstanceWarmup value for auto-scaling instance refresh'
required: false
type: number
default: 1200

secrets:
ENCRYPTION_KEY:
required: false
Expand Down Expand Up @@ -478,8 +538,8 @@ jobs:
ParameterKey=HighlyAvailableNat,ParameterValue="${{ needs.CONSTANTS.outputs.highlyAvailableNat }}" \
ParameterKey=EnableVpcFlowLogs,ParameterValue="${{ needs.CONSTANTS.outputs.enableVpcFlowLogs }}"

echo "Sleeping for 240 seconds to allow VPC stack exports to propagate. We've seen timing issues with the VPC stack exports not being available immediately for networkshares."
sleep 240
echo "Sleeping for ${{ inputs.vpcStackPropagationSeconds }} seconds to allow VPC stack exports to propagate. We've seen timing issues with the VPC stack exports not being available immediately for networkshares."
sleep ${{ inputs.vpcStackPropagationSeconds }}

else
echo "The VPC stack already exists on the AWS network account."
Expand Down Expand Up @@ -678,8 +738,8 @@ jobs:

echo "Waiting for certificate to be validated for $domain ($cert)"

timeout=300 # seconds
interval=10 # seconds
timeout=${{ inputs.dnsPropagationTimeoutSeconds }} # seconds
interval=${{ inputs.dnsPropagationCheckIntervalSeconds }} # seconds
elapsed=0

while ! dig +short "$CNAME" >/dev/null; do
Expand Down Expand Up @@ -765,7 +825,7 @@ jobs:
set -eEBx
php "./.github/assets/php/createAlbYaml.php" "${{ env.certificates }}" > ./CloudFormation/alb.yaml
cat ./CloudFormation/alb.yaml
sleep 3
sleep ${{ inputs.albStackPauseSeconds }}
./.github/assets/shell/createUpdateCFStack.sh ${{ matrix.aws-region }} alb \
--template-body file://./CloudFormation/alb.yaml \
--parameters 'ParameterKey=PublicSubnets,ParameterValue="${{ env.publicSubnet }}"'
Expand Down Expand Up @@ -1057,7 +1117,7 @@ jobs:
echo "❌ Cluster failed to become active"
exit 1
fi
sleep 30
sleep ${{ inputs.eksClusterCheckIntervalSeconds }}
done

- name: Add new node group
Expand Down Expand Up @@ -1255,7 +1315,7 @@ jobs:
aws imagebuilder create-distribution-configuration \
--name ${{ inputs.accountName }}-distribution-configuration \
"${DISTRIBUTION_ARGUMENTS[@]}"
sleep 240
sleep ${{ inputs.imageBuilderDistributionSleepSeconds }}
else
aws imagebuilder update-distribution-configuration \
--distribution-configuration-arn \
Expand Down Expand Up @@ -1287,6 +1347,7 @@ jobs:
IMAGE-BUILDER-VALIDATE-EOF

chmod +x ./.github/assets/shell/createUpdateImageBuilder.sh
export IMAGE_BUILDER_STEP_TIMEOUT_SECONDS="${{ inputs.imageBuilderStepTimeoutSeconds }}"
./.github/assets/shell/createUpdateImageBuilder.sh \
"${{ env.distribution }}" \
"${{ matrix.aws-region }}" \
Expand Down Expand Up @@ -1471,7 +1532,7 @@ jobs:
if: ${{ steps.web.outputs.refresh == '1' && needs.IMAGE-BUILDER.outputs.image_rebuilt == '1' }}
run: |
REFRESH_ID=$(aws autoscaling start-instance-refresh \
--preferences '{"InstanceWarmup": 1200, "MinHealthyPercentage": 100}' \
--preferences '{"InstanceWarmup": ${{ inputs.instanceWarmupSeconds }}, "MinHealthyPercentage": 100}' \
--strategy Rolling \
--auto-scaling-group-name "${{ inputs.environment }}-${{ needs.CONSTANTS.outputs.repositoryNicename }}-${{ env.version }}.${{ github.run_number }}-asg" --output text)

Expand All @@ -1497,12 +1558,12 @@ jobs:

getStatus

sleep 240
sleep ${{ inputs.instanceRefreshInitialSleepSeconds }}

while [[ "$STATUS" == "Pending" || "$STATUS" == "InProgress" ]]; do
getLog ${{ env.version }}
echo "Waiting 60 seconds... <$STATUS> (attempt:$TRY)"
sleep 60
echo "Waiting ${{ inputs.instanceRefreshCheckIntervalSeconds }} seconds... <$STATUS> (attempt:$TRY)"
sleep ${{ inputs.instanceRefreshCheckIntervalSeconds }}
getStatus
done
if [[ "$STATUS" == "Successful" ]]; then
Expand Down