diff --git a/lib/aws-semaphore-agent-stack.js b/lib/aws-semaphore-agent-stack.js index d8b90b2..31e6634 100644 --- a/lib/aws-semaphore-agent-stack.js +++ b/lib/aws-semaphore-agent-stack.js @@ -94,6 +94,33 @@ class AwsSemaphoreAgentStack extends Stack { new PolicyStatement({ effect: Effect.ALLOW, actions: [ + "ec2:CreateReplaceRootVolumeTask" + ], + resources: [ + `arn:aws:ec2:${this.region}:${this.account}:instance/*`, + ], + conditions: { + "StringLike": { + "aws:userid": "*:${ec2:InstanceID}" + } + } + }), + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "ec2:CreateReplaceRootVolumeTask" + ], + resources: [ + `arn:aws:ec2:${this.region}:${this.account}:replace-root-volume-task/*`, + `arn:aws:ec2:${this.region}:${this.account}:volume/*`, + `arn:aws:ec2:${this.region}:${this.account}:image/*`, + ] + }), + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "autoscaling:EnterStandby", + "autoscaling:ExitStandby", "autoscaling:SetInstanceHealth", "autoscaling:TerminateInstanceInAutoScalingGroup" ], @@ -323,6 +350,7 @@ class AwsSemaphoreAgentStack extends Stack { blockDevices: this.createBlockDevices(), securityGroup: securityGroup, instanceInitiatedShutdownBehavior: InstanceInitiatedShutdownBehavior.TERMINATE, + instanceMetadataTags: true, keyName: keyName != "" ? keyName : undefined }); diff --git a/packer/macos/files/start-agent.sh b/packer/macos/files/start-agent.sh index 054aabb..ecccf6b 100755 --- a/packer/macos/files/start-agent.sh +++ b/packer/macos/files/start-agent.sh @@ -204,6 +204,24 @@ region=$(curl \ --show-error \ --location "http://169.254.169.254/latest/meta-data/placement/region" ) +instance_id=$(curl \ + -H "X-aws-ec2-metadata-token: $token" \ + --fail \ + --silent \ + --show-error \ + --location "http://169.254.169.254/latest/meta-data/instance-id" +) +asg_name=$(curl \ + -H "X-aws-ec2-metadata-token: $token" \ + --fail \ + --silent \ + --show-error \ + --location "http://169.254.169.254/latest/meta-data/tags/instance/aws:autoscaling:groupName" +) + +# If the instance is in the "Standby" LifecycleState (after root volume replacement) we want to exit so the health-check runs +# If it's the first boot the instance may already be in "InService" LifecycleState so we ignore any errors from this command +aws autoscaling exit-standby --region "$region" --instance-ids "$instance_id" --auto-scaling-group-name "$asg_name" || true # The parameters required for the agent configuration are stored in an SSM parameter. # We need to fetch them before proceeding with anything else. diff --git a/packer/macos/files/terminate-instance.sh b/packer/macos/files/terminate-instance.sh index b1e2ce1..2548a63 100755 --- a/packer/macos/files/terminate-instance.sh +++ b/packer/macos/files/terminate-instance.sh @@ -27,8 +27,20 @@ if [[ $SEMAPHORE_AGENT_SHUTDOWN_REASON == "IDLE" ]]; then --instance-id "$instance_id" \ --should-decrement-desired-capacity else - aws autoscaling terminate-instance-in-auto-scaling-group \ + # Enter standby LifecycleState because the EC2 health check will fail while we're replacing the root volume + # We also decrement desired capacity so the ASG doesn't create a new replacement instance in the meantime + # The instance will exit standby automatically in start-agent.sh after reboot + asg_name=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/tags/instance/aws:autoscaling:groupName") + aws autoscaling enter-standby \ + --region "$region" \ + --instance-ids "$instance_id" \ + --auto-scaling-group-name "$asg_name" \ + --should-decrement-desired-capacity + # https://aws.amazon.com/blogs/compute/new-reset-amazon-ec2-mac-instances-to-a-known-state-using-replace-root-volume-capability/ + ami_id=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/ami-id") + aws ec2 create-replace-root-volume-task \ --region "$region" \ --instance-id "$instance_id" \ - --no-should-decrement-desired-capacity + --image-id "$ami_id" \ + --delete-replaced-root-volume fi diff --git a/packer/macos/macos.pkr.hcl b/packer/macos/macos.pkr.hcl index 112fd1b..f0c9bf3 100644 --- a/packer/macos/macos.pkr.hcl +++ b/packer/macos/macos.pkr.hcl @@ -76,7 +76,7 @@ source "amazon-ebs" "macos" { owners = ["amazon"] filters = { - name = "amzn-ec2-macos-12.6*" + name = "amzn-ec2-macos-13.7*" architecture = "${var.arch}*" root-device-type = "ebs" virtualization-type = "hvm"