From 472a3f8a5fc224bd0d904c2ebea5894286bbb53d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Sep 2024 09:57:28 -0400 Subject: [PATCH 01/13] fix: ami runner runs out of space with larger builds --- stage2-nix-psql.pkr.hcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 57ddfd9e7..34f7fe97a 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -78,6 +78,13 @@ source "amazon-ebs" "ubuntu" { ena_support = true + + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 50 + volume_type = "gp3" + delete_on_termination = true + } run_tags = { creator = "packer" From af9acc107e92abf862bc34c959fbc57718d4a65a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Sep 2024 12:43:23 -0400 Subject: [PATCH 02/13] fix: ami_block_device_mappings --- stage2-nix-psql.pkr.hcl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 34f7fe97a..86bbb4d6e 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -85,7 +85,11 @@ source "amazon-ebs" "ubuntu" { volume_type = "gp3" delete_on_termination = true } - + ami_block_device_mappings { + device_name = "/dev/sda1" + volume_type = "gp3" + delete_on_termination = true + } run_tags = { creator = "packer" appType = "postgres" From 6cec39ea11516d2d414870d6fcadeedb37bea374 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Sep 2024 13:32:12 -0400 Subject: [PATCH 03/13] fix: try a default size on amu_block_device_mappings --- stage2-nix-psql.pkr.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 86bbb4d6e..5a8f21ed8 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -88,6 +88,7 @@ source "amazon-ebs" "ubuntu" { ami_block_device_mappings { device_name = "/dev/sda1" volume_type = "gp3" + volume_size = 8 delete_on_termination = true } run_tags = { From 9d2599da4fdf2adfe436447f8a704bdd08cc03e1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Sep 2024 16:11:32 -0400 Subject: [PATCH 04/13] fix: a different approach to building and resizing/reset default --- .github/workflows/testinfra-nix.yml | 46 ++++++++++++++++++++++++++--- stage2-nix-psql.pkr.hcl | 6 ---- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 3835a9a00..8f556d3e7 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -53,12 +53,48 @@ jobs: GIT_SHA=${{github.sha}} packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl + - name: Get pre-resize AMI ID + id: get-ami + run: | + AMI_ID=$(aws ec2 describe-images --owners self --filters "Name=name,Values=supabase-postgres-${{ steps.random.outputs.random_string }}-pre-resize" --query 'Images[0].ImageId' --output text) + echo "AMI_ID=${AMI_ID}" >> $GITHUB_OUTPUT + + - name: Launch temporary instance + id: launch-instance + run: | + INSTANCE_ID=$(aws ec2 run-instances --image-id ${{ steps.get-ami.outputs.AMI_ID }} --instance-type t4g.micro --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=AMI-Resize-Temp}]' --query 'Instances[0].InstanceId' --output text) + echo "INSTANCE_ID=${INSTANCE_ID}" >> $GITHUB_OUTPUT + + - name: Wait for instance to be running + run: aws ec2 wait instance-running --instance-ids ${{ steps.launch-instance.outputs.INSTANCE_ID }} + + - name: Resize filesystem + run: | + aws ec2 start-session --target ${{ steps.launch-instance.outputs.INSTANCE_ID }} --document-name AWS-StartInteractiveCommand --parameters '{"command":["sudo e2fsck -f /dev/sda1 && sudo resize2fs /dev/sda1 8G && sudo sync"]}' + + - name: Create new AMI + id: create-ami + run: | + NEW_AMI_NAME="supabase-postgres-${{ steps.random.outputs.random_string }}" + NEW_AMI_ID=$(aws ec2 create-image --instance-id ${{ steps.launch-instance.outputs.INSTANCE_ID }} --name "${NEW_AMI_NAME}" --description "Resized AMI" --query 'ImageId' --output text) + echo "NEW_AMI_ID=${NEW_AMI_ID}" >> $GITHUB_OUTPUT + echo "NEW_AMI_NAME=${NEW_AMI_NAME}" >> $GITHUB_OUTPUT + + - name: Wait for AMI to be available + run: aws ec2 wait image-available --image-ids ${{ steps.create-ami.outputs.NEW_AMI_ID }} + + - name: Modify AMI block device mapping + run: | + aws ec2 modify-image-attribute --image-id ${{ steps.create-ami.outputs.NEW_AMI_ID }} --block-device-mappings '[{"DeviceName": "/dev/sda1","Ebs":{"VolumeSize":8}}]' + + - name: Terminate temporary instance + run: aws ec2 terminate-instances --instance-ids ${{ steps.launch-instance.outputs.INSTANCE_ID }} + - name: Run tests timeout-minutes: 10 env: - AMI_NAME: "supabase-postgres-${{ steps.random.outputs.random_string }}" + AMI_NAME: ${{ steps.create-ami.outputs.NEW_AMI_NAME }} run: | - # TODO: use poetry for pkg mgmt pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests pytest -vv -s testinfra/test_ami_nix.py @@ -77,7 +113,8 @@ jobs: run: | # Define AMI name patterns STAGE1_AMI_NAME="supabase-postgres-ci-ami-test-stage-1" - STAGE2_AMI_NAME="${{ steps.random.outputs.random_string }}" + PRE_RESIZE_AMI_NAME="supabase-postgres-${{ steps.random.outputs.random_string }}-pre-resize" + FINAL_AMI_NAME="supabase-postgres-${{ steps.random.outputs.random_string }}" # Function to deregister AMIs by name pattern deregister_ami_by_name() { @@ -91,4 +128,5 @@ jobs: # Deregister AMIs deregister_ami_by_name "$STAGE1_AMI_NAME" - deregister_ami_by_name "$STAGE2_AMI_NAME" \ No newline at end of file + deregister_ami_by_name "$PRE_RESIZE_AMI_NAME" + deregister_ami_by_name "$FINAL_AMI_NAME" \ No newline at end of file diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 5a8f21ed8..ef0196c4e 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -85,12 +85,6 @@ source "amazon-ebs" "ubuntu" { volume_type = "gp3" delete_on_termination = true } - ami_block_device_mappings { - device_name = "/dev/sda1" - volume_type = "gp3" - volume_size = 8 - delete_on_termination = true - } run_tags = { creator = "packer" appType = "postgres" From 0df352605c482c2213d4320ac1e14508dbed6019 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 09:04:51 -0400 Subject: [PATCH 05/13] fox: use ami name --- .github/workflows/testinfra-nix.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 8f556d3e7..5b2ad7334 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -53,16 +53,11 @@ jobs: GIT_SHA=${{github.sha}} packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - - name: Get pre-resize AMI ID - id: get-ami - run: | - AMI_ID=$(aws ec2 describe-images --owners self --filters "Name=name,Values=supabase-postgres-${{ steps.random.outputs.random_string }}-pre-resize" --query 'Images[0].ImageId' --output text) - echo "AMI_ID=${AMI_ID}" >> $GITHUB_OUTPUT - - name: Launch temporary instance id: launch-instance run: | - INSTANCE_ID=$(aws ec2 run-instances --image-id ${{ steps.get-ami.outputs.AMI_ID }} --instance-type t4g.micro --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=AMI-Resize-Temp}]' --query 'Instances[0].InstanceId' --output text) + AMI_NAME="supabase-postgres-${{ steps.random.outputs.random_string }}-pre-resize" + INSTANCE_ID=$(aws ec2 run-instances --image-id $(aws ec2 describe-images --owners self --filters "Name=name,Values=${AMI_NAME}" --query 'Images[0].ImageId' --output text) --instance-type t4g.micro --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=AMI-Resize-Temp}]' --query 'Instances[0].InstanceId' --output text) echo "INSTANCE_ID=${INSTANCE_ID}" >> $GITHUB_OUTPUT - name: Wait for instance to be running From bf321ecb1b76692ed697b9276df9f1ce4f1b300c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 12:09:52 -0400 Subject: [PATCH 06/13] chore: create a script to resize using boto3 --- .github/workflows/testinfra-nix.yml | 41 ++++--------- scripts/aws_resize.py | 95 +++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 29 deletions(-) create mode 100644 scripts/aws_resize.py diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 5b2ad7334..eafe71e4f 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -53,37 +53,20 @@ jobs: GIT_SHA=${{github.sha}} packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - - name: Launch temporary instance - id: launch-instance - run: | - AMI_NAME="supabase-postgres-${{ steps.random.outputs.random_string }}-pre-resize" - INSTANCE_ID=$(aws ec2 run-instances --image-id $(aws ec2 describe-images --owners self --filters "Name=name,Values=${AMI_NAME}" --query 'Images[0].ImageId' --output text) --instance-type t4g.micro --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=AMI-Resize-Temp}]' --query 'Instances[0].InstanceId' --output text) - echo "INSTANCE_ID=${INSTANCE_ID}" >> $GITHUB_OUTPUT - - - name: Wait for instance to be running - run: aws ec2 wait instance-running --instance-ids ${{ steps.launch-instance.outputs.INSTANCE_ID }} - - - name: Resize filesystem - run: | - aws ec2 start-session --target ${{ steps.launch-instance.outputs.INSTANCE_ID }} --document-name AWS-StartInteractiveCommand --parameters '{"command":["sudo e2fsck -f /dev/sda1 && sudo resize2fs /dev/sda1 8G && sudo sync"]}' - - - name: Create new AMI - id: create-ami - run: | - NEW_AMI_NAME="supabase-postgres-${{ steps.random.outputs.random_string }}" - NEW_AMI_ID=$(aws ec2 create-image --instance-id ${{ steps.launch-instance.outputs.INSTANCE_ID }} --name "${NEW_AMI_NAME}" --description "Resized AMI" --query 'ImageId' --output text) - echo "NEW_AMI_ID=${NEW_AMI_ID}" >> $GITHUB_OUTPUT - echo "NEW_AMI_NAME=${NEW_AMI_NAME}" >> $GITHUB_OUTPUT - - - name: Wait for AMI to be available - run: aws ec2 wait image-available --image-ids ${{ steps.create-ami.outputs.NEW_AMI_ID }} + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' - - name: Modify AMI block device mapping + - name: Run AMI resize script + env: + PRE_AMI_NAME: "supabase-postgres-${{ steps.random.outputs.random_string }}-pre-resize" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ap-southeast-1 run: | - aws ec2 modify-image-attribute --image-id ${{ steps.create-ami.outputs.NEW_AMI_ID }} --block-device-mappings '[{"DeviceName": "/dev/sda1","Ebs":{"VolumeSize":8}}]' - - - name: Terminate temporary instance - run: aws ec2 terminate-instances --instance-ids ${{ steps.launch-instance.outputs.INSTANCE_ID }} + pip install boto3 boto3-stubs[essential] ec2instanceconnectcli + python scripts/ami_resize_script.py - name: Run tests timeout-minutes: 10 diff --git a/scripts/aws_resize.py b/scripts/aws_resize.py new file mode 100644 index 000000000..6d5b06264 --- /dev/null +++ b/scripts/aws_resize.py @@ -0,0 +1,95 @@ +import os +import boto3 +import time +from ec2instanceconnectcli.ec2instanceconnectcli import EC2InstanceConnectCLI + +# Initialize boto3 clients +ec2_client = boto3.client('ec2') +ec2_resource = boto3.resource('ec2') + +# Get the AMI name from environment variable +AMI_NAME = os.environ.get('PRE_AMI_NAME') + +def launch_temporary_instance(): + # Describe the AMI + images = ec2_client.describe_images(Owners=['self'], Filters=[{'Name': 'name', 'Values': [AMI_NAME]}]) + if not images['Images']: + raise Exception(f"No AMI found with name: {AMI_NAME}") + image_id = images['Images'][0]['ImageId'] + + # Launch the instance + instance = ec2_resource.create_instances( + ImageId=image_id, + InstanceType='t4g.micro', + MinCount=1, + MaxCount=1, + TagSpecifications=[ + { + 'ResourceType': 'instance', + 'Tags': [{'Key': 'Name', 'Value': 'AMI-Resize-Temp'}] + } + ] + )[0] + + print(f"Launched instance: {instance.id}") + return instance.id + +def wait_for_instance_running(instance_id): + instance = ec2_resource.Instance(instance_id) + instance.wait_until_running() + print(f"Instance {instance_id} is now running") + +def resize_filesystem(instance_id): + cli = EC2InstanceConnectCLI() + command = "sudo e2fsck -f /dev/sda1 && sudo resize2fs /dev/sda1 8G && sudo sync" + cli.start_session(instance_id=instance_id, command=command) + +def create_new_ami(instance_id): + new_ami_name = f"supabase-postgres-{os.environ.get('GITHUB_RUN_ID', 'resized')}" + response = ec2_client.create_image( + InstanceId=instance_id, + Name=new_ami_name, + Description="Resized AMI" + ) + new_ami_id = response['ImageId'] + print(f"Created new AMI: {new_ami_id} with name: {new_ami_name}") + return new_ami_id, new_ami_name + +def wait_for_ami_available(ami_id): + waiter = ec2_client.get_waiter('image_available') + waiter.wait(ImageIds=[ami_id]) + print(f"AMI {ami_id} is now available") + +def modify_ami_block_device_mapping(ami_id): + ec2_client.modify_image_attribute( + ImageId=ami_id, + BlockDeviceMappings=[ + { + 'DeviceName': '/dev/sda1', + 'Ebs': {'VolumeSize': 8} + } + ] + ) + print(f"Modified block device mapping for AMI {ami_id}") + +def terminate_instance(instance_id): + ec2_resource.Instance(instance_id).terminate() + print(f"Terminated instance {instance_id}") + +def main(): + try: + instance_id = launch_temporary_instance() + wait_for_instance_running(instance_id) + resize_filesystem(instance_id) + new_ami_id, new_ami_name = create_new_ami(instance_id) + wait_for_ami_available(new_ami_id) + modify_ami_block_device_mapping(new_ami_id) + finally: + if 'instance_id' in locals(): + terminate_instance(instance_id) + + print(f"NEW_AMI_ID={new_ami_id}") + print(f"NEW_AMI_NAME={new_ami_name}") + +if __name__ == "__main__": + main() \ No newline at end of file From 6d42f68237df3ee369a1a2109f80dfd1df6a3653 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 12:59:05 -0400 Subject: [PATCH 07/13] fix: writing values back to gh action env vars --- scripts/aws_resize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/aws_resize.py b/scripts/aws_resize.py index 6d5b06264..03d69eac1 100644 --- a/scripts/aws_resize.py +++ b/scripts/aws_resize.py @@ -88,8 +88,9 @@ def main(): if 'instance_id' in locals(): terminate_instance(instance_id) - print(f"NEW_AMI_ID={new_ami_id}") - print(f"NEW_AMI_NAME={new_ami_name}") + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + print(f'NEW_AMI_ID={new_ami_id}', file=fh) + print(f'NEW_AMI_NAME={new_ami_name}', file=fh) if __name__ == "__main__": main() \ No newline at end of file From e450675959d9ee9b981f0d8484a3bef336dbdf95 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 13:46:28 -0400 Subject: [PATCH 08/13] chore: pip3 --- .github/workflows/testinfra-nix.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index eafe71e4f..a589c9638 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -53,11 +53,6 @@ jobs: GIT_SHA=${{github.sha}} packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - name: Run AMI resize script env: PRE_AMI_NAME: "supabase-postgres-${{ steps.random.outputs.random_string }}-pre-resize" @@ -65,7 +60,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ap-southeast-1 run: | - pip install boto3 boto3-stubs[essential] ec2instanceconnectcli + pip3 install boto3 boto3-stubs[essential] ec2instanceconnectcli python scripts/ami_resize_script.py - name: Run tests From 619c9e83b61aedfbafa4947d017a4f83772d1b0b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 14:47:25 -0400 Subject: [PATCH 09/13] fix: cd to dir --- .github/workflows/testinfra-nix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index a589c9638..1a27db32a 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -60,8 +60,9 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ap-southeast-1 run: | + cd scripts pip3 install boto3 boto3-stubs[essential] ec2instanceconnectcli - python scripts/ami_resize_script.py + python3 aws_resize.py - name: Run tests timeout-minutes: 10 From f84fc1f4be27f06bd66c3abd25bb4d7bdf3ccafd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Sep 2024 13:27:39 -0400 Subject: [PATCH 10/13] fix: py module usage --- scripts/aws_resize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws_resize.py b/scripts/aws_resize.py index 03d69eac1..64514bbfc 100644 --- a/scripts/aws_resize.py +++ b/scripts/aws_resize.py @@ -1,7 +1,7 @@ import os import boto3 import time -from ec2instanceconnectcli.ec2instanceconnectcli import EC2InstanceConnectCLI +from ec2instanceconnectcli import EC2InstanceConnectCLI # Initialize boto3 clients ec2_client = boto3.client('ec2') From f3d4d96e07e685764f5a6c49d0c184e0cd93153d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Sep 2024 14:23:27 -0400 Subject: [PATCH 11/13] fix: try ami name in scope --- scripts/aws_resize.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/aws_resize.py b/scripts/aws_resize.py index 64514bbfc..f7d3ce11f 100644 --- a/scripts/aws_resize.py +++ b/scripts/aws_resize.py @@ -7,11 +7,9 @@ ec2_client = boto3.client('ec2') ec2_resource = boto3.resource('ec2') -# Get the AMI name from environment variable -AMI_NAME = os.environ.get('PRE_AMI_NAME') - def launch_temporary_instance(): # Describe the AMI + AMI_NAME = os.environ.get('PRE_AMI_NAME') images = ec2_client.describe_images(Owners=['self'], Filters=[{'Name': 'name', 'Values': [AMI_NAME]}]) if not images['Images']: raise Exception(f"No AMI found with name: {AMI_NAME}") From 5564892ea6af660d6cc8343f910bbd80cd74a413 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Sep 2024 15:19:34 -0400 Subject: [PATCH 12/13] fix: complete ami_name --- stage2-nix-psql.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index ef0196c4e..32187cc19 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -56,7 +56,7 @@ packer { } source "amazon-ebs" "ubuntu" { - ami_name = "${var.ami_name}-${var.postgres-version}" + ami_name = "${var.ami_name}-${var.postgres-version}-pre-resize" instance_type = "c6g.4xlarge" region = "${var.region}" source_ami_filter { From 5f3691063a9fd4344b768f9228d0a7ecfb0fad4f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Sep 2024 16:20:32 -0400 Subject: [PATCH 13/13] fix: refining resize script --- scripts/aws_resize.py | 77 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/scripts/aws_resize.py b/scripts/aws_resize.py index f7d3ce11f..b4f7491db 100644 --- a/scripts/aws_resize.py +++ b/scripts/aws_resize.py @@ -1,21 +1,33 @@ import os import boto3 import time -from ec2instanceconnectcli import EC2InstanceConnectCLI +import socket +import logging +from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger +from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey # Initialize boto3 clients -ec2_client = boto3.client('ec2') -ec2_resource = boto3.resource('ec2') +ec2_client = boto3.client('ec2', region_name="ap-southeast-1") +ec2_resource = boto3.resource('ec2', region_name="ap-southeast-1") + +# Set up logging +logger = logging.getLogger("ami-resize") +handler = logging.StreamHandler() +formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) +logger.setLevel(logging.DEBUG) def launch_temporary_instance(): - # Describe the AMI AMI_NAME = os.environ.get('PRE_AMI_NAME') + logger.info(f"Searching for AMI with name: {AMI_NAME}") images = ec2_client.describe_images(Owners=['self'], Filters=[{'Name': 'name', 'Values': [AMI_NAME]}]) + logger.debug(f"Describe images response: {images}") if not images['Images']: raise Exception(f"No AMI found with name: {AMI_NAME}") image_id = images['Images'][0]['ImageId'] + logger.info(f"Found AMI: {image_id}") - # Launch the instance instance = ec2_resource.create_instances( ImageId=image_id, InstanceType='t4g.micro', @@ -24,20 +36,57 @@ def launch_temporary_instance(): TagSpecifications=[ { 'ResourceType': 'instance', - 'Tags': [{'Key': 'Name', 'Value': 'AMI-Resize-Temp'}] + 'Tags': [ + {'Key': 'Name', 'Value': 'AMI-Resize-Temp'}, + {'Key': 'creator', 'Value': 'ami-resize-script'}, + {'Key': 'resize-run-id', 'Value': os.environ.get("GITHUB_RUN_ID", "local-run")} + ] } - ] + ], + NetworkInterfaces=[ + { + 'DeviceIndex': 0, + 'AssociatePublicIpAddress': True, + 'Groups': ["sg-0a883ca614ebfbae0", "sg-014d326be5a1627dc"], + } + ], )[0] - print(f"Launched instance: {instance.id}") + logger.info(f"Launched instance: {instance.id}") return instance.id def wait_for_instance_running(instance_id): instance = ec2_resource.Instance(instance_id) instance.wait_until_running() - print(f"Instance {instance_id} is now running") + logger.info(f"Instance {instance_id} is now running") + + + while not instance.public_ip_address: + logger.warning("Waiting for IP to be available") + time.sleep(5) + instance.reload() + + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if sock.connect_ex((instance.public_ip_address, 22)) == 0: + break + else: + logger.warning("Waiting for SSH to be available") + time.sleep(10) + + return instance.public_ip_address def resize_filesystem(instance_id): + ec2logger = EC2InstanceConnectLogger(debug=False) + temp_key = EC2InstanceConnectKey(ec2logger.get_logger()) + ec2ic = boto3.client("ec2-instance-connect", region_name="ap-southeast-1") + response = ec2ic.send_ssh_public_key( + InstanceId=instance_id, + InstanceOSUser="ubuntu", + SSHPublicKey=temp_key.get_pub_key(), + ) + assert response["Success"] + cli = EC2InstanceConnectCLI() command = "sudo e2fsck -f /dev/sda1 && sudo resize2fs /dev/sda1 8G && sudo sync" cli.start_session(instance_id=instance_id, command=command) @@ -50,13 +99,13 @@ def create_new_ami(instance_id): Description="Resized AMI" ) new_ami_id = response['ImageId'] - print(f"Created new AMI: {new_ami_id} with name: {new_ami_name}") + logger.info(f"Created new AMI: {new_ami_id} with name: {new_ami_name}") return new_ami_id, new_ami_name def wait_for_ami_available(ami_id): waiter = ec2_client.get_waiter('image_available') waiter.wait(ImageIds=[ami_id]) - print(f"AMI {ami_id} is now available") + logger.info(f"AMI {ami_id} is now available") def modify_ami_block_device_mapping(ami_id): ec2_client.modify_image_attribute( @@ -68,16 +117,16 @@ def modify_ami_block_device_mapping(ami_id): } ] ) - print(f"Modified block device mapping for AMI {ami_id}") + logger.info(f"Modified block device mapping for AMI {ami_id}") def terminate_instance(instance_id): ec2_resource.Instance(instance_id).terminate() - print(f"Terminated instance {instance_id}") + logger.info(f"Terminated instance {instance_id}") def main(): try: instance_id = launch_temporary_instance() - wait_for_instance_running(instance_id) + public_ip = wait_for_instance_running(instance_id) resize_filesystem(instance_id) new_ami_id, new_ami_name = create_new_ami(instance_id) wait_for_ami_available(new_ami_id)