fix(ci): replace EC2 Instance Connect with cloud-init SSH key injection

Tests are often failing due to the SSH access to the instance.

EC2 Instance Connect push the temporary SSH key which is then available
only for 60 seconds. Recently, errors often occur when the SSH key is
sent to the instance, resulting in a timeout.

We replace runtime SSH key injection via EC2 Instance Connect API with
cloud-init configuration to add the SSH public key during instance initialization.

Note that we are still using EC2 Instance Connect to create the SSH key
pair, but we are not using it to push the key to the instance.

Author: jfroche

testinfra/test_ami_nix.py

@@ -9,7 +9,6 @@
 from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger
 from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey
 from time import sleep
-import subprocess
 import paramiko
 
 # if GITHUB_RUN_ID is not set, use a default value that includes the user and hostname
@@ -233,6 +232,10 @@ def host():
     def gzip_then_base64_encode(s: str) -> str:
         return base64.b64encode(gzip.compress(s.encode())).decode()
 
+    # Create temporary SSH key pair
+    ec2logger = EC2InstanceConnectLogger(debug=False)
+    temp_key = EC2InstanceConnectKey(ec2logger.get_logger())
+
     instance = list(
         ec2.create_instances(
             BlockDeviceMappings=[
@@ -279,6 +282,10 @@ def gzip_then_base64_encode(s: str) -> str:
     - 'bash init.sh "staging"'
     - 'touch /var/lib/init-complete'
     - 'rm -rf /tmp/*'
+users:
+  - name: ubuntu
+    ssh_authorized_keys:
+      - {temp_key.get_pub_key()}
 """,
             TagSpecifications=[
                 {
@@ -297,16 +304,6 @@ def gzip_then_base64_encode(s: str) -> str:
     # Increase wait time before starting health checks
     sleep(30)  # Wait for 30 seconds to allow services to start
 
-    ec2logger = EC2InstanceConnectLogger(debug=False)
-    temp_key = EC2InstanceConnectKey(ec2logger.get_logger())
-    ec2ic = boto3.client("ec2-instance-connect", region_name="ap-southeast-1")
-    response = ec2ic.send_ssh_public_key(
-        InstanceId=instance.id,
-        InstanceOSUser="ubuntu",
-        SSHPublicKey=temp_key.get_pub_key(),
-    )
-    assert response["Success"]
-
     # Wait for instance to have public IP
     while not instance.public_ip_address:
         logger.warning("waiting for ip to be available")