ci: Add multi-az support

This change:
* Refactors existing scripts to re-use common code
* Adds support for building CI devstack on OpenStack
* Captures devstack logs as ci artifacts
* Adds a second worker node in a separate AZ

The worker node is built asynchronously, so tests which don't require
multi-az can proceed immediately the controller node is built as normal.
The worker node will be automatically registered with the controller
when it completes, so tests requiring multi-az can poll for creation of
the AZ before starting.

This change heavily touches AWS, but it has not been tested. We should
assume that AWS support is broken.

Author: mdbooth

hack/ci/.gitignore

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# hack script for preparing AWS to run cluster-api-provider-openstack e2e
+
+set -x -o errexit -o nounset -o pipefail
+
+function cloud_init {
+  AWS_REGION=${AWS_REGION:-"eu-central-1"}
+  AWS_ZONE=${AWS_ZONE:-"eu-central-1a"}
+  # AMIs:
+  # * capa-ami-ubuntu-20.04-1.20.4-00-1613898574 id: ami-0120656d38c206057
+  # * ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20210223 id: ami-0767046d1677be5a0
+  AWS_AMI=${AWS_AMI:-"ami-0767046d1677be5a0"}
+  # Choose via: https://eu-central-1.console.aws.amazon.com/ec2/v2/home?region=eu-central-1#InstanceTypes:
+  AWS_MACHINE_TYPE=${AWS_MACHINE_TYPE:-"c5.metal"}
+  AWS_NETWORK_NAME=${AWS_NETWORK_NAME:-"${CLUSTER_NAME}-mynetwork"}
+  # prepare with:
+  # * create key pair:
+  # aws ec2 create-key-pair --key-name capo-e2e --query 'KeyMaterial' --region "${AWS_REGION}" --output text > ~/.ssh/aws-capo-e2e
+  # * add to local agent and generate public key:
+  # ssh-add ~/.ssh/aws-capo-e2e
+  # ssh-keygen -y -f ~/.ssh/aws-capo-e2e > ~/.ssh/aws-capo-e2e.pub
+  AWS_KEY_PAIR=${AWS_KEY_PAIR:-"capo-e2e"}
+  # disable pagination of AWS cli
+  export AWS_PAGER=""
+
+  echo "Using: AWS_REGION: ${AWS_REGION} AWS_NETWORK_NAME: ${AWS_NETWORK_NAME}"
+}
+
+function init_infrastructure() {
+  if [[ ${AWS_NETWORK_NAME} != "default" ]]; then
+    if [[ $(aws ec2 describe-vpcs --filters Name=tag:Name,Values=capo-e2e-mynetwork --region="${AWS_REGION}" --query 'length(*[0])') = "0" ]];
+    then
+      aws ec2 create-vpc --cidr-block "$PRIVATE_NETWORK_CIDR" --tag-specifications "ResourceType=vpc,Tags=[{Key=Name,Value=${AWS_NETWORK_NAME}}]" --region="${AWS_REGION}"
+      AWS_VPC_ID=$(aws ec2 describe-vpcs --filters Name=tag:Name,Values=capo-e2e-mynetwork --region "${AWS_REGION}" --query '*[0].VpcId' --output text)
+
+      aws ec2 create-subnet --cidr-block "$PRIVATE_NETWORK_CIDR" --vpc-id "${AWS_VPC_ID}" --tag-specifications "ResourceType=subnet,Tags=[{Key=Name,Value=${AWS_NETWORK_NAME}}]" --region "${AWS_REGION}" --availability-zone "${AWS_ZONE}"
+      AWS_SUBNET_ID=$(aws ec2 describe-subnets --filters Name=tag:Name,Values=capo-e2e-mynetwork --region "${AWS_REGION}" --query '*[0].SubnetId' --output text)
+      # It's also the route table of the VPC
+      AWS_SUBNET_ROUTE_TABLE_ID=$(aws ec2 describe-route-tables --filters "Name=vpc-id,Values=${AWS_VPC_ID}" --region "${AWS_REGION}" --query '*[0].RouteTableId' --output text)
+
+      aws ec2 create-security-group --group-name "${AWS_NETWORK_NAME}" --description "${AWS_NETWORK_NAME}" --vpc-id "${AWS_VPC_ID}" --tag-specifications "ResourceType=security-group,Tags=[{Key=Name,Value=${AWS_NETWORK_NAME}}]" --region="${AWS_REGION}"
+      AWS_SECURITY_GROUP_ID=$(aws ec2 describe-security-groups --filters Name=tag:Name,Values=capo-e2e-mynetwork --region "${AWS_REGION}" --query '*[0].GroupId' --output text)
+
+      aws ec2 authorize-security-group-ingress --group-id "${AWS_SECURITY_GROUP_ID}" --protocol tcp --port 22 --cidr 0.0.0.0/0 --region="${AWS_REGION}"
+
+      # Documentation to enable internet access for subnet:
+      # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/TroubleshootingInstancesConnecting.html#TroubleshootingInstancesConnectionTimeout
+      aws ec2 create-internet-gateway --tag-specifications "ResourceType=internet-gateway,Tags=[{Key=Name,Value=${AWS_NETWORK_NAME}}]" --region="${AWS_REGION}"
+      aws ec2 attach-internet-gateway --internet-gateway-id "${AWS_INTERNET_GATEWAY_ID}" --vpc-id "${AWS_VPC_ID}" --region="${AWS_REGION}"
+      AWS_INTERNET_GATEWAY_ID=$(aws ec2 describe-internet-gateways --filters Name=tag:Name,Values=capo-e2e-mynetwork --region "${AWS_REGION}" --query '*[0].InternetGatewayId' --output text)
+
+      aws ec2 create-route --route-table-id "${AWS_SUBNET_ROUTE_TABLE_ID}" --destination-cidr-block 0.0.0.0/0 --gateway-id "${AWS_INTERNET_GATEWAY_ID}" --region "${AWS_REGION}"
+      aws ec2 create-route --route-table-id "${AWS_SUBNET_ROUTE_TABLE_ID}" --destination-ipv6-cidr-block ::/0 --gateway-id "${AWS_INTERNET_GATEWAY_ID}" --region "${AWS_REGION}"
+    fi
+  fi
+}
+
+function create_vm {
+  local name=$1 && shift
+  local ip=$1 && shift
+  local userdata=$1 && shift
+  local public=$1 && shift # Unused by AWS
+
+  if [[ $(aws ec2 describe-instances --filters Name=tag:Name,Values="${name}" --region="${AWS_REGION}" --query 'length(*[0])') = "0" ]];
+  then
+    AWS_SUBNET_ID=$(aws ec2 describe-subnets --filters Name=tag:Name,Values=capo-e2e-mynetwork --region "${AWS_REGION}" --query '*[0].SubnetId' --output text)
+    AWS_SECURITY_GROUP_ID=$(aws ec2 describe-security-groups --filters Name=tag:Name,Values=capo-e2e-mynetwork --region "${AWS_REGION}" --query '*[0].GroupId' --output text)
+
+    # /dev/sda1 is renamed to /dev/nvme0n1 by AWS
+    aws ec2 run-instances --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=${name}}]" \
+      --region "${AWS_REGION}" \
+      --placement "AvailabilityZone=${AWS_ZONE}" \
+      --image-id "${AWS_AMI}" \
+      --instance-type "${AWS_MACHINE_TYPE}" \
+      --block-device-mappings 'DeviceName=/dev/sda1,Ebs={VolumeSize=300}' \
+      --subnet-id "${AWS_SUBNET_ID}"  \
+      --private-ip-address "${ip}" \
+      --count 1 \
+      --associate-public-ip-address \
+      --security-group-ids "${AWS_SECURITY_GROUP_ID}"  \
+      --key-name "${AWS_KEY_PAIR}" \
+      --user-data "file://${userdata}" \
+      --no-paginate
+  fi
+
+  # wait a bit so the server has time to get a public ip
+  sleep 30
+}
+
+function get_public_ip {
+  aws ec2 describe-instances --filters "Name=tag:Name,Values=${CLUSTER_NAME}-controller" --region "${AWS_REGION}" \
+      --query 'Reservations[*].Instances[*].PublicIpAddress' --output text
+}
+
+function get_mtu {
+    # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/network_mtu.html
+    echo 1300
+}
+
+function get_ssh_public_key {
+  cat "${SSH_PUBLIC_KEY_FILE}"
+}
+
+function get_ssh_private_key_file {
+  echo "${SSH_PRIVATE_KEY_FILE}"
+}
+
+function cloud_cleanup {
+  echo Not implemented
+  exit 1
+}

hack/ci/aws-project.sh

@@ -0,0 +1,51 @@
+#cloud-config
+runcmd:
+- sysctl -p /etc/sysctl.d/devstack.conf
+- /root/devstack.sh
+final_message: "The system is finally up, after $UPTIME seconds"
+users:
+- name: cloud
+  lock_passwd: true
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  ssh_authorized_keys:
+  - ${SSH_PUBLIC_KEY}
+# Infrastructure packages required:
+#   python3 - required by sshuttle
+#   git - required to obtain devstack
+#   jq - required by devstack-common.sh
+packages:
+- python3
+- git
+- jq
+package_upgrade: true
+write_files:
+- path: /etc/sysctl.d/devstack.conf
+  permissions: 0644
+  content: |
+    net.ipv4.ip_forward=1
+    net.ipv4.conf.default.rp_filter=0
+    net.ipv4.conf.all.rp_filter=0
+- path: /tmp/devstack-common.sh
+  permissions: 0644
+  content: |
+    # ensure nested virtualization
+    function ensure_kvm {
+      sudo modprobe kvm-intel
+      if [ ! -c /dev/kvm ]; then
+          echo /dev/kvm is not present
+          exit 1
+      fi
+    }
+
+    function run_devstack {
+      su - stack -c "TERM=vt100 /opt/stack/devstack/stack.sh"
+    }
+
+    function upload_images {
+      # Add environment variables for auth/endpoints
+      echo 'source /opt/stack/devstack/openrc admin admin' >> /opt/stack/.bashrc
+
+      # Upload the images so we don't have to upload them from Prow
+      su - stack -c "source /opt/stack/devstack/openrc admin admin && /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2021-03-27/ubuntu-2004-kube-v1.18.15.qcow2"
+      su - stack -c "source /opt/stack/devstack/openrc admin admin && /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2021-03-27/cirros-0.5.1-x86_64-disk.img"
+    }

hack/ci/cloud-init/common.yaml.tpl

@@ -0,0 +1,155 @@
+- path: /tmp/local.conf
+  permissions: 0644
+  content: |
+    [[local|localrc]]
+    GIT_BASE=https://github.com
+    HOST_IP=${HOST_IP}
+    SERVICE_TIMEOUT=240
+    FLOATING_RANGE=${FLOATING_RANGE}
+
+    # Enable Logging
+    LOGFILE=/opt/stack/logs/stack.sh.log
+    VERBOSE=True
+    LOG_COLOR=True
+
+    # Neutron
+    enable_plugin neutron https://github.com/openstack/neutron stable/${OPENSTACK_RELEASE}
+
+    # Octavia
+    enable_plugin octavia https://github.com/openstack/octavia stable/${OPENSTACK_RELEASE}
+    enable_plugin octavia-dashboard https://github.com/openstack/octavia-dashboard stable/${OPENSTACK_RELEASE}
+
+    DATABASE_PASSWORD=secretdatabase
+    RABBIT_PASSWORD=secretrabbit
+    ADMIN_PASSWORD=secretadmin
+    SERVICE_PASSWORD=secretservice
+    SERVICE_TOKEN=111222333444
+
+    # Pre-requisite
+    ENABLED_SERVICES=key,rabbit,mysql
+    # Nova
+    ENABLED_SERVICES+=,n-api,n-obj,n-cpu,n-cond,n-sch,n-novnc,n-api-meta
+    # Placement service needed for Nova
+    ENABLED_SERVICES+=,placement-api,placement-client
+    # Glance
+    ENABLED_SERVICES+=,g-api,g-reg
+
+    # Octavia-Neutron
+    ENABLED_SERVICES+=,neutron-api,neutron-agent,neutron-dhcp,neutron-l3
+    ENABLED_SERVICES+=,neutron-metadata-agent,neutron-qos
+    # Octavia
+    ENABLED_SERVICES+=,octavia,o-api,o-cw,o-hm,o-hk,o-da
+
+    # Horizon (enable for manual tests)
+    # ENABLED_SERVICES+=,horizon
+
+    # Cinder
+    ENABLED_SERVICES+=,c-sch,c-api,c-vol
+
+    # Additional services
+    ENABLED_SERVICES+=${OPENSTACK_ADDITIONAL_SERVICES}
+
+    LIBVIRT_TYPE=kvm
+
+    # Don't download default images, just our test images
+    DOWNLOAD_DEFAULT_IMAGES=False
+    # We upload the Amphora image so it doesn't have to be build
+    IMAGE_URLS="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/amphora/2021-03-27/amphora-x64-haproxy.qcow2"
+
+    [[post-config|$NOVA_CONF]]
+    [DEFAULT]
+    # On GCE's n2-standard-16 an allocation ratio of 2.0 gives us 32 vCPUS,
+    # which is enough to run any 2 test clusters concurrently.
+    cpu_allocation_ratio = 2.0
+
+    # We ensure that the controller has capacity to run all workloads, and that
+    # all workloads run on the controller unless explicitly scheduled to the
+    # worker. This prevents non-deterministic failures of multi-AZ tests due to
+    # capacity on the worker.
+    default_schedule_zone = ${PRIMARY_AZ}
+
+    [scheduler]
+    # query_placement_for_availability_zone is the default from Xena
+    query_placement_for_availability_zone = True
+
+    [[post-config|$CINDER_CONF]]
+    [DEFAULT]
+    storage_availability_zone = ${PRIMARY_AZ}
+
+    [[post-config|/$NEUTRON_CORE_PLUGIN_CONF]]
+    [ml2]
+    path_mtu = ${MTU}
+- path: /tmp/register-worker.sh
+  permissions: 0755
+  content: |
+    #!/bin/bash
+
+    source /opt/stack/devstack/openrc admin admin
+
+    # Wait until the worker shows up as a second compute service
+    while [ $(openstack compute service list --service nova-compute -f value | wc -l) -lt 2 ]
+    do
+      sleep 60
+    done
+
+    nova-manage cell_v2 discover_hosts
+    
+    # Look for hypervisors other than the current host and add them to a
+    # secondary AZ
+    if ! openstack aggregate show ${SECONDARY_AZ} > /dev/null 2>&1; then
+      openstack aggregate create --zone ${SECONDARY_AZ} ${SECONDARY_AZ}
+    fi
+
+    for hypervisor in $(openstack hypervisor list -f value -c "Hypervisor Hostname" 2>/dev/null | grep -v $(hostname)); do
+      openstack aggregate add host ${SECONDARY_AZ} ${hypervisor}
+    done
+- path: /etc/systemd/system/register-worker.service
+  permissions: 0644
+  content: |
+    [Unit]
+    Description=Register devstack worker node
+
+    [Service]
+    Type=oneshot
+    User=stack
+    ExecStart=/tmp/register-worker.sh
+    Environment=TERM=ansi
+
+    [Install]
+    WantedBy=multi-user.target
+- path: /root/devstack.sh
+  permissions: 0755
+  content: |
+    #!/bin/bash
+
+    set -x -o errexit -o nounset -o pipefail
+
+    source /tmp/devstack-common.sh
+
+    ensure_kvm    
+
+    # from https://raw.githubusercontent.com/openstack/octavia/master/devstack/contrib/new-octavia-devstack.sh
+    git clone -b stable/${OPENSTACK_RELEASE} https://github.com/openstack/devstack.git /tmp/devstack
+    cp /tmp/local.conf /tmp/devstack/
+
+    # Create the stack user
+    HOST_IP=${HOST_IP} /tmp/devstack/tools/create-stack-user.sh
+    chmod 0755 /opt/stack
+
+    # Move everything into place (/opt/stack is the $HOME folder of the stack user)
+    mv /tmp/devstack /opt/stack/
+    chown -R stack:stack /opt/stack/devstack/
+
+    run_devstack
+    upload_images
+
+    # When using ML2/OVS all public traffic will be routed via the L3 agent,
+    # which is only running on the controller
+    INTERFACE=$(ip -j addr show | jq -re 'map(select(.addr_info | map(.local == "${HOST_IP}") | any)) | first | .ifname')
+    sudo iptables -t nat -I POSTROUTING -o ${INTERFACE} -s ${FLOATING_RANGE} -j MASQUERADE
+    sudo iptables -I FORWARD -s ${FLOATING_RANGE} -j ACCEPT
+
+    # Start polling for the worker node
+    # We defined the register-worker unit above
+    systemctl daemon-reload
+    systemctl start --no-block register-worker