Merge pull request #2681 from Nordix/nuhakala/e2e_irso_deployment

🌱 Replace ironic deployment with IrSO in e2e

Author: metal3-io-bot

config/overlays/e2e/ironic.env

@@ -1 +1 @@
-IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/
+IRONIC_ENDPOINT=https://192.168.222.2:6385/v1/

hack/ci-e2e.sh

@@ -33,7 +33,7 @@ echo "BMO_E2E_EMULATOR=${BMO_E2E_EMULATOR}"
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml"
 export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMC_PROTOCOL}.yaml"
 
- case "${GINKGO_FOCUS,,}" in
+case "${GINKGO_FOCUS,,}" in
   *upgrade*)
     export DEPLOY_IRONIC="false"
     export DEPLOY_BMO="false"
@@ -59,29 +59,57 @@ sudo apt-get install -y libvirt-dev pkg-config
 # Build the container image with e2e tag (used in tests)
 IMG=quay.io/metal3-io/baremetal-operator:e2e make docker
 
-virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml"
-virsh -c qemu:///system net-start baremetal-e2e
+if ! sudo virsh net-list --all | grep baremetal-e2e; then
+    virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml"
+    virsh -c qemu:///system net-start baremetal-e2e
+fi
+
+# We need to create veth pair to connect metal3 net (defined above) and kind
+# docker subnet. Let us start by creating a docker network with pre-defined
+# name for bridge, so that we can configure the veth pair correctly.
+# Also assume that if kind net exists, it is created by us.
+if ! docker network list | grep kind; then
+    # These options are used by kind itself. It uses docker default mtu and
+    # generates ipv6 subnet ULA, but we can fix the ULA. Only addition to kind
+    # options is the network bridge name.
+    docker network create -d=bridge \
+        -o com.docker.network.bridge.enable_ip_masquerade=true \
+        -o com.docker.network.driver.mtu=1500 \
+        -o com.docker.network.bridge.name="kind-bridge" \
+        --ipv6 --subnet "fc00:f853:ccd:e793::/64" \
+        kind
+fi
+docker network list
+
+# Next create the veth pair
+if ! ip a | grep metalend; then
+    sudo ip link add metalend type veth peer name kindend
+    sudo ip link set metalend master metal3
+    sudo ip link set kindend master kind-bridge
+    sudo ip link set metalend up
+    sudo ip link set kindend up
+fi
+ip a
 
-# Allow traffic between docker bridges and the metal3 interface
-sudo iptables -I FORWARD -i br-+ -o metal3 -j ACCEPT
-sudo iptables -I FORWARD -i metal3 -o br-+ -j ACCEPT
+# Then we need to set routing rules as well
+if ! sudo iptables -L FORWARD -v -n | grep kind-bridge; then
+    sudo iptables -I FORWARD -i kind-bridge -o metal3 -j ACCEPT
+    sudo iptables -I FORWARD -i metal3 -o kind-bridge -j ACCEPT
+fi
 sudo iptables -L FORWARD -n -v
 
-# This IP is defined by the network we created above.
+# This IP is defined by the network we created above. It is sushy-tools / image
+# server endpoint, not ironic.
 IP_ADDRESS="192.168.222.1"
 
-# This IP is also defined by the network above, and is used consistently in all of
-# our e2e overlays
-export IRONIC_PROVISIONING_IP="${IP_ADDRESS}"
-
 # Build vbmctl
 make build-vbmctl
 # Create VMs to act as BMHs in the tests.
 ./bin/vbmctl --yaml-source-file "${E2E_BMCS_CONF_FILE}"
 
 if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
   # Start VBMC
-  docker run --name vbmc --network host -d \
+  docker start vbmc || docker run --name vbmc --network host -d \
     -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
     -v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \
     quay.io/metal3-io/vbmc
@@ -98,7 +126,7 @@ elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then
   # Sushy-tools variables
   SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/sushy-tools/sushy-emulator.conf
   # Start sushy-tools
-  docker run --name sushy-tools -d --network host \
+  docker start sushy-tools || docker run --name sushy-tools -d --network host \
     -v "${SUSHY_EMULATOR_FILE}":/etc/sushy/sushy-emulator.conf:Z \
     -v /var/run/libvirt:/var/run/libvirt:Z \
     -e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf \
@@ -118,29 +146,33 @@ IMAGE_DIR="${REPO_ROOT}/test/e2e/images"
 mkdir -p "${IMAGE_DIR}"
 
 ## Download disk images
-wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}"
-wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/sysrescue/systemrescue-11.00-amd64.iso
+if [[ ! -f "${IMAGE_DIR}/${IMAGE_FILE}" ]]; then
+    wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}"
+    wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/sysrescue/systemrescue-11.00-amd64.iso
+fi
 
 ## Start the image server
-docker run --name image-server-e2e -d \
+docker start image-server-e2e || docker run --name image-server-e2e -d \
   -p 80:8080 \
   -v "${IMAGE_DIR}:/usr/share/nginx/html" nginxinc/nginx-unprivileged
 
 # Generate ssh key pair for verifying provisioned BMHs
-ssh-keygen -t ed25519 -f "${IMAGE_DIR}/ssh_testkey" -q -N ""
+if [[ ! -f "${IMAGE_DIR}/ssh_testkey" ]]; then
+    ssh-keygen -t ed25519 -f "${IMAGE_DIR}/ssh_testkey" -q -N ""
+fi
+pub_ssh_key=$(cut -d " " -f "1,2" "${IMAGE_DIR}/ssh_testkey.pub")
 
 # Build an ISO image with baked ssh key
 # See https://www.system-rescue.org/scripts/sysrescue-customize/
 # We use the systemrescue ISO and their script for customizing it.
-pushd "${IMAGE_DIR}"
-wget -O sysrescue-customize https://gitlab.com/systemrescue/systemrescue-sources/-/raw/main/airootfs/usr/share/sysrescue/bin/sysrescue-customize?inline=false
-chmod +x sysrescue-customize
-
-pub_ssh_key=$(cut -d " " -f "1,2" "ssh_testkey.pub")
-
-mkdir -p recipe/iso_add/sysrescue.d
-# Reference: https://www.system-rescue.org/manual/Configuring_SystemRescue/
-cat << EOF > recipe/iso_add/sysrescue.d/90-config.yaml
+if [[ ! -f "${IMAGE_DIR}/sysrescue-out.iso" ]];then
+    pushd "${IMAGE_DIR}"
+    wget -O sysrescue-customize https://gitlab.com/systemrescue/systemrescue-sources/-/raw/main/airootfs/usr/share/sysrescue/bin/sysrescue-customize?inline=false
+    chmod +x sysrescue-customize
+
+    mkdir -p recipe/iso_add/sysrescue.d
+    # Reference: https://www.system-rescue.org/manual/Configuring_SystemRescue/
+    cat << EOF > recipe/iso_add/sysrescue.d/90-config.yaml
 ---
 global:
     nofirewall: true
@@ -149,9 +181,10 @@ sysconfig:
         "[email protected]": "${pub_ssh_key}"
 EOF
 
-./sysrescue-customize --auto --recipe-dir recipe --source systemrescue-11.00-amd64.iso --dest=sysrescue-out.iso
+    ./sysrescue-customize --auto --recipe-dir recipe --source systemrescue-11.00-amd64.iso --dest=sysrescue-out.iso
+    popd
+fi
 export ISO_IMAGE_URL="http://${IP_ADDRESS}/sysrescue-out.iso"
-popd
 
 # Generate credentials
 BMO_OVERLAYS=(
@@ -189,6 +222,12 @@ for overlay in "${IRONIC_OVERLAYS[@]}"; do
   "${overlay}/ironic-auth-config"
 done
 
+IRSO_IRONIC_AUTH_DIR="${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/components/basic-auth"
+echo "${IRONIC_USERNAME}" > "${IRSO_IRONIC_AUTH_DIR}/ironic-username"
+echo "${IRONIC_PASSWORD}" > "${IRSO_IRONIC_AUTH_DIR}/ironic-password"
+
+sed -i "s|SSH_PUB_KEY_CONTENT|${pub_ssh_key}|" "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/base/ironic.yaml
+
 # We need to gather artifacts/logs before exiting also if there are errors
 set +e
 

hack/e2e/net.xml

@@ -8,8 +8,8 @@
   <bridge name='metal3'/>
   <ip address='192.168.222.1' netmask='255.255.255.0'>
     <dhcp>
-      <range start='192.168.222.2' end='192.168.222.99'/>
-      <bootp file='http://192.168.222.1:6180/boot.ipxe'/>
+      <range start='192.168.222.3' end='192.168.222.99'/>
+      <bootp file='http://192.168.222.2:6180/boot.ipxe'/>
     </dhcp>
   </ip>
 </network>

test/e2e/common.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
+	irsov1alpha1 "github.com/metal3-io/ironic-standalone-operator/api/v1alpha1"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"golang.org/x/crypto/ssh"
@@ -686,3 +687,36 @@ func dumpIronicNodes(ctx context.Context, e2eConfig *Config, artifactFolder stri
 	_, err = file.Write(logOutput.Bytes())
 	Expect(err).ToNot(HaveOccurred(), "Error writing JSON to file")
 }
+
+// WaitForIronicReady waits until the given Ironic resource has Ready condition = True.
+func WaitForIronicReady(ctx context.Context, input WaitForIronicInput) {
+	Logf("Waiting for Ironic %q to be Ready", input.Name)
+
+	Eventually(func(g Gomega) {
+		ironic := &irsov1alpha1.Ironic{}
+		err := input.Client.Get(ctx, client.ObjectKey{
+			Namespace: input.Namespace,
+			Name:      input.Name,
+		}, ironic)
+		g.Expect(err).ToNot(HaveOccurred())
+
+		ready := false
+		for _, cond := range ironic.Status.Conditions {
+			if cond.Type == string(irsov1alpha1.IronicStatusReady) && cond.Status == metav1.ConditionTrue && ironic.Status.InstalledVersion != "" {
+				ready = true
+				break
+			}
+		}
+		g.Expect(ready).To(BeTrue(), "Ironic %q is not Ready yet", input.Name)
+	}, input.Intervals...).Should(Succeed())
+
+	Logf("Ironic %q is Ready", input.Name)
+}
+
+// WaitForIronicInput bundles the parameters for WaitForIronicReady.
+type WaitForIronicInput struct {
+	Client    client.Client
+	Name      string
+	Namespace string
+	Intervals []interface{} // e.g. []interface{}{time.Minute * 15, time.Second * 5}
+}

test/e2e/config/ironic.yaml

@@ -17,6 +17,10 @@ images:
   loadBehavior: tryLoad
 - name: quay.io/jetstack/cert-manager-controller:v1.17.1
   loadBehavior: tryLoad
+- name: quay.io/metal3-io/ironic-standalone-operator:v0.5.1
+  loadBehavior: tryLoad
+- name: quay.io/metal3-io/ironic-ipa-downloader:main
+  loadBehavior: tryLoad
 
 # These variables can be overridden with environment variables.
 variables:
@@ -47,8 +51,15 @@ variables:
   FETCH_IRONIC_NODES: "true"
   IRONIC_USERNAME: "changeme"
   IRONIC_PASSWORD: "changeme"
-  IRONIC_PROVISIONING_IP: "localhost"
+  IRONIC_PROVISIONING_IP: "192.168.222.2"
   IRONIC_PROVISIONING_PORT: "6385"
+  IRSO_OPERATOR_LATEST: "data/ironic-standalone-operator/operator"
+  # IRSO_IRONIC_27.0: "data/ironic-standalone-operator/ironic/overlays/e2e-release-27.0"
+  # IRSO_IRONIC_28.0: "data/ironic-standalone-operator/ironic/overlays/e2e-release-28.0"
+  # IRSO_IRONIC_29.0: "data/ironic-standalone-operator/ironic/overlays/e2e-release-29.0"
+  # IRSO_IRONIC_30.0: "data/ironic-standalone-operator/ironic/overlays/e2e-release-30.0"
+  # IRSO_IRONIC_31.0: "data/ironic-standalone-operator/ironic/overlays/e2e-release-31.0"
+  IRSO_IRONIC_MAIN: "data/ironic-standalone-operator/ironic/overlays/e2e"
 
   NAMESPACE_SCOPED: "true"
 

test/e2e/data/ironic-standalone-operator/components/basic-auth/auth-patch.yaml

@@ -0,0 +1,7 @@
+apiVersion: ironic.metal3.io/v1alpha1
+kind: Ironic
+metadata:
+  name: ironic
+  namespace: baremetal-operator-system
+spec:
+  apiCredentialsName: ironic-auth

test/e2e/data/ironic-standalone-operator/components/basic-auth/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+patches:
+  - path: auth-patch.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+# NOTE: These credentials are generated automatically in hack/ci-e2e.sh
+secretGenerator:
+- name: ironic-auth
+  behavior: create
+  files:
+    - username=ironic-username
+    - password=ironic-password
+  type: Opaque

test/e2e/data/ironic-standalone-operator/components/tls/certificate.yaml

@@ -0,0 +1,43 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: baremetal-operator-system
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ironic-cacert
+  namespace: baremetal-operator-system
+spec:
+  commonName: ironic-ca
+  isCA: true
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: ironic-cacert
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: ca-issuer
+  namespace: baremetal-operator-system
+spec:
+  ca:
+    secretName: ironic-cacert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ironic-cert
+  namespace: baremetal-operator-system
+spec:
+  commonName: ironic-cert
+  ipAddresses:
+  - 192.168.222.2
+  issuerRef:
+    kind: Issuer
+    name: ca-issuer
+  secretName: ironic-cert

test/e2e/data/ironic-standalone-operator/components/tls/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+- certificate.yaml
+
+patches:
+  - path: tls.yaml

test/e2e/data/ironic-standalone-operator/components/tls/tls.yaml

@@ -0,0 +1,8 @@
+apiVersion: ironic.metal3.io/v1alpha1
+kind: Ironic
+metadata:
+  name: ironic
+  namespace: baremetal-operator-system
+spec:
+  tls:
+    certificateName: ironic-cert