Make baremetal targets more resilient to re-running

This refactors some of the baremetal targets and scripts so that the
targets can be re-used without completely removing everything (resources
and instances) that were created on previous runs. This enables:

• creating different sets of edpm compute nodes, some preprovisioned and
  some baremetal, without the baremetal target completely deleting all
  edpm instances that existed previously.
• adding additional baremetal nodes to create different
  NodeSets/Deployments without deleting all previous resources.
• Setting specific baremetal node prefixes and suffixes to control the
  naming so that specific new nodes can be added without overwriting
  others.
• Removes over eager rm's, such as for out/edpm, which completely wipes
  out anything edpm related already deployed.

Signed-off-by: James Slagle <[email protected]>

Author: slagle

devsetup/Makefile

@@ -127,7 +127,8 @@ BMAAS_ROUTE_LIBVIRT_NETWORKS ?= ${BMAAS_NETWORK_NAME},crc,default
 DATAPLANE_PLAYBOOK ?= osp.edpm.download_cache
 DATAPLANE_CUSTOM_SERVICE_RUNNER_IMG ?=quay.io/openstack-k8s-operators/openstack-ansibleee-runner:latest
 BM_NETWORK_NAME            ?=default
-BM_INSTANCE_NAME_PREFIX    ?=edpm-compute
+BM_INSTANCE_NAME_PREFIX    ?=edpm-compute-baremetal
+BM_INSTANCE_NAME_SUFFIX    ?=0
 BM_INSTANCE_MEMORY         ?=8192
 BM_NODE_COUNT              ?=1
 BM_ROOT_PASSWORD_SECRET    ?=
@@ -379,6 +380,7 @@ edpm_baremetal_compute: export OPERATOR_NAME=openstack
 edpm_baremetal_compute: export NAMESPACE=${BMH_NAMESPACE}
 edpm_baremetal_compute: export BMAAS_NETWORK_NAME=${BM_NETWORK_NAME}
 edpm_baremetal_compute: export BMAAS_INSTANCE_NAME_PREFIX=${BM_INSTANCE_NAME_PREFIX}
+edpm_baremetal_compute: export BMAAS_INSTANCE_NAME_SUFFIX=${BM_INSTANCE_NAME_SUFFIX}
 edpm_baremetal_compute: export BMAAS_INSTANCE_MEMORY=${BM_INSTANCE_MEMORY}
 edpm_baremetal_compute: export BMAAS_NODE_COUNT=${BM_NODE_COUNT}
 edpm_baremetal_compute: export NODE_COUNT=${BM_NODE_COUNT}
@@ -392,13 +394,12 @@ edpm_baremetal_compute: ## Create virtual baremetal for the dataplane
 edpm_baremetal_compute_cleanup: export BMAAS_NETWORK_NAME=${BM_NETWORK_NAME}
 edpm_baremetal_compute_cleanup: export NAMESPACE=${BMH_NAMESPACE}
 edpm_baremetal_compute_cleanup: export BMAAS_INSTANCE_NAME_PREFIX=${BM_INSTANCE_NAME_PREFIX}
+edpm_baremetal_compute_cleanup: export BMAAS_INSTANCE_NAME_SUFFIX=${BM_INSTANCE_NAME_SUFFIX}
 edpm_baremetal_compute_cleanup: export BMAAS_NODE_COUNT=${BM_NODE_COUNT}
 edpm_baremetal_compute_cleanup: export NODE_COUNT=${BM_NODE_COUNT}
 edpm_baremetal_compute_cleanup:  ## Cleanup dataplane with BMAAS
 	$(eval $(call vars))
 	scripts/edpm-compute-baremetal.sh --cleanup
-	pushd .. && rm -Rf out/edpm || true && popd
-	make bmaas_sushy_emulator_cleanup || true
 	make bmaas_virtual_bms_cleanup || true
 
 .PHONY: edpm_compute
@@ -667,6 +668,7 @@ bmaas_metallb_cleanup:
 bmaas_virtual_bms: export NODE_COUNT = ${BMAAS_NODE_COUNT}
 bmaas_virtual_bms: export NETWORK_NAME = ${BMAAS_NETWORK_NAME}
 bmaas_virtual_bms: export NODE_NAME_PREFIX = ${BMAAS_INSTANCE_NAME_PREFIX}
+bmaas_virtual_bms: export NODE_NAME_SUFFIX = ${BMAAS_INSTANCE_NAME_SUFFIX}
 bmaas_virtual_bms: export MEMORY = ${BMAAS_INSTANCE_MEMORY}
 bmaas_virtual_bms: export VCPUS = ${BMAAS_INSTANCE_VCPUS}
 bmaas_virtual_bms: export DISK_SIZE = ${BMAAS_INSTANCE_DISK_SIZE}
@@ -680,6 +682,7 @@ bmaas_virtual_bms: ## Create libvirt VM for BMaaS
 bmaas_virtual_bms_cleanup: export NODE_COUNT = ${BMAAS_NODE_COUNT}
 bmaas_virtual_bms_cleanup: export NETWORK_NAME = ${BMAAS_NETWORK_NAME}
 bmaas_virtual_bms_cleanup: export NODE_NAME_PREFIX = ${BMAAS_INSTANCE_NAME_PREFIX}
+bmaas_virtual_bms_cleanup: export NODE_NAME_SUFFIX = ${BMAAS_INSTANCE_NAME_SUFFIX}
 bmaas_virtual_bms_cleanup: export MEMORY = ${BMAAS_INSTANCE_MEMORY}
 bmaas_virtual_bms_cleanup: export VCPUS = ${BMAAS_INSTANCE_VCPUS}
 bmaas_virtual_bms_cleanup: export DISK_SIZE = ${BMAAS_INSTANCE_DISK_SIZE}
@@ -702,17 +705,20 @@ bmaas_sushy_emulator: export SUSHY_EMULATOR_OS_CLOUD = ${BMAAS_SUSHY_EMULATOR_OS
 bmaas_sushy_emulator: export SUSHY_EMULATOR_OS_CLIENT_CONFIG_FILE = ${BMAAS_SUSHY_EMULATOR_OS_CLIENT_CONFIG_FILE}
 bmaas_sushy_emulator: export REDFISH_USERNAME = ${BMAAS_REDFISH_USERNAME}
 bmaas_sushy_emulator: export REDFISH_PASSWORD = ${BMAAS_REDFISH_PASSWORD}
+bmaas_sushy_emulator: export DEPLOY_DIR=../out/bmaas
 bmaas_sushy_emulator: ## Create BMaaS sushy-emulator (Virtual RedFish)
 	scripts/bmaas/sushy-emulator.sh --create
 
 .PHONY: bmaas_sushy_emulator_cleanup
 bmaas_sushy_emulator_cleanup: export NODE_NAME_PREFIX = ${BMAAS_INSTANCE_NAME_PREFIX}
+bmaas_sushy_emulator_cleanup: export NODE_NAME_SUFFIX = ${BMAAS_INSTANCE_NAME_SUFFIX}
 bmaas_sushy_emulator_cleanup: export LIBVIRT_USER = ${BMAAS_LIBVIRT_USER}
 bmaas_sushy_emulator_cleanup: export SUSHY_EMULATOR_NAMESPACE = ${BMAAS_SUSHY_EMULATOR_NAMESPACE}
 bmaas_sushy_emulator_cleanup: export SUSHY_EMULATOR_DRIVER = ${BMAAS_SUSHY_EMULATOR_DRIVER}
 bmaas_sushy_emulator_cleanup: export SUSHY_EMULATOR_OS_CLOUD = ${BMAAS_SUSHY_EMULATOR_OS_CLOUD}
 bmaas_sushy_emulator_cleanup: export REDFISH_USERNAME = ${BMAAS_REDFISH_USERNAME}
 bmaas_sushy_emulator_cleanup: export REDFISH_PASSWORD = ${BMAAS_REDFISH_PASSWORD}
+bmaas_sushy_emulator_cleanup: export DEPLOY_DIR=../out/bmaas
 bmaas_sushy_emulator_cleanup: ## Cleanup BMaaS sushy-emulator (Virtual RedFish)
 	scripts/bmaas/sushy-emulator.sh --cleanup
 

devsetup/README.md

@@ -164,12 +164,21 @@ This requires controlplane to be deployed before dataplane:
 ```
 pushd ..
 make openstack_deploy
+make openstack_init
 popd
 ```
 
 Create and manage the virtual machines:
 ```
 BM_NODE_COUNT=1 make edpm_baremetal_compute
+
+# optional, create more virtual machines later:
+# creates edpm-baremetal-compute-01 and edpm-baremetal-compute-02
+BM_NODE_COUNT=2 BM_NODE_SUFFIX=1 make edpm_baremetal_compute
+
+# optional, create more virtual machines with differet names:
+# creates edpm-bootc-00
+BM_NODE_PREFIX=edpm-bootc make edpm_baremetal_compute
 ```
 
 The dataplane can then be deployed on these nodes as for other baremetal
@@ -187,6 +196,14 @@ make edpm_deploy_cleanup
 popd
 # Will delete VM's!:
 BM_NODE_COUNT=1 make edpm_baremetal_compute_cleanup
+
+# optional, cleanup other virtual machines:
+# cleans up edpm-baremetal-compute-01 and edpm-baremetal-compute-02
+BM_NODE_COUNT=2 BM_NODE_SUFFIX=1 make edpm_baremetal_compute_cleanup
+
+# optional, cleanup virtual machines with differet names:
+# cleans up edpm-bootc-00
+BM_NODE_PREFIX=edpm-bootc make edpm_baremetal_compute_cleanup
 ```
 
 ### BMaaS LAB

devsetup/scripts/bmaas/sushy-emulator.sh

@@ -14,9 +14,6 @@ function usage {
     echo
 }
 
-MY_TMP_DIR="$(mktemp -d)"
-trap 'rm -rf -- "$MY_TMP_DIR"' EXIT
-
 NODE_NAME_PREFIX=${NODE_NAME_PREFIX:-"crc-bmaas"}
 NAMESPACE=${SUSHY_EMULATOR_NAMESPACE:-"sushy-emulator"}
 DRIVER=${SUSHY_EMULATOR_DRIVER:-"libvirt"}
@@ -27,6 +24,9 @@ IMAGE=${SUSHY_EMULATOR_IMAGE:-"quay.io/metal3-io/sushy-tools:latest"}
 CRC_NETWORK_NAME=${CRC_NETWORK_NAME:-crc}
 BM_IPV6=${BM_IPV6:-false}
 BM_IPV4=${BM_IPV4:-false}
+SUSHY_DIR=${SUSHY_DIR:-"${DEPLOY_DIR}/sushy"}
+
+mkdir -p ${SUSHY_DIR}
 
 
 if [ $DRIVER = "libvirt" ]; then
@@ -49,7 +49,12 @@ if [ $DRIVER = "libvirt" ]; then
         LIBVIRT_URI="'qemu+ssh://${LIBVIRT_USER}@${LIBVIRT_IP_ADDRESS}/system${LIBVIRT_SOCKET}'"
         SUSHY_EMULATOR_LISTEN_IP="'0.0.0.0'"
     fi
-    INSTANCES=$(virsh --connect=qemu:///system list --all --uuid --name | grep "${NODE_NAME_PREFIX}" | awk 'BEGIN{ printf "[" }; { printf "%s\"%s\"", sep, $1, sep=","}; END{ printf "]" }')
+    BMHS=$(oc get -o name bmh | cut -d/ -f2)
+    LIBVIRT_INSTANCES=$(virsh --connect=qemu:///system list --all --uuid --name | grep "${NODE_NAME_PREFIX}" | awk '{print $1}')
+    for bmh in ${BMHS}; do
+        LIBVIRT_INSTANCES="${LIBVIRT_INSTANCES} $(virsh --connect=qemu:///system domuuid ${bmh})"
+    done
+    INSTANCES=$(echo ${LIBVIRT_INSTANCES} | awk 'BEGIN{ printf "[" }; { printf "%s\"%s\"", sep, $1, sep=","}; END{ printf "]" }')
     EMULATOR_OS_CLOUD="None"
 elif [ $DRIVER = "openstack" ]; then
     OS_CLIENT_CONFIG_FILE=${SUSHY_EMULATOR_OS_CLIENT_CONFIG_FILE:-/etc/openstack/clouds.yaml}
@@ -60,15 +65,15 @@ fi
 
 function create_sushy_emulator_namespace {
     echo "Creating namespace ${NAMESPACE}"
-    cat <<EOF > "${MY_TMP_DIR}/namespace.yaml"
+    cat <<EOF > "${SUSHY_DIR}/namespace.yaml"
 apiVersion: v1
 kind: Namespace
 metadata:
     name: ${NAMESPACE}
 EOF
 
-    # cat ${MY_TMP_DIR}/namespace.yaml
-    oc apply -f "${MY_TMP_DIR}/namespace.yaml"
+    # cat ${SUSHY_DIR}/namespace.yaml
+    oc apply -f "${SUSHY_DIR}/namespace.yaml"
 }
 
 function create_sushy_emulator_user {
@@ -88,28 +93,39 @@ function generate_ssh_keypair {
         echo "PANIC, unable to get ${LIBVIRT_USER} home directory."
         exit 1
     fi
-    ssh-keygen -q -f "${MY_TMP_DIR}/${SSH_KEY_FILE}" -N "" -t "${SSH_ALGORITHM}" -b "${SSH_KEY_SIZE}"
+    if [ ! -f "${SUSHY_DIR}/${SSH_KEY_FILE}" ]; then
+        ssh-keygen -q -f "${SUSHY_DIR}/${SSH_KEY_FILE}" -N "" -t "${SSH_ALGORITHM}" -b "${SSH_KEY_SIZE}"
+    else
+        echo "${SUSHY_DIR}/${SSH_KEY_FILE} already exists, not re-generating"
+    fi
     sudo mkdir -p "${homedir}/.ssh"
-    sudo cp "${MY_TMP_DIR}/${SSH_KEY_FILE}" "${homedir}/.ssh/${SSH_KEY_FILE}"
-    sudo cp "${MY_TMP_DIR}/${SSH_KEY_FILE}.pub" "${homedir}/.ssh/${SSH_KEY_FILE}.pub"
+    sudo cp "${SUSHY_DIR}/${SSH_KEY_FILE}" "${homedir}/.ssh/${SSH_KEY_FILE}"
+    sudo cp "${SUSHY_DIR}/${SSH_KEY_FILE}.pub" "${homedir}/.ssh/${SSH_KEY_FILE}.pub"
     sudo touch "${homedir}/.ssh/authorized_keys"
-    cat "${MY_TMP_DIR}/${SSH_KEY_FILE}.pub" | sudo tee "${homedir}/.ssh/authorized_keys" > /dev/null
+    cat "${SUSHY_DIR}/${SSH_KEY_FILE}.pub" | sudo tee "${homedir}/.ssh/authorized_keys" > /dev/null
     sudo chown -R "${LIBVIRT_USER}":"${LIBVIRT_USER}" "${homedir}/.ssh"
     sudo chmod 700 "${homedir}/.ssh"
     sudo chmod -R og-rwx "${homedir}/.ssh"
 }
 
 function create_sushy_emulator_config {
     echo "Creating sushy-emulator-config"
-    cat << EOF > "${MY_TMP_DIR}/config-map.yaml"
+    local htpasswd
+    if [ -f "${SUSHY_DIR}/htpasswd" ]; then
+        htpasswd=$(cat ${SUSHY_DIR}/htpasswd)
+    else
+        htpasswd=$(htpasswd -nbB "${REDFISH_USERNAME}" "${REDFISH_PASSWORD}")
+        echo ${htpasswd} > ${SUSHY_DIR}/htpasswd
+    fi
+    cat << EOF > "${SUSHY_DIR}/config-map.yaml"
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: sushy-emulator-config
   namespace: ${NAMESPACE}
 data:
   htpasswd: |
-$(htpasswd -nbB "${REDFISH_USERNAME}" "${REDFISH_PASSWORD}" | sed 's/^/    /')
+    ${htpasswd}
   config: |
     # Listen on all local IP interfaces
     SUSHY_EMULATOR_LISTEN_IP = ${SUSHY_EMULATOR_LISTEN_IP}
@@ -146,24 +162,24 @@ $(htpasswd -nbB "${REDFISH_USERNAME}" "${REDFISH_PASSWORD}" | sed 's/^/    /')
     SUSHY_EMULATOR_ALLOWED_INSTANCES = ${INSTANCES}
 EOF
 
-    # cat ${MY_TMP_DIR}/config-map.yaml
-    oc apply -f "${MY_TMP_DIR}/config-map.yaml"
+    # cat ${SUSHY_DIR}/config-map.yaml
+    oc apply -f "${SUSHY_DIR}/config-map.yaml"
 }
 
 function create_sushy_emulator_secret {
     echo "Creating sushy-emulator-secret"
     if [ $DRIVER = "libvirt" ]; then
-        cat << EOF > "${MY_TMP_DIR}/secret.yaml"
+        cat << EOF > "${SUSHY_DIR}/secret.yaml"
 apiVersion: v1
 kind: Secret
 metadata:
     name: sushy-emulator-secret
     namespace: ${NAMESPACE}
 data:
     ssh-publickey: |
-$(base64 < "${MY_TMP_DIR}/${SSH_KEY_FILE}.pub" | sed 's/^/        /')
+$(base64 < "${SUSHY_DIR}/${SSH_KEY_FILE}.pub" | sed 's/^/        /')
     ssh-privatekey: |
-$(base64 < "${MY_TMP_DIR}/${SSH_KEY_FILE}" | sed 's/^/        /')
+$(base64 < "${SUSHY_DIR}/${SSH_KEY_FILE}" | sed 's/^/        /')
     ssh-known-hosts: |
 $(ssh-keyscan -H "${LIBVIRT_IP_ADDRESS}" | base64 | sed 's/^/        /')
 ---
@@ -176,7 +192,7 @@ data:
     openstack-clouds-yaml: ""
 EOF
     elif [ $DRIVER = "openstack" ]; then
-        cat << EOF > "${MY_TMP_DIR}/secret.yaml"
+        cat << EOF > "${SUSHY_DIR}/secret.yaml"
 apiVersion: v1
 kind: Secret
 metadata:
@@ -198,13 +214,13 @@ $(cat ${OS_CLIENT_CONFIG_FILE} | base64 | sed 's/^/        /')
 EOF
     fi
 
-    # cat ${MY_TMP_DIR}/secret.yaml
-    oc apply -f "${MY_TMP_DIR}/secret.yaml"
+    # cat ${SUSHY_DIR}/secret.yaml
+    oc apply -f "${SUSHY_DIR}/secret.yaml"
 }
 
 function create_sushy_emulator_pod {
     echo "Creating sushy-emulator pod"
-    cat << EOF > "${MY_TMP_DIR}/sushy-emulator-pod.yaml"
+    cat << EOF > "${SUSHY_DIR}/sushy-emulator-pod.yaml"
 ---
 apiVersion: v1
 kind: Pod
@@ -284,13 +300,15 @@ spec:
   restartPolicy: OnFailure
 EOF
 
-    # cat ${MY_TMP_DIR}/sushy-emulator-pod.yaml
-    oc apply -f "${MY_TMP_DIR}/sushy-emulator-pod.yaml"
+    # cat ${SUSHY_DIR}/sushy-emulator-pod.yaml
+    # delete existing pod to force reloading of any config changes
+    oc delete -n sushy-emulator --wait pod sushy-emulator
+    oc apply -f "${SUSHY_DIR}/sushy-emulator-pod.yaml"
 }
 
 function create_sushy_emulator_service {
     echo "Creating sushy-emulator-service"
-    cat << EOF > "${MY_TMP_DIR}/sushy-emulator-service.yaml"
+    cat << EOF > "${SUSHY_DIR}/sushy-emulator-service.yaml"
 ---
 apiVersion: v1
 kind: Service
@@ -308,13 +326,13 @@ spec:
     targetPort: 8000
 EOF
 
-    # cat ${MY_TMP_DIR}/sushy-emulator-service.yaml
-    oc apply -f "${MY_TMP_DIR}/sushy-emulator-service.yaml"
+    # cat ${SUSHY_DIR}/sushy-emulator-service.yaml
+    oc apply -f "${SUSHY_DIR}/sushy-emulator-service.yaml"
 }
 
 function create_sushy_emulator_route {
     echo "Creating sushy-emulator-route"
-    cat << EOF > "${MY_TMP_DIR}/sushy-emulator-route.yaml"
+    cat << EOF > "${SUSHY_DIR}/sushy-emulator-route.yaml"
 ---
 apiVersion: route.openshift.io/v1
 kind: Route
@@ -332,8 +350,8 @@ spec:
     name: sushy-emulator-service
 EOF
 
-    # cat ${MY_TMP_DIR}/sushy-emulator-route.yaml
-    oc apply -f "${MY_TMP_DIR}/sushy-emulator-route.yaml"
+    # cat ${SUSHY_DIR}/sushy-emulator-route.yaml
+    oc apply -f "${SUSHY_DIR}/sushy-emulator-route.yaml"
 }
 
 function create {
@@ -366,6 +384,8 @@ function cleanup {
             echo "Not deleting user ${LIBVIRT_USER}, user does not exist"
         fi
     fi
+
+    rm -rf ${SUSHY_DIR}
 }
 
 case "$1" in

devsetup/scripts/bmaas/vbm-setup.sh

@@ -14,8 +14,9 @@ fi
 
 SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 NETWORK_NAME=${NETWORK_NAME:-"crc-bmaas"}
-NODE_NAME_PREFIX=${NODE_NAME_PREFIX:-"crc-bmaas"}
 NODE_COUNT=${NODE_COUNT:-"1"}
+NODE_NAME_PREFIX=${NODE_NAME_PREFIX:-"crc-bmaas"}
+NODE_NAME_SUFFIX=${NODE_NAME_SUFFIX:-"0"}
 ACTION=""
 
 # Virtual Machine spec
@@ -105,7 +106,7 @@ function create_vm {
     local temp_file
     local name
     temp_file=$(mktemp -p "$MY_TMP_DIR")
-    name="$NODE_NAME_PREFIX-$(printf "%02d" "$i")"
+    name=$1
     echo "Creating VM: $name"
     virt-install --connect qemu:///system \
         --name "$name" \
@@ -142,15 +143,17 @@ function create {
         echo "Network $NETWORK_NAME does not exist, please create it"
         exit 1
     fi
-    for (( i=1; i<=NODE_COUNT; i++ )); do
-        create_vm "$i"
+    for (( i=0; i<NODE_COUNT; i++ )); do
+        node_suffix=$(($i + ${NODE_NAME_SUFFIX}))
+        vm="$NODE_NAME_PREFIX-$(printf "%02d" "${node_suffix}")"
+        create_vm "$vm"
     done
 }
 
 function cleanup {
-    local vms
-    vms=$(virsh --connect=qemu:///system list --all --name | grep "$NODE_NAME_PREFIX")
-    for vm in $vms; do
+    for (( i=0; i<NODE_COUNT; i++ )); do
+        node_suffix=$(($i + ${NODE_NAME_SUFFIX}))
+        vm="$NODE_NAME_PREFIX-$(printf "%02d" "${node_suffix}")"
         delete_vm "$vm"
     done
 }