Add script to generate multiple fake nodes

Author: Huy Mai

Support/Multitenancy/ironic-env/.gitignore

@@ -0,0 +1,9 @@
+*.xml
+*.json
+*.log
+ironicclient.sh
+_clouds_yaml/*
+kubectl
+minikube-linux-amd64
+macgen
+uuids

Support/Multitenancy/ironic-env/03-images-and-run-local-services.sh

@@ -1,14 +1,19 @@
 # Set variables
+N_NODES=${1:-1000}
 REGISTRY_NAME="registry"
 REGISTRY_PORT="5000"
 IMAGE_NAMES=(
-    "quay.io/metal3-io/sushy-tools"
+    "quay.io/metal3-io/ironic-python-agent"
+# For now, sushy-tools needs to be compiled locally with https://review.opendev.org/c/openstack/sushy-tools/+/875366
+    # "quay.io/metal3-io/sushy-tools"
     "quay.io/metal3-io/ironic-ipa-downloader"
     "quay.io/metal3-io/ironic:latest"
     "quay.io/metal3-io/ironic-client"
-    "quay.io/metal3-io/keepalived"
+    "quay.io/metal3-io/keepalived:v0.2.0"
+    "docker.io/nicolaka/netshoot"
 )
 
+
 # Attach provisioning and baremetal network interfaces to minikube domain
 virsh attach-interface --domain minikube --model virtio --source provisioning --type network --config
 virsh attach-interface --domain minikube --model virtio --source baremetal --type network --config
@@ -18,6 +23,8 @@ if ! podman ps | grep -q "$REGISTRY_NAME"; then
     podman run -d -p "$REGISTRY_PORT":"$REGISTRY_PORT" --name "$REGISTRY_NAME" docker.io/library/registry:2.7.1
 fi
 
+podman pod create -n infra-pod || true
+podman pod create -n ironic-pod || true
 # Pull images, tag to local registry, and push to registry
 for NAME in "${IMAGE_NAMES[@]}"; do
     # Pull and tag the image
@@ -27,11 +34,13 @@ for NAME in "${IMAGE_NAMES[@]}"; do
     podman push --tls-verify=false 127.0.0.1:5000/localimages/"${NAME##*/}"
 done
 
+podman push --tls-verify=false 127.0.0.1:5000/localimages/sushy-tools
+
 # Define variables for repeated values
 IRONIC_IMAGE="127.0.0.1:5000/localimages/ironic:latest"
 SUSHY_TOOLS_IMAGE="127.0.0.1:5000/localimages/sushy-tools"
 LIBVIRT_URI="qemu+ssh://[email protected]/system?&keyfile=/root/ssh/id_rsa_virt_power&no_verify=1&no_tty=1"
-API_URL="http://172.22.0.2:6385"
+API_URL="http://173.22.0.2:6385"
 CALLBACK_URL="http://172.22.0.2:5050/v1/continue"
 ADVERTISE_HOST="192.168.111.1"
 ADVERTISE_PORT="9999"
@@ -46,6 +55,9 @@ for DIR in "${DIRECTORIES[@]}"; do
     chmod -R 755 "$DIR"
 done
 
+rm -f nodes.json
+echo '[]' > nodes.json
+
 # Run httpd container
 podman run -d --net host --name httpd-infra \
     --pod infra-pod \
@@ -54,6 +66,46 @@ podman run -d --net host --name httpd-infra \
     -e LISTEN_ALL_INTERFACES=false \
     --entrypoint /bin/runhttpd \
     "$IRONIC_IMAGE"
+
+rm -rf macaddrs uuids
+
+function macgen {
+    hexdump -n 6 -ve '1/1 "%.2x "' /dev/random | awk -v a="2,6,a,e" -v r="$RANDOM" 'BEGIN{srand(r);}NR==1{split(a,b,",");r=int(rand()*4+1);printf "%s%s:%s:%s:%s:%s:%s\n",substr($1,0,1),b[r],$2,$3,$4,$5,$6}'
+}
+
+function generate_unique {
+    func=$1
+    store_file=$2
+    newgen=$($func)
+    if [[ ! -f "$store_file" || $(grep "$newgen" "$store_file") == "" ]]; then
+	echo "$newgen" >> "$store_file"
+	echo "$newgen"
+	return
+    fi
+    $func
+}
+
+for i in $(seq 1 "$N_NODES"); do
+  uuid=$(generate_unique uuidgen uuids)
+  macaddr=$(generate_unique macgen macgen)
+  name="fake${i}" 
+  jq --arg node_name "${name}" \
+    --arg uuid "${uuid}" \
+    --arg macaddr "${macaddr}" \
+    '{
+      "uuid": $uuid,
+      "name": $node_name,
+      "power_state": "Off",
+      "nics": [
+	{"mac": $macaddr, "ip": "172.0.0.100"}
+      ]
+    }' nodes_template.json > node.json
+
+  jq -s '.[0] + [.[1] ]' nodes.json node.json > tmp.json
+  rm -f nodes.json
+  mv tmp.json nodes.json
+done
+
 # Set configuration options
 cat <<EOF >/opt/metal3-dev-env/ironic/virtualbmc/sushy-tools/conf.py
 import collections
@@ -65,23 +117,28 @@ SUSHY_EMULATOR_IGNORE_BOOT_DEVICE = False
 SUSHY_EMULATOR_VMEDIA_VERIFY_SSL = False
 SUSHY_EMULATOR_AUTH_FILE = "/root/sushy/htpasswd"
 SUSHY_EMULATOR_FAKE_DRIVER = True
-
 FAKE_IPA_API_URL = "${API_URL}"
 FAKE_IPA_INSPECTION_CALLBACK_URL = "${CALLBACK_URL}"
 FAKE_IPA_ADVERTISE_ADDRESS = Host(hostname="${ADVERTISE_HOST}", port="${ADVERTISE_PORT}")
+SUSHY_EMULATOR_FAKE_SYSTEMS = $(cat nodes.json)
 EOF
 
 # Create an htpasswd file
-cat <<EOF >/opt/metal3-dev-env/ironic/virtualbmc/sushy-tools/htpasswd
-admin:$2b${12}$/dVOBNatORwKpF.ss99KB.vESjfyONOxyH.UgRwNyZi1Xs/W2pGVS
+cat <<'EOF' > /opt/metal3-dev-env/ironic/virtualbmc/sushy-tools/htpasswd
+admin:$2b$12$/dVOBNatORwKpF.ss99KB.vESjfyONOxyH.UgRwNyZi1Xs/W2pGVS
 EOF
 
 # Generate ssh keys to use for virtual power and add them to authorized_keys
 sudo ssh-keygen -f /root/.ssh/id_rsa_virt_power -P "" -q -y
 sudo cat /root/.ssh/id_rsa_virt_power.pub | sudo tee -a /root/.ssh/authorized_keys
 
-# Create and start a container for sushy-tools
 podman run -d --net host --name sushy-tools --pod infra-pod \
     -v /opt/metal3-dev-env/ironic/virtualbmc/sushy-tools:/root/sushy \
     -v /root/.ssh:/root/ssh \
     "${SUSHY_TOOLS_IMAGE}"
+
+podman run --entrypoint='["sushy-fake-ipa", "--config", "/root/sushy/conf.py"]' \
+    -d --net host --name fake-ipa --pod infra-pod \
+    -v /opt/metal3-dev-env/ironic/virtualbmc/sushy-tools:/root/sushy \
+    -v /root/.ssh:/root/ssh \
+    "${SUSHY_TOOLS_IMAGE}"

Support/Multitenancy/ironic-env/05-apply-manifests.sh

@@ -2,48 +2,41 @@ set -e
 # Apply ironic
 kubectl apply -f manifests/ironic.yaml -n baremetal-operator-system
 kubectl -n baremetal-operator-system wait --for=condition=available deployment/baremetal-operator-ironic --timeout=300s
-cat <<'EOF' >ironicclient.sh
+
+openstack_dir="${PWD}/_clouds_yaml"
+ironic_client="ironicclient.sh"
+ironic_port=6385
+
+  cat << EOT >"${ironic_client}"
 #!/bin/bash
 
 DIR="$(dirname "$(readlink -f "$0")")"
 
-if [ -d "${PWD}/_clouds_yaml" ]; then
-  MOUNTDIR="${PWD}/_clouds_yaml"
+if [ -d $openstack_dir ]; then
+  MOUNTDIR=$openstack_dir
 else
-  echo "cannot find _clouds_yaml"
+  echo 'cannot find '$openstack_dir
   exit 1
 fi
 
-if [ "$1" == "baremetal" ] ; then
+if [ \$1 == "baremetal" ] ; then
   shift 1
 fi
 
 # shellcheck disable=SC2086
 sudo podman run --net=host --tls-verify=false \
-  -v "${MOUNTDIR}:/etc/openstack" --rm \
-  -e OS_CLOUD="${OS_CLOUD:-metal3}" "172.22.0.1:5000/localimages/ironic-client" "$@"
-EOF
+  -v "${openstack_dir}:/etc/openstack" --rm \
+  -e OS_CLOUD="${OS_CLOUD:-metal3}" "172.22.0.1:5000/localimages/ironic-client" "\$@"
+EOT
 
-mkdir _clouds_yaml
+  mkdir -p "${openstack_dir}"
 
-cat <<'EOF' >_clouds_yaml/clouds.yaml
+  cat << EOT >"${openstack_dir}/clouds.yaml"
 clouds:
   metal3:
     auth_type: none
-    baremetal_endpoint_override: http://172.22.0.2:6385
+    baremetal_endpoint_override: http://172.22.0.2:${ironic_port}
     baremetal_introspection_endpoint_override: http://172.22.0.2:5050
-EOF
-sudo chmod a+x ironicclient.sh
-sudo ln -sf "$PWD/ironicclient.sh" "/usr/local/bin/baremetal"
-
-# Create ironic node
-
-baremetal node create --driver redfish --driver-info \
-  redfish_address=http://192.168.111.1:8000 --driver-info \
-  redfish_system_id=/redfish/v1/Systems/27946b59-9e44-4fa7-8e91-f3527a1ef094 --driver-info \
-  redfish_username=admin --driver-info redfish_password=password \
-  --name default-node
-
-# baremetal node manage $NODE_UUID
-# get mac : virsh domiflist vmname
-# baremetal port create 00:5c:52:31:3a:9c --node $NODE_UUID
+EOT
+sudo chmod a+x "${ironic_client}"
+sudo ln -sf "$PWD/${ironic_client}" "/usr/local/bin/baremetal"

Support/Multitenancy/ironic-env/06-create-nodes.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+ironic_client="baremetal"
+nodes=($(jq -c -r '.[]' nodes.json))
+n_nodes=${#nodes[@]}
+start_idx=${1:-0}
+
+function create_nodes() {
+  start_idx=$1
+  end_idx=$2
+  if [[ $end_idx -ge $n_nodes ]]; then
+    end_idx=$(( n_nodes - 1 ))
+  fi
+  jq ".[$start_idx:$(( end_idx + 1 ))]" nodes.json > batch.json
+  for idx in $(seq $start_idx $end_idx); do
+    node=${nodes[$idx]}
+    uuid=$(echo ${node} | jq -r '.uuid')
+    node_name=$(echo ${node} | jq -r '.name')
+  ${ironic_client} node create --driver redfish --driver-info \
+    redfish_address=http://192.168.111.1:8000 --driver-info \
+    redfish_system_id=/redfish/v1/Systems/${uuid} --driver-info \
+    redfish_username=admin --driver-info redfish_password=password \
+    --uuid ${uuid} \
+    --name ${node_name} > /dev/null
+    echo "Created node ${node_name} on ironic"
+  done
+}
+
+batch_size=${2:-200}
+inspect_batch_size=${3:-30}
+while true; do
+  end_idx=$((start_idx + batch_size - 1))
+  create_nodes $start_idx $end_idx
+  ./07-inspect-nodes.sh $inspect_batch_size
+  start_idx=$(( end_idx + 1 ))
+  if [[ $start_idx -ge $(( n_nodes - 1 )) ]]; then
+    exit 0
+  fi
+done

Support/Multitenancy/ironic-env/07-inspect-nodes.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+ironic_client="baremetal"
+node_names=($(jq -c -r '.[].name' batch.json))
+n_nodes=${#node_names[@]}
+batch_size=${1:-30}
+
+function inspect_batch() {
+  start_idx=$1
+  end_idx=$2
+  if [[ "$end_idx" -ge "$n_nodes" ]]; then
+    end_idx=$(( n_nodes - 1 ))
+  fi
+  current_batch_size=$(( end_idx - start_idx + 1 ))
+  inspected_nodes=""
+  echo "${ironic_client}: Inspecting nodes batch ${node_names[$start_idx]} - ${node_names[$end_idx]}"
+  while true; do
+    for idx in $(seq "$start_idx" "$end_idx"); do
+      node_name=${node_names[$idx]}
+      if [[ "$inspected_nodes" == *$node_name* ]]; then
+        continue
+      fi
+      node_info=$("${ironic_client}" node show "$node_name" -f json)
+      provisioning_state=$(echo "${node_info}" | jq -r '.provision_state')
+      if [[ "$provisioning_state" == "enroll" ]]; then
+        "${ironic_client}" node manage "${node_name}"
+        continue
+      fi 
+      if [[ "$provisioning_state" == "verifying" || "$provisioning_state" == "inspect wait" || "$provisioning_state" == "inspecting" ]]; then
+        continue
+      fi
+      inspection_info=$(echo "${node_info}" | jq -r '.inspection_finished_at')
+      if [[ "$provisioning_state" == "manageable" ]]; then
+        if [[ "$inspection_info" == "null" ]]; then
+          "${ironic_client}" node inspect "${node_name}"
+        else
+          inspected_nodes="${inspected_nodes} ${node_name}"
+          echo "${ironic_client}: ${node_name} was inspected at ${inspection_info}"
+          # ${ironic_client} node delete $node_name
+          if [[ $(echo "$inspected_nodes" | wc -w) == "$current_batch_size" ]]; then
+            echo "${ironic_client}: Done batch"
+            return
+          fi
+        fi
+      fi
+      if [[ "$provisioning_state" == "inspect failed" ]]; then
+        echo "${ironic_client}: ${node_name} was failed in inspection"
+        # ${ironic_client} node inspect $node_name
+        inspected_nodes="${inspected_nodes} ${node_name}"
+        if [[ $(echo "$inspected_nodes" | wc -w) == "$current_batch_size" ]]; then
+          echo "${ironic_client}: Done batch"
+          return
+        fi
+      fi
+    done
+  done
+}
+
+start_idx=0
+while true; do
+  end_idx=$((start_idx + batch_size - 1))
+  inspect_batch "$start_idx" "$end_idx"
+  start_idx=$(( end_idx + 1 ))
+  if [[ "$start_idx" -ge "$n_nodes" ]]; then
+    exit 0
+  fi
+done

Support/Multitenancy/ironic-env/Init-environment.sh

@@ -1,6 +1,13 @@
 set -e
+trap "trap - SIGTERM && kill -- -'$$'" SIGINT SIGTERM EXIT
+__dir__=$(realpath $(dirname $0))
+
+source "$__dir__/config.sh"
+# This is temporarily required since https://review.opendev.org/c/openstack/sushy-tools/+/875366 has not been merged.
+./build-sushy-tools-image.sh
 sudo ./01-vm-setup.sh
 ./02-configure-minikube.sh
-sudo ./03-images-and-run-local-services.sh
+sudo ./03-images-and-run-local-services.sh "$N_NODES"
 ./04-start-minikube.sh
 ./05-apply-manifests.sh
+./06-create-nodes.sh 0 "$NODE_CREATE_BATCH_SIZE" "$NODE_INSPECT_BATCH_SIZE"

Support/Multitenancy/ironic-env/build-sushy-tools-image.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+SUSHYTOOLS_IMAGE="127.0.0.1:5000/localimages/sushy-tools"
+
+# if [[ $(sudo podman images | grep "$SUSHYTOOLS_IMAGE") != "" ]]4; then
+#     exit 0
+# fi
+
+SUSHYTOOLS_DIR="$HOME/sushy-tools"
+rm -rf "$SUSHYTOOLS_DIR"
+git clone https://opendev.org/openstack/sushy-tools.git "$SUSHYTOOLS_DIR"
+cd "$SUSHYTOOLS_DIR"
+git fetch https://review.opendev.org/openstack/sushy-tools refs/changes/66/875366/18 && git cherry-pick FETCH_HEAD
+
+pip3 install build
+python3 -m build
+
+cd dist
+WHEEL_FILENAME=$(ls *.whl)
+echo $WHEEL_FILENAME
+
+cd ..
+
+cat <<EOF > "${SUSHYTOOLS_DIR}/Dockerfile"
+# Use the official Centos image as the base image
+FROM ubuntu:22.04
+
+# Install necessary packages
+RUN apt update -y && \
+    apt install -y python3 python3-pip python3-venv && \
+    apt clean all
+
+WORKDIR /opt
+
+# RUN python3 setup.py install
+
+# Copy the application code to the container
+COPY dist/${WHEEL_FILENAME} .
+
+RUN pip3 install ${WHEEL_FILENAME}
+
+ENV FLASK_DEBUG=1
+
+RUN mkdir -p /root/sushy
+
+# Set the default command to run when starting the container
+# CMD ["python3", "app.py"]
+# CMD ["sleep", "infinity"]
+CMD ["sushy-emulator", "-i", "::", "--config", "/root/sushy/conf.py"]
+EOF
+
+sudo podman build -t 127.0.0.1:5000/localimages/sushy-tools .
+rm -rf "$SUSHYTOOLS_DIR"