Merge pull request ceph#57895 from rhcs-dashboard/kcli-multi-cluster

mgr/dashboard: adapt kcli plan to improve multi-cluster deployment

Reviewed-by: Pedro Gonzalez Gomez <[email protected]>
Reviewed-by: Aashish Sharma <[email protected]>

Author: nizamial09

src/pybind/mgr/dashboard/.gitignore

@@ -5,6 +5,7 @@ coverage.xml
 junit*xml
 .cache
 ceph.conf
+ceph_image.tar
 
 # IDE
 .vscode

src/pybind/mgr/dashboard/ci/cephadm/bootstrap-cluster.sh

@@ -28,8 +28,14 @@ quick_install_options=''
   quick_install_options="--image localhost:5000/ceph"
 {% endif %}
 
-{% if nodes < 3 %}
+if [[ ${NODES} -lt 2 ]]; then
   bootstrap_extra_options+=" --config /root/initial-ceph.conf"
+fi
+
+{% if prefix is not defined %}
+  PREFIX="ceph"
+{% else %}
+  PREFIX="{{ prefix }}"
 {% endif %}
 
 {% if ceph_dev_folder is defined %}
@@ -41,13 +47,13 @@ $CEPHADM ${quick_install_options} bootstrap --mon-ip $mon_ip --initial-dashboard
 fsid=$(cat /etc/ceph/ceph.conf | grep fsid | awk '{ print $3}')
 cephadm_shell="$CEPHADM shell --fsid ${fsid} -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring"
 
-
-{% for number in range(1, nodes) %}
-  ssh-copy-id -f -i /etc/ceph/ceph.pub  -o StrictHostKeyChecking=no [email protected].10{{ number }}
+for number in $(seq 1 $((NODES - 1))); do
+  LAST_OCTET=$((NODE_IP_OFFSET + $number))
+  ssh-copy-id -f -i /etc/ceph/ceph.pub  -o StrictHostKeyChecking=no [email protected].${LAST_OCTET}
   {% if expanded_cluster is defined %}
-    ${cephadm_shell} ceph orch host add {{ prefix }}-node-0{{ number }} 192.168.100.10{{ number }}
+    ${cephadm_shell} ceph orch host add ${PREFIX}-node-0${number} 192.168.100.${LAST_OCTET}
   {% endif %}
-{% endfor %}
+done
 
 {% if expanded_cluster is defined %}
   ${cephadm_shell} ceph orch apply osd --all-available-devices

src/pybind/mgr/dashboard/ci/cephadm/ceph_cluster.yml

@@ -1,11 +1,8 @@
 parameters:
- nodes: 4
- node_ip_offset: 100
  pool: ceph-dashboard
  network: ceph-dashboard
  gateway: 192.168.100.1
  netmask: 255.255.255.0
- prefix: ceph
  numcpus: 1
  memory: 2048
  image: fedora40
@@ -16,6 +13,10 @@ parameters:
  - 5
  - 5
 
+{% set nodes = (nodes | default(4)) | int %}
+{% set node_ip_offset = node_ip_offset | default(100) %}
+{% set prefix = prefix | default('ceph') %}
+
 {% for number in range(0, nodes) %}
 {{ prefix }}-node-0{{ number }}:
  image: {{ image }}
@@ -49,6 +50,7 @@ parameters:
  - sed -i "s/SELINUX=enforcing/SELINUX=permissive/" /etc/selinux/config
  - setenforce 0
  {% if number == 0 %}
+ - export NODES={{nodes}} && export NODE_IP_OFFSET={{node_ip_offset}}
  - bash /root/bootstrap-cluster.sh
  {% endif %}
 {% endfor %}

src/pybind/mgr/dashboard/ci/cephadm/load-podman-image.sh

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 
+PODMAN_IMG_LOC="/mnt/{{ ceph_dev_folder }}/src/pybind/mgr/dashboard/ci/cephadm/ceph_image.tar"
 echo -e "[registries.insecure]\n\
 registries = ['localhost:5000']" | sudo tee /etc/containers/registries.conf
 
 podman run -d -p 5000:5000 --name my-registry registry:2
 # Load the image and capture the output
-output=$(podman load -i /root/ceph_image.tar)
+output=$(podman load -i "${PODMAN_IMG_LOC}")
 
 # Extract image name from output
 image_name=$(echo "$output" | grep -oP '(?<=^Loaded image: ).*')
@@ -20,4 +21,3 @@ else
 fi
 
 podman push localhost:5000/ceph
-rm -f /root/ceph_image.tar

src/pybind/mgr/dashboard/ci/cephadm/quick-bootstrap.sh

@@ -11,14 +11,21 @@ show_help() {
   echo "  -u, --use-cached-image     Uses the existing podman image in local. Only use this if there is such an image present."
   echo "  -dir, --ceph-dir             Use this to provide the local ceph directory. eg. --ceph-dir=/path/to/ceph"
   echo "  -e, --expanded-cluster     To add all the hosts and deploy OSDs on top of it."
+  echo "  -c, --clusters          Number of clusters to be created. Default is 1."
+  echo "  -n, --nodes           Number of nodes to be created per cluster. Default is 3."
   echo "  -h, --help             Display this help message."
   echo ""
   echo "Example:"
   echo "  ./quick-bootstrap.sh --use-cached-image"
 }
 
+GREEN='\033[0;32m'
+RESET='\033[0m'
+
 use_cached_image=false
 extra_args="-P quick_install=True"
+CLUSTERS=1
+NODES=3
 
 for arg in "$@"; do
   case "$arg" in
@@ -31,6 +38,12 @@ for arg in "$@"; do
     -e|--expanded-cluster)
       extra_args+=" -P expanded_cluster=True"
       ;;
+    -n=*|--nodes=*)
+      NODES="${arg#*=}"
+      ;;
+    -c=*|--clusters=*)
+      CLUSTERS="${arg#*=}"
+      ;;
     -h|--help)
       show_help
       exit 0
@@ -44,8 +57,8 @@ for arg in "$@"; do
 done
 
 image_name=$(echo "$CEPHADM_IMAGE")
-ceph_cluster_yml='ceph_cluster.yml'
-node_count=$(awk '/nodes:/ {print $2}' "${ceph_cluster_yml}")
+
+extra_args+=" -P nodes=${NODES}"
 
 if [[ ${use_cached_image} == false ]]; then
     printf "Pulling the image: %s\n" "$image_name"
@@ -57,30 +70,44 @@ rm -f ceph_image.tar
 printf "Saving the image: %s\n" "$image_name"
 podman save -o ceph_image.tar quay.ceph.io/ceph-ci/ceph:main
 
-printf "Creating the plan\n"
-kcli create plan -f ceph_cluster.yml ${extra_args} ceph
+NODE_IP_OFFSET=100
+PREFIX="ceph"
+for cluster in $(seq 1 $CLUSTERS); do
+    if [[ $cluster -gt 1 ]]; then
+        PREFIX="ceph${cluster}"
+    fi
+    printf "\nCreating cluster: %s\n" "${PREFIX}"
+    kcli create plan -f ceph_cluster.yml ${extra_args} -P node_ip_offset=${NODE_IP_OFFSET} \
+      -P prefix=${PREFIX} ${PREFIX}
+    NODE_IP_OFFSET=$((NODE_IP_OFFSET + 10))
+done
 
 attempt=0
 
 MAX_ATTEMPTS=10
 SLEEP_INTERVAL=5
 
-printf "Waiting for the host to be reachable\n"
-while [[ ${attempt} -lt ${MAX_ATTEMPTS} ]]; do
-    if ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10 [email protected] exit; then
-        break
-    else
-        echo "Waiting for ssh connection to be available..., attempt: ${attempt}"
-        ((attempt++))
-        sleep ${SLEEP_INTERVAL}
-    fi
+NODE_IP_OFFSET=100
+PREFIX="ceph"
+for cluster in $(seq 1 $CLUSTERS); do
+  if [[ $cluster -gt 1 ]]; then
+      PREFIX="ceph${cluster}"
+  fi
+  printf "\nWaiting for the host to be reachable on cluster ${PREFIX}...\n"
+
+  while [[ ${attempt} -lt ${MAX_ATTEMPTS} ]]; do
+      if ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10 [email protected]."${NODE_IP_OFFSET}" exit; then
+          printf "\n${GREEN}Host is reachable on cluster %s\n${RESET}" "${PREFIX}"
+          break
+      else
+          echo "Waiting for ssh connection to be available..., attempt: ${attempt}"
+          ((attempt++))
+          sleep ${SLEEP_INTERVAL}
+      fi
+  done
+
+  NODE_IP_OFFSET=$((NODE_IP_OFFSET + 10))
 done
 
-printf "Copying the image to the hosts\n"
-
-for node in $(seq 0 $((node_count - 1))); do
-    scp -o StrictHostKeyChecking=no ceph_image.tar [email protected]"${node}":/root/
-done
-
-rm -f ceph_image.tar
+printf "\nOpening logs for first cluster"
 kcli ssh -u root -- ceph-node-00 'journalctl -n all -ft cloud-init'