Add ceph config wrapper with existence check (resolves #15)

Fix config sync bug between nodes (fixes #16)

Improve workflow messages

Author: Sheridan

.github/workflows/build.yml

@@ -1,6 +1,6 @@
-name: Build CephOS installer
+name: Build CephOS Installer
 
-run-name: Building CephOS installer ${{ github.ref_name }} (@${{ github.actor }})
+run-name: Building CephOS Installer ${{ github.ref_name }} (@${{ github.actor }})
 
 on:
   release:
@@ -16,16 +16,16 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up docker
+      - name: Set up Docker
         uses: docker/setup-buildx-action@v3
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install -y make
 
-      - name: Building CephOS
+      - name: Build CephOS
         run: make build
 
-      - name: Rename artifact with release tag
+      - name: Rename artifacts with release tag
         run: |
           TAG=${GITHUB_REF_NAME}
           echo "Using tag: $TAG"
@@ -34,7 +34,7 @@ jobs:
           mv tmp/work/cephos.img           "tmp/work/cephos_${TAG}.img"
           ls -lh tmp/work/
 
-      - name: Get fresh Access Token from Yandex
+      - name: Get fresh access token from Yandex
         id: get_token
         run: |
           RESPONSE=$(curl -s -X POST https://oauth.yandex.ru/token \
@@ -68,12 +68,12 @@ jobs:
               --no-progress-meter \
               "$UPLOAD_URL"
           done
-          echo "Uploading artefacts done."
+          echo "Artifacts upload completed successfully."
 
-      - name: Send build complete message to Telegram
+      - name: Send build completion message to Telegram
         run: |
           TAG=${GITHUB_REF_NAME}
-          MESSAGE="💾 Version ${TAG} build and upload to [storage](https://disk.yandex.ru/d/VrbJWGtHjB90QQ) is complete"
+          MESSAGE="💾 Version ${TAG} build and upload to [storage](https://disk.yandex.ru/d/VrbJWGtHjB90QQ) completed successfully"
           curl -s -X POST \
             "https://api.telegram.org/bot${{ secrets.TELEGRAM_TOKEN }}/sendMessage" \
             -d chat_id="${{ secrets.TELEGRAM_CHAT_ID }}" \

.github/workflows/notify-commits.yml

@@ -1,6 +1,6 @@
 name: Notify on Commit
 
-run-name: Commit ${{ github.sha }} notify (@${{ github.actor }})
+run-name: Commit ${{ github.sha }} notification (@${{ github.actor }})
 
 on:
   push:
@@ -11,10 +11,10 @@ jobs:
   notify:
     runs-on: ubuntu-latest
     steps:
-      - name: Send commit info to Telegram
+      - name: Send commit information to Telegram
         run: |
           COMMIT_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}"
-          MESSAGE="📝 New [commit](${COMMIT_URL}) to branch \`${GITHUB_REF_NAME}\` from author ${GITHUB_ACTOR} with message: \`${{ github.event.head_commit.message }}\`"
+          MESSAGE="✒️ New [commit](${COMMIT_URL}) pushed to branch \`${GITHUB_REF_NAME}\` by ${GITHUB_ACTOR} with message: \`${{ github.event.head_commit.message }}\`"
           curl -s -X POST \
             "https://api.telegram.org/bot${{ secrets.TELEGRAM_TOKEN }}/sendMessage" \
             -d chat_id="${{ secrets.TELEGRAM_CHAT_ID }}" \

.github/workflows/notify-issues.yml

@@ -1,6 +1,6 @@
 name: Notify on Issue
 
-run-name: Issue notify (@${{ github.actor }})
+run-name: Issue notification (@${{ github.actor }})
 
 on:
   issues:
@@ -10,7 +10,7 @@ jobs:
   notify:
     runs-on: ubuntu-latest
     steps:
-      - name: Send issue info to Telegram
+      - name: Send issue information to Telegram
         run: |
           ISSUE_URL="${{ github.event.issue.html_url }}"
           TITLE="${{ github.event.issue.title }}"
@@ -19,7 +19,7 @@ jobs:
 
           if [ "$ACTION" = "opened" ]
           then
-            MESSAGE="🔔 New [Issue](${ISSUE_URL}) from author ${USER} with title: \`${TITLE}\`"
+            MESSAGE="🔔 New [Issue](${ISSUE_URL}) opened by ${USER} with title: \`${TITLE}\`"
           else
             MESSAGE="✅ [Issue](${ISSUE_URL}) closed by ${USER} (Title: \`${TITLE}\`)"
           fi

.github/workflows/notify-releases.yml

@@ -1,6 +1,6 @@
 name: Notify on Release
 
-run-name: Release ${{ github.ref_name }} notify (@${{ github.actor }})
+run-name: Release ${{ github.ref_name }} notification (@${{ github.actor }})
 
 on:
   release:
@@ -10,13 +10,13 @@ jobs:
   notify:
     runs-on: ubuntu-latest
     steps:
-      - name: Send release info to Telegram
+      - name: Send release information to Telegram
         run: |
           RELEASE_URL="${{ github.event.release.html_url }}"
           TAG="${{ github.event.release.tag_name }}"
           NAME="${{ github.event.release.name }}"
 
-          MESSAGE="🚀 New [Release](${RELEASE_URL}). Version: *${TAG}*, name: ${NAME}"
+          MESSAGE="🚀 New [Release](${RELEASE_URL}) published. Version: *${TAG}*, Name: ${NAME}"
 
           curl -s -X POST \
             "https://api.telegram.org/bot${{ secrets.TELEGRAM_TOKEN }}/sendMessage" \

live_build_config/includes.chroot_after_packages/usr/local/bin/cephos-conf-sync

@@ -32,17 +32,19 @@ done
 
 function update_nodes()
 {
-  log_info "Updating /etc/hosts"
-  while read -r node_name public_ip cluster_ip
+  log_info "Sync cluster nodes configuration"
+  mapfile -t ceph_hosts < <(get_ceph_hosts)
+  for line in "${ceph_hosts[@]}"
   do
+    read -r node_name public_ip cluster_ip <<< "${line}"
     if [[ "${node_name}" == "$(hostname -s)" ]]
     then
       cephos-conf-reconcile $(verbose_flag)
     else
       spread_ssh_key "${user_home}" "${current_user}" "${public_ip}"
       exec_ssh "${public_ip}" "${current_user}" "cephos-conf-reconcile $(verbose_flag)"
     fi
-  done < <(get_ceph_hosts)
+  done
 }
 
 # --- functions ---

live_build_config/includes.chroot_after_packages/usr/local/bin/cephos-init-cluster

@@ -33,10 +33,6 @@ set_cluster_environment "fsid" "${ceph_fsid}"
 log_info "Initializing cluster with [fsid:${ceph_fsid}] and main configuration file ${ceph_main_conf}"
 create_directories
 
-log_verbose "Creating keys"
-${sudo_ceph} ceph-authtool --create-keyring ${ceph_conf_dir}/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mgr 'allow *' --cap mds 'allow *'
-${sudo_ceph} ceph-authtool --create-keyring ${ceph_data_dir}/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
-
 log_verbose "Creating configuration ${ceph_main_conf}"
 
 ceph_conf_add_to_list "mon_initial_members" "$(hostname -s)"
@@ -64,6 +60,10 @@ ceph_conf_set "global" "auth_client_required"  "cephx"
 ceph_conf_set "global" "auth_cluster_required" "cephx"
 ceph_conf_set "global" "auth_service_required" "cephx"
 
+log_verbose "Creating keys"
+${sudo_ceph} ceph-authtool --create-keyring ${ceph_conf_dir}/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mgr 'allow *' --cap mds 'allow *'
+${sudo_ceph} ceph-authtool --create-keyring ${ceph_data_dir}/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
+
 set_fs_permissions
 ceph_crash_up
 

live_build_config/includes.chroot_after_packages/usr/local/bin/cephos-init-metrics

@@ -42,6 +42,6 @@ manage_service_state "enable" "ceph-exporter"
 
 log_info "Ceph internal metrics port: 9108"
 ${sudo_ceph} ceph mgr module enable prometheus
-${sudo_ceph} ceph config set mgr mgr/prometheus/server_addr "::"
-${sudo_ceph} ceph config set mgr mgr/prometheus/server_port 9108
+ceph_config_set "mgr" "mgr/prometheus/server_addr" "::"
+ceph_config_set "mgr" "mgr/prometheus/server_port" "9108"
 sudo systemctl restart ceph-mgr@${mgr_id}.service

live_build_config/includes.chroot_after_packages/usr/local/bin/cephos-init-mgr

@@ -48,10 +48,10 @@ then
   log_info "Configuration"
   ${sudo_ceph} ceph mgr module enable dashboard
   ${sudo_ceph} ceph mgr module enable stats
-  ${sudo_ceph} ceph config set mgr mgr/dashboard/server_port 8080
-  ${sudo_ceph} ceph config set mgr mgr/dashboard/ssl false
-  ${sudo_ceph} ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false
-  # ${sudo_ceph} ceph restful create-self-signed-cert
+
+  ceph_config_set "mgr" "mgr/dashboard/server_port"                      "8080"
+  ceph_config_set "mgr" "mgr/dashboard/ssl"                              "false"
+  ceph_config_set "mon" "mon_warn_on_insecure_global_id_reclaim_allowed" "false"
 
   if ! dashboard_user_exists "cephos"
   then

live_build_config/includes.chroot_after_packages/usr/local/lib/cephos/cephos.sh.lib

@@ -61,7 +61,7 @@ function dashboard_user_exists()
 
 function get_ceph_hosts()
 {
-  ceph osd metadata -f json \
+  ${sudo_ceph} ceph osd metadata -f json \
   | jq -r '.[] | {host: .hostname, front: .front_addr, back: .back_addr}
             | [.host, (.front | capture("v2:(?<ip>[^:]+)") .ip // "-"),
                       (.back  | capture("v2:(?<ip>[^:]+)") .ip // "-")]
@@ -129,7 +129,6 @@ function disk_already_in_ceph()
   fi
 }
 
-
 function prepare_ceph_data()
 {
   log_verbose "Preparing data"
@@ -163,7 +162,6 @@ function prepare_ceph_data()
 
   sudo_ceph="sudo --preserve-env"
 
-  # log_info "Main conf file: ${ceph_main_conf}"
   log_info "Ceph fsid: ${ceph_fsid}"
   log_info "Ceph id: [mon:${mon_id}], [mgr:${mgr_id}], [mds:${mds_id}]"
 }

live_build_config/includes.chroot_after_packages/usr/local/lib/cephos/config.sh.lib

@@ -11,7 +11,7 @@ function write_to_ini()
 
   sudo mkdir -p "$(dirname $inifile)"
 
-  log_verbose "Write '${section}.${variable} = ${value}' to ${inifile}"
+  log_verbose "Write '${value}' to '${section}.${variable}' in ${inifile}"
   sudo crudini --set ${options} "${inifile}" "${section}" "${variable}" "${value}"
 
   if (( $? ))
@@ -147,13 +147,29 @@ function ceph_conf_get()
   read_from_ini "${ceph_main_conf}" "${section}" "${variable}" "none"
 }
 
+# Set value to ceph.conf
 function ceph_conf_set()
 {
   local section="$1"
   local variable="$2"
   local value="$3"
   write_to_ini "${ceph_main_conf}" "${section}" "${variable}" "${value}" ""
 }
+
+# Set value to ceph distributed configuration
+function ceph_config_set()
+{
+  local entity="$1"
+  local key="$2"
+  local value="$3"
+  local force_flag="${4:-}"
+
+  if [[ "$force_flag" == "force" ]] || ! ceph config dump | grep -qE "^[[:space:]]*$entity[[:space:]].*[[:space:]]$key[[:space:]]"
+  then
+    log_verbose "Applying to [$entity:$key] new value: $value"
+    ceph config set "$entity" "$key" "$value"
+  fi
+}
 # ---=---
 function ceph_conf_get_list()
 {