RBD cluster 구분에 대응하는 HA heartbeat 커맨드 수정

Author: jschoiRR

scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh

@@ -38,122 +38,119 @@ SourceHostIP=
 interval=0
 rflag=0
 cflag=0
-UUIDList=""
+UUIDList=
 skeyPath="/var/lib/libvirt/images/"
 
-while getopts 'p:n:s:h:i:t:u:r:c' OPTION; do
+while getopts 'p:n:s:h:i:t:u:r:c' OPTION
+do
   case $OPTION in
   p)
-    PoolName="$OPTARG"
-    ;;
+     PoolName="$OPTARG"
+     ;;
   n)
-    PoolAuthUserName="$OPTARG"
-    ;;
+     PoolAuthUserName="$OPTARG"
+     ;;
   s)
-    PoolAuthSecret="$OPTARG"
-    ;;
+     PoolAuthSecret="$OPTARG"
+     ;;
   h)
-    HostIP="$OPTARG"
-    ;;
+     HostIP="$OPTARG"
+     ;;
   i)
-    SourceHostIP="$OPTARG"
-    ;;
+     SourceHostIP="$OPTARG"
+     ;;
   t)
-    interval="$OPTARG"
-    ;;
+     interval="$OPTARG"
+     ;;
   u)
-    UUIDList="$OPTARG"
-    ;;
+     UUIDList="$OPTARG"
+     ;;
   r)
-    rflag=1
-    ;;
+     rflag=1
+     ;;
   c)
-    cflag=1
-    ;;
+     cflag=1
+     ;;
   *)
-    help
-    ;;
+     help
+     ;;
   esac
 done
 
 if [ -z "$PoolName" ]; then
   exit 2
 fi
 
+# rados object touch action for vol list
+res=$(rbd -p $PoolName ls --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret | grep MOLD-AC)
+if [ $? -gt 0 ]; then
+  rbd -p $PoolName create --size 1 --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret MOLD-AC
+fi
+
+timestamp=$(date +%s)
+
+if [ -n "$UUIDList" ]; then
+    for uuid in $(echo $UUIDList | sed 's/,/ /g'); do
+      objId=$(rbd -p $PoolName info $uuid --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret | grep 'id:')
+      objId=${objId#*id: }
+      res=$(timeout 3s bash -c "rados -p $PoolName touch rbd_object_map.$objId")
+      if [ $? -eq 0 ]; then
+        # 정상적인 touch 상태면 image meta에 key: uuid / value : timestamp 입력
+        rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta set MOLD-AC $uuid $HostIP:$timestamp
+      else
+        # 정상적으로 touch 상태가 아니면 image meta에 key : uuid 삭제
+        rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta rm MOLD-AC $uuid
+      fi
+    done
+fi
+
 #write the heart beat log
 write_hbLog() {
   Timestamp=$(date +%s)
-  CurrentTime=$(date +"%Y-%m-%d %H:%M:%S")
-
-  obj=$(rbd -p $PoolName ls --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret | grep MOLD-HB-$HostIP)
+  obj=$(rbd -p $PoolName ls --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret | grep MOLD-HB)
 
   if [ $? -gt 0 ]; then
-    rbd -p $PoolName create --size 1 --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret MOLD-HB-$HostIP
+     rbd -p $PoolName create --size 1 --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret MOLD-HB
   fi
 
-  rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta set MOLD-HB-$HostIP $HostIP $Timestamp
-  ret=$?
-  if [ $ret -eq 0 ]; then
-    logger -p user.info -t MOLD-HA-HB "[Writing]  호스트:$HostIP | HB 파일 갱신(RBD, 스토리지:$PoolName) > [현 시간:$CurrentTime]"
-  else
-    logger -p user.info -t MOLD-HA-HB "[Writing]  호스트:$HostIP | HB 파일 갱신(RBD, 스토리지:$PoolName) > HB 갱신 실패!!!"
-  fi
-  return $ret
-
+  obj=$(rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta set MOLD-HB $HostIP $Timestamp)
   if [ $? -gt 0 ]; then
-    printf "Failed to create rbd file and set image-meta"
+   	printf "Failed to create rbd file and set image-meta"
     return 2
   fi
   return 0
 }
 
 #check the heart beat log
 check_hbLog() {
-  Timestamp=$(date +%s)
-  CurrentTime=$(date +"%Y-%m-%d %H:%M:%S")
-
-  getHbTime=$(rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta get MOLD-HB-$HostIP $HostIP)
+  now=$(date +%s)
+  getHbTime=$(rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta get MOLD-HB $HostIP)
   if [ $? -gt 0 ] || [ -z "$getHbTime" ]; then
     return 1
   fi
 
-  diff=$(expr $Timestamp - $getHbTime)
-  getHbTimeFmt=$(date -d @${getHbTime} '+%Y-%m-%d %H:%M:%S')
-  logger -p user.info -t MOLD-HA-HB "[Checking] 호스트:$HostIP | HB 파일 체크(RBD, 스토리지:$PoolName) > [현 시간:$CurrentTime | HB 파일 시간:$getHbTimeFmt | 시간 차이:$diff초]"
-  if { [ "$diff" -gt 30 ] && [ "$diff" -le 45 ]; } || { [ "$diff" -gt 60 ] && [ "$diff" -le 75 ]; }; then
-    timeout 1 ssh ccvm "
-      mysql -u cloud -pAblecloud1! -D cloud -e \"
-        INSERT INTO event (
-          uuid, type, state, description, user_id, account_id, domain_id,
-          resource_id, resource_type, created, level, start_id,
-          parameters, archived, display
-        ) VALUES (
-          UUID(), 'HA.STATE.TRANSITION', 'Completed',
-          '[Heartbeat Checking] Host: $HostIP | Storage: $PoolName [Current Time: $CurrentTime | HB File Time: $getHbTimeFmt | Time Difference: $diff seconds]', 1, 1, 1, 0, 'Host',
-          UTC_TIMESTAMP(), 'WARN', 0, NULL, 0, 1
-        );
-      \"
-    "
-  fi
+  diff=$(expr $now - $getHbTime)
+
   if [ $diff -gt $interval ]; then
-    logger -p user.info -t MOLD-HA-HB "[Result]   호스트:$HostIP | HB 체크 결과(RBD, 스토리지:$PoolName) > [HOST STATE : DEAD]"
-    echo "### [HOST STATE : DEAD] Set maximum interval: ($interval seconds), Actual difference: ($diff seconds) => Considered host down in [PoolType : RBD] ###"
-    return 0
-  else
-    logger -p user.info -t MOLD-HA-HB "[Result]   호스트:$HostIP | HB 체크 결과(RBD, 스토리지:$PoolName) > [HOST STATE : ALIVE]"
-    echo "### [HOST STATE : ALIVE] in [PoolType : RBD] ###"
+    return $diff
   fi
   return 0
 }
 
 if [ "$rflag" == "1" ]; then
   check_hbLog
-  exit 0
+  diff=$?
+  if [ $diff == 0 ]; then
+    echo "### [HOST STATE : ALIVE] in [PoolType : RBD] ###"
+  else
+    echo "### [HOST STATE : DEAD] Set maximum interval: ($interval seconds), Actual difference: ($diff seconds) => Considered host down in [PoolType : RBD] ###"
+  fi
+    exit 0
 elif [ "$cflag" == "1" ]; then
   /usr/bin/logger -t heartbeat "kvmheartbeat_rbd.sh will reboot system because it was unable to write the heartbeat to the storage."
   sync &
   sleep 5
-  echo b >/proc/sysrq-trigger
+  echo b > /proc/sysrq-trigger
   exit $?
 else
   write_hbLog