Fix cloud-sysvmadmin hang (#12355)

Author: abh1sar

setup/bindir/cloud-sysvmadm.in

@@ -184,59 +184,38 @@ stop_start_system() {
 stop_start_router() {
     prepare_ids_clause
     router=(`mysql -h $db --user=$user --password=$password --skip-column-names -U cloud -e "select uuid from vm_instance where state=\"Running\" and type=\"DomainRouter\"$zone$vmidsclause"`)
-    length_router=(${#router[@]})
+    length_router=${#router[@]}
 
     echo -e "\nStopping and starting $length_router running routing vm(s)$inzone$withids... "
     echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Stopping and starting $length_router running routing vm(s)$inzone$withids... " >>$LOGFILE
 
     #Spawn reboot router in parallel - run commands in <n> chunks - number of threads is configurable
 
+    if [ $maxthreads -gt $length_router ]; then
+        maxthreads=$length_router
+    fi
     pids=()
     for d in "${router[@]}"; do
 
         reboot_router $d &
 
         pids=( "${pids[@]}" $! )
-
-        length_pids=(${#pids[@]})
-        unfinishedPids=(${#pids[@]})
-
-        if [ $maxthreads -gt $length_router ]; then
-          maxthreads=$length_router
-        fi
+        length_pids=${#pids[@]}
 
         if [ $length_pids -ge $maxthreads ]; then
-            while [ $unfinishedPids -gt 0 ]; do
-                sleep 10
-                count=0
-                for ((  i = 0 ;  i < $length_pids;  i++  )); do
-                    if ! ps ax | grep -v grep | grep ${pids[$i]} > /dev/null; then
-                        count=`expr $count + 1`
-                    fi
-                done
-
-                if [ $count -eq $unfinishedPids ]; then
-                  unfinishedPids=0
-                fi
-
-            done
-
-            #remove all elements from pids
-            if [ $unfinishedPids -eq 0 ]; then
-                pids=()
-                length_pids=(${#pids[@]})
-            fi
-
+            # Wait for $maxthreads number of processes to finish
+            wait
+            # Clear the pids array for the next batch
+            pids=()
         fi
 
     done
 
     if [ "$length_router" == "0" ];then
         echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] No running router vms found \n" >>$LOGFILE
     else
-        while [ $unfinishedPids -gt 0 ]; do
-          sleep 10
-        done
+        # Wait for the remaining background processes to finish
+        wait
 
         echo -e "Done restarting router(s)$inzone$withids. \n"
         echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Done restarting router(s)$inzone$withids. \n" >>$LOGFILE
@@ -288,49 +267,29 @@ reboot_router(){
 restart_networks(){
     networks=(`mysql -h $db --user=$user --password=$password --skip-column-names -U cloud -e "select n.id
     from networks n, network_offerings no where n.network_offering_id = no.id and no.system_only = 0 and n.removed is null$zone"`)
-    length_networks=(${#networks[@]})
+    length_networks=${#networks[@]}
 
     echo -e "\nRestarting $length_networks networks$inzone... "
     echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Restarting $length_networks networks$inzone... " >>$LOGFILE
 
     #Spawn restart network in parallel - run commands in <n> chunks - number of threads is configurable
 
+    if [ $maxthreads -gt $length_networks ]; then
+        maxthreads=$length_networks
+    fi
     pids=()
     for d in "${networks[@]}"; do
 
         restart_network $d &
 
         pids=( "${pids[@]}" $! )
-
-        length_pids=(${#pids[@]})
-        unfinishedPids=(${#pids[@]})
-
-        if [ $maxthreads -gt $length_networks ]; then
-            maxthreads=$length_networks
-        fi
+        length_pids=${#pids[@]}
 
         if [ $length_pids -ge $maxthreads ]; then
-            while [ $unfinishedPids -gt 0 ]; do
-                sleep 10
-                count=0
-                for ((  i = 0 ;  i < $length_pids;  i++  )); do
-                    if ! ps ax | grep -v grep | grep ${pids[$i]} > /dev/null; then
-                        count=`expr $count + 1`
-                    fi
-                done
-
-                if [ $count -eq $unfinishedPids ]; then
-                    unfinishedPids=0
-                fi
-
-            done
-
-            #remove all elements from pids
-            if [ $unfinishedPids -eq 0 ]; then
-                pids=()
-                length_pids=(${#pids[@]})
-            fi
-
+            # Wait for $maxthreads number of processes to finish
+            wait
+            # Clear the pids array for the next batch
+            pids=()
         fi
 
     done
@@ -339,9 +298,8 @@ restart_networks(){
     if [ "$length_networks" == "0" ];then
         echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] No networks found \n" >>$LOGFILE
     else
-        while [ $unfinishedPids -gt 0 ]; do
-            sleep 10
-        done
+        # Wait for the remaining background processes to finish
+        wait
 
         echo -e "Done restarting networks$inzone. \n"
         echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Done restarting networks$inzone. \n" >>$LOGFILE
@@ -392,49 +350,29 @@ restart_vpc(){
 
 restart_vpcs(){
     vpcs=(`mysql -h $db --user=$user --password=$password --skip-column-names -U cloud -e "select uuid from vpc WHERE removed is null$zone"`)
-    length_vpcs=(${#vpcs[@]})
+    length_vpcs=${#vpcs[@]}
 
     echo -e "\nRestarting $length_vpcs vpcs... "
     echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Restarting $length_vpcs vpcs... " >>$LOGFILE
 
     #Spawn restart vpcs in parallel - run commands in <n> chunks - number of threads is configurable
 
+    if [ $maxthreads -gt $length_vpcs ]; then
+        maxthreads=$length_vpcs
+    fi
     pids=()
     for d in "${vpcs[@]}"; do
 
         restart_vpc $d &
 
         pids=( "${pids[@]}" $! )
-
-        length_pids=(${#pids[@]})
-        unfinishedPids=(${#pids[@]})
-
-        if [ $maxthreads -gt $length_vpcs ]; then
-            maxthreads=$length_vpcs
-        fi
+        length_pids=${#pids[@]}
 
         if [ $length_pids -ge $maxthreads ]; then
-            while [ $unfinishedPids -gt 0 ]; do
-                sleep 10
-                count=0
-                for ((  i = 0 ;  i < $length_pids;  i++  )); do
-                    if ! ps ax | grep -v grep | grep ${pids[$i]} > /dev/null; then
-                        count=`expr $count + 1`
-                    fi
-                done
-
-                if [ $count -eq $unfinishedPids ]; then
-                    unfinishedPids=0
-                fi
-
-            done
-
-            #remove all elements from pids
-            if [ $unfinishedPids -eq 0 ]; then
-                pids=()
-                length_pids=(${#pids[@]})
-            fi
-
+            # Wait for $maxthreads number of processes to finish
+            wait
+            # Clear the pids array for the next batch
+            pids=()
         fi
 
     done
@@ -443,9 +381,8 @@ restart_vpcs(){
     if [ "$length_vpcs" == "0" ];then
         echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] No vpcs found \n" >>$LOGFILE
     else
-        while [ $unfinishedPids -gt 0 ]; do
-            sleep 10
-        done
+        # Wait for the remaining background processes to finish
+        wait
 
         echo -e "Done restarting vpcs$inzone. \n"
         echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Done restarting vpcs$inzone. \n" >>$LOGFILE