run gather in background

run_bg can only be used to run executables in the backgorund
not bash functions. This change adds a wait_for_bg_slot funciton
and refactors the sos report gathering to run in parallel up to
the concurrancy limit.

Author: SeanMooney

collection-scripts/bg.sh

@@ -2,6 +2,28 @@
 
 CONCURRENCY=${CONCURRENCY:-5}
 
+# Function to wait for a background slot to be available
+# This function is used by run_bg to wait for a slot to be available before
+# running a new command in the background.
+# This function is can also be used to limit background functions
+# to a certain number of concurrent processes.
+# Example:
+#    CONCURRENCY=2
+#    function my_function {
+#        echo "my_function: $@"
+#        sleep 1
+#    }
+#    for i in {1..10}; do
+#        wait_for_bg_slot
+#        my_function $i &
+#    done
+#    wait_bg
+function wait_for_bg_slot {
+    while [[ $(jobs -r | wc -l) -ge $CONCURRENCY ]]; do
+        wait -n
+    done
+}
+
 # Function to run commands in background without exceeding $CONCURRENCY
 # processes in parallel.
 # The recommendation is to use this function at the deepest level that can be
@@ -16,11 +38,8 @@ CONCURRENCY=${CONCURRENCY:-5}
 #
 # For now these methods ignore errors on the calls that are made in the
 # background.
-
 function run_bg {
-    while [[ $(jobs -r | wc -l) -ge $CONCURRENCY ]]; do
-        wait -n
-    done
+    wait_for_bg_slot
 
     # Cannot use the alternative suggested by SC2294 which is just "$@"&
     # because that doesn't accomplish what we want, as it executes the first

collection-scripts/gather_edpm_sos

@@ -127,7 +127,10 @@ data=$(oc get openstackdataplanenodesets --all-namespaces -o json | jq -j '
 while read -r node address username secret namespace; do
     [[ -z "$node" ]] && continue
     if [[ "${SOS_EDPM[0]}" == "all" || "${SOS_EDPM[*]}" == *"${node}"* ]]; then
-        run_bg gather_edpm_sos $node $address $username $secret $namespace
+        # run_bg cannot be used here as that only support invoking external scripts
+        # or executables and not functions
+        wait_for_bg_slot
+        gather_edpm_sos $node $address $username $secret $namespace &
     fi
 done <<< "$data"
 

collection-scripts/gather_sos

@@ -174,7 +174,10 @@ echo "Will retrieve SOS reports from nodes ${nodes//$'\n'/ }"
 for node in $nodes; do
     [[ -z "$node" ]] && continue
     # Gather SOS report for the node in background
-    run_bg gather_node_sos "$node"
+    # run_bg cannot be used here as that only support invoking external scripts
+    # or executables and not functions
+    wait_for_bg_slot
+    gather_node_sos "$node" &
 done
 
 [[ $CALLED -eq 1 ]] && wait_bg