Add TOTAL_MIN_COUNT of a cluster as comment

* Run static fleet checks if there are any static nodes

Adding it as condition

Author: Himani Anil Deshpande

cookbooks/aws-parallelcluster-computefleet/templates/compute_fleet_status/is_fleet_ready.erb

@@ -1,7 +1,16 @@
 #!/bin/bash
 
-sinfo_output=$(<%= node['cluster']['slurm']['install_dir'] %>/bin/sinfo -h -o '%N %t' | grep -v -E '(idle|alloc|mix|maint)$')
-while IFS= read -r line; do
-    nodelist=$(echo "$line" | awk '{print $1}')
-    <%= node['cluster']['slurm']['install_dir'] %>/bin/scontrol show hostnames "$nodelist" | { grep -E '^[a-z0-9\-]+\-st\-[a-z0-9\-]+\-[0-9]+.*' || true; }
-done <<< "$sinfo_output"
+
+
+cluster_static_node_count=$1
+if [[ -z "$cluster_static_node_count" ]]; then
+ cluster_static_node_count=1
+fi
+
+if [[ "$cluster_static_node_count" -ge "1" ]]; then
+  sinfo_output=$(<%= node['cluster']['slurm']['install_dir'] %>/bin/sinfo -h -o '%N %t' | grep -v -E '(idle|alloc|mix|maint)$')
+  while IFS= read -r line; do
+      nodelist=$(echo "$line" | awk '{print $1}')
+      <%= node['cluster']['slurm']['install_dir'] %>/bin/scontrol show hostnames "$nodelist" | { grep -E '^[a-z0-9\-]+\-st\-[a-z0-9\-]+\-[0-9]+.*' || true; }
+  done <<< "$sinfo_output"
+fi

cookbooks/aws-parallelcluster-slurm/files/default/head_node_slurm/slurm/templates/slurm_parallelcluster.conf

@@ -36,16 +36,20 @@ include {{ output_dir }}/pcluster/slurm_parallelcluster_{{ queue.Name }}_partiti
 {% endfor %}
 
 {% if ns.has_static %}
+{%- set ns.total_min_count = 0 %}
 SuspendExcNodes=
     {%- set ns.is_first = True %}
     {%- for queue in queues %}
         {% for compute_resource in queue.ComputeResources %}
             {% if compute_resource.MinCount > 0 %}
 {{- "," if not ns.is_first else "" -}}
 {{ queue.Name }}-st-{{ compute_resource.Name }}-[1-{{ compute_resource.MinCount }}]
+                {%- set ns.total_min_count = ns.total_min_count +  compute_resource.MinCount %}
                 {%- set ns.is_first = False %}
             {%- endif %}
         {% endfor %}
     {% endfor %}
 
 {% endif %}
+
+#TOTAL_MIN_COUNT={{ ns.total_min_count }}

cookbooks/aws-parallelcluster-slurm/libraries/helpers.rb

@@ -178,6 +178,11 @@ def wait_cluster_ready
   end
 end
 
+def get_static_node_count
+  cmd = Mixlib::ShellOut.new("cat #{node['cluster']['slurm']['install_dir']}/etc/slurm_parallelcluster.conf | grep -o '#TOTAL_MIN_COUNT=\([0-9]*\)' | cut -d'=' -f2")
+  cmd.run_command.stdout.strip
+end
+
 def wait_static_fleet_running
   ruby_block "wait for static fleet capacity" do
     block do
@@ -203,11 +208,12 @@ def check_for_protected_mode(fleet_status_command) # rubocop:disable Lint/Nested
       fleet_status_command = Shellwords.escape(
         "/usr/local/bin/get-compute-fleet-status.sh"
       )
+
       # Example output for sinfo
       # sinfo -h -o '%N %t'
       # queue-0-dy-compute-resource-g4dn-0-[1-10],queue-1-dy-compute-resource-g4dn-1-[1-10] idle~
       # queue-2-dy-compute-resource-g4dn-2-[1-10],queue-3-dy-compute-resource-g4dn-3-[1-10] idle
-      until shell_out!("/bin/bash -c /usr/local/bin/is_fleet_ready.sh").stdout.strip.empty?
+      until shell_out!("/bin/bash -c /usr/local/bin/is_fleet_ready.sh #{get_static_node_count.to_i}").stdout.strip.empty?
         check_for_protected_mode(fleet_status_command)
 
         Chef::Log.info("Waiting for static fleet capacity provisioning")