[Fault Tolerance] Add retry with delay to the block to copy Munge key and the blocks to start Chronyd and Munge services.

Signed-off-by: Giacomo Marciani <[email protected]>

Author: gmarciani

cookbooks/aws-parallelcluster-platform/resources/chrony/partial/_chrony_common.rb

@@ -39,6 +39,8 @@
     supports restart: false
     reload_command chrony_reload_command
     action %i(enable start)
+    retries 5
+    retry_delay 10
   end unless redhat_on_docker?
 end
 

cookbooks/aws-parallelcluster-slurm/libraries/helpers.rb

@@ -60,6 +60,8 @@ def enable_munge_service
   service "munge" do
     supports restart: true
     action %i(enable start)
+    retries 5
+    retry_delay 10
   end
 end
 
@@ -111,6 +113,8 @@ def setup_munge_compute_node
       # Enforce correct permission on the key
       chmod 0600 /etc/munge/munge.key
     COMPUTE_MUNGE_KEY
+    retries 5
+    retry_delay 10
   end
 
   enable_munge_service