Merge branch 'develop' into versionbumpdevelop3.14.0

Author: himani2411

CHANGELOG.md

@@ -13,11 +13,10 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 ------
 **ENHANCEMENTS**
 - Add support for Ubuntu 24.04.
-  Notice that ParallelCluster official AMI for Ubuntu 24.04 does not support Lustre.
 - Disable unused services like cups and wpa_supplicant from Official ParallelCluster AMIs to improve security.
 
 **CHANGES**
-- Upgrade Slurm to version 24.05.6.
+- Upgrade Slurm to version 24.05.7.
 - Upgrade NVIDIA driver to version 570.86.15 (from 550.127.08) for all OSs except AL2.
 - Upgrade CUDA Toolkit to version 12.8.0 (from 12.4.1) for all OSs except AL2.
 - Upgrade Python to 3.12.8 for all OSs except AL2 (from 3.9.20).
@@ -43,6 +42,10 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 - Remove generation of DSA keys for login nodes as DSA, which became unsupported in OpenSSH 9.7+.
 - Set instance ID and instance type information in Slurm upon compute nodes launch.
 - Install NVIDIA drivers without the option 'no-cc-version-check', which is now deprecated in the NVIDIA installer.
+- Reduce RHEL/Rocky Linux boot time by the following network customization:
+  - Configuring higher priority to IPv4 than IPv6
+  - Disabling Internet connectivity check
+  - Configuring only IPv4 IMDS endpoint to cloud-init
 
 **BUG FIXES**
 - Remove usage of cfn-init for compute node bootstrapping to reduce node scale-up time.

cookbooks/aws-parallelcluster-environment/libraries/fsx.rb

@@ -10,5 +10,5 @@ def aws_domain_for_fsx(region)
 end
 
 def lustre_enabled?
-  ['yes', true].include?(node['cluster']['lustre']['enabled'])
+  ['yes', true, 'true'].include?(node['cluster']['lustre']['enabled'])
 end

cookbooks/aws-parallelcluster-environment/resources/lustre/partial/_install_lustre_debian.rb

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 action :setup do
-  return if node['platform_version'].to_i == 24
   apt_repository 'fsxlustreclientrepo' do
     uri          "https://fsx-lustre-client-repo.s3.amazonaws.com/ubuntu"
     components   ['main']

cookbooks/aws-parallelcluster-environment/test/controls/cloudwatch_spec.rb

@@ -20,14 +20,13 @@
 
   describe 'Check the presence of the cloudwatch package gpg key'
   # In Ubuntu >20.04 due to environment variable the keyring is placed under home of the user ubuntu with the permission of root
-  ubuntu2004 = os_properties.ubuntu2004?
-  ubuntu2204 = os_properties.ubuntu2204?
-  keyring = (ubuntu2004 || ubuntu2204) && !os_properties.on_docker? ? '--keyring /home/ubuntu/.gnupg/pubring.kbx' : ''
+  is_ubuntu = os_properties.ubuntu?
+  keyring = is_ubuntu && !os_properties.on_docker? ? '--keyring /home/ubuntu/.gnupg/pubring.kbx' : ''
   sudo = os_properties.redhat_on_docker? ? '' : 'sudo'
   describe bash("#{sudo} gpg --list-keys #{keyring}") do
     # Don't check exit status for Ubuntu20 because it returns 2 when executed in the validate phase of a created AMI
     # os_properties cannot be used in the describe block level.  It can be used within an it{} block
-    its('exit_status') { should eq 0 } unless ubuntu2004 || ubuntu2204
+    its('exit_status') { should eq 0 } unless is_ubuntu
     its('stdout') { should match /3B789C72/ }
     its('stdout') { should match /Amazon CloudWatch Agent/ }
   end

cookbooks/aws-parallelcluster-environment/test/controls/lustre_spec.rb

@@ -27,7 +27,7 @@
     end
   end
 
-  if os_properties.redhat? && inspec.os.release.to_f >= 8.2 && !os_properties.on_docker? && !os_properties.ubuntu2404?
+  if os_properties.redhat? && inspec.os.release.to_f >= 8.2 && !os_properties.on_docker?
     # TODO: restore installation and check on docker when Lustre is available for RH8.9
     # See: https://docs.aws.amazon.com/fsx/latest/LustreGuide/install-lustre-client.html
     unless inspec.os.release.to_f == 8.7 && (node['cluster']['kernel_release'].include?("4.18.0-425.3.1.el8") || node['cluster']['kernel_release'].include?("4.18.0-425.13.1.el8_7"))
@@ -55,7 +55,7 @@
     end
   end
 
-  if os_properties.debian_family? && !os_properties.ubuntu2404?
+  if os_properties.debian_family?
     describe apt('https://fsx-lustre-client-repo.s3.amazonaws.com/ubuntu') do
       it { should exist }
       it { should be_enabled }
@@ -89,7 +89,7 @@
 
 control 'tag:install_lustre_lnet_kernel_module_enabled' do
   title "Verify that lnet kernel module is enabled"
-  only_if { !os_properties.on_docker? && !os_properties.alinux? && !os_properties.ubuntu2404? }
+  only_if { !os_properties.on_docker? && !os_properties.alinux? }
   describe kernel_module("lnet") do
     it { should be_loaded }
     it { should_not be_disabled }
@@ -98,15 +98,15 @@
 end
 
 control 'lustre_mounted' do
-  only_if { !os_properties.on_docker? && !os_properties.ubuntu2404? }
+  only_if { !os_properties.on_docker? }
   describe mount('/shared_dir') do
     it { should be_mounted }
     its('type') { should eq 'lustre' }
   end
 end
 
 control 'lustre_unmounted' do
-  only_if { !os_properties.on_docker? && !os_properties.ubuntu2404? }
+  only_if { !os_properties.on_docker? }
 
   describe mount('/shared_dir') do
     it { should_not be_mounted }

cookbooks/aws-parallelcluster-platform/files/ami_cleanup.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+IS_OFFICIAL_AMI_BUILD=${1:-"false"}
+
 # clean up cloud init artifacts https://cloudinit.readthedocs.io/en/latest/topics/cli.html#clean
 cloud-init clean -s
 
@@ -20,5 +22,12 @@ if [ "${ID}${VERSION_ID}" == "centos7" ]; then
     rm -f /etc/sysconfig/network-scripts/ifcfg-eth0
 fi
 
+# Clean resolv.conf if it's not managed by system
+if [ "${IS_OFFICIAL_AMI_BUILD}" == "true" ]; then
+    echo "Clean resolv.conf for official AMIs"
+    echo -n > /etc/resolv.conf
+    rm -f /run/systemd/resolve/resolv.conf
+fi
+
 find /var/log -type f -exec /bin/rm -v {} \;
 touch /var/log/lastlog

cookbooks/aws-parallelcluster-platform/libraries/nvidia.rb

@@ -1,5 +1,5 @@
 def nvidia_enabled?
-  ['yes', true].include?(node['cluster']['nvidia']['enabled'])
+  ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled'])
 end
 
 #

cookbooks/aws-parallelcluster-platform/recipes/config/cluster_user.rb

@@ -73,7 +73,9 @@
   bash "copy_auth_file" do
     code <<-PERMS
       set -e
-      cp -p #{node['cluster']['shared_dir']}/authorized_keys #{node['cluster']['cluster_user_home']}/.ssh/authorized_keys
+      cp #{node['cluster']['shared_dir']}/authorized_keys #{node['cluster']['cluster_user_home']}/.ssh/authorized_keys
+      chmod --reference=#{node['cluster']['shared_dir']}/authorized_keys #{node['cluster']['cluster_user_home']}/.ssh/authorized_keys
+      chown --reference=#{node['cluster']['shared_dir']}/authorized_keys #{node['cluster']['cluster_user_home']}/.ssh/authorized_keys
     PERMS
     only_if { node['cluster']['default_user_home'] == 'local' }
   end
@@ -90,7 +92,9 @@
   bash "copy_auth_file" do
     code <<-PERMS
       set -e
-      cp -p #{node['cluster']['shared_dir_login_nodes']}/authorized_keys #{node['cluster']['cluster_user_home']}/.ssh/authorized_keys
+      cp #{node['cluster']['shared_dir_login_nodes']}/authorized_keys #{node['cluster']['cluster_user_home']}/.ssh/authorized_keys
+      chmod --reference=#{node['cluster']['shared_dir_login_nodes']}/authorized_keys #{node['cluster']['cluster_user_home']}/.ssh/authorized_keys
+      chown --reference=#{node['cluster']['shared_dir_login_nodes']}/authorized_keys #{node['cluster']['cluster_user_home']}/.ssh/authorized_keys
     PERMS
     only_if { node['cluster']['default_user_home'] == 'local' }
   end

cookbooks/aws-parallelcluster-platform/resources/fabric_manager/partial/_fabric_manager_common.rb

@@ -45,7 +45,7 @@ def _fabric_manager_enabled
 end
 
 def _nvidia_enabled
-  nvidia_enabled.nil? ? ['yes', true].include?(node['cluster']['nvidia']['enabled']) : nvidia_enabled
+  nvidia_enabled.nil? ? ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled']) : nvidia_enabled
 end
 
 def _nvidia_driver_version

cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/partial/_nvidia_dcgm_common.rb

@@ -24,7 +24,7 @@
 end
 
 def _nvidia_enabled
-  nvidia_enabled.nil? ? ['yes', true].include?(node['cluster']['nvidia']['enabled']) : nvidia_enabled
+  nvidia_enabled.nil? ? ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled']) : nvidia_enabled
 end
 
 def package_version