Merge branch 'develop' into wip/mgiacomo/develop/slurm-upgrade-1

Author: gmarciani

CHANGELOG.md

@@ -17,6 +17,17 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 **CHANGES**
 - Upgrade Slurm to version 24.05.8.
+- Upgrade EFA installer to 1.41.0 (from 1.38.1).
+  - Efa-driver: efa-2.15.0-1
+  - Efa-config: efa-config-1.18-1
+  - Efa-profile: efa-profile-1.7-1
+  - Libfabric-aws: libfabric-aws-2.1.0-1
+  - Rdma-core: rdma-core-57.0-1
+  - Open MPI: openmpi40-aws-4.1.7-2 and openmpi50-aws-5.0.6
+
+**BUG FIXES**
+- Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments.
+- Upgrade amazon-efs-utils to version 2.3.1 (from v2.1.0) for non-Amazon Linux AMI's.
 
 3.13.0
 ------

cookbooks/aws-parallelcluster-environment/attributes/environment.rb

@@ -70,9 +70,14 @@
 
 default['cluster']['head_node_private_ip'] = nil
 
-default['cluster']['efa']['version'] = '1.38.1'
-default['cluster']['efa']['sha256'] = '83923374afd388b1cfcf4b3a21a2b1ba7cf46a01a587f7b519b8386cb95e4f81'
+default['cluster']['efa']['version'] = '1.41.0'
+default['cluster']['efa']['sha256'] = '3506354cdfbe31ff552fe75f5d0d9bb7efd29cf79bd99457347d29c751c38f9f'
 
+default['cluster']['efs']['version'] = '2.3.1'
+default['cluster']['efs']['sha256'] = 'ced12f82e76f9740476b63f30c49bd76cc00b6375e12a9f5f7ba852635c49e15'
+if platform?('amazon')
+  default['cluster']['efs']['version'] = '2.1.0'
+end
 # TODO: Move to platform cookbook
 default['cluster']['spack_shared_dir'] = "#{node['cluster']['shared_dir']}/spack"
 default['cluster']['spack']['version'] = '0.20.2'

cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb

@@ -20,9 +20,9 @@
 
 action :install_utils do
   package_name = "amazon-efs-utils"
-  full_package = "#{package_name}-#{new_resource.efs_utils_version}"
+  full_package = "#{package_name}-#{_efs_utils_version}"
   # Do not install efs-utils if a same or newer version is already installed.
-  return if already_installed?(package_name, new_resource.efs_utils_version)
+  return if already_installed?(package_name, _efs_utils_version)
 
   # On Amazon Linux 2, amazon-efs-utils and stunnel are installed from OS repo.
   package full_package do

cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb

@@ -21,10 +21,10 @@
 use 'partial/_mount_umount'
 
 action :install_utils do
-  package_name = "amazon-efs-utils-#{new_resource.efs_utils_version}"
+  package_name = "amazon-efs-utils-#{_efs_utils_version}"
 
   # Do not install efs-utils if a same or newer version is already installed.
-  return if already_installed?(package_name, new_resource.efs_utils_version)
+  return if already_installed?(package_name, _efs_utils_version)
 
   # On Amazon Linux 2, amazon-efs-utils and stunnel are installed from OS repo.
   package package_name do

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb

@@ -2,8 +2,16 @@
 
 default_action :install_utils
 
-property :efs_utils_version, String, default: '2.1.0'
-property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24'
+property :efs_utils_version, String
+property :efs_utils_checksum, String
+
+def _efs_utils_version
+  efs_utils_version || node['cluster']['efs']['version']
+end
+
+def _efs_utils_checksum
+  efs_utils_checksum || node['cluster']['efs']['sha256']
+end
 
 def already_installed?(package_name, expected_version)
   Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version)

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb

@@ -30,7 +30,7 @@
   return if redhat_on_docker?
 
   package_name = "amazon-efs-utils"
-  package_version = new_resource.efs_utils_version
+  package_version = _efs_utils_version
   efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
   efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
 
@@ -46,7 +46,7 @@
     mode '0644'
     retries 3
     retry_delay 5
-    checksum new_resource.efs_utils_checksum
+    checksum _efs_utils_checksum
     action :create_if_missing
   end
 

cookbooks/aws-parallelcluster-environment/spec/unit/resources/efa_spec.rb

@@ -2,8 +2,8 @@
 
 # parallelcluster default source dir defined in attributes
 source_dir = '/opt/parallelcluster/sources'
-efa_version = '1.38.1'
-efa_checksum = '83923374afd388b1cfcf4b3a21a2b1ba7cf46a01a587f7b519b8386cb95e4f81'
+efa_version = '1.41.0'
+efa_checksum = '3506354cdfbe31ff552fe75f5d0d9bb7efd29cf79bd99457347d29c751c38f9f'
 
 class ConvergeEfa
   def self.setup(chef_run, efa_version: nil, efa_checksum: nil)

cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb

@@ -4,9 +4,9 @@
 
   only_if { !os_properties.redhat_on_docker? }
 
-  describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do
+  describe file("#{node['cluster']['sources_dir']}/efs-utils-2.3.1.tar.gz") do
     it { should exist }
-    its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' }
+    its('sha256sum') { should eq 'ced12f82e76f9740476b63f30c49bd76cc00b6375e12a9f5f7ba852635c49e15' }
     its('owner') { should eq 'root' }
     its('group') { should eq 'root' }
     its('mode') { should cmp '0644' }

cookbooks/aws-parallelcluster-platform/resources/arm_pl/partial/_arm_pl_common.rb

@@ -125,8 +125,10 @@
         rm -rf gcc-#{gcc_version}
         tar -xf #{gcc_tarball}
         cd gcc-#{gcc_version}
-        # Patch the download_prerequisites script to download over https and not ftp. This works better in China regions.
-        sed -i "s#ftp://gcc\.gnu\.org/pub/gcc/infrastructure##{node['cluster']['artifacts_s3_url']}/dependencies/gcc/prerequisites#g" ./contrib/download_prerequisites
+        # Patch the download_prerequisites script to download GCC dependencies from our public bucket.
+        # This is required to support build image in isolated environments.
+        # Note: gcc 9.3 uses ftp, whereas gcc 11.3 uses http.
+        sed -i "s#\\(ftp\\|http\\)://gcc\.gnu\.org/pub/gcc/infrastructure##{node['cluster']['artifacts_s3_url']}/dependencies/gcc/prerequisites#g" ./contrib/download_prerequisites
         ./contrib/download_prerequisites
         mkdir build && cd build
         ../configure --prefix=/opt/arm/armpl/gcc/#{gcc_version} --disable-bootstrap --enable-checking=release --enable-languages=c,c++,fortran --disable-multilib

cookbooks/aws-parallelcluster-platform/test/controls/c_states_spec.rb

@@ -4,8 +4,8 @@
   only_if { !os_properties.on_docker? && os_properties.x86? }
   ## cpupower is installed for Ubuntu >=22
   describe bash('cpupower idle-info') do
-    its('stdout') { should match(/Number of idle states: 2/) }
-    its('stdout') { should match(/Available idle states: POLL C1/) }
+    its('stdout') { should match(/Number of idle states: 2|No idle states/) }
+    its('stdout') { should match(/Available idle states: POLL C1|No idle states/) }
   end
 end