Merge branch 'develop' into develop-min-count-ad

himani2411 · web-flow · commit 86b8eb45f7c2 · 2025-06-06T14:37:23.000-04:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -26,9 +26,8 @@ This file is used to list changes made in each version of the AWS ParallelCluste
   - Open MPI: openmpi40-aws-4.1.7-2 and openmpi50-aws-5.0.6
 
 **BUG FIXES**
-- Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments 
-due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket.
-- Use patched version of efs-utils v2.1.0 which pins backtrace version to v3.0.74 to resolve build image failure.
+- Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments.
+- Upgrade amazon-efs-utils to version 2.3.1 (from v2.1.0) for non-Amazon Linux AMI's.
 
 3.13.0
 ------
diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/parallelcluster_node.rb
@@ -36,7 +36,7 @@
 if aws_region.start_with?("us-iso") && !is_custom_node?
   node_package = "aws-parallelcluster-node-#{node['cluster']['parallelcluster-node-version']}.tgz"
 
-  node.default['cluster']['custom_node_package'] = "#{node['cluster']['s3_url']}/parallelcluster/#{node['cluster']['parallelcluster-node-version']}/node/#{node_package}"
+  node.override['cluster']['custom_node_package'] = "#{node['cluster']['s3_url']}/parallelcluster/#{node['cluster']['parallelcluster-node-version']}/node/#{node_package}"
 end
 
 if is_custom_node?
diff --git a/cookbooks/aws-parallelcluster-environment/attributes/environment.rb b/cookbooks/aws-parallelcluster-environment/attributes/environment.rb
@@ -73,6 +73,11 @@
 default['cluster']['efa']['version'] = '1.41.0'
 default['cluster']['efa']['sha256'] = '3506354cdfbe31ff552fe75f5d0d9bb7efd29cf79bd99457347d29c751c38f9f'
 
+default['cluster']['efs']['version'] = '2.3.1'
+default['cluster']['efs']['sha256'] = 'ced12f82e76f9740476b63f30c49bd76cc00b6375e12a9f5f7ba852635c49e15'
+if platform?('amazon')
+  default['cluster']['efs']['version'] = '2.1.0'
+end
 # TODO: Move to platform cookbook
 default['cluster']['spack_shared_dir'] = "#{node['cluster']['shared_dir']}/spack"
 default['cluster']['spack']['version'] = '0.20.2'
diff --git a/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml b/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml
diff --git a/cookbooks/aws-parallelcluster-environment/libraries/directory_service_utils.rb b/cookbooks/aws-parallelcluster-environment/libraries/directory_service_utils.rb
@@ -21,7 +21,7 @@ def parse_arn(arn_string)
 def _get_cn_subparam(param)
   tokens = param.split(",") unless param.blank?
   tokens.each do |token|
-    return token if token.include? "cn"
+    return token if token.downcase.start_with? "cn"
   end
   ""
 end
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb
@@ -20,9 +20,9 @@
 
 action :install_utils do
   package_name = "amazon-efs-utils"
-  full_package = "#{package_name}-#{new_resource.efs_utils_version}"
+  full_package = "#{package_name}-#{_efs_utils_version}"
   # Do not install efs-utils if a same or newer version is already installed.
-  return if already_installed?(package_name, new_resource.efs_utils_version)
+  return if already_installed?(package_name, _efs_utils_version)
 
   # On Amazon Linux 2, amazon-efs-utils and stunnel are installed from OS repo.
   package full_package do
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb
@@ -21,10 +21,10 @@
 use 'partial/_mount_umount'
 
 action :install_utils do
-  package_name = "amazon-efs-utils-#{new_resource.efs_utils_version}"
+  package_name = "amazon-efs-utils-#{_efs_utils_version}"
 
   # Do not install efs-utils if a same or newer version is already installed.
-  return if already_installed?(package_name, new_resource.efs_utils_version)
+  return if already_installed?(package_name, _efs_utils_version)
 
   # On Amazon Linux 2, amazon-efs-utils and stunnel are installed from OS repo.
   package package_name do
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb
@@ -2,8 +2,16 @@
 
 default_action :install_utils
 
-property :efs_utils_version, String, default: '2.1.0'
-property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24'
+property :efs_utils_version, String
+property :efs_utils_checksum, String
+
+def _efs_utils_version
+  efs_utils_version || node['cluster']['efs']['version']
+end
+
+def _efs_utils_checksum
+  efs_utils_checksum || node['cluster']['efs']['sha256']
+end
 
 def already_installed?(package_name, expected_version)
   Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version)
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb
@@ -13,9 +13,10 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version)
+def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
   <<-EFSUTILSINSTALL
       set -e
+      tar xf #{efs_utils_tarball}
       cd efs-utils-#{efs_utils_version}
       ./build-deb.sh
       apt-get -y install ./build/#{efs_utils_package}*deb
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb
@@ -30,7 +30,7 @@
   return if redhat_on_docker?
 
   package_name = "amazon-efs-utils"
-  package_version = new_resource.efs_utils_version
+  package_version = _efs_utils_version
   efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
   efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
 
@@ -46,26 +46,10 @@
     mode '0644'
     retries 3
     retry_delay 5
-    checksum new_resource.efs_utils_checksum
+    checksum _efs_utils_checksum
     action :create_if_missing
   end
 
-  bash "Untar the efs-utils" do
-    cwd node['cluster']['sources_dir']
-    code <<-EFSUTILSUNTAR
-      set -e
-      tar xf #{efs_utils_tarball}
-    EFSUTILSUNTAR
-  end
-
-  cookbook_file "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}/src/proxy/Cargo.toml" do
-    source 'efs/Cargo.toml'
-    owner 'root'
-    group 'root'
-    mode '0755'
-    action :create
-  end
-
   # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
   bash "install efs utils" do
     cwd node['cluster']['sources_dir']
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb
@@ -13,9 +13,10 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version)
+def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
   <<-EFSUTILSINSTALL
       set -e
+      tar xf #{efs_utils_tarball}
       cd efs-utils-#{efs_utils_version}
       make rpm
       yum -y install ./build/#{efs_utils_package}*rpm
diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb
@@ -88,15 +88,10 @@ def mock_already_installed(package, expected_version, installed)
       cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" }
       cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" }
       cached(:tarball_checksum) { 'TARBALL CHECKSUM' }
-      cached(:bash_untar_code) do
-        <<-EFSUTILSUNTAR
-      set -e
-      tar xf #{tarball_path}
-        EFSUTILSUNTAR
-      end
       cached(:bash_code) do
         <<-EFSUTILSINSTALL
       set -e
+      tar xf #{tarball_path}
       cd efs-utils-#{utils_version}
       ./build-deb.sh
       apt-get -y install ./build/amazon-efs-utils*deb
@@ -132,12 +127,6 @@ def mock_already_installed(package, expected_version, installed)
             .with(checksum: tarball_checksum)
         end
 
-        it 'it untars the downloaded tarball' do
-          is_expected.to run_bash('Untar the efs-utils')
-            .with(cwd: source_dir)
-            .with(code: bash_untar_code)
-        end
-
         it 'installs package from downloaded tarball' do
           is_expected.to run_bash('install efs utils')
             .with(cwd: source_dir)
@@ -179,15 +168,10 @@ def mock_already_installed(package, expected_version, installed)
       cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" }
       cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" }
       cached(:tarball_checksum) { 'TARBALL CHECKSUM' }
-      cached(:bash_untar_code) do
-        <<-EFSUTILSUNTAR
-      set -e
-      tar xf #{tarball_path}
-        EFSUTILSUNTAR
-      end
       cached(:bash_code) do
         <<-EFSUTILSINSTALL
       set -e
+      tar xf #{tarball_path}
       cd efs-utils-#{utils_version}
       make rpm
       yum -y install ./build/amazon-efs-utils*rpm
@@ -234,12 +218,6 @@ def mock_already_installed(package, expected_version, installed)
             .with(checksum: tarball_checksum)
         end
 
-        it 'it untars the downloaded tarball' do
-          is_expected.to run_bash('Untar the efs-utils')
-            .with(cwd: source_dir)
-            .with(code: bash_untar_code)
-        end
-
         it 'installs package from downloaded tarball' do
           is_expected.to run_bash('install efs utils')
             .with(cwd: source_dir)
diff --git a/cookbooks/aws-parallelcluster-environment/templates/directory_service/update_directory_service_password.sh.erb b/cookbooks/aws-parallelcluster-environment/templates/directory_service/update_directory_service_password.sh.erb
@@ -27,9 +27,8 @@ echo "[INFO] Reading password from AWS: ${SECRET_ARN}"
 service=$(echo "${SECRET_ARN}" | cut -d ':' -f 3)
 resource=$(echo "${SECRET_ARN}" | cut -d ':' -f 6-)
 if [ "$service" == "secretsmanager" ]; then
-  secret_name=$(echo "$resource" | cut -d ':' -f 2)
-  echo "[INFO] Reading password as a secret from AWS Secrets Manager: ${secret_name}"
-  password_from_secret_store=$(aws secretsmanager get-secret-value --secret-id "${secret_name}" --region "${REGION}" --query "SecretString" --output text)
+  echo "[INFO] Reading password as a secret from AWS Secrets Manager: ${SECRET_ARN}"
+  password_from_secret_store=$(aws secretsmanager get-secret-value --secret-id "${SECRET_ARN}" --region "${REGION}" --query "SecretString" --output text)
 elif [ "$service" == "ssm" ]; then
   parameter_name=$(echo "$resource" | cut -d '/' -f 2)
   echo "[INFO] Reading password as a parameter from AWS SSM: ${SECRET_ARN}"
diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb
@@ -4,9 +4,9 @@
 
   only_if { !os_properties.redhat_on_docker? }
 
-  describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do
+  describe file("#{node['cluster']['sources_dir']}/efs-utils-2.3.1.tar.gz") do
     it { should exist }
-    its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' }
+    its('sha256sum') { should eq 'ced12f82e76f9740476b63f30c49bd76cc00b6375e12a9f5f7ba852635c49e15' }
     its('owner') { should eq 'root' }
     its('group') { should eq 'root' }
     its('mode') { should cmp '0644' }
diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/nfs_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/nfs_spec.rb
@@ -1,21 +1,22 @@
-control 'tag:install_nfs_installed_with_right_version' do
-  title 'Check NFS process is running and installed version'
-
-  only_if { !os_properties.on_docker? }
-
-  # Check nfsd process is running
-  describe command('ps aux') do
-    its('stdout') { should match(/nfsd/) }
-  end
-
-  # Check version of NFS
-  describe "Verify installed NFS version is 4\n" do
-    nfs_version = command("rpcinfo -p localhost | awk '{print $5$2}' | grep nfs4")
-    describe nfs_version do
-      its('stdout') { should match "nfs4" }
-    end
-  end
-end
+# FIXME: Re-enabled the following check and fix failures
+# control 'tag:install_nfs_installed_with_right_version' do
+#   title 'Check NFS process is running and installed version'
+#
+#   only_if { !os_properties.on_docker? }
+#
+#   # Check nfsd process is running
+#   describe command('ps aux') do
+#     its('stdout') { should match(/nfsd/) }
+#   end
+#
+#   # Check version of NFS
+#   describe "Verify installed NFS version is 4\n" do
+#     nfs_version = command("rpcinfo -p localhost | awk '{print $5$2}' | grep nfs4")
+#     describe nfs_version do
+#       its('stdout') { should match "nfs4" }
+#     end
+#   end
+# end
 
 control 'tag:config_nfs_configured_on_head_node' do
   title 'Check that nfs is configured correctly'
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_alinux2023.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_alinux2023.rb
@@ -33,5 +33,6 @@ def compiler_path
 end
 
 def extra_packages
-  %w(kernel-modules-extra kernel-modules-extra-common)
+  kernel_version = node['kernel']['release'].chomp('.x86_64').chomp('.aarch64')
+  ["kernel-modules-extra-#{kernel_version}", "kernel-modules-extra-common-#{kernel_version}"]
 end
diff --git a/cookbooks/aws-parallelcluster-slurm/attributes/versions.rb b/cookbooks/aws-parallelcluster-slurm/attributes/versions.rb
@@ -1,8 +1,8 @@
 # Slurm
-default['cluster']['slurm']['version'] = '24-05-7-1'
+default['cluster']['slurm']['version'] = '24-05-8-1'
 default['cluster']['slurm']['commit'] = ''
 default['cluster']['slurm']['branch'] = ''
-default['cluster']['slurm']['sha256'] = '297e85853314a0a4d227ca66bb44179c099f0de5d86e83ffe21cb464b9ad3709'
+default['cluster']['slurm']['sha256'] = '06f4b837c3f7ec523a8924a5fee23d8305f20ca8ef3ad6128e128e87986821e4'
 default['cluster']['slurm']['base_url'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/slurm"
 # Munge
 default['cluster']['munge']['munge_version'] = '0.5.16'
diff --git a/cookbooks/aws-parallelcluster-slurm/recipes/install/install_pyxis.rb b/cookbooks/aws-parallelcluster-slurm/recipes/install/install_pyxis.rb
@@ -15,7 +15,7 @@
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
 
-return unless nvidia_installed?
+return unless nvidia_enabled? || nvidia_installed?
 return if pyxis_installed?
 
 pyxis_version = node['cluster']['pyxis']['version']
diff --git a/cookbooks/aws-parallelcluster-slurm/spec/unit/recipes/install_pyxis_spec.rb b/cookbooks/aws-parallelcluster-slurm/spec/unit/recipes/install_pyxis_spec.rb
@@ -33,6 +33,7 @@
           node.override['cluster']['pyxis']['version'] = pyxis_version
           node.override['cluster']['pyxis']['runtime_path'] = pyxis_runtime_dir
         end
+        allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true)
         allow_any_instance_of(Object).to receive(:nvidia_installed?).and_return(true)
         allow_any_instance_of(Object).to receive(:pyxis_installed?).and_return(false)
         runner.converge(described_recipe)
@@ -93,6 +94,7 @@
           runner = runner(platform: platform, version: version) do |_node|
             RSpec::Mocks.configuration.allow_message_expectations_on_nil = true
           end
+          allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true)
           allow_any_instance_of(Object).to receive(:nvidia_installed?).and_return(true)
           allow_any_instance_of(Object).to receive(:pyxis_installed?).and_return(true)
           runner.converge(described_recipe)
diff --git a/cookbooks/aws-parallelcluster-slurm/test/controls/munge_spec.rb b/cookbooks/aws-parallelcluster-slurm/test/controls/munge_spec.rb
@@ -47,11 +47,6 @@
     it { should exist }
     it { should be_directory }
   end
-
-  describe file('/var/run/munge') do
-    it { should exist }
-    it { should be_directory }
-  end
 end unless os_properties.redhat_on_docker?
 
 control 'tag:config_munge_service_enabled' do
diff --git a/cookbooks/aws-parallelcluster-slurm/test/controls/pyxis_spec.rb b/cookbooks/aws-parallelcluster-slurm/test/controls/pyxis_spec.rb
@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 control 'tag:install_pyxis_installed' do
-  only_if { instance.nvidia_installed? }
+  only_if { ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled']) || instance.nvidia_installed? }
 
   title 'Checks Pyxis has been installed'
 
