Fix unit tests

hgreebe · hgreebe · commit 27b1519e1a6d · 2025-02-19T09:25:34.000-05:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,10 +2,12 @@ aws-parallelcluster-cookbook CHANGELOG
 ======================================
 
 This file is used to list changes made in each version of the AWS ParallelCluster cookbook.
+
 3.13.0
 ------
 
 **CHANGES**
+- Upgrade Python to 3.12.8 (from 3.9.20).
 - On Ubuntu 22.04, install the Nvidia driver with the same compiler version used to compile the kernel.
 - Upgrade `aws-cfn-bootstrap` to version 2.0-32.
 - Upgrade amazon-efs-utils to version 2.1.0.
@@ -16,6 +18,10 @@ This file is used to list changes made in each version of the AWS ParallelCluste
   - openssh-2.11.14 (from openssh-2.11.12)
   - yum-7.4.20 (from yum-7.4.13)
   - yum-epel-5.0.8 (from yum-epel-5.0.2)
+- Upgrade Pmix to 5.0.6 (from 5.0.3).
+- Upgrade ARM PL to version 24.10 (from 23.10).
+- Remove generation of DSA keys for login nodes as DSA, which became unsupported in OpenSSH 9.7+.
+- Set instance ID and instance type information in Slurm upon compute nodes launch.
 
 3.12.0
 ------
diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt
diff --git a/cookbooks/aws-parallelcluster-environment/files/imds/imds-access.sh b/cookbooks/aws-parallelcluster-environment/files/imds/imds-access.sh
@@ -32,6 +32,24 @@ function help() {
   --unset <user1,...,userN>   Remove iptables rules related to IMDS for the given list of users
   --flush                     Restore default IMDS access
   --help                      Print this help message
+
+  Examples:
+
+    Allow access for USER1 and USER2, without changing other users
+      ${cmd} --allow USER1,USER2
+
+    Deny access for USER1 and USER2, without changing other users
+      ${cmd} --deny USER1,USER2
+
+    Allow access only for USER1 and USER2, removing every other user from the allowlist
+      ${cmd} --flush
+      ${cmd} --allow USER1,USER2
+
+    Remove rules for USER1 and USER2, without changing other users
+      ${cmd} --unset USER1,USER2
+
+    Remove IMDS lockdown, so that all users will be able to access IMDS
+      ${cmd} --flush
 EOF
 }
 
diff --git a/cookbooks/aws-parallelcluster-environment/files/login_nodes/keys-manager.sh b/cookbooks/aws-parallelcluster-environment/files/login_nodes/keys-manager.sh
@@ -38,9 +38,6 @@ function create_keys() {
   ssh-keygen -t ecdsa -f "$FOLDER_PATH/ssh_host_ecdsa_key" -q -P ""
   ssh-keygen -t ed25519 -f "$FOLDER_PATH/ssh_host_ed25519_key" -q -P ""
   ssh-keygen -t rsa -f "$FOLDER_PATH/ssh_host_rsa_key" -q -P ""
-  if is_ubuntu; then
-    ssh-keygen -t dsa -f "$FOLDER_PATH/ssh_host_dsa_key" -q -P ""
-  fi
 }
 
 function import_keys() {
@@ -50,7 +47,6 @@ function import_keys() {
   cp "$FOLDER_PATH/ssh_host_ed25519"* /etc/ssh/
   cp "$FOLDER_PATH/ssh_host_rsa"* /etc/ssh/
   if is_ubuntu; then
-    cp "$FOLDER_PATH/ssh_host_dsa"* /etc/ssh/
     chown root:root /etc/ssh/ssh_host_*
     chmod 600 /etc/ssh/ssh_host_*_key
   else
diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb
@@ -5,7 +5,7 @@
     context "on #{platform}#{version}" do
       cached(:cfnbootstrap_version) { '2.0-32' }
       cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" }
-      cached(:python_version) { '3.12.8' }
+      cached(:python_version) { "#{node['cluster']['python-version']}" }
       cached(:system_pyenv_root) { 'system_pyenv_root' }
       cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" }
       cached(:timeout) { 1800 }
@@ -32,14 +32,6 @@
           )
         end
 
-        it 'installs python packages' do
-          is_expected.to run_bash("pip install").with(
-            user: 'root',
-            group: 'root',
-            cwd: "#{node['cluster']['base_dir']}"
-          )
-        end
-
         it 'sets virtualenv path' do
           expect(node.default['cluster']['cfn_bootstrap_virtualenv_path']).to eq(virtualenv_path)
           is_expected.to write_node_attributes('dump node attributes')
diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/login_nodes_keys_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/login_nodes_keys_spec.rb
@@ -11,9 +11,6 @@
 
 key_types = %w(ecdsa ed25519 rsa)
 is_ubuntu = os_properties.ubuntu?
-if is_ubuntu
-  key_types << 'dsa'
-end
 
 control 'head_node_directory_initialized' do
   only_if { instance.head_node? && node['cluster']['scheduler'] != 'awsbatch' }
diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb
@@ -12,6 +12,10 @@
 # limitations under the License.
 
 virtualenv_path = cookbook_virtualenv_path
+pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz"
+if platform?('amazon') && node['platform_version'] == "2"
+  pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
+end
 
 node.default['cluster']['cookbook_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"
@@ -28,7 +32,7 @@
 end
 
 remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
+  source pypi_s3_uri
   mode '0644'
   retries 3
   retry_delay 5
@@ -43,8 +47,9 @@
 bash 'pip install' do
   user 'root'
   group 'root'
+  cwd "#{node['cluster']['base_dir']}"
   code <<-REQ
-    #{virtualenv_path}/bin/pip install -r #{cookbook_virtualenv_path}/requirements.txt
+    #{virtualenv_path}/bin/pip install -r fail-pls
     if [ $? -ne 0 ]; then
       tar xzf cookbook-dependencies.tgz
       cd dependencies
diff --git a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/partial/_arm_pl_common.rb b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/partial/_arm_pl_common.rb
@@ -27,13 +27,9 @@
 # We upload ArmPL to a ParallelCluster bucket (account for it in scope of the upgrade) and download it from there
 # to install ArmPL on the AMI.
 # We download gcc directly from gnu.org repository to install correct gcc version on the AMI.
-property :armpl_major_minor_version, String, default: '23.10'
+property :armpl_major_minor_version, String, default: '24.10'
 property :gcc_patch_version, String, default: '0'
 
-action :arm_pl_prerequisite do
-  # Do nothing
-end
-
 action :setup do
   return unless node['conditions']['arm_pl_supported']
 
@@ -49,10 +45,8 @@
   build_tools 'Prerequisite: build tools'
   package %w(wget bzip2)
 
-  action_arm_pl_prerequisite
-
   armpl_version = "#{new_resource.armpl_major_minor_version}"
-  armpl_tarball_name = "arm-performance-libraries_#{armpl_version}_#{armpl_platform}_gcc-#{gcc_major_minor_version}.tar"
+  armpl_tarball_name = "arm-performance-libraries_#{armpl_version}_#{package_manager}_gcc.tar"
 
   armpl_url = %W(
     #{node['cluster']['artifacts_s3_url']}
@@ -62,7 +56,7 @@
 
   armpl_installer = "#{new_resource.sources_dir}/#{armpl_tarball_name}"
 
-  armpl_name = "arm-performance-libraries_#{armpl_version}_#{armpl_platform}"
+  armpl_name = "arm-performance-libraries_#{armpl_version}_#{package_manager}"
 
   # download ArmPL tarball
   remote_file armpl_installer do
@@ -89,11 +83,7 @@
   # create armpl module directory
   directory "#{modulefile_dir}/armpl"
 
-  armpl_license_dir = if new_resource.armpl_major_minor_version == "21.0"
-                        "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{new_resource.armpl_major_minor_version}_gcc-#{gcc_major_minor_version}/license_terms"
-                      else
-                        "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{armpl_version}_gcc-#{gcc_major_minor_version}/license_terms"
-                      end
+  armpl_license_dir = "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{armpl_version}_gcc/license_terms"
 
   # arm performance library modulefile configuration
   template "#{modulefile_dir}/armpl/#{armpl_version}" do
@@ -104,7 +94,6 @@
     mode '0755'
     variables(
       armpl_version: armpl_version,
-      armpl_major_minor_version: new_resource.armpl_major_minor_version,
       armpl_license_dir: armpl_license_dir,
       gcc_major_minor_version: gcc_major_minor_version
     )
@@ -167,7 +156,6 @@
   # save ArmPL and gcc versions on the node environment so that they will be available
   # to dependencies (for instance, test code)
   # Complete versions are intentionally redundant.
-  node.default['cluster']['armpl']['major_minor_version'] = new_resource.armpl_major_minor_version
   node.default['cluster']['armpl']['version'] = armpl_version
   node.default['cluster']['armpl']['gcc']['major_minor_version'] = gcc_major_minor_version
   node.default['cluster']['armpl']['gcc']['patch_version'] = new_resource.gcc_patch_version
@@ -180,4 +168,12 @@
   def modulefile_dir
     '/usr/share/Modules/modulefiles'
   end
+
+  def package_manager
+    if platform_family?('debian')
+      'deb'
+    else
+      'rpm'
+    end
+  end
 end
diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb
@@ -15,7 +15,7 @@
         it 'downloads awscli-bundle from s3' do
           is_expected.to create_remote_file('download awscli bundle from s3').with(
             path: "#{file_cache_path}/awscli-bundle.zip",
-            source: 'https://s3.amazonaws.com/aws-cli/awscli-bundle.zip',
+            source: "https://awscli.test_aws_domain/awscli-exe-linux-#{node['kernel']['machine']}.zip",
             retries: 5,
             retry_delay: 5
           )
@@ -31,7 +31,7 @@
 
         it 'installs awscli into cookbook virtualev path' do
           is_expected.to run_bash('install awscli')
-            .with_code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws"
+            .with_code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
         end
       end
 
diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb
@@ -16,7 +16,7 @@ def self.setup(chef_run)
       cached(:aws_region) { 'test_region' }
       cached(:aws_domain) { 'test_domain' }
       cached(:armpl_major_minor_version) do
-        '23.10'
+        '24.10'
       end
 
       cached(:armpl_platform) do
@@ -43,11 +43,12 @@ def self.setup(chef_run)
       cached(:gcc_patch_version) { '0' }
       cached(:sources_dir) { 'sources_test_dir' }
       cached(:modulefile_dir) { platform == 'ubuntu' ? '/usr/share/modules/modulefiles' : '/usr/share/Modules/modulefiles' }
+      cached(:package_manager) { platform == 'ubuntu' ? 'deb' : 'rpm' }
       cached(:armpl_version) { "#{armpl_major_minor_version}" }
-      cached(:armpl_tarball_name) { "arm-performance-libraries_#{armpl_version}_#{armpl_platform}_gcc-#{gcc_major_minor_version}.tar" }
+      cached(:armpl_tarball_name) { "arm-performance-libraries_#{armpl_version}_#{package_manager}_gcc.tar" }
       cached(:armpl_url) { "https://bucket.s3.amazonaws.com/archives/armpl/#{armpl_platform}/#{armpl_tarball_name}" }
       cached(:armpl_installer) { "#{sources_dir}/#{armpl_tarball_name}" }
-      cached(:armpl_name) { "arm-performance-libraries_#{armpl_version}_#{armpl_platform}" }
+      cached(:armpl_name) { "arm-performance-libraries_#{armpl_version}_#{package_manager}" }
       cached(:gcc_version) { "#{gcc_major_minor_version}.#{gcc_patch_version}" }
       cached(:gcc_url) { "https://bucket.s3.amazonaws.com/archives/dependencies/gcc/gcc-#{gcc_version}.tar.gz" }
       cached(:gcc_tarball) { "#{sources_dir}/gcc-#{gcc_version}.tar.gz" }
@@ -128,19 +129,14 @@ def self.setup(chef_run)
         end
 
         it 'creates arm performance library modulefile configuration' do
-          armpl_license_dir = if armpl_major_minor_version == "21.0"
-                                "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{armpl_major_minor_version}_gcc-#{gcc_major_minor_version}/license_terms"
-                              else
-                                "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{armpl_version}_gcc-#{gcc_major_minor_version}/license_terms"
-                              end
+          armpl_license_dir = "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{armpl_version}_gcc/license_terms"
           is_expected.to create_template("#{modulefile_dir}/armpl/#{armpl_version}").with(
             source: 'arm_pl/armpl_modulefile.erb',
             user: 'root',
             group: 'root',
             mode: '0755',
             variables: {
               armpl_version: armpl_version,
-              armpl_major_minor_version: armpl_major_minor_version,
               armpl_license_dir: armpl_license_dir,
               gcc_major_minor_version: gcc_major_minor_version,
             }
@@ -179,7 +175,6 @@ def self.setup(chef_run)
         end
 
         it 'sets node attributes' do
-          expect(node['cluster']['armpl']['major_minor_version']).to eq(armpl_major_minor_version)
           expect(node['cluster']['armpl']['version']).to eq(armpl_version)
           expect(node['cluster']['armpl']['gcc']['major_minor_version']).to eq(gcc_major_minor_version)
           expect(node['cluster']['armpl']['gcc']['patch_version']).to eq(gcc_patch_version)
diff --git a/cookbooks/aws-parallelcluster-platform/templates/arm_pl/armpl_modulefile.erb b/cookbooks/aws-parallelcluster-platform/templates/arm_pl/armpl_modulefile.erb
@@ -4,7 +4,7 @@
 set category arm
 set name     armpl
 set version  <%= @armpl_version %>
-set major_minor_version  <%= @armpl_major_minor_version %>
+set major_minor_version  <%= @armpl_version %>
 set gcc_version <%= @gcc_major_minor_version %>
 set root     /opt/$category/$name/$version
 
@@ -40,7 +40,7 @@ puts stderr "At compile time add '-I<armpl_include>' and at link time"
   }
 
   # Load the pre-installed armpl module
-  module load $root/modulefiles/armpl/${full_version}_gcc-${gcc_version}
+  module load $root/modulefiles/armpl/${full_version}_gcc
 
   # EULA
   if [ module-info mode load ] {
diff --git a/cookbooks/aws-parallelcluster-platform/test/controls/arm_pl_spec.rb b/cookbooks/aws-parallelcluster-platform/test/controls/arm_pl_spec.rb
@@ -13,21 +13,15 @@
   title "Check ARM Performance libraries installation"
   only_if { os_properties.arm? && !os_properties.on_docker? }
 
-  armpl_major_minor_version = node['cluster']['armpl']['major_minor_version']
   armpl_version = node['cluster']['armpl']['version']
   gcc_major_minor_version = node['cluster']['armpl']['gcc']['major_minor_version']
 
   armpl_module_general_name = "armpl/#{armpl_version}"
-  armpl_module_name = "armpl/#{armpl_version}.0_gcc-#{gcc_major_minor_version}"
+  armpl_module_name = "armpl/#{armpl_version}.0_gcc"
   gcc_module_name = "armpl/gcc-#{gcc_major_minor_version}"
 
-  if os_properties.ubuntu2204?
-    armpl_script_dir = "/opt/arm/#{armpl_module_general_name}/arm-performance-libraries_#{armpl_version}_gcc-#{gcc_major_minor_version}"
-    armpl_install_dir = "/opt/arm/#{armpl_module_general_name}/armpl_#{armpl_version}_gcc-#{gcc_major_minor_version}"
-  else
-    armpl_script_dir = "/opt/arm/#{armpl_module_general_name}/arm-performance-libraries_#{armpl_major_minor_version}_gcc-#{gcc_major_minor_version}"
-    armpl_install_dir = "/opt/arm/#{armpl_module_general_name}/armpl_#{armpl_major_minor_version}_gcc-#{gcc_major_minor_version}"
-  end
+  armpl_script_dir = "/opt/arm/#{armpl_module_general_name}/arm-performance-libraries_#{armpl_version}_gcc"
+  armpl_install_dir = "/opt/arm/#{armpl_module_general_name}/armpl_#{armpl_version}_gcc"
 
   setup = "unset MODULEPATH && source /etc/profile.d/modules.sh"
 
@@ -50,14 +44,14 @@
   scl_centos7 = "scl enable devtoolset-8" if os_properties.centos?
 
   describe bash("#{setup} && module load #{armpl_module_general_name} && "\
-                "cd #{armpl_install_dir}/examples && "\
+                "cd #{armpl_install_dir}/examples_lp64 && "\
                 "make clean && #{scl_centos7} make") do
     its('exit_status') { should eq(0) }
     its('stdout') { should match /testing: no example difference files were generated/i }
     its('stdout') { should match /test passed ok/i }
   end
 
-  describe bash("sudo bash -c 'unset MODULEPATH && source /etc/profile.d/modules.sh && module load armpl && cd #{armpl_install_dir}/examples &&  \
+  describe bash("sudo bash -c 'unset MODULEPATH && source /etc/profile.d/modules.sh && module load armpl && cd #{armpl_install_dir}/examples_lp64 &&  \
     gcc -c -I#{armpl_install_dir}/include #{test_software}.c -o #{test_software}.o && \
     gcc #{test_software}.o -L#{armpl_install_dir}/lib -o #{test_software}.exe -larmpl_lp64 -lm && \
     ./#{test_software}.exe'") do
diff --git a/cookbooks/aws-parallelcluster-shared/resources/install_pyenv.rb b/cookbooks/aws-parallelcluster-shared/resources/install_pyenv.rb
@@ -14,7 +14,7 @@
 
 action :run do
   python_version = new_resource.python_version || node['cluster']['python-version']
-  python_url =  "https://www.python.org/ftp/python/#{python_version}/Python-#{python_version}.tgz"
+  python_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/python/Python-#{python_version}.tgz"
 
   if !aws_region.start_with?("us-iso") && new_resource.python_version
     python_url = "https://www.python.org/ftp/python/#{python_version}/Python-#{python_version}.tgz"
diff --git a/cookbooks/aws-parallelcluster-slurm/attributes/slurm_attributes.rb b/cookbooks/aws-parallelcluster-slurm/attributes/slurm_attributes.rb
@@ -13,8 +13,8 @@
 default['cluster']['enable_nss_slurm'] = node['cluster']['directory_service']['enabled']
 
 # PMIX Version and Checksum
-default['cluster']['pmix']['version'] = '5.0.3'
-default['cluster']['pmix']['sha256'] = 'd9b0b81ba45f58f44573796eb05858061ed7f47ea6b29af8058090023e35dfa9'
+default['cluster']['pmix']['version'] = '5.0.6'
+default['cluster']['pmix']['sha256'] = '5a5e0cd36067144e2171d59164d59ea478a2e540ccf4eee4530f55fc6e8cf78b'
 
 # Slurmdbd
 default['cluster']['slurmdbd_service_enabled'] = "true"
diff --git a/cookbooks/aws-parallelcluster-slurm/attributes/versions.rb b/cookbooks/aws-parallelcluster-slurm/attributes/versions.rb
@@ -1,8 +1,8 @@
 # Slurm
-default['cluster']['slurm']['version'] = '23-11-10-1'
+default['cluster']['slurm']['version'] = '24-05-2-1'
 default['cluster']['slurm']['commit'] = ''
 default['cluster']['slurm']['branch'] = ''
-default['cluster']['slurm']['sha256'] = 'ca880fcd44a9e0303a36e05ede75913d16f254d88d4ef05595cd135a29aa3071'
+default['cluster']['slurm']['sha256'] = '21aa526f20e8c84e46a34b8f31c8f9500395e5e828f5189f38aca3836e5270a1'
 default['cluster']['slurm']['base_url'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/slurm"
 # Munge
 default['cluster']['munge']['munge_version'] = '0.5.16'
diff --git a/cookbooks/aws-parallelcluster-slurm/spec/unit/recipes/config_slurmd_systemd_service_spec.rb b/cookbooks/aws-parallelcluster-slurm/spec/unit/recipes/config_slurmd_systemd_service_spec.rb
@@ -21,7 +21,11 @@
   for_all_oses do |platform, version|
     context "on #{platform}#{version}" do
       cached(:chef_run) do
-        runner(platform: platform, version: version).converge(described_recipe)
+        runner = runner(platform: platform, version: version) do |node|
+          node.override['ec2']['instance_id'] = "i-xxx"
+          node.override['ec2']['instance_type'] = "fake-instance-type"
+        end
+        runner.converge(described_recipe)
       end
 
       it 'creates the service definition for slurmd' do
diff --git a/cookbooks/aws-parallelcluster-slurm/templates/default/slurm/compute/slurmd.service.erb b/cookbooks/aws-parallelcluster-slurm/templates/default/slurm/compute/slurmd.service.erb
@@ -7,7 +7,7 @@ ConditionPathExists=<%= node['cluster']['slurm']['install_dir'] %>/etc/slurm.con
 [Service]
 Type=simple
 EnvironmentFile=-/etc/sysconfig/slurmd
-ExecStart=<%= node['cluster']['slurm']['install_dir'] %>/sbin/slurmd -D -s $SLURMD_OPTIONS
+ExecStart=<%= node['cluster']['slurm']['install_dir'] %>/sbin/slurmd -D -s $SLURMD_OPTIONS --instance-id <%= node['ec2']['instance_id'] %> --instance-type <%= node['ec2']['instance_type'] %>
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=process
 LimitNOFILE=131072
