DCGM upgrade

Signed-off-by: Hanwen <[email protected]>

Author: hanwen-cluster

CHANGELOG.md

@@ -11,7 +11,15 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 **CHANGES**
 - Ubuntu 20.04 is no longer supported.
-- Upgrade Slurm to version 24.11.5.
+- Upgrade Slurm to version 24.11.6 (from 24.05.8).
+- Upgrade EFA installer to 1.42.0 (from 1.41.0).
+  - Efa-driver: efa-2.15.3-1
+  - Efa-config: efa-config-1.18-1
+  - Efa-profile: efa-profile-1.7-1
+  - Libfabric-aws: libfabric-aws-2.1.0-3
+  - Rdma-core: rdma-core-57.0-1
+  - Open MPI: openmpi40-aws-4.1.7-2 and openmpi50-aws-5.0.6-11
+- Upgrade Cinc Client to version to 18.4.12 from 18.2.7.
 - Addressed cluster id mismatch known issue by deleting the file `/var/spool/slurm.state/clustername` before configuring Slurm accounting.
 - Upgrade DCV to version 2024.0-19030.
 - Remove `berkshelf`. All cookbooks are local and do not need `berkshelf` dependency management.

cookbooks/aws-parallelcluster-platform/attributes/platform.rb

@@ -17,10 +17,10 @@
 # NVidia
 default['cluster']['nvidia']['enabled'] = 'no'
 default['cluster']['nvidia']['driver_version'] = '570.172.08'
-default['cluster']['nvidia']['dcgm_version'] = '4-cuda12-4.2.3-2.'
+default['cluster']['nvidia']['dcgm_version'] = '4.2.3-2'
 if platform?('amazon') && node['platform_version'] == "2"
   default['cluster']['nvidia']['driver_version'] = '550.127.08'
-  default['cluster']['nvidia']['dcgm_version'] = '3.3.6-1-'
+  default['cluster']['nvidia']['dcgm_version'] = '3.3.6-1'
 end
 
 # DCV

cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/partial/_nvidia_dcgm_debian.rb

@@ -13,34 +13,45 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 action :install_package do
-  remote_file "#{node['cluster']['sources_dir']}/#{dcgm_package}-#{package_version}.deb" do
-    source "#{dcgm_url}"
-    mode '0644'
-    retries 3
-    retry_delay 5
-    action :create_if_missing
+  if package_version.start_with?("3.")
+    packages_urls_list = [dcgm_package]
+  else
+    packages_urls_list = [dcgm4_core_package, dcgm4_package]
   end
+  for package in packages_urls_list
+    remote_file "#{node['cluster']['sources_dir']}/#{package}-#{package_version}.deb" do
+      source "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_dcgm/#{platform}/#{package}_#{package_version}_#{arch_suffix}.deb"
+      mode '0644'
+      retries 3
+      retry_delay 5
+      action :create_if_missing
+    end
 
-  bash "Install #{dcgm_package}" do
-    user 'root'
-    cwd node['cluster']['sources_dir']
-    code <<-DCGM_INSTALL
-    set -e
-    dpkg -i #{dcgm_package}-#{package_version}.deb
-    DCGM_INSTALL
-    retries 3
-    retry_delay 5
+    bash "Install #{package}" do
+      user 'root'
+      cwd node['cluster']['sources_dir']
+      code <<-DCGM_INSTALL
+      set -e
+      dpkg -i #{package}-#{package_version}.deb
+      DCGM_INSTALL
+      retries 3
+      retry_delay 5
+    end
   end
 end
 
-def dcgm_url
-  "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_dcgm/#{platform}/#{dcgm_package}_#{package_version}_#{arch_suffix}.deb"
-end
-
 def dcgm_package
   'datacenter-gpu-manager'
 end
 
+def dcgm4_package
+  "#{dcgm_package}-4-cuda12"
+end
+
+def dcgm4_core_package
+  "#{dcgm_package}-4-core"
+end
+
 def arch_suffix
   arm_instance? ? 'arm64' : 'amd64'
 end

cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/partial/_nvidia_dcgm_rhel.rb

@@ -13,34 +13,47 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 action :install_package do
-  remote_file "#{node['cluster']['sources_dir']}/#{dcgm_package}-#{package_version}.rpm" do
-    source "#{dcgm_url}"
-    mode '0644'
-    retries 3
-    retry_delay 5
-    action :create_if_missing
+  if package_version.start_with?("3.")
+    packages_urls_list = [dcgm_package]
+    package_url_separator = "-"
+  else
+    packages_urls_list = [dcgm4_core_package, dcgm4_package]
+    package_url_separator = "."
   end
+  for package in packages_urls_list
+    remote_file "#{node['cluster']['sources_dir']}/#{package}-#{package_version}.rpm" do
+      source "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_dcgm/#{platform}/#{package}-#{package_version}#{package_url_separator}#{arch_suffix}.rpm"
+      mode '0644'
+      retries 3
+      retry_delay 5
+      action :create_if_missing
+    end
 
-  bash "Install #{dcgm_package}" do
-    user 'root'
-    cwd node['cluster']['sources_dir']
-    code <<-DCGM_INSTALL
-    set -e
-    yum install -y #{dcgm_package}-#{package_version}.rpm
-    DCGM_INSTALL
-    retries 3
-    retry_delay 5
+    bash "Install #{package}" do
+      user 'root'
+      cwd node['cluster']['sources_dir']
+      code <<-DCGM_INSTALL
+      set -e
+      yum install -y #{package}-#{package_version}.rpm
+      DCGM_INSTALL
+      retries 3
+      retry_delay 5
+    end
   end
 end
 
-def dcgm_url
-  "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_dcgm/#{platform}/#{dcgm_package}-#{package_version}#{arch_suffix}.rpm"
-end
-
 def dcgm_package
   'datacenter-gpu-manager'
 end
 
+def dcgm4_package
+  "#{dcgm_package}-4-cuda12"
+end
+
+def dcgm4_core_package
+  "#{dcgm_package}-4-core"
+end
+
 def arch_suffix
   arm_instance? ? 'aarch64' : 'x86_64'
 end

cookbooks/aws-parallelcluster-platform/test/controls/nvidia_dcgm_spec.rb

@@ -15,7 +15,7 @@
       (!os_properties.arm? || !(os_properties.alinux2? || os_properties.centos?))
   end
 
-  describe package('datacenter-gpu-manager') do
+  describe package('datacenter-gpu-manager-4-cuda12') do
     it { should be_installed }
   end
 end