[B200] Debugging

Author: gmarciani

cookbooks/aws-parallelcluster-platform/recipes/install.rb

@@ -14,33 +14,34 @@
 include_recipe "aws-parallelcluster-platform::sudo_install"
 include_recipe "aws-parallelcluster-platform::users"
 include_recipe "aws-parallelcluster-platform::disable_services"
-bash "Check kernel before repo setup" do
-  code <<-CODE
-    set -x
-    uname -a
-    yum info infiniband-diags libibumad 
-    yum provides infiniband-diags libibumad
-    CODE
-end
+# bash "Check kernel before repo setup" do
+#   code <<-CODE
+#     set -x
+#     uname -a
+#     yum info infiniband-diags libibumad
+#     yum provides infiniband-diags libibumad
+#     CODE
+# end
 package_repos 'setup the repositories'
-bash "Check kernel after repo setup" do
-  code <<-CODE
-    set -x
-    uname -a
-    yum info infiniband-diags libibumad 
-    yum provides infiniband-diags libibumad
-    CODE
-end
+# bash "Check kernel after repo setup" do
+#   code <<-CODE
+#     set -x
+#     uname -a
+#     yum info infiniband-diags libibumad
+#     yum provides infiniband-diags libibumad
+#     CODE
+# end
 include_recipe "aws-parallelcluster-platform::directories"
+# nvidia_nvlsm 'Install Nvidia NVLink Subnet Manager'
 install_packages 'Install OS and extra packages'
-bash "Check kernel after extra packages" do
-  code <<-CODE
-    set -x
-    uname -a
-    yum info infiniband-diags libibumad 
-    yum provides infiniband-diags libibumad
-    CODE
-end
+# bash "Check kernel after extra packages" do
+#   code <<-CODE
+#     set -x
+#     uname -a
+#     yum info infiniband-diags libibumad
+#     yum provides infiniband-diags libibumad
+#     CODE
+# end
 include_recipe "aws-parallelcluster-platform::cookbook_virtualenv"
 include_recipe "aws-parallelcluster-platform::awscli"
 unless alinux2023_on_docker? # Running this recipe on Alinux 2023 docker generates false failure.

cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_common.rb

@@ -29,7 +29,7 @@
     user 'root'
     cwd node['cluster']['sources_dir']
     code <<-CODE
-    set -e
+    set -ex
     #{nvidia_nvlsm_install_preconditions_commands}
     CODE
     retries 3
@@ -38,10 +38,20 @@
 end
 
 action :install_nvlsm_dependencies do
-  package nvidia_nvlsm_dependencies do
+  bash "Install nvlsm dependencies" do
+    user 'root'
+    code <<-CODE
+    set -ex
+    #{nvidia_nvlsm_install_dependencies_commands}
+    CODE
     retries 3
     retry_delay 5
   end
+  # package nvidia_nvlsm_dependencies do
+  #   options '--verbose'
+  #   retries 3
+  #   retry_delay 5
+  # end
 
   # Make sure kernel module for Infiniband is loaded at instance boot time
   cookbook_file 'infiniband.conf' do
@@ -67,7 +77,7 @@
     user 'root'
     cwd node['cluster']['sources_dir']
     code <<-CODE
-    set -e
+    set -ex
     #{nvidia_nvlsm_install_commands}
     CODE
     retries 3
@@ -103,6 +113,10 @@ def nvidia_nvlsm_install_preconditions_commands
   # OS dependent
 end
 
+def nvidia_nvlsm_install_dependencies_commands
+  # OS dependent
+end
+
 def nvidia_nvlsm_dependencies
   # OS dependent
 end

cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_debian.rb

@@ -40,3 +40,7 @@ def nvidia_nvlsm_dependencies
 def nvidia_nvlsm_install_preconditions_commands
   "uname -a ; apt-cache policy ; apt-mark showhold"
 end
+
+def nvidia_nvlsm_install_dependencies_commands
+  "apt -o Debug::pkgProblemResolver=1 install -y infiniband-diags ibutils"
+end

cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_rhel.rb

@@ -39,3 +39,7 @@ def nvidia_nvlsm_dependencies
 def nvidia_nvlsm_install_preconditions_commands
   "uname -a ; yum repolist all ; yum versionlock list ; yum info kernel-modules-extra-aws infiniband-diags libibumad ; yum provides kernel-modules-extra-aws infiniband-diags libibumad"
 end
+
+def nvidia_nvlsm_install_dependencies_commands
+  "yum install -yv infiniband-diags libibumad"
+end

cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_nvlsm_spec.rb

@@ -144,9 +144,9 @@ def self.install(chef_run)
 
           cached(:nvlsm_installation_commands) do
             if %(redhat rocky amazon).include?(platform)
-              "    set -e\n    yum install -y #{nvlsm_package_full_name} && yum versionlock nvlsm\n"
+              "    set -ex\n    yum install -y #{nvlsm_package_full_name} && yum versionlock nvlsm\n"
             else
-              "    set -e\n    dpkg -i #{nvlsm_package_full_name} && apt-mark hold nvlsm\n"
+              "    set -ex\n    dpkg -i #{nvlsm_package_full_name} && apt-mark hold nvlsm\n"
             end
           end
           cached(:nvlsm_dependencies) do
@@ -156,11 +156,24 @@ def self.install(chef_run)
               %(infiniband-diags ibutils)
             end
           end
+          cached(:nvlsm_dependencies_installation_commands) do
+            if %(redhat rocky amazon).include?(platform)
+              "    set -ex\n    yum install -yv #{nvlsm_dependencies}\n"
+            else
+              "    set -ex\n    apt -o Debug::pkgProblemResolver=1 install -y #{nvlsm_dependencies}\n"
+            end
+          end
 
           it 'installs dependencies of nvlsm' do
-            is_expected.to install_package(nvlsm_dependencies).with(
+            # is_expected.to install_package(nvlsm_dependencies).with(
+            #   retries: 3,
+            #   retry_delay: 5
+            # )
+            is_expected.to run_bash("Install nvlsm dependencies").with(
+              user: 'root',
               retries: 3,
-              retry_delay: 5
+              retry_delay: 5,
+              code: nvlsm_dependencies_installation_commands
             )
           end