[Isolated] Install new node pypi dependencies and move efs-proxy installation

Author: hgreebe

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -25,9 +25,16 @@
   command = "pip install ."
 end
 
-if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2"
+if aws_region.start_with?("us-iso")
+  dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
+  dependency_folder_name = dependency_package_name
+  if platform?('amazon') && node['platform_version'] == "2"
+    dependency_package_name = "node-dependencies"
+    dependency_folder_name = "node"
+  end
+
   remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
-    source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
+    source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
     mode '0644'
     retries 3
     retry_delay 5
@@ -41,7 +48,7 @@
     code <<-REQ
       set -e
       tar xzf node-dependencies.tgz
-      cd node
+      cd #{dependency_folder_name}
       #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
       REQ
   end

cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb

@@ -19,7 +19,7 @@
 use 'partial/_get_package_version_rpm'
 use 'partial/_common'
 # use 'partial/_redhat_based'
-use 'partial/_install_from_tar'
+# use 'partial/_install_from_tar'
 use 'partial/_mount_umount'
 
 def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
@@ -39,3 +39,59 @@ def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
 def prerequisites
   %w(rpm-build make rust cargo openssl-devel)
 end
+
+action :install_utils do
+  package_repos 'update package repositories' do
+    action :update
+  end
+
+  package prerequisites do
+    retries 3
+    retry_delay 5
+  end
+
+  directory node['cluster']['sources_dir'] do
+    recursive true
+  end
+
+  return if redhat_on_docker?
+
+  package_name = "amazon-efs-utils"
+  package_version = new_resource.efs_utils_version
+  efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
+  efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
+
+  # Do not install efs-utils if a same or newer version is already installed.
+  return if already_installed?(package_name, package_version)
+
+  # On all OSes but Amazon Linux 2, amazon-efs-utils and stunnel are installed from source,
+  # because their OS repos do not have amazon-efs-utils and new stunnel
+
+  # Get EFS Utils tarball
+  remote_file efs_utils_tarball do
+    source efs_utils_url
+    mode '0644'
+    retries 3
+    retry_delay 5
+    checksum new_resource.efs_utils_checksum
+    action :create_if_missing
+  end
+
+  efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz"
+  efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}"
+  efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}"
+  remote_file efs_proxy_deps_tarball do
+    source efs_proxy_deps_url
+    mode '0644'
+    retries 3
+    retry_delay 5
+    action :create_if_missing
+  end
+
+  # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
+  bash "install efs utils" do
+    cwd node['cluster']['sources_dir']
+    code install_script_code(efs_utils_tarball, package_name, package_version)
+  end
+  action_increase_poll_interval
+end

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb

@@ -50,20 +50,6 @@
     action :create_if_missing
   end
 
-  if aws_region.start_with?("us-iso") && platform?('redhat') && node['platform_version'] == "8"
-    efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz"
-    efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}"
-    efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}"
-    remote_file efs_proxy_deps_tarball do
-      source efs_proxy_deps_url
-      mode '0644'
-      retries 3
-      retry_delay 5
-      checksum new_resource.efs_utils_checksum
-      action :create_if_missing
-    end
-  end
-
   # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
   bash "install efs utils" do
     cwd node['cluster']['sources_dir']

cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb

@@ -33,22 +33,24 @@
   not_if { ::File.exist?("#{cookbook_virtualenv_path}/bin/activate") }
 end
 
-remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
-  source pypi_s3_uri
-  mode '0644'
-  retries 3
-  retry_delay 5
-  action :create_if_missing
-end
+if aws_region.start_with?("us-iso")
+  remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
+    source pypi_s3_uri
+    mode '0644'
+    retries 3
+    retry_delay 5
+    action :create_if_missing
+  end
 
-bash 'pip install' do
-  user 'root'
-  group 'root'
-  cwd "#{node['cluster']['base_dir']}"
-  code <<-REQ
+  bash 'pip install' do
+    user 'root'
+    group 'root'
+    cwd "#{node['cluster']['base_dir']}"
+    code <<-REQ
     set -e
     tar xzf cookbook-dependencies.tgz
     cd #{dependency_package_name}
     #{virtualenv_path}/bin/pip install * -f ./ --no-index
     REQ
+  end
 end