[Isolated] Remove use of new_resource as this will be empty without a default value set

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -19,39 +19,33 @@
 
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
-command = if aws_region.start_with?("us-iso")
-            "pip install . --no-build-isolation"
-          else
-            "pip install ."
-          end
+command = "pip install . --no-build-isolation"
 
-if aws_region.start_with?("us-iso")
-  dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
-  dependency_folder_name = dependency_package_name
-  if platform?('amazon') && node['platform_version'] == "2"
-    dependency_package_name = "node-dependencies"
-    dependency_folder_name = "node"
-  end
+dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
+dependency_folder_name = dependency_package_name
+if platform?('amazon') && node['platform_version'] == "2"
+  dependency_package_name = "node-dependencies"
+  dependency_folder_name = "node"
+end
 
-  remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
-    source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
-    mode '0644'
-    retries 3
-    retry_delay 5
-    action :create_if_missing
-  end
+remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
 
-  bash 'pip install' do
-    user 'root'
-    group 'root'
-    cwd "#{node['cluster']['base_dir']}"
-    code <<-REQ
-      set -e
-      tar xzf node-dependencies.tgz
-      cd #{dependency_folder_name}
-      #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
-      REQ
-  end
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf node-dependencies.tgz
+    cd #{dependency_folder_name}
+    #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
 end
 
 bash "install custom aws-parallelcluster-node" do

cookbooks/aws-parallelcluster-computefleet/spec/unit/recipes/custom_parallelcluster_node_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+
+describe 'aws-parallelcluster-computefleet::custom_parallelcluster_node' do
+  for_all_oses do |platform, version|
+    context "on #{platform}#{version}" do
+      cached(:s3_url) { 's3://url' }
+      cached(:base_dir) { 'base_dir' }
+      cached(:arch) { 'x86_64' }
+      cached(:region) { 'any-region' }
+      cached(:python_version) { 'python_version' }
+      cached(:dependency_pkg_name_suffix) do
+        if platform == 'amazon' && version == '2'
+          'node-dependencies'
+        else
+          "pypi-node-dependencies-#{python_version}-#{arch}"
+        end
+      end
+      cached(:dependency_folder_name_suffix) do
+        if platform == 'amazon' && version == '2'
+          "node"
+        else
+          dependency_pkg_name_suffix
+        end
+      end
+      cached(:virtualenv_path) { "#{base_dir}/pyenv/versions/#{python_version}/envs/node_virtualenv" }
+      cached(:cookbook_virtualenv_path) { "#{base_dir}/pyenv/versions/#{python_version}/envs/cookbook_virtualenv" }
+      cached(:custom_node_s3_url) { "#{s3_url}/pyenv/versions/#{python_version}/envs/node_virtualenv" }
+      cached(:pip_install_bash_code) do
+        <<-REQ
+  set -e
+  tar xzf node-dependencies.tgz
+  cd #{dependency_folder_name_suffix}
+  #{virtualenv_path}/bin/pip install * -f ./ --no-index
+        REQ
+      end
+      cached(:node_bash_code) do
+        <<-NODE
+  set -e
+  [[ ":$PATH:" != *":/usr/local/bin:"* ]] && PATH="/usr/local/bin:${PATH}"
+  echo "PATH is $PATH"
+  source #{virtualenv_path}/bin/activate
+  pip uninstall --yes aws-parallelcluster-node
+  if [[ "#{custom_node_s3_url}" =~ ^s3:// ]]; then
+    custom_package_url=$(#{cookbook_virtualenv_path}/bin/aws s3 presign #{custom_node_s3_url} --region #{region})
+  else
+    custom_package_url=#{custom_node_s3_url}
+  fi
+  curl --retry 3 -L -o aws-parallelcluster-node.tgz ${custom_package_url}
+  rm -fr aws-parallelcluster-custom-node
+  mkdir aws-parallelcluster-custom-node
+  tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node
+  cd aws-parallelcluster-custom-node/*aws-parallelcluster-node*
+  pip install . --no-build-isolation
+  deactivate
+        NODE
+      end
+      cached(:chef_run) do
+        runner = runner(platform: platform, version: version) do |node|
+          node.override['kernel']['machine'] = arch
+          node.override['cluster']['python-major-minor-version'] = python_version
+          node.override['cluster']['python-version'] = python_version
+          node.override['cluster']['base_dir'] = base_dir
+          node.override['cluster']['region'] = region
+          node.override['cluster']['artifacts_s3_url'] = s3_url
+          node.override['cluster']['custom_node_package'] = custom_node_s3_url
+        end
+        allow(File).to receive(:exist?).with("#{virtualenv_path}/bin/activate").and_return(true)
+        runner.converge(described_recipe)
+      end
+
+      it 'downloads tarball' do
+        is_expected.to create_if_missing_remote_file("base_dir/node-dependencies.tgz")
+          .with(source: "#{s3_url}/dependencies/PyPi/#{arch}/#{dependency_pkg_name_suffix}.tgz")
+          .with(mode: '0644')
+          .with(retries: 3)
+          .with(retry_delay: 5)
+      end
+
+      it 'pip installs' do
+        is_expected.to run_bash('pip install')
+          .with(cwd: base_dir)
+          .with(code: pip_install_bash_code.gsub(/^  /, '    '))
+      end
+
+      it 'install custom aws-parallelcluster-node' do
+        is_expected.to run_bash('install custom aws-parallelcluster-node')
+          .with(code: node_bash_code.gsub(/^  /, '    '))
+      end
+    end
+  end
+end

cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb

@@ -33,32 +33,31 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
-if aws_region.start_with?("us-iso")
-  dependency_package_name = "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
-  dependency_folder_name = dependency_package_name
-  if platform?('amazon') && node['platform_version'] == "2"
-    dependency_package_name = "cfn-dependencies"
-    dependency_folder_name = "cfn"
-  end
-  remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
-    source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
-    mode '0644'
-    retries 3
-    retry_delay 5
-    action :create_if_missing
-  end
-
-  bash 'pip install' do
-    user 'root'
-    group 'root'
-    cwd "#{node['cluster']['base_dir']}"
-    code <<-REQ
-      set -e
-      tar xzf cfn-dependencies.tgz
-      cd #{dependency_folder_name}
-      #{virtualenv_path}/bin/pip install * -f ./ --no-index
-      REQ
-  end
+dependency_package_name = "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
+dependency_folder_name = dependency_package_name
+if platform?('amazon') && node['platform_version'] == "2"
+  dependency_package_name = "cfn-dependencies"
+  dependency_folder_name = "cfn"
+end
+
+remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf cfn-dependencies.tgz
+    cd #{dependency_folder_name}
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
 end
 
 cfnbootstrap_version = '2.0-33'
@@ -79,11 +78,8 @@
   retry_delay 5
 end
 
-command = if aws_region.start_with?("us-iso")
-            "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation"
-          else
-            "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}"
-          end
+command = "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation"
+
 bash "Install CloudFormation helpers from #{cfnbootstrap_package}" do
   user 'root'
   group 'root'

cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb

@@ -42,7 +42,7 @@ def prerequisites
 
 action :install_efs_utils do
   package_name = "amazon-efs-utils"
-  package_version = new_resource.efs_utils_version
+  package_version = _efs_utils_version
   efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
 
   if aws_region.start_with?("us-iso")

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb

@@ -2,15 +2,12 @@
 
 default_action :install_utils
 
-property :efs_utils_version, String
-property :efs_utils_checksum, String
-
 def _efs_utils_version
-  efs_utils_version || node['cluster']['efs']['version']
+  node['cluster']['efs']['version']
 end
 
 def _efs_utils_checksum
-  efs_utils_checksum || node['cluster']['efs']['sha256']
+  node['cluster']['efs']['sha256']
 end
 
 def already_installed?(package_name, expected_version)

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb

@@ -15,7 +15,7 @@
 package_name = "amazon-efs-utils"
 
 action :install_utils do
-  package_version = new_resource.efs_utils_version
+  package_version = _efs_utils_version
   efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
   efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
 
@@ -46,7 +46,7 @@
     mode '0644'
     retries 3
     retry_delay 5
-    checksum new_resource.efs_utils_checksum
+    checksum _efs_utils_checksum
     action :create_if_missing
   end
 
@@ -61,9 +61,9 @@
 end
 
 action :install_efs_utils do
-  package_version = new_resource.efs_utils_version
+  package_version = _efs_utils_version
   efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
-
+  # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
   bash "install efs utils" do
     cwd node['cluster']['sources_dir']
     code install_script_code(efs_utils_tarball, package_name, package_version)

cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb

@@ -4,18 +4,46 @@
   for_all_oses do |platform, version|
     context "on #{platform}#{version}" do
       cached(:cfnbootstrap_version) { '2.0-33' }
-      cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" }
+      cached(:arch) { 'x86_64' }
+      cached(:s3_url) { 's3://url' }
+      cached(:base_dir) { 'base_dir' }
       cached(:python_version) { "#{node['cluster']['python-version']}" }
+      cached(:dependecy_package_name_suffix) do
+        if platform == 'amazon' && version == '2'
+          "cfn-dependencies"
+        else
+          "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{arch}"
+        end
+      end
+      cached(:dependecy_folder_name) do
+        if platform == 'amazon' && version == '2'
+          "cfn"
+        else
+          dependecy_package_name_suffix
+        end
+      end
+      cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" }
       cached(:system_pyenv_root) { 'system_pyenv_root' }
       cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" }
       cached(:timeout) { 1800 }
+      cached(:dependency_bash_code) do
+        <<-REQ
+    set -e
+    tar xzf cfn-dependencies.tgz
+    cd #{dependecy_folder_name}
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+        REQ
+      end
 
       context "when cfn_bootstrap virtualenv not installed yet" do
         cached(:chef_run) do
           runner = runner(platform: platform, version: version) do |node|
             node.override['cluster']['system_pyenv_root'] = system_pyenv_root
             node.override['cluster']['region'] = 'non_china'
+            node.override['cluster']['base_dir'] = base_dir
             node.override['cluster']['compute_node_bootstrap_timeout'] = timeout
+            node.override['cluster']['artifacts_s3_url'] = s3_url
+            node.override['kernel']['machine'] = arch
           end
           runner.converge(described_recipe)
         end
@@ -37,6 +65,22 @@
           is_expected.to write_node_attributes('dump node attributes')
         end
 
+        it 'downloads cfn_dependecies package from s3' do
+          is_expected.to create_if_missing_remote_file("#{base_dir}/cfn-dependencies.tgz")
+            .with(source: "#{s3_url}/dependencies/PyPi/#{arch}/#{dependecy_package_name_suffix}.tgz")
+            .with(mode: '0644')
+            .with(retries: 3)
+            .with(retry_delay: 5)
+        end
+
+        it 'pip installs dependencies' do
+          is_expected.to run_bash('pip install')
+            .with(user: 'root')
+            .with(group: 'root')
+            .with(cwd: base_dir)
+            .with(code: dependency_bash_code)
+        end
+
         it 'downloads cfn_bootstrap package from s3' do
           is_expected.to create_remote_file("/tmp/#{cfnbootstrap_package}").with(
             source: "https://s3.amazonaws.com/cloudformation-examples/#{cfnbootstrap_package}"
@@ -48,7 +92,7 @@
             user: 'root',
             group: 'root',
             cwd: '/tmp',
-            code: "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}",
+            code: "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation",
             creates: "#{virtualenv_path}/bin/cfn-hup"
           )
         end