Upgrade python version to 3.12 for all OSs except AL2 and remove unneccesary python dep installation (#2869)

Author: hgreebe

CHANGELOG.md

@@ -9,6 +9,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 - Add support for Ubuntu24.
 
 **CHANGES**
+- Upgrade Python to 3.12.8 for all OSs except AL2 (from 3.9.20).
 - On Ubuntu 22.04, install the Nvidia driver with the same compiler version used to compile the kernel.
 - Upgrade `aws-cfn-bootstrap` to version 2.0-32.
 - Upgrade amazon-efs-utils to version 2.1.0.

cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb

@@ -28,25 +28,5 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
-remote_file "#{node['cluster']['base_dir']}/awsbatch-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/awsbatch-dependencies.tgz"
-  mode '0644'
-  retries 3
-  retry_delay 5
-  action :create_if_missing
-end
-
-bash 'pip install' do
-  user 'root'
-  group 'root'
-  cwd "#{node['cluster']['base_dir']}"
-  code <<-REQ
-    set -e
-    tar xzf awsbatch-dependencies.tgz
-    cd awsbatch
-    #{virtualenv_path}/bin/pip install * -f ./ --no-index
-    REQ
-end
-
 node.default['cluster']['awsbatch_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"

cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb

@@ -9,7 +9,7 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-python_version = '3.9.20'
+python_version = '3.12.8'
 base_dir = "/opt/parallelcluster"
 pyenv_dir = "#{base_dir}/pyenv"
 

cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml

@@ -31,7 +31,7 @@ suites:
     attributes:
       cluster:
         custom_node_package: https://github.com/aws/aws-parallelcluster-node/archive/develop.tar.gz
-        python-version: 3.9.20
+        python-version: 3.12.8
         node_virtualenv_path: /opt/parallelcluster/pyenv/versions/node_virtualenv
   - name: fleet_status
     run_list:

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -19,25 +19,6 @@
 
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
-remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
-  mode '0644'
-  retries 3
-  retry_delay 5
-  action :create_if_missing
-end
-
-bash 'pip install' do
-  user 'root'
-  group 'root'
-  cwd "#{node['cluster']['base_dir']}"
-  code <<-REQ
-    set -e
-    tar xzf node-dependencies.tgz
-    cd node
-    #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
-    REQ
-end
 
 bash "install custom aws-parallelcluster-node" do
   cwd Chef::Config[:file_cache_path]

cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb

@@ -15,7 +15,7 @@
 virtualenv_name = 'cfn_bootstrap_virtualenv'
 pyenv_root = node['cluster']['system_pyenv_root']
 # FIXME: Python Version cfn_bootstrap_virtualenv due to a bug with cfn-hup
-python_version = '3.9.20'
+python_version = node['cluster']['python-version']
 virtualenv_path = "#{pyenv_root}/versions/#{python_version}/envs/#{virtualenv_name}"
 
 node.default['cluster']['cfn_bootstrap_virtualenv_path'] = virtualenv_path
@@ -33,26 +33,6 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
-remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz"
-  mode '0644'
-  retries 3
-  retry_delay 5
-  action :create_if_missing
-end
-
-bash 'pip install' do
-  user 'root'
-  group 'root'
-  cwd "#{node['cluster']['base_dir']}"
-  code <<-REQ
-    set -e
-    tar xzf cfn-dependencies.tgz
-    cd cfn
-    #{virtualenv_path}/bin/pip install * -f ./ --no-index
-    REQ
-end
-
 cfnbootstrap_version = '2.0-32'
 cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz"
 

cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb

@@ -5,7 +5,7 @@
     context "on #{platform}#{version}" do
       cached(:cfnbootstrap_version) { '2.0-32' }
       cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" }
-      cached(:python_version) { '3.9.20' }
+      cached(:python_version) { "#{node['cluster']['python-version']}" }
       cached(:system_pyenv_root) { 'system_pyenv_root' }
       cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" }
       cached(:timeout) { 1800 }
@@ -32,14 +32,6 @@
           )
         end
 
-        it 'installs python packages' do
-          is_expected.to run_bash("pip install").with(
-            user: 'root',
-            group: 'root',
-            cwd: "#{node['cluster']['base_dir']}"
-          )
-        end
-
         it 'sets virtualenv path' do
           expect(node.default['cluster']['cfn_bootstrap_virtualenv_path']).to eq(virtualenv_path)
           is_expected.to write_node_attributes('dump node attributes')

cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb

@@ -9,7 +9,7 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-cfn_python_version = '3.9.20'
+cfn_python_version = '3.12.8'
 base_dir = "/opt/parallelcluster"
 pyenv_dir = "#{base_dir}/pyenv"
 

cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb

@@ -20,7 +20,7 @@
 
 file_cache_path = Chef::Config[:file_cache_path]
 region = aws_region
-awscli_url = "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip"
+awscli_url = "https://awscli.#{aws_domain}/awscli-exe-linux-#{node['kernel']['machine']}.zip"
 
 if region.start_with?("us-iso-")
   awscli_url = "https://aws-sdk-common-infra-dca-prod-deployment-bucket.s3.#{aws_region}.#{aws_domain}/aws-cli-v2/linux/x86_64/awscli-exe-linux-x86_64.zip"
@@ -42,10 +42,11 @@
   overwrite true
 end
 
+bash 'install awscli' do
+  code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
+end
+
 if region.start_with?("us-iso")
-  bash 'install awscli' do
-    code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
-  end
 
   cookbook_file "#{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" do
     source 'isolated/iso-ca-bundle-config.sh'
@@ -59,8 +60,4 @@
   execute "patch ca bundle" do
     command "sh #{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh"
   end
-else
-  bash 'install awscli' do
-    code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws"
-  end
 end

cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb

@@ -12,6 +12,10 @@
 # limitations under the License.
 
 virtualenv_path = cookbook_virtualenv_path
+pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz"
+if platform?('amazon') && node['platform_version'] == "2"
+  pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
+end
 
 node.default['cluster']['cookbook_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"
@@ -28,7 +32,7 @@
 end
 
 remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
+  source pypi_s3_uri
   mode '0644'
   retries 3
   retry_delay 5