Upgrade python version to 3.12 for all OSs except AL2

CHANGELOG.md

@@ -13,8 +13,9 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 - Upgrade Slurm to version 24.05.6.
 - Upgrade NVIDIA driver to version 570.86.15 (from 550.127.08) for all OSs except AL2.
 - Upgrade CUDA Toolkit to version 12.8.0 (from 12.4.1) for all OSs except AL2.
+- Upgrade Python to 3.12.8 for all OSs except AL2 (from 3.9.20).
 - On Ubuntu 22.04, install the Nvidia driver with the same compiler version used to compile the kernel.
-- Upgrade `aws-cfn-bootstrap` to version 2.0-32.
+- Upgrade `aws-cfn-bootstrap` to version 2.0-33.
 - Upgrade amazon-efs-utils to version 2.1.0.
 - Remove third-party cookbook: apt-7.5.22 and pyenv-4.2.3.
 - Upgrade third-party cookbook dependencies:

cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb

@@ -28,25 +28,5 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
-remote_file "#{node['cluster']['base_dir']}/awsbatch-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/awsbatch-dependencies.tgz"
-  mode '0644'
-  retries 3
-  retry_delay 5
-  action :create_if_missing
-end
-
-bash 'pip install' do
-  user 'root'
-  group 'root'
-  cwd "#{node['cluster']['base_dir']}"
-  code <<-REQ
-    set -e
-    tar xzf awsbatch-dependencies.tgz
-    cd awsbatch
-    #{virtualenv_path}/bin/pip install * -f ./ --no-index
-    REQ
-end
-
 node.default['cluster']['awsbatch_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"

cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb

@@ -9,11 +9,11 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-python_version = '3.9.20'
 base_dir = "/opt/parallelcluster"
 pyenv_dir = "#{base_dir}/pyenv"
 
 control 'tag:install_awsbatch_virtualenv_created' do
+  python_version = os_properties.alinux2? ? '3.9.20' : '3.12.8'
   title "awsbatch virtualenv should be created on #{python_version}"
   only_if { !os_properties.redhat? }
 

cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml

@@ -31,7 +31,7 @@ suites:
     attributes:
       cluster:
         custom_node_package: https://github.com/aws/aws-parallelcluster-node/archive/develop.tar.gz
-        python-version: 3.9.20
+        python-version: 3.12.8
         node_virtualenv_path: /opt/parallelcluster/pyenv/versions/node_virtualenv
   - name: fleet_status
     run_list:

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -19,25 +19,6 @@
 
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
-remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
-  mode '0644'
-  retries 3
-  retry_delay 5
-  action :create_if_missing
-end
-
-bash 'pip install' do
-  user 'root'
-  group 'root'
-  cwd "#{node['cluster']['base_dir']}"
-  code <<-REQ
-    set -e
-    tar xzf node-dependencies.tgz
-    cd node
-    #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
-    REQ
-end
 
 bash "install custom aws-parallelcluster-node" do
   cwd Chef::Config[:file_cache_path]

cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb

@@ -15,7 +15,7 @@
 virtualenv_name = 'cfn_bootstrap_virtualenv'
 pyenv_root = node['cluster']['system_pyenv_root']
 # FIXME: Python Version cfn_bootstrap_virtualenv due to a bug with cfn-hup
-python_version = '3.9.20'
+python_version = node['cluster']['python-version']
 virtualenv_path = "#{pyenv_root}/versions/#{python_version}/envs/#{virtualenv_name}"
 
 node.default['cluster']['cfn_bootstrap_virtualenv_path'] = virtualenv_path
@@ -33,27 +33,7 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
-remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz"
-  mode '0644'
-  retries 3
-  retry_delay 5
-  action :create_if_missing
-end
-
-bash 'pip install' do
-  user 'root'
-  group 'root'
-  cwd "#{node['cluster']['base_dir']}"
-  code <<-REQ
-    set -e
-    tar xzf cfn-dependencies.tgz
-    cd cfn
-    #{virtualenv_path}/bin/pip install * -f ./ --no-index
-    REQ
-end
-
-cfnbootstrap_version = '2.0-32'
+cfnbootstrap_version = '2.0-33'
 cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz"
 
 region = node['cluster']['region']

cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb

@@ -3,9 +3,9 @@
 describe 'aws-parallelcluster-environment::cfn_bootstrap' do
   for_all_oses do |platform, version|
     context "on #{platform}#{version}" do
-      cached(:cfnbootstrap_version) { '2.0-32' }
+      cached(:cfnbootstrap_version) { '2.0-33' }
       cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" }
-      cached(:python_version) { '3.9.20' }
+      cached(:python_version) { "#{node['cluster']['python-version']}" }
       cached(:system_pyenv_root) { 'system_pyenv_root' }
       cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" }
       cached(:timeout) { 1800 }
@@ -32,14 +32,6 @@
           )
         end
 
-        it 'installs python packages' do
-          is_expected.to run_bash("pip install").with(
-            user: 'root',
-            group: 'root',
-            cwd: "#{node['cluster']['base_dir']}"
-          )
-        end
-
         it 'sets virtualenv path' do
           expect(node.default['cluster']['cfn_bootstrap_virtualenv_path']).to eq(virtualenv_path)
           is_expected.to write_node_attributes('dump node attributes')

cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb

@@ -9,11 +9,11 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-cfn_python_version = '3.9.20'
 base_dir = "/opt/parallelcluster"
 pyenv_dir = "#{base_dir}/pyenv"
 
 control 'tag:install_cfnbootstrap_virtualenv_created' do
+  cfn_python_version = os_properties.alinux2? ? '3.9.20' : '3.12.8'
   title "cfnbootstrap virtualenv should be created on #{cfn_python_version}"
   only_if { !os_properties.redhat_on_docker? }
 

cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb

@@ -20,7 +20,7 @@
 
 file_cache_path = Chef::Config[:file_cache_path]
 region = aws_region
-awscli_url = "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip"
+awscli_url = "https://awscli.#{aws_domain}/awscli-exe-linux-#{node['kernel']['machine']}.zip"
 
 if region.start_with?("us-iso-")
   awscli_url = "https://aws-sdk-common-infra-dca-prod-deployment-bucket.s3.#{aws_region}.#{aws_domain}/aws-cli-v2/linux/x86_64/awscli-exe-linux-x86_64.zip"
@@ -42,10 +42,11 @@
   overwrite true
 end
 
+bash 'install awscli' do
+  code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
+end
+
 if region.start_with?("us-iso")
-  bash 'install awscli' do
-    code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
-  end
 
   cookbook_file "#{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" do
     source 'isolated/iso-ca-bundle-config.sh'
@@ -59,8 +60,4 @@
   execute "patch ca bundle" do
     command "sh #{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh"
   end
-else
-  bash 'install awscli' do
-    code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws"
-  end
 end

cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb

@@ -12,6 +12,10 @@
 # limitations under the License.
 
 virtualenv_path = cookbook_virtualenv_path
+pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz"
+if platform?('amazon') && node['platform_version'] == "2"
+  pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
+end
 
 node.default['cluster']['cookbook_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"
@@ -28,7 +32,7 @@
 end
 
 remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
-  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
+  source pypi_s3_uri
   mode '0644'
   retries 3
   retry_delay 5

cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb

@@ -15,7 +15,7 @@
         it 'downloads awscli-bundle from s3' do
           is_expected.to create_remote_file('download awscli bundle from s3').with(
             path: "#{file_cache_path}/awscli-bundle.zip",
-            source: 'https://s3.amazonaws.com/aws-cli/awscli-bundle.zip',
+            source: "https://awscli.test_aws_domain/awscli-exe-linux-#{node['kernel']['machine']}.zip",
             retries: 5,
             retry_delay: 5
           )
@@ -31,7 +31,7 @@
 
         it 'installs awscli into cookbook virtualev path' do
           is_expected.to run_bash('install awscli')
-            .with_code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws"
+            .with_code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
         end
       end
 

cookbooks/aws-parallelcluster-shared/attributes/versions.rb

@@ -1,6 +1,10 @@
 # Python Version
-default['cluster']['python-version'] = '3.9.20'
-default['cluster']['python-major-minor-version'] = '3.9'
+default['cluster']['python-version'] = '3.12.8'
+default['cluster']['python-major-minor-version'] = '3.12'
+if platform?('amazon') && node['platform_version'] == "2"
+  default['cluster']['python-version'] = '3.9.20'
+  default['cluster']['python-major-minor-version'] = '3.9'
+end
 
 # ParallelCluster versions
 default['cluster']['parallelcluster-version'] = '3.13.0'