diff --git a/CHANGELOG.md b/CHANGELOG.md index bfaef50d9c..66a96c3f06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste - Add support for Ubuntu24. **CHANGES** +- Upgrade Python to 3.12.8 for all OSs except AL2 (from 3.9.20). - On Ubuntu 22.04, install the Nvidia driver with the same compiler version used to compile the kernel. - Upgrade `aws-cfn-bootstrap` to version 2.0-32. - Upgrade amazon-efs-utils to version 2.1.0. diff --git a/cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb b/cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb index cbcd61cf00..936a15edd9 100644 --- a/cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb @@ -28,25 +28,5 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end -remote_file "#{node['cluster']['base_dir']}/awsbatch-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/awsbatch-dependencies.tgz" - mode '0644' - retries 3 - retry_delay 5 - action :create_if_missing -end - -bash 'pip install' do - user 'root' - group 'root' - cwd "#{node['cluster']['base_dir']}" - code <<-REQ - set -e - tar xzf awsbatch-dependencies.tgz - cd awsbatch - #{virtualenv_path}/bin/pip install * -f ./ --no-index - REQ -end - node.default['cluster']['awsbatch_virtualenv_path'] = virtualenv_path node_attributes "dump node attributes" diff --git a/cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb b/cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb index de691649c1..f9d3c33f2e 100644 --- a/cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb +++ b/cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb @@ -9,7 +9,7 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -python_version = '3.9.20' +python_version = '3.12.8' base_dir = "/opt/parallelcluster" pyenv_dir = "#{base_dir}/pyenv" diff --git a/cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml b/cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml index 97702b4943..3bcf169b8b 100644 --- a/cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml +++ b/cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml @@ -31,7 +31,7 @@ suites: attributes: cluster: custom_node_package: https://github.com/aws/aws-parallelcluster-node/archive/develop.tar.gz - python-version: 3.9.20 + python-version: 3.12.8 node_virtualenv_path: /opt/parallelcluster/pyenv/versions/node_virtualenv - name: fleet_status run_list: diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb index 2e74aa2827..c2bdf0930f 100644 --- a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb +++ b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb @@ -19,25 +19,6 @@ # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the # bash block to a recipe that uses the pyenv resource. -remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz" - mode '0644' - retries 3 - retry_delay 5 - action :create_if_missing -end - -bash 'pip install' do - user 'root' - group 'root' - cwd "#{node['cluster']['base_dir']}" - code <<-REQ - set -e - tar xzf node-dependencies.tgz - cd node - #{node_virtualenv_path}/bin/pip install * -f ./ --no-index - REQ -end bash "install custom aws-parallelcluster-node" do cwd Chef::Config[:file_cache_path] diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index 41fb845c4f..ecd70ff9f2 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -15,7 +15,7 @@ virtualenv_name = 'cfn_bootstrap_virtualenv' pyenv_root = node['cluster']['system_pyenv_root'] # FIXME: Python Version cfn_bootstrap_virtualenv due to a bug with cfn-hup -python_version = '3.9.20' +python_version = node['cluster']['python-version'] virtualenv_path = "#{pyenv_root}/versions/#{python_version}/envs/#{virtualenv_name}" node.default['cluster']['cfn_bootstrap_virtualenv_path'] = virtualenv_path @@ -33,26 +33,6 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end -remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" - mode '0644' - retries 3 - retry_delay 5 - action :create_if_missing -end - -bash 'pip install' do - user 'root' - group 'root' - cwd "#{node['cluster']['base_dir']}" - code <<-REQ - set -e - tar xzf cfn-dependencies.tgz - cd cfn - #{virtualenv_path}/bin/pip install * -f ./ --no-index - REQ -end - cfnbootstrap_version = '2.0-32' cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb index 73bd8cd4e5..e584e64400 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb @@ -5,7 +5,7 @@ context "on #{platform}#{version}" do cached(:cfnbootstrap_version) { '2.0-32' } cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" } - cached(:python_version) { '3.9.20' } + cached(:python_version) { "#{node['cluster']['python-version']}" } cached(:system_pyenv_root) { 'system_pyenv_root' } cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" } cached(:timeout) { 1800 } @@ -32,14 +32,6 @@ ) end - it 'installs python packages' do - is_expected.to run_bash("pip install").with( - user: 'root', - group: 'root', - cwd: "#{node['cluster']['base_dir']}" - ) - end - it 'sets virtualenv path' do expect(node.default['cluster']['cfn_bootstrap_virtualenv_path']).to eq(virtualenv_path) is_expected.to write_node_attributes('dump node attributes') diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb index 00be636af1..db924c8807 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb @@ -9,7 +9,7 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -cfn_python_version = '3.9.20' +cfn_python_version = '3.12.8' base_dir = "/opt/parallelcluster" pyenv_dir = "#{base_dir}/pyenv" diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb index 677a80ba3f..38d87bcd24 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb @@ -20,7 +20,7 @@ file_cache_path = Chef::Config[:file_cache_path] region = aws_region -awscli_url = "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" +awscli_url = "https://awscli.#{aws_domain}/awscli-exe-linux-#{node['kernel']['machine']}.zip" if region.start_with?("us-iso-") awscli_url = "https://aws-sdk-common-infra-dca-prod-deployment-bucket.s3.#{aws_region}.#{aws_domain}/aws-cli-v2/linux/x86_64/awscli-exe-linux-x86_64.zip" @@ -42,10 +42,11 @@ overwrite true end +bash 'install awscli' do + code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin" +end + if region.start_with?("us-iso") - bash 'install awscli' do - code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin" - end cookbook_file "#{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" do source 'isolated/iso-ca-bundle-config.sh' @@ -59,8 +60,4 @@ execute "patch ca bundle" do command "sh #{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" end -else - bash 'install awscli' do - code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws" - end end diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb index 63fe8b5ecf..cdc557df5f 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb @@ -12,6 +12,10 @@ # limitations under the License. virtualenv_path = cookbook_virtualenv_path +pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz" +if platform?('amazon') && node['platform_version'] == "2" + pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz" +end node.default['cluster']['cookbook_virtualenv_path'] = virtualenv_path node_attributes "dump node attributes" @@ -28,7 +32,7 @@ end remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz" + source pypi_s3_uri mode '0644' retries 3 retry_delay 5 diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb index 4f3c1136ec..9940fae2eb 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb @@ -15,7 +15,7 @@ it 'downloads awscli-bundle from s3' do is_expected.to create_remote_file('download awscli bundle from s3').with( path: "#{file_cache_path}/awscli-bundle.zip", - source: 'https://s3.amazonaws.com/aws-cli/awscli-bundle.zip', + source: "https://awscli.test_aws_domain/awscli-exe-linux-#{node['kernel']['machine']}.zip", retries: 5, retry_delay: 5 ) @@ -31,7 +31,7 @@ it 'installs awscli into cookbook virtualev path' do is_expected.to run_bash('install awscli') - .with_code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws" + .with_code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin" end end diff --git a/cookbooks/aws-parallelcluster-shared/attributes/versions.rb b/cookbooks/aws-parallelcluster-shared/attributes/versions.rb index f23c0091fa..19c231a1bd 100644 --- a/cookbooks/aws-parallelcluster-shared/attributes/versions.rb +++ b/cookbooks/aws-parallelcluster-shared/attributes/versions.rb @@ -1,6 +1,10 @@ # Python Version -default['cluster']['python-version'] = '3.9.20' -default['cluster']['python-major-minor-version'] = '3.9' +default['cluster']['python-version'] = '3.12.8' +default['cluster']['python-major-minor-version'] = '3.12' +if platform?('amazon') && node['platform_version'] == "2" + default['cluster']['python-version'] = '3.9.20' + default['cluster']['python-major-minor-version'] = '3.9' +end # ParallelCluster versions default['cluster']['parallelcluster-version'] = '3.13.0'