diff --git a/CHANGELOG.md b/CHANGELOG.md index 66a96c3f06..bfaef50d9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ This file is used to list changes made in each version of the AWS ParallelCluste - Add support for Ubuntu24. **CHANGES** -- Upgrade Python to 3.12.8 for all OSs except AL2 (from 3.9.20). - On Ubuntu 22.04, install the Nvidia driver with the same compiler version used to compile the kernel. - Upgrade `aws-cfn-bootstrap` to version 2.0-32. - Upgrade amazon-efs-utils to version 2.1.0. diff --git a/cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb b/cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb index 936a15edd9..cbcd61cf00 100644 --- a/cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb @@ -28,5 +28,25 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end +remote_file "#{node['cluster']['base_dir']}/awsbatch-dependencies.tgz" do + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/awsbatch-dependencies.tgz" + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing +end + +bash 'pip install' do + user 'root' + group 'root' + cwd "#{node['cluster']['base_dir']}" + code <<-REQ + set -e + tar xzf awsbatch-dependencies.tgz + cd awsbatch + #{virtualenv_path}/bin/pip install * -f ./ --no-index + REQ +end + node.default['cluster']['awsbatch_virtualenv_path'] = virtualenv_path node_attributes "dump node attributes" diff --git a/cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb b/cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb index 7b877b10f5..de691649c1 100644 --- a/cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb +++ b/cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb @@ -9,11 +9,11 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. +python_version = '3.9.20' base_dir = "/opt/parallelcluster" pyenv_dir = "#{base_dir}/pyenv" control 'tag:install_awsbatch_virtualenv_created' do - python_version = os_properties.alinux2? ? '3.9.20' : '3.12.8' title "awsbatch virtualenv should be created on #{python_version}" only_if { !os_properties.redhat? } diff --git a/cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml b/cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml index 3bcf169b8b..97702b4943 100644 --- a/cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml +++ b/cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml @@ -31,7 +31,7 @@ suites: attributes: cluster: custom_node_package: https://github.com/aws/aws-parallelcluster-node/archive/develop.tar.gz - python-version: 3.12.8 + python-version: 3.9.20 node_virtualenv_path: /opt/parallelcluster/pyenv/versions/node_virtualenv - name: fleet_status run_list: diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb index c2bdf0930f..2e74aa2827 100644 --- a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb +++ b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb @@ -19,6 +19,25 @@ # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the # bash block to a recipe that uses the pyenv resource. +remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz" + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing +end + +bash 'pip install' do + user 'root' + group 'root' + cwd "#{node['cluster']['base_dir']}" + code <<-REQ + set -e + tar xzf node-dependencies.tgz + cd node + #{node_virtualenv_path}/bin/pip install * -f ./ --no-index + REQ +end bash "install custom aws-parallelcluster-node" do cwd Chef::Config[:file_cache_path] diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index ecd70ff9f2..41fb845c4f 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -15,7 +15,7 @@ virtualenv_name = 'cfn_bootstrap_virtualenv' pyenv_root = node['cluster']['system_pyenv_root'] # FIXME: Python Version cfn_bootstrap_virtualenv due to a bug with cfn-hup -python_version = node['cluster']['python-version'] +python_version = '3.9.20' virtualenv_path = "#{pyenv_root}/versions/#{python_version}/envs/#{virtualenv_name}" node.default['cluster']['cfn_bootstrap_virtualenv_path'] = virtualenv_path @@ -33,6 +33,26 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end +remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing +end + +bash 'pip install' do + user 'root' + group 'root' + cwd "#{node['cluster']['base_dir']}" + code <<-REQ + set -e + tar xzf cfn-dependencies.tgz + cd cfn + #{virtualenv_path}/bin/pip install * -f ./ --no-index + REQ +end + cfnbootstrap_version = '2.0-32' cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb index e584e64400..73bd8cd4e5 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb @@ -5,7 +5,7 @@ context "on #{platform}#{version}" do cached(:cfnbootstrap_version) { '2.0-32' } cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" } - cached(:python_version) { "#{node['cluster']['python-version']}" } + cached(:python_version) { '3.9.20' } cached(:system_pyenv_root) { 'system_pyenv_root' } cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" } cached(:timeout) { 1800 } @@ -32,6 +32,14 @@ ) end + it 'installs python packages' do + is_expected.to run_bash("pip install").with( + user: 'root', + group: 'root', + cwd: "#{node['cluster']['base_dir']}" + ) + end + it 'sets virtualenv path' do expect(node.default['cluster']['cfn_bootstrap_virtualenv_path']).to eq(virtualenv_path) is_expected.to write_node_attributes('dump node attributes') diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb index 132c3a94ed..00be636af1 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb @@ -9,11 +9,11 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. +cfn_python_version = '3.9.20' base_dir = "/opt/parallelcluster" pyenv_dir = "#{base_dir}/pyenv" control 'tag:install_cfnbootstrap_virtualenv_created' do - cfn_python_version = os_properties.alinux2? ? '3.9.20' : '3.12.8' title "cfnbootstrap virtualenv should be created on #{cfn_python_version}" only_if { !os_properties.redhat_on_docker? } diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb index 38d87bcd24..677a80ba3f 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb @@ -20,7 +20,7 @@ file_cache_path = Chef::Config[:file_cache_path] region = aws_region -awscli_url = "https://awscli.#{aws_domain}/awscli-exe-linux-#{node['kernel']['machine']}.zip" +awscli_url = "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" if region.start_with?("us-iso-") awscli_url = "https://aws-sdk-common-infra-dca-prod-deployment-bucket.s3.#{aws_region}.#{aws_domain}/aws-cli-v2/linux/x86_64/awscli-exe-linux-x86_64.zip" @@ -42,11 +42,10 @@ overwrite true end -bash 'install awscli' do - code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin" -end - if region.start_with?("us-iso") + bash 'install awscli' do + code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin" + end cookbook_file "#{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" do source 'isolated/iso-ca-bundle-config.sh' @@ -60,4 +59,8 @@ execute "patch ca bundle" do command "sh #{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" end +else + bash 'install awscli' do + code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws" + end end diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb index cdc557df5f..63fe8b5ecf 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb @@ -12,10 +12,6 @@ # limitations under the License. virtualenv_path = cookbook_virtualenv_path -pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz" -if platform?('amazon') && node['platform_version'] == "2" - pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz" -end node.default['cluster']['cookbook_virtualenv_path'] = virtualenv_path node_attributes "dump node attributes" @@ -32,7 +28,7 @@ end remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do - source pypi_s3_uri + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz" mode '0644' retries 3 retry_delay 5 diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb index 9940fae2eb..4f3c1136ec 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/awscli_spec.rb @@ -15,7 +15,7 @@ it 'downloads awscli-bundle from s3' do is_expected.to create_remote_file('download awscli bundle from s3').with( path: "#{file_cache_path}/awscli-bundle.zip", - source: "https://awscli.test_aws_domain/awscli-exe-linux-#{node['kernel']['machine']}.zip", + source: 'https://s3.amazonaws.com/aws-cli/awscli-bundle.zip', retries: 5, retry_delay: 5 ) @@ -31,7 +31,7 @@ it 'installs awscli into cookbook virtualev path' do is_expected.to run_bash('install awscli') - .with_code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin" + .with_code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws" end end diff --git a/cookbooks/aws-parallelcluster-shared/attributes/versions.rb b/cookbooks/aws-parallelcluster-shared/attributes/versions.rb index 19c231a1bd..f23c0091fa 100644 --- a/cookbooks/aws-parallelcluster-shared/attributes/versions.rb +++ b/cookbooks/aws-parallelcluster-shared/attributes/versions.rb @@ -1,10 +1,6 @@ # Python Version -default['cluster']['python-version'] = '3.12.8' -default['cluster']['python-major-minor-version'] = '3.12' -if platform?('amazon') && node['platform_version'] == "2" - default['cluster']['python-version'] = '3.9.20' - default['cluster']['python-major-minor-version'] = '3.9' -end +default['cluster']['python-version'] = '3.9.20' +default['cluster']['python-major-minor-version'] = '3.9' # ParallelCluster versions default['cluster']['parallelcluster-version'] = '3.13.0'