Revert "Upgrade python version to 3.12 for all OSs except AL2 and remove unneccesary python dep installation (aws#2869)"

This reverts commit 45431e6.

Author: Himani Anil Deshpande

CHANGELOG.md

@@ -9,7 +9,6 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 - Add support for Ubuntu24.
 
 **CHANGES**
-- Upgrade Python to 3.12.8 for all OSs except AL2 (from 3.9.20).
 - On Ubuntu 22.04, install the Nvidia driver with the same compiler version used to compile the kernel.
 - Upgrade `aws-cfn-bootstrap` to version 2.0-32.
 - Upgrade amazon-efs-utils to version 2.1.0.

cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb

@@ -28,5 +28,25 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
+remote_file "#{node['cluster']['base_dir']}/awsbatch-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/awsbatch-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf awsbatch-dependencies.tgz
+    cd awsbatch
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
+
 node.default['cluster']['awsbatch_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"

cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb

@@ -9,7 +9,7 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-python_version = '3.12.8'
+python_version = '3.9.20'
 base_dir = "/opt/parallelcluster"
 pyenv_dir = "#{base_dir}/pyenv"
 

cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml

@@ -31,7 +31,7 @@ suites:
     attributes:
       cluster:
         custom_node_package: https://github.com/aws/aws-parallelcluster-node/archive/develop.tar.gz
-        python-version: 3.12.8
+        python-version: 3.9.20
         node_virtualenv_path: /opt/parallelcluster/pyenv/versions/node_virtualenv
   - name: fleet_status
     run_list:

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -19,6 +19,25 @@
 
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
+remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf node-dependencies.tgz
+    cd node
+    #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
 
 bash "install custom aws-parallelcluster-node" do
   cwd Chef::Config[:file_cache_path]

cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb

@@ -15,7 +15,7 @@
 virtualenv_name = 'cfn_bootstrap_virtualenv'
 pyenv_root = node['cluster']['system_pyenv_root']
 # FIXME: Python Version cfn_bootstrap_virtualenv due to a bug with cfn-hup
-python_version = node['cluster']['python-version']
+python_version = '3.9.20'
 virtualenv_path = "#{pyenv_root}/versions/#{python_version}/envs/#{virtualenv_name}"
 
 node.default['cluster']['cfn_bootstrap_virtualenv_path'] = virtualenv_path
@@ -33,6 +33,26 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
+remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf cfn-dependencies.tgz
+    cd cfn
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
+
 cfnbootstrap_version = '2.0-32'
 cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz"
 

cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb

@@ -5,7 +5,7 @@
     context "on #{platform}#{version}" do
       cached(:cfnbootstrap_version) { '2.0-32' }
       cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" }
-      cached(:python_version) { "#{node['cluster']['python-version']}" }
+      cached(:python_version) { '3.9.20' }
       cached(:system_pyenv_root) { 'system_pyenv_root' }
       cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" }
       cached(:timeout) { 1800 }
@@ -32,6 +32,14 @@
           )
         end
 
+        it 'installs python packages' do
+          is_expected.to run_bash("pip install").with(
+            user: 'root',
+            group: 'root',
+            cwd: "#{node['cluster']['base_dir']}"
+          )
+        end
+
         it 'sets virtualenv path' do
           expect(node.default['cluster']['cfn_bootstrap_virtualenv_path']).to eq(virtualenv_path)
           is_expected.to write_node_attributes('dump node attributes')

cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb

@@ -9,7 +9,7 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-cfn_python_version = '3.12.8'
+cfn_python_version = '3.9.20'
 base_dir = "/opt/parallelcluster"
 pyenv_dir = "#{base_dir}/pyenv"
 

cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb

@@ -20,7 +20,7 @@
 
 file_cache_path = Chef::Config[:file_cache_path]
 region = aws_region
-awscli_url = "https://awscli.#{aws_domain}/awscli-exe-linux-#{node['kernel']['machine']}.zip"
+awscli_url = "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip"
 
 if region.start_with?("us-iso-")
   awscli_url = "https://aws-sdk-common-infra-dca-prod-deployment-bucket.s3.#{aws_region}.#{aws_domain}/aws-cli-v2/linux/x86_64/awscli-exe-linux-x86_64.zip"
@@ -42,11 +42,10 @@
   overwrite true
 end
 
-bash 'install awscli' do
-  code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
-end
-
 if region.start_with?("us-iso")
+  bash 'install awscli' do
+    code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
+  end
 
   cookbook_file "#{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" do
     source 'isolated/iso-ca-bundle-config.sh'
@@ -60,4 +59,8 @@
   execute "patch ca bundle" do
     command "sh #{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh"
   end
+else
+  bash 'install awscli' do
+    code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws"
+  end
 end

cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb

@@ -12,10 +12,6 @@
 # limitations under the License.
 
 virtualenv_path = cookbook_virtualenv_path
-pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz"
-if platform?('amazon') && node['platform_version'] == "2"
-  pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
-end
 
 node.default['cluster']['cookbook_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"
@@ -32,7 +28,7 @@
 end
 
 remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
-  source pypi_s3_uri
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
   mode '0644'
   retries 3
   retry_delay 5