Download http-parser from s3 (#2896)

* Download http-parser from s3

* Fix isolated spec test

Author: hgreebe

cookbooks/aws-parallelcluster-environment/spec/unit/recipes/isolated_install_spec.rb

@@ -26,7 +26,7 @@
 
       it 'has the correct content' do
         is_expected.to render_file("#{node['cluster']['scripts_dir']}/patch-iso-instance.sh")
-          .with_content("USERS=(root #{node['cluster']['cluster_admin_user']} #{node['cluster']['cluster_user']})")
+          .with_content("source /etc/profile.d/aws-cli-default-config.sh")
       end
     end
   end

cookbooks/aws-parallelcluster-environment/templates/isolated/patch-iso-instance.sh.erb

@@ -19,15 +19,41 @@ REGION="$(get_instance_region)"
 
 source /etc/os-release
 OS="${ID}${VERSION_ID}"
-[[ "${OS}" != "amzn2" ]] && echo "[ERROR] Unsupported OS '${OS}'. Configuration supported only on Amazon Linux 2." && exit 1
+
+if [[ "${OS}" != "amzn2" && "${OS}" != "amzn2023" ]]; then
+  echo "[ERROR] Unsupported OS '${OS}'. Configuration supported only on Amazon Linux 2 and Amazon Linux 2023."
+  exit 1
+fi
 
 echo "[INFO] Starting: instance configuration for US isolated region"
 
 echo "[INFO] Starting: installation of packages from amazon Linux 2 repository for US isolated region"
 
-REPOSITORY_DEFINITION_FILE="/etc/yum.repos.d/tmp-amzn2-iso.repo"
+REPOSITORY_DEFINITION_FILE="/etc/yum.repos.d/tmp-${OS}-iso.repo"
+
+if [[ "${OS}" == "amzn2023" ]]; then
+  cat > ${REPOSITORY_DEFINITION_FILE} <<REPO_DEFINITION
+[amzn2023-iso]
+name=Amazon Linux 2023 isolated Region repository
+mirrorlist=http://al2023-repos-\$awsregion-de612dc2.s3.\$awsregion.\$awsdomain/core-iso/mirrors/\$releasever/\$basearch/mirror.list
+priority=10
+enabled=1
+repo_gpgcheck=0
+type=rpm
+gpgcheck=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-amazon-linux-2023
+REPO_DEFINITION
 
-cat > ${REPOSITORY_DEFINITION_FILE} <<REPO_DEFINITION
+  echo "sslverify=0" >> /etc/dnf/dnf.conf
+  echo -n "" | sudo tee /etc/dnf/vars/dualstack
+  if [[ ${REGION} == us-isob* ]]; then
+    dnf install -y amazon-linux-repo-iso ca-certificates-isob
+  else
+    dnf install -y amazon-linux-repo-iso ca-certificates-iso
+  fi
+  sed -i "s/sslverify=0//g" /etc/dnf/dnf.conf
+else
+  cat > ${REPOSITORY_DEFINITION_FILE} <<REPO_DEFINITION
 [amzn2-iso]
 name=Amazon Linux 2 isolated region repository
 mirrorlist=http://amazonlinux.\$awsregion.\$awsdomain/\$releasever/core-\$awsregion/latest/\$basearch/mirror.list
@@ -39,30 +65,35 @@ mirrorlist_expire=300
 report_instanceid=yes
 REPO_DEFINITION
 
-yum --disablerepo="*" --enablerepo="amzn2-iso" install -y "*-${REGION}"
+  yum --disablerepo="*" --enablerepo="${OS}-iso" install -y "*-${REGION}"
+fi
+
 rm -f ${REPOSITORY_DEFINITION_FILE}
 
-echo "[INFO] Complete: installation of packages from amazon Linux 2 repository for US isolated region"
+echo "[INFO] Complete: installation of packages from ${OS} repository for US isolated region"
 
 echo "[INFO] Starting: CA bundle configuration for AWS CLI in US isolated region"
 
-USERS=(<%= @users %>)
 CA_BUNDLE="/etc/pki/${REGION}/certs/ca-bundle.pem"
 
-for user in "${USERS[@]}"; do
-  echo "[INFO] Setting CA bundle ${CA_BUNDLE} for user ${user}"
-  sudo mkhomedir_helper $user
-  sudo -u $user aws configure set ca_bundle "$CA_BUNDLE"
-done
+sudo aws configure set ca_bundle "$CA_BUNDLE"
 
 echo "[INFO] Complete: CA bundle configuration for AWS CLI in US isolated region"
 
 echo "[INFO] Starting: Setting system-wide environment variables for AWS CLI in US isolated region"
 
+echo "export AWS_CA_BUNDLE=/etc/pki/${REGION}/certs/ca-bundle.pem" >> /etc/profile.d/aws-cli-default-config.sh
+
 echo "export AWS_DEFAULT_REGION=${REGION}" >> /etc/profile.d/aws-cli-default-config.sh
 
-echo "Defaults env_keep += \"AWS_DEFAULT_REGION AWS_CA_BUNDLE\"" > /etc/sudoers.d/pcluster-aws-cli-envkeep
+echo "export REQUESTS_CA_BUNDLE=${AWS_CA_BUNDLE}" >> /etc/profile.d/aws-cli-default-config.sh
 
-echo "[INFO] Complete: Setting system-wide environment variables for AWS CLI in US isolated region"
+echo "export SSL_CERT_FILE=${AWS_CA_BUNDLE}" >> /etc/profile.d/aws-cli-default-config.sh
+
+echo "Defaults env_keep += \"AWS_DEFAULT_REGION AWS_CA_BUNDLE REQUESTS_CA_BUNDLE SSL_CERT_FILE\"" > /etc/sudoers.d/pcluster-aws-cli-envkeep
 
-echo "[INFO] Complete: instance configuration for US isolated region"
+source /etc/profile.d/aws-cli-default-config.sh
+
+sudo aws configure set ca_bundle "$CA_BUNDLE"
+
+echo "[INFO] Complete: Setting system-wide environment variables for AWS CLI in US isolated region"

cookbooks/aws-parallelcluster-environment/test/controls/isolated_spec.rb

@@ -17,6 +17,5 @@
     its('owner') { should cmp 'root' }
     its('group') { should cmp 'root' }
     its('mode') { should cmp '0744' }
-    its('content') { should match /USERS=\(root #{node['cluster']['cluster_admin_user']} #{node['cluster']['cluster_user']}\)/ }
   end
 end

cookbooks/aws-parallelcluster-platform/files/isolated/iso-ca-bundle-config.sh

@@ -9,6 +9,8 @@ function get_instance_region {
 
 REGION="$(get_instance_region)"
 
+CA_BUNDLE="/etc/pki/${REGION}/certs/ca-bundle.pem"
+
 echo "export AWS_CA_BUNDLE=/etc/pki/${REGION}/certs/ca-bundle.pem" >> /etc/profile.d/aws-cli-default-config.sh
 
 echo "export AWS_DEFAULT_REGION=${REGION}" >> /etc/profile.d/aws-cli-default-config.sh

cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_alinux2023.rb

@@ -14,6 +14,10 @@
 
 use 'partial/_slurm_dependencies_common'
 
+http_parser_version = "2.9.4"
+http_parser_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/http_parser/v#{http_parser_version}.tar.gz"
+http_parser_tarball = "#{node['cluster']['sources_dir']}/http-parser-#{http_parser_version}.tar.gz"
+
 def dependencies
   %w(json-c-devel perl perl-Switch lua-devel dbus-devel)
 end
@@ -22,13 +26,25 @@ def dependencies
   # http parser is no longer maintained, therefore Amazon Linux 2023 does have have the package in OS repos
   # https://docs.aws.amazon.com/linux/al2023/release-notes/removed-AL2023.4-AL2.html
   # Following https://slurm.schedmd.com/related_software.html#jwt for Installing Http-parser
-  bash 'Install http-parser' do
-    code <<-HTTP_PARSER
-    set -e
-    git clone --depth 1 --single-branch -b v2.9.4 https://github.com/nodejs/http-parser.git http_parser
-    cd http_parser
-    make
-    make install
-    HTTP_PARSER
+
+  remote_file "#{http_parser_tarball}" do
+    source "#{http_parser_url}"
+    mode '0644'
+    retries 3
+    retry_delay 5
+    action :create_if_missing
+  end
+
+  bash 'make install' do
+    user 'root'
+    group 'root'
+    cwd "#{node['cluster']['sources_dir']}"
+    code <<-HTTP
+      set -e
+      tar xf #{http_parser_tarball}
+      cd http-parser-#{http_parser_version}
+      make
+      make install
+    HTTP
   end
 end