Update the efs-utils to use a different Cargo.toml

Author: Himani Anil Deshpande

CHANGELOG.md

@@ -18,6 +18,11 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 **CHANGES**
 - Upgrade Slurm to version 24.05.8.
 
+**BUG FIXES**
+- Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments 
+due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket.
+- Use patched version of efs-utils v2.1.0 which pins backtrace version to v3.0.74 to resolve build image failure.
+
 3.13.0
 ------
 **ENHANCEMENTS**

cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "efs-proxy"
+edition = "2021"
+build = "build.rs"
+# The version of efs-proxy is tied to efs-utils.
+version = "2.1.0"
+publish = false
+
+[dependencies]
+anyhow = "1.0.72"
+async-trait = "0.1"
+bytes = { version = "1.4.0" }
+chrono = "0.4"
+clap = { version = "=4.0.0", features = ["derive"] }
+fern = "0.6"
+futures = "0.3"
+log = "0.4"
+log4rs = { version = "0", features = ["rolling_file_appender", "compound_policy", "size_trigger", "fixed_window_roller"]}
+nix = { version = "0.26.2", features = ["signal"]}
+onc-rpc = "0.2.3"
+rand = "0.8.5"
+s2n-tls = "0.0"
+s2n-tls-tokio = "0.0"
+s2n-tls-sys = "0.0"
+serde = {version="1.0.175",features=["derive"]}
+serde_ini = "0.2.0"
+thiserror = "1.0.44"
+tokio = { version = "1.29.0, <1.39", features = ["full"] }
+tokio-util = "0.7.8"
+uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"]}
+xdr-codec = "0.4.4"
+backtrace = "=0.3.74"
+
+[dev-dependencies]
+test-case = "*"
+tokio = { version = "1.29.0", features = ["test-util"] }
+tempfile = "3.10.1"
+
+[build-dependencies]
+xdrgen = "0.4.4"

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb

@@ -13,10 +13,9 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
+def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version)
   <<-EFSUTILSINSTALL
       set -e
-      tar xf #{efs_utils_tarball}
       cd efs-utils-#{efs_utils_version}
       ./build-deb.sh
       apt-get -y install ./build/#{efs_utils_package}*deb

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb

@@ -50,6 +50,22 @@
     action :create_if_missing
   end
 
+  bash "Untar the efs-utils" do
+    cwd node['cluster']['sources_dir']
+    code <<-EFSUTILSUNTAR
+      set -e
+      tar xf #{efs_utils_tarball}
+    EFSUTILSUNTAR
+  end
+
+  cookbook_file "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}/src/proxy/Cargo.toml" do
+    source 'efs/Cargo.toml'
+    owner 'root'
+    group 'root'
+    mode '0755'
+    action :create
+  end
+
   # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
   bash "install efs utils" do
     cwd node['cluster']['sources_dir']

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb

@@ -13,10 +13,9 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
+def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version)
   <<-EFSUTILSINSTALL
       set -e
-      tar xf #{efs_utils_tarball}
       cd efs-utils-#{efs_utils_version}
       make rpm
       yum -y install ./build/#{efs_utils_package}*rpm

cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb

@@ -88,10 +88,15 @@ def mock_already_installed(package, expected_version, installed)
       cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" }
       cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" }
       cached(:tarball_checksum) { 'TARBALL CHECKSUM' }
+      cached(:bash_untar_code) do
+        <<-EFSUTILSUNTAR
+      set -e
+      tar xf #{tarball_path}
+        EFSUTILSUNTAR
+      end
       cached(:bash_code) do
         <<-EFSUTILSINSTALL
       set -e
-      tar xf #{tarball_path}
       cd efs-utils-#{utils_version}
       ./build-deb.sh
       apt-get -y install ./build/amazon-efs-utils*deb
@@ -127,6 +132,12 @@ def mock_already_installed(package, expected_version, installed)
             .with(checksum: tarball_checksum)
         end
 
+        it 'it untars the downloaded tarball' do
+          is_expected.to run_bash('Untar the efs-utils')
+            .with(cwd: source_dir)
+            .with(code: bash_untar_code)
+        end
+
         it 'installs package from downloaded tarball' do
           is_expected.to run_bash('install efs utils')
             .with(cwd: source_dir)
@@ -168,10 +179,15 @@ def mock_already_installed(package, expected_version, installed)
       cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" }
       cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" }
       cached(:tarball_checksum) { 'TARBALL CHECKSUM' }
+      cached(:bash_untar_code) do
+        <<-EFSUTILSUNTAR
+      set -e
+      tar xf #{tarball_path}
+        EFSUTILSUNTAR
+      end
       cached(:bash_code) do
         <<-EFSUTILSINSTALL
       set -e
-      tar xf #{tarball_path}
       cd efs-utils-#{utils_version}
       make rpm
       yum -y install ./build/amazon-efs-utils*rpm
@@ -218,6 +234,12 @@ def mock_already_installed(package, expected_version, installed)
             .with(checksum: tarball_checksum)
         end
 
+        it 'it untars the downloaded tarball' do
+          is_expected.to run_bash('Untar the efs-utils')
+            .with(cwd: source_dir)
+            .with(code: bash_untar_code)
+        end
+
         it 'installs package from downloaded tarball' do
           is_expected.to run_bash('install efs utils')
             .with(cwd: source_dir)