Update the efs-utils to use a different Cargo.toml

Author: Himani Anil Deshpande

CHANGELOG.md

@@ -9,9 +9,6 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 **CHANGES**
 - Upgrade Slurm to version 24.05.8.
 
-**BUG FIXES**
-- Use `v2.1.0-patched.tar.gz` which pins backtrace version to v3.0.74
-
 **BUG FIXES**
 - Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments 
 due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket.

cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "efs-proxy"
+edition = "2021"
+build = "build.rs"
+# The version of efs-proxy is tied to efs-utils.
+version = "2.1.0"
+publish = false
+
+[dependencies]
+anyhow = "1.0.72"
+async-trait = "0.1"
+bytes = { version = "1.4.0" }
+chrono = "0.4"
+clap = { version = "=4.0.0", features = ["derive"] }
+fern = "0.6"
+futures = "0.3"
+log = "0.4"
+log4rs = { version = "0", features = ["rolling_file_appender", "compound_policy", "size_trigger", "fixed_window_roller"]}
+nix = { version = "0.26.2", features = ["signal"]}
+onc-rpc = "0.2.3"
+rand = "0.8.5"
+s2n-tls = "0.0"
+s2n-tls-tokio = "0.0"
+s2n-tls-sys = "0.0"
+serde = {version="1.0.175",features=["derive"]}
+serde_ini = "0.2.0"
+thiserror = "1.0.44"
+tokio = { version = "1.29.0, <1.39", features = ["full"] }
+tokio-util = "0.7.8"
+uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"]}
+xdr-codec = "0.4.4"
+backtrace = "=0.3.74"
+
+[dev-dependencies]
+test-case = "*"
+tokio = { version = "1.29.0", features = ["test-util"] }
+tempfile = "3.10.1"
+
+[build-dependencies]
+xdrgen = "0.4.4"

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb

@@ -3,7 +3,7 @@
 default_action :install_utils
 
 property :efs_utils_version, String, default: '2.1.0'
-property :efs_utils_checksum, String, default: 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887'
+property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24'
 
 def already_installed?(package_name, expected_version)
   Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version)

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb

@@ -13,10 +13,9 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
+def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version)
   <<-EFSUTILSINSTALL
       set -e
-      tar xf #{efs_utils_tarball}
       cd efs-utils-#{efs_utils_version}
       ./build-deb.sh
       apt-get -y install ./build/#{efs_utils_package}*deb

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb

@@ -32,7 +32,7 @@
   package_name = "amazon-efs-utils"
   package_version = new_resource.efs_utils_version
   efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
-  efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}-patched.tar.gz"
+  efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
 
   # Do not install efs-utils if a same or newer version is already installed.
   return if already_installed?(package_name, package_version)
@@ -50,6 +50,22 @@
     action :create_if_missing
   end
 
+  bash "Untar the efs-utils" do
+    cwd node['cluster']['sources_dir']
+    code <<-EFSUTILSUNTAR
+      set -e
+      tar xf #{efs_utils_tarball}
+    EFSUTILSUNTAR
+  end
+
+  cookbook_file "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}/src/proxy/Cargo.toml" do
+    source 'efs/Cargo.toml'
+    owner 'root'
+    group 'root'
+    mode '0755'
+    action :create
+  end
+
   # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
   bash "install efs utils" do
     cwd node['cluster']['sources_dir']

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb

@@ -13,10 +13,9 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
+def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version)
   <<-EFSUTILSINSTALL
       set -e
-      tar xf #{efs_utils_tarball}
       cd efs-utils-#{efs_utils_version}
       make rpm
       yum -y install ./build/#{efs_utils_package}*rpm

cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb

@@ -88,10 +88,15 @@ def mock_already_installed(package, expected_version, installed)
       cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" }
       cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" }
       cached(:tarball_checksum) { 'TARBALL CHECKSUM' }
+      cached(:bash_untar_code) do
+        <<-EFSUTILSUNTAR
+      set -e
+      tar xf #{tarball_path}
+        EFSUTILSUNTAR
+      end
       cached(:bash_code) do
         <<-EFSUTILSINSTALL
       set -e
-      tar xf #{tarball_path}
       cd efs-utils-#{utils_version}
       ./build-deb.sh
       apt-get -y install ./build/amazon-efs-utils*deb
@@ -127,6 +132,12 @@ def mock_already_installed(package, expected_version, installed)
             .with(checksum: tarball_checksum)
         end
 
+        it 'it untars the downloaded tarball' do
+          is_expected.to run_bash('Untar the efs-utils')
+            .with(cwd: source_dir)
+            .with(code: bash_untar_code)
+        end
+
         it 'installs package from downloaded tarball' do
           is_expected.to run_bash('install efs utils')
             .with(cwd: source_dir)
@@ -168,10 +179,15 @@ def mock_already_installed(package, expected_version, installed)
       cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" }
       cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" }
       cached(:tarball_checksum) { 'TARBALL CHECKSUM' }
+      cached(:bash_untar_code) do
+        <<-EFSUTILSUNTAR
+      set -e
+      tar xf #{tarball_path}
+        EFSUTILSUNTAR
+      end
       cached(:bash_code) do
         <<-EFSUTILSINSTALL
       set -e
-      tar xf #{tarball_path}
       cd efs-utils-#{utils_version}
       make rpm
       yum -y install ./build/amazon-efs-utils*rpm
@@ -218,6 +234,12 @@ def mock_already_installed(package, expected_version, installed)
             .with(checksum: tarball_checksum)
         end
 
+        it 'it untars the downloaded tarball' do
+          is_expected.to run_bash('Untar the efs-utils')
+            .with(cwd: source_dir)
+            .with(code: bash_untar_code)
+        end
+
         it 'installs package from downloaded tarball' do
           is_expected.to run_bash('install efs utils')
             .with(cwd: source_dir)

cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb

@@ -6,7 +6,7 @@
 
   describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do
     it { should exist }
-    its('sha256sum') { should eq 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' }
+    its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' }
     its('owner') { should eq 'root' }
     its('group') { should eq 'root' }
     its('mode') { should cmp '0644' }