Configure Enroot and Pyxis only on HeadNode

* Replacing sed commands with templates

* Replacing creation of directories with chef resource

* Adding Chef Attributes for enabling Pyxis and enroot

* Creating pyxis conf in Config Phase gives no such file found error

* Changing Kitchen test for Enroot and Pyxis

Author: Himani Deshpande

cookbooks/aws-parallelcluster-platform/attributes/platform.rb

@@ -11,7 +11,9 @@
 
 # Enroot + Pyxis
 default['cluster']['enroot']['version'] = '3.4.1'
+default['cluster']['enroot']['enabled'] = 'no'
 default['cluster']['pyxis']['version'] = '0.20.0'
+default['cluster']['pyxis']['enabled'] = 'no'
 
 # NVidia
 default['cluster']['nvidia']['enabled'] = 'no'

cookbooks/aws-parallelcluster-platform/recipes/config.rb

@@ -26,6 +26,8 @@
 include_recipe 'aws-parallelcluster-platform::supervisord_config'
 fetch_config 'Fetch and load cluster configs'
 include_recipe 'aws-parallelcluster-platform::config_login' if node['cluster']['node_type'] == 'LoginNode'
-enroot 'Configure Enroot' do
-  action :configure
+if node['cluster']['node_type'] == 'HeadNode'
+  enroot 'Configure Enroot' do
+    action :configure
+  end
 end

cookbooks/aws-parallelcluster-platform/resources/enroot/partial/_enroot_common.rb

@@ -23,6 +23,8 @@
 action :configure do
   return if on_docker?
   return unless enroot_installed
+  return unless configure_enroot
+  return unless configure_pyxis
 
   cookbook_file "/tmp/enroot.template.conf" do
     source 'enroot/enroot.template.conf'
@@ -33,34 +35,57 @@
     action :create_if_missing
   end
 
+  directory "#{node['cluster']['shared_dir']}/enroot" do
+    owner node['cluster']['cluster_user']
+    # group node['cluster']['cluster_user']
+    mode '1777'
+    action :create
+  end
+
+  directory "#{node['cluster']['shared_dir']}/pyxis" do
+    owner node['cluster']['cluster_user']
+    # group node['cluster']['cluster_user']
+    # mode '0755'
+    action :create
+  end
+
+  directory "/tmp/enroot/data" do
+    mode '1777'
+    action :create
+    recursive true
+  end
+
   bash "Configure enroot" do
     user 'root'
     code <<-ENROOT_CONFIGURE
       set -e
       ENROOT_CONFIG_RELEASE=pyxis
       SHARED_DIR=#{node['cluster']['shared_dir']}
-      NONROOT_USER=#{node['cluster']['cluster_user']}
-      mkdir -p ${SHARED_DIR}/enroot
-      chown ${NONROOT_USER} ${SHARED_DIR}/enroot
       ENROOT_CACHE_PATH=${SHARED_DIR}/enroot envsubst < /tmp/enroot.template.conf > /tmp/enroot.conf
       mv /tmp/enroot.conf /etc/enroot/enroot.conf
       chmod 0644 /etc/enroot/enroot.conf
+    ENROOT_CONFIGURE
+    retries 3
+    retry_delay 5
+  end
 
-      mkdir -p /tmp/enroot
-      chmod 1777 /tmp/enroot
-      mkdir -p /tmp/enroot/data
-      chmod 1777 /tmp/enroot/data
-
-      chmod 1777 ${SHARED_DIR}/enroot
+  template "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf.d/pyxis.conf " do
+    source 'pyxis/pyxis.conf.erb'
+    cookbook 'aws-parallelcluster-platform'
+    owner 'root'
+    group 'root'
+    mode '0755' # TODO: Chnage the permission
+  end
 
-      mkdir -p ${SHARED_DIR}/pyxis/
-      chown ${NONROOT_USER} ${SHARED_DIR}/pyxis/
-      sed -i '${s/$/ runtime_path=${SHARED_DIR}\\/pyxis/}' /opt/slurm/etc/plugstack.conf.d/pyxis.conf
+  bash "Configure Pyxis" do
+    user 'root'
+    code <<-PYXIS_CONFIGURE
+      set -e
+      SHARED_DIR=#{node['cluster']['shared_dir']}
       SHARED_DIR=${SHARED_DIR} envsubst < /opt/slurm/etc/plugstack.conf.d/pyxis.conf > /opt/slurm/etc/plugstack.conf.d/pyxis.tmp.conf
       mv /opt/slurm/etc/plugstack.conf.d/pyxis.tmp.conf /opt/slurm/etc/plugstack.conf.d/pyxis.conf
-
-    ENROOT_CONFIGURE
-    retries 3
+    PYXIS_CONFIGURE
+    retries 10
     retry_delay 5
   end
 end
@@ -72,3 +97,11 @@ def package_version
 def enroot_installed
   ::File.exist?('/usr/bin/enroot')
 end
+
+def configure_enroot
+  ['yes', true].include?(node['cluster']['enroot']['enabled'])
+end
+
+def configure_pyxis
+  ['yes', true].include?(node['cluster']['pyxis']['enabled'])
+end

cookbooks/aws-parallelcluster-platform/templates/pyxis/pyxis.conf.erb

@@ -0,0 +1 @@
+required /usr/local/lib/slurm/spank_pyxis.so runtime_path=<%= node['cluster']['shared_dir'] %>/pyxis

cookbooks/aws-parallelcluster-platform/test/controls/enroot_spec.rb

@@ -14,7 +14,7 @@
 
   expected_enroot_version = node['cluster']['enroot']['version']
 
-  describe "gdrcopy version is expected to be #{expected_enroot_version}" do
+  describe "enroot version is expected to be #{expected_enroot_version}" do
     subject { command('enroot version').stdout.strip() }
     it { should eq expected_enroot_version }
   end

cookbooks/aws-parallelcluster-slurm/recipes/install/install_pyxis.rb

@@ -38,9 +38,19 @@
     CPPFLAGS='-I /opt/slurm/include/' make
     CPPFLAGS='-I /opt/slurm/include/' make install
     mkdir -p /opt/slurm/etc/plugstack.conf.d
-    echo -e 'include /opt/slurm/etc/plugstack.conf.d/*' | tee /opt/slurm/etc/plugstack.conf
-    ln -fs /usr/local/share/pyxis/pyxis.conf /opt/slurm/etc/plugstack.conf.d/pyxis.conf
   PYXIS_INSTALL
   retries 3
   retry_delay 5
 end
+
+template "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf" do
+  source 'pyxis/plugstack.conf.erb'
+  cookbook 'aws-parallelcluster-slurm'
+  owner 'root'
+  group 'root'
+  mode '0644'
+end
+
+link '/usr/local/share/pyxis/pyxis.conf' do
+  to "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf.d/pyxis.conf"
+end

cookbooks/aws-parallelcluster-slurm/templates/default/pyxis/plugstack.conf.erb

@@ -0,0 +1 @@
+include /opt/slurm/etc/plugstack.conf.d/*

cookbooks/aws-parallelcluster-slurm/test/controls/pyxis_spec.rb

@@ -14,6 +14,10 @@
 
   title 'Checks Pyxis has been installed'
 
+  describe directory('/opt/slurm/etc/plugstack.conf.d') do
+    it { should exist }
+  end
+
   describe file("/opt/slurm/etc/plugstack.conf.d/pyxis.conf") do
     it { should exist }
   end