Configure Enroot and Pyxis only on HeadNode

* Replacing sed commands with templates

* Replacing creation of directories with chef resource

* Adding Chef Attributes for enabling Pyxis and enroot

* Creating pyxis conf in Config Phase gives no such file found error

* Changing Kitchen test for Enroot and Pyxis

Author: Himani Deshpande

cookbooks/aws-parallelcluster-platform/attributes/platform.rb

@@ -12,6 +12,7 @@
 # Enroot + Pyxis
 default['cluster']['enroot']['version'] = '3.4.1'
 default['cluster']['pyxis']['version'] = '0.20.0'
+default['cluster']['pyxis']['enabled'] = 'no'
 
 # NVidia
 default['cluster']['nvidia']['enabled'] = 'no'

cookbooks/aws-parallelcluster-platform/recipes/config.rb

@@ -26,6 +26,8 @@
 include_recipe 'aws-parallelcluster-platform::supervisord_config'
 fetch_config 'Fetch and load cluster configs'
 include_recipe 'aws-parallelcluster-platform::config_login' if node['cluster']['node_type'] == 'LoginNode'
-enroot 'Configure Enroot' do
-  action :configure
+if node['cluster']['node_type'] == 'HeadNode'
+  enroot 'Configure Enroot' do
+    action :configure
+  end
 end

cookbooks/aws-parallelcluster-platform/resources/enroot/partial/_enroot_common.rb

@@ -23,45 +23,47 @@
 action :configure do
   return if on_docker?
   return unless enroot_installed
+  return unless configure_pyxis
 
-  cookbook_file "/tmp/enroot.template.conf" do
-    source 'enroot/enroot.template.conf'
+  template "/etc/enroot/enroot.conf" do
+    source 'enroot/enroot.conf.erb'
     cookbook 'aws-parallelcluster-platform'
     owner 'root'
     group 'root'
-    mode '0755'
-    action :create_if_missing
+    mode '0644'
+    action :create
   end
 
-  bash "Configure enroot" do
-    user 'root'
-    code <<-ENROOT_CONFIGURE
-      set -e
-      ENROOT_CONFIG_RELEASE=pyxis
-      SHARED_DIR=#{node['cluster']['shared_dir']}
-      NONROOT_USER=#{node['cluster']['cluster_user']}
-      mkdir -p ${SHARED_DIR}/enroot
-      chown ${NONROOT_USER} ${SHARED_DIR}/enroot
-      ENROOT_CACHE_PATH=${SHARED_DIR}/enroot envsubst < /tmp/enroot.template.conf > /tmp/enroot.conf
-      mv /tmp/enroot.conf /etc/enroot/enroot.conf
-      chmod 0644 /etc/enroot/enroot.conf
+  directory "#{node['cluster']['shared_dir']}/enroot" do
+    owner node['cluster']['cluster_user']
+    # group node['cluster']['cluster_user']
+    mode '1777'
+    action :create
+  end
 
-      mkdir -p /tmp/enroot
-      chmod 1777 /tmp/enroot
-      mkdir -p /tmp/enroot/data
-      chmod 1777 /tmp/enroot/data
+  directory "#{node['cluster']['shared_dir']}/pyxis" do
+    owner node['cluster']['cluster_user']
+    # group node['cluster']['cluster_user']
+    # mode '0755'
+    action :create
+  end
 
-      chmod 1777 ${SHARED_DIR}/enroot
+  directory "/tmp/enroot" do
+    mode '1777'
+    action :create
+  end
 
-      mkdir -p ${SHARED_DIR}/pyxis/
-      chown ${NONROOT_USER} ${SHARED_DIR}/pyxis/
-      sed -i '${s/$/ runtime_path=${SHARED_DIR}\\/pyxis/}' /opt/slurm/etc/plugstack.conf.d/pyxis.conf
-      SHARED_DIR=${SHARED_DIR} envsubst < /opt/slurm/etc/plugstack.conf.d/pyxis.conf > /opt/slurm/etc/plugstack.conf.d/pyxis.tmp.conf
-      mv /opt/slurm/etc/plugstack.conf.d/pyxis.tmp.conf /opt/slurm/etc/plugstack.conf.d/pyxis.conf
+  directory "/tmp/enroot/data" do
+    mode '1777'
+    action :create
+  end
 
-    ENROOT_CONFIGURE
-    retries 3
-    retry_delay 5
+  template "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf.d/pyxis.conf " do
+    source 'pyxis/pyxis.conf.erb'
+    cookbook 'aws-parallelcluster-platform'
+    owner 'root'
+    group 'root'
+    mode '0755' # TODO: Chnage the permission
   end
 end
 
@@ -72,3 +74,7 @@ def package_version
 def enroot_installed
   ::File.exist?('/usr/bin/enroot')
 end
+
+def configure_pyxis
+  ['yes', true].include?(node['cluster']['pyxis']['enabled'])
+end

cookbooks/aws-parallelcluster-platform/files/enroot/enroot.template.conf renamed to cookbooks/aws-parallelcluster-platform/templates/enroot/enroot.conf.erb

@@ -1,8 +1,8 @@
 #ENROOT_LIBRARY_PATH        /usr/lib/enroot
 #ENROOT_SYSCONF_PATH        /etc/enroot
 ENROOT_RUNTIME_PATH        /tmp/enroot/user-$(id -u)
-ENROOT_CONFIG_PATH         ${ENROOT_CONFIG_PATH}
-ENROOT_CACHE_PATH          ${ENROOT_CACHE_PATH}
+ENROOT_CONFIG_PATH         
+ENROOT_CACHE_PATH          <%= node['cluster']['shared_dir'] %>/enroot
 ENROOT_DATA_PATH           /tmp/enroot/data/user-$(id -u)
 #ENROOT_TEMP_PATH           ${TMPDIR:-/tmp}
 
@@ -68,4 +68,4 @@ ENROOT_RESTRICT_DEV        no
 #all_proxy
 #no_proxy
 #http_proxy
-#https_proxy
+#https_proxy

cookbooks/aws-parallelcluster-platform/templates/pyxis/pyxis.conf.erb

@@ -0,0 +1 @@
+required /usr/local/lib/slurm/spank_pyxis.so runtime_path=<%= node['cluster']['shared_dir'] %>/pyxis

cookbooks/aws-parallelcluster-platform/test/controls/enroot_spec.rb

@@ -14,7 +14,7 @@
 
   expected_enroot_version = node['cluster']['enroot']['version']
 
-  describe "gdrcopy version is expected to be #{expected_enroot_version}" do
+  describe "enroot version is expected to be #{expected_enroot_version}" do
     subject { command('enroot version').stdout.strip() }
     it { should eq expected_enroot_version }
   end

cookbooks/aws-parallelcluster-slurm/recipes/install/install_pyxis.rb

@@ -38,9 +38,19 @@
     CPPFLAGS='-I /opt/slurm/include/' make
     CPPFLAGS='-I /opt/slurm/include/' make install
     mkdir -p /opt/slurm/etc/plugstack.conf.d
-    echo -e 'include /opt/slurm/etc/plugstack.conf.d/*' | tee /opt/slurm/etc/plugstack.conf
-    ln -fs /usr/local/share/pyxis/pyxis.conf /opt/slurm/etc/plugstack.conf.d/pyxis.conf
   PYXIS_INSTALL
   retries 3
   retry_delay 5
 end
+
+template "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf" do
+  source 'pyxis/plugstack.conf.erb'
+  cookbook 'aws-parallelcluster-slurm'
+  owner 'root'
+  group 'root'
+  mode '0644'
+end
+
+link '/usr/local/share/pyxis/pyxis.conf' do
+  to "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf.d/pyxis.conf"
+end

cookbooks/aws-parallelcluster-slurm/templates/default/pyxis/plugstack.conf.erb

@@ -0,0 +1 @@
+include /opt/slurm/etc/plugstack.conf.d/*

cookbooks/aws-parallelcluster-slurm/test/controls/pyxis_spec.rb

@@ -14,6 +14,10 @@
 
   title 'Checks Pyxis has been installed'
 
+  describe directory('/opt/slurm/etc/plugstack.conf.d') do
+    it { should exist }
+  end
+
   describe file("/opt/slurm/etc/plugstack.conf.d/pyxis.conf") do
     it { should exist }
   end