Running Config Phase of Enroot and Pyxis only on HeadNode

Author: Himani Deshpande

cookbooks/aws-parallelcluster-platform/recipes/config.rb

@@ -26,6 +26,8 @@
 include_recipe 'aws-parallelcluster-platform::supervisord_config'
 fetch_config 'Fetch and load cluster configs'
 include_recipe 'aws-parallelcluster-platform::config_login' if node['cluster']['node_type'] == 'LoginNode'
-enroot 'Configure Enroot' do
-  action :configure
+if node['cluster']['node_type'] == 'HeadNode'
+  enroot 'Configure Enroot' do
+    action :configure
+  end
 end

cookbooks/aws-parallelcluster-platform/resources/enroot/partial/_enroot_common.rb

@@ -33,15 +33,28 @@
     action :create_if_missing
   end
 
+  directory "#{node['cluster']['shared_dir']}/enroot" do
+    owner node['cluster']['cluster_user']
+    # group node['cluster']['cluster_user']
+    # mode '0755'
+    action :create
+  end
+
+  directory "#{node['cluster']['shared_dir']}/pyxis" do
+    owner node['cluster']['cluster_user']
+    # group node['cluster']['cluster_user']
+    # mode '0755'
+    action :create
+  end
+
   bash "Configure enroot" do
     user 'root'
     code <<-ENROOT_CONFIGURE
       set -e
       ENROOT_CONFIG_RELEASE=pyxis
       SHARED_DIR=#{node['cluster']['shared_dir']}
       NONROOT_USER=#{node['cluster']['cluster_user']}
-      mkdir -p ${SHARED_DIR}/enroot
-      chown ${NONROOT_USER} ${SHARED_DIR}/enroot
+      
       ENROOT_CACHE_PATH=${SHARED_DIR}/enroot envsubst < /tmp/enroot.template.conf > /tmp/enroot.conf
       mv /tmp/enroot.conf /etc/enroot/enroot.conf
       chmod 0644 /etc/enroot/enroot.conf
@@ -53,13 +66,41 @@
 
       chmod 1777 ${SHARED_DIR}/enroot
 
-      mkdir -p ${SHARED_DIR}/pyxis/
-      chown ${NONROOT_USER} ${SHARED_DIR}/pyxis/
-      sed -i '${s/$/ runtime_path=${SHARED_DIR}\\/pyxis/}' /opt/slurm/etc/plugstack.conf.d/pyxis.conf
+    ENROOT_CONFIGURE
+    retries 3
+    retry_delay 5
+  end
+
+  template "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf" do
+    source 'pyxis/plugstack.conf.erb'
+    cookbook 'aws-parallelcluster-platform'
+    owner 'root'
+    group 'root'
+    mode '0644'
+  end
+
+
+  link '/usr/local/share/pyxis/pyxis.conf' do
+    to "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf.d/pyxis.conf"
+  end
+  
+  template "#{node['cluster']['slurm']['install_dir']}/etc/plugstack.conf.d/pyxis.conf " do
+    source 'pyxis/pyxis.conf.erb'
+    cookbook 'aws-parallelcluster-platform'
+    owner 'root'
+    group 'root'
+    mode '0644'
+  end
+
+  
+  bash "Configure Pyxis" do
+    user 'root'
+    code <<-PYXIS_CONFIGURE 
+      SHARED_DIR=#{node['cluster']['shared_dir']}
       SHARED_DIR=${SHARED_DIR} envsubst < /opt/slurm/etc/plugstack.conf.d/pyxis.conf > /opt/slurm/etc/plugstack.conf.d/pyxis.tmp.conf
       mv /opt/slurm/etc/plugstack.conf.d/pyxis.tmp.conf /opt/slurm/etc/plugstack.conf.d/pyxis.conf
 
-    ENROOT_CONFIGURE
+    PYXIS_CONFIGURE
     retries 3
     retry_delay 5
   end

cookbooks/aws-parallelcluster-platform/templates/pyxis/plugstack.conf.erb

@@ -0,0 +1 @@
+include /opt/slurm/etc/plugstack.conf.d/*

cookbooks/aws-parallelcluster-platform/templates/pyxis/pyxis.conf.erb

@@ -0,0 +1 @@
+ runtime_path=<%= node['cluster']['shared_dir'] %>/pyxis/

cookbooks/aws-parallelcluster-slurm/recipes/install/install_pyxis.rb

@@ -38,8 +38,6 @@
     CPPFLAGS='-I /opt/slurm/include/' make
     CPPFLAGS='-I /opt/slurm/include/' make install
     mkdir -p /opt/slurm/etc/plugstack.conf.d
-    echo -e 'include /opt/slurm/etc/plugstack.conf.d/*' | tee /opt/slurm/etc/plugstack.conf
-    ln -fs /usr/local/share/pyxis/pyxis.conf /opt/slurm/etc/plugstack.conf.d/pyxis.conf
   PYXIS_INSTALL
   retries 3
   retry_delay 5