Add global= option for plugstack.conf to configure singularity global options (#9)

Author: dennisklein

.gitignore

@@ -3,3 +3,5 @@ singularity-exec.so
 build/
 install/
 build-*
+_codeql_build_dir/
+_codeql_detected_source_root

CMakeLists.txt

@@ -17,6 +17,7 @@ set(SLURM_PLUGSTACK_CONF_D ${SLURM_SYSCONFDIR}/plugstack.conf.d
 set(PLUGIN_DEFAULT_ARG "" CACHE STRING "Plugin default= arg")
 set(PLUGIN_BIND_ARG ${SLURM_SYSCONFDIR},/var/spool/slurm,/var/run/munge
   CACHE STRING "Plugin bind= arg")
+set(PLUGIN_GLOBAL_ARG "" CACHE STRING "Plugin global= arg")
 set(PLUGIN_EXTRA_ARGS "" CACHE STRING "Plugin args= arg")
 
 find_path(SLURM_INCLUDE_DIR

README.md

@@ -49,6 +49,7 @@ CMake option                     | Default
 `-DSLURM_PLUGSTACK_CONF_D=...`   | `${SLURM_SYSCONFDIR}/plugstack.conf.d`                | Slurm plugstack conf dir
 `-DPLUGIN_DEFAULT_ARG=...`       | `""`                                                  | Plugin default= arg
 `-DPLUGIN_BIND_ARG=...`          | `${SLURM_SYSCONFDIR},/var/spool/slurm,/var/run/munge` | Plugin bind= arg
+`-DPLUGIN_GLOBAL_ARG=...`        | `""`                                                  | Plugin global= arg
 `-DPLUGIN_EXTRA_ARGS=...`        | `""`                                                  | Plugin args= arg
 `-DSLURM_INCLUDE_DIR=...`        | Detected by CMake, typically `/usr/include`           | Slurm include dir passed to compiler via `-I` so `#include <slurm/spank.h>` resolves
 `-DCMAKE_INSTALL_LIBEXECDIR=...` | `libexec` on RHEL-based systems                       | FHS "internal binaries" directory [^sSrfT]
@@ -82,7 +83,7 @@ include /etc/slurm/plugstack.conf.d/*.conf'
 EOF
 # reference the path to the plug-in and the wrapper script
 cat > /etc/slurm/plugstack.conf.d/singularity-exec.conf <<EOF
-required /usr/lib64/slurm/singularity-exec.so default= script=/usr/libexec/slurm-singularity-wrapper.sh bind= args=disabled
+required /usr/lib64/slurm/singularity-exec.so default= script=/usr/libexec/slurm-singularity-wrapper.sh bind= global= args=disabled
 EOF
 ```
 
@@ -95,6 +96,7 @@ Option                 | Description
 `default=<path>`       | Path to the Singularity container launched by default. If this is set user require to explicitly use an empty `--singularity-container=` option to prevent the start of a container.
 `script=<path>`        | Path to the wrapper script which consumes the input arguments and environment variables set by the plugin to launch the Singularity container.
 `bind=<spec>`          | List of paths to bind-mount into the container by default. Please reference the section about [User-defined bind paths][95] in the Singularity User Documentation [^E9F6O].
+`global=<options>`     | List of [global command-line options][93] passed to the `singularity` command itself (e.g., `--silent`, `--quiet`). Equivalent to using the environment variable `SLURM_SINGULARITY_GLOBAL`.
 `args=<string>`        | List of [command-line arguments][94] passed to `singularity exec`. Disable support for this feature by setting `args=disabled`. This will prompt an error for an unrecognized option if the user adds the `--singularity-args=` option. Use an empty string `args=""` to enable support for singularity arguments without a default configuration. Supply default for all users by adding a list of options i.e. `args="--home /network/$USER"`
 
 Passing `-DINSTALL_PLUGSTACK_CONF=ON` to the CMake configure command will automate the above configuration.

main.cpp

@@ -210,6 +210,7 @@ struct singularity_exec
   inline static std::string s_container_name = {};
   inline static std::string s_singularity_script = "/usr/lib/slurm/slurm-singularity-wrapper.sh";
   inline static std::string s_singularity_args = {};
+  inline static std::string s_singularity_global = {};
   inline static std::string s_bind_defaults = {};
   inline static std::string s_bind_mounts = {};
   inline static bool s_no_args_option = false;
@@ -323,6 +324,8 @@ struct singularity_exec
               s_singularity_script = arg.substr(7);
             else if (starts_with(arg, "bind="))
               s_bind_defaults = arg.substr(5);
+            else if (starts_with(arg, "global="))
+              s_singularity_global = arg.substr(7);
             else if (arg == "args=disabled")
               s_no_args_option = true;
             else if (starts_with(arg, "args=\""))
@@ -343,6 +346,7 @@ struct singularity_exec
                   "/usr/lib/slurm/slurm-singularity-wrapper.sh\n"
                   "bind=src[:dest[:opts]][,src[:dest[:opts]]]*\n"
                   "                              set default bind mounts\n"
+                  "global=<global options>       set default global singularity options\n"
                   "args=disabled                 Disable custom arguments\n"
                   "args=\"<singulary args>\"       quotes are mandatory; "
                   "string may be empty\n",
@@ -424,10 +428,11 @@ struct singularity_exec
               s_bind_mounts = s_bind_defaults + ',' + s_bind_mounts;
           }
 
-        // unconditionally set these two variables so they don't become an
+        // unconditionally set these three variables so they don't become an
         // accidental user interface
         s.setenv("SLURM_SINGULARITY_BIND", s_bind_mounts.c_str());
         s.setenv("SLURM_SINGULARITY_ARGS", s_singularity_args.c_str());
+        s.setenv("SLURM_SINGULARITY_GLOBAL", s_singularity_global.c_str());
         std::vector<char*> argv = s.job_argument_vector();
         argv.insert(argv.begin(),
                     { s_singularity_script.data(), s_container_name.data() });

singularity-exec.conf.in

@@ -1,3 +1,3 @@
 # Singularity Plug-in
 #
-required @CMAKE_INSTALL_FULL_LIBEXECDIR@/@PROJECT_NAME@.so default=@PLUGIN_DEFAULT_ARG@ script=@CMAKE_INSTALL_FULL_LIBEXECDIR@/@WRAPPER@ bind=@PLUGIN_BIND_ARG@ args="@PLUGIN_EXTRA_ARGS@"
+required @CMAKE_INSTALL_FULL_LIBEXECDIR@/@PROJECT_NAME@.so default=@PLUGIN_DEFAULT_ARG@ script=@CMAKE_INSTALL_FULL_LIBEXECDIR@/@WRAPPER@ bind=@PLUGIN_BIND_ARG@ global=@PLUGIN_GLOBAL_ARG@ args="@PLUGIN_EXTRA_ARGS@"

tests/runtime/entrypoint-plugin-builder.sh

@@ -12,7 +12,8 @@ cmake -GNinja -S "$SRC_DIR" -B "$BUILD_DIR" \
     -DCMAKE_INSTALL_PREFIX=/usr \
     -DSLURM_SYSCONFDIR=/etc/slurm \
     -DINSTALL_PLUGSTACK_CONF=ON \
-    -DPLUGIN_BIND_ARG="/etc/slurm,/var/spool/slurm,/var/spool/slurmd,/var/run/munge"
+    -DPLUGIN_BIND_ARG="/etc/slurm,/var/spool/slurm,/var/spool/slurmd,/var/run/munge" \
+    -DPLUGIN_GLOBAL_ARG="--silent"
 
 cmake --build "$BUILD_DIR"
 

tests/runtime/test-integration.sh

@@ -280,5 +280,128 @@ else
     echo
 fi
 
+# Test 12: Verify global= option in plugstack.conf is propagated correctly
+echo "Test 12: Verifying global= option configuration..."
+
+if [ ! -f "$PLUGSTACK_CONF" ]; then
+    echo "✗ ERROR: Plugin config not found at $PLUGSTACK_CONF"
+    exit 1
+fi
+
+# Check if global= parameter exists in the config
+if ! grep -q "global=" "$PLUGSTACK_CONF"; then
+    echo "⚠ Warning: global= parameter not found in $PLUGSTACK_CONF"
+    echo "  This may indicate the plugin was built without PLUGIN_GLOBAL_ARG configured"
+    echo
+    echo "=== All tests passed! ==="
+    exit 0
+fi
+
+echo "✓ Found global= parameter in $PLUGSTACK_CONF"
+
+# Extract the global= value from config
+GLOBAL_VALUE=$(grep "global=" "$PLUGSTACK_CONF" | sed -n 's/.*global=\([^ ]*\).*/\1/p')
+echo "  Configuration value: global=${GLOBAL_VALUE}"
+
+# If container tests are not available, skip runtime verification
+if [ "$SKIP_CONTAINER_TEST" = "true" ]; then
+    echo "  Skipping runtime verification (no container available)"
+    echo
+    echo "=== All tests passed! ==="
+    exit 0
+fi
+
+# Verify the value is used in a containerized job
+echo "  Testing global option propagation with containerized job..."
+
+# Create a test job script that checks if SLURM_SINGULARITY_GLOBAL is set and prints its value
+TEST_GLOBAL_SCRIPT=$(mktemp /tmp/test_global.XXXXXX.sh)
+cat > "$TEST_GLOBAL_SCRIPT" <<'GLOBALEOF'
+#!/bin/bash
+# Check if SLURM_SINGULARITY_GLOBAL is set and print its value
+if [ -n "${SLURM_SINGULARITY_GLOBAL+x}" ]; then
+    echo "SLURM_SINGULARITY_GLOBAL is set to: '${SLURM_SINGULARITY_GLOBAL}'"
+else
+    echo "SLURM_SINGULARITY_GLOBAL is NOT set"
+fi
+hostname
+GLOBALEOF
+chmod +x "$TEST_GLOBAL_SCRIPT"
+
+# Submit job with container
+GLOBAL_JOB_ID=$(sbatch --singularity-container="$TEST_CONTAINER" \
+                       --output="${SLURM_JOB_SPOOL}/test_global_%j.out" \
+                       --error="${SLURM_JOB_SPOOL}/test_global_%j.err" \
+                       "$TEST_GLOBAL_SCRIPT" 2>&1 | awk '{print $NF}')
+
+if [ -z "$GLOBAL_JOB_ID" ]; then
+    echo "⚠ Warning: Failed to submit global option test job"
+    rm -f "$TEST_GLOBAL_SCRIPT"
+    echo
+    echo "=== All tests passed! ==="
+    exit 0
+fi
+
+echo "  Job submitted: $GLOBAL_JOB_ID"
+
+# Wait for job to complete
+retry --times="$RETRY_TIMES" --delay="$JOB_RETRY_DELAY" -- \
+    bash -c "scontrol show job $GLOBAL_JOB_ID 2>/dev/null | grep -qE 'JobState=(COMPLETED|FAILED|CANCELLED)'" >/dev/null 2>&1
+
+JOB_STATE=$(scontrol show job "$GLOBAL_JOB_ID" 2>/dev/null | grep "JobState" | awk '{print $1}' | cut -d= -f2)
+
+if [ "$JOB_STATE" != "COMPLETED" ] && [ "$JOB_STATE" != "COMPLETING" ]; then
+    echo "⚠ Warning: Global option test job did not complete successfully (State: $JOB_STATE)"
+    rm -f "$TEST_GLOBAL_SCRIPT"
+    echo
+    echo "=== All tests passed! ==="
+    exit 0
+fi
+
+# Check stdout for the SLURM_SINGULARITY_GLOBAL environment variable check
+GLOBAL_OUT_FILE="${SLURM_JOB_SPOOL}/test_global_${GLOBAL_JOB_ID}.out"
+
+if [ ! -f "$GLOBAL_OUT_FILE" ]; then
+    echo "⚠ Warning: Job output file not found, cannot verify global option propagation"
+    rm -f "$TEST_GLOBAL_SCRIPT"
+    echo
+    echo "=== All tests passed! ==="
+    exit 0
+fi
+
+# Check if the variable is set
+if grep -q "SLURM_SINGULARITY_GLOBAL is NOT set" "$GLOBAL_OUT_FILE"; then
+    echo "✗ ERROR: SLURM_SINGULARITY_GLOBAL was not set by plugin"
+    cat "$GLOBAL_OUT_FILE"
+    rm -f "$TEST_GLOBAL_SCRIPT" "${SLURM_JOB_SPOOL}/test_global_${GLOBAL_JOB_ID}."*
+    exit 1
+fi
+
+if ! grep -q "SLURM_SINGULARITY_GLOBAL is set to:" "$GLOBAL_OUT_FILE"; then
+    echo "⚠ Warning: Could not verify SLURM_SINGULARITY_GLOBAL in job output"
+    rm -f "$TEST_GLOBAL_SCRIPT" "${SLURM_JOB_SPOOL}/test_global_${GLOBAL_JOB_ID}."*
+    echo
+    echo "=== All tests passed! ==="
+    exit 0
+fi
+
+echo "✓ SLURM_SINGULARITY_GLOBAL environment variable was set by plugin"
+
+# Show the actual value that was set
+ACTUAL_GLOBAL=$(grep "SLURM_SINGULARITY_GLOBAL is set to:" "$GLOBAL_OUT_FILE" | head -1)
+echo "  ${ACTUAL_GLOBAL}"
+
+# Extract just the value for comparison (between quotes)
+ACTUAL_VALUE=$(echo "$ACTUAL_GLOBAL" | sed -n "s/.*SLURM_SINGULARITY_GLOBAL is set to: '\(.*\)'/\1/p")
+if [ "$ACTUAL_VALUE" = "$GLOBAL_VALUE" ]; then
+    echo "✓ Value matches configuration: '${GLOBAL_VALUE}'"
+else
+    echo "⚠ Warning: Value mismatch (config: '${GLOBAL_VALUE}', actual: '${ACTUAL_VALUE}')"
+fi
+
+# Cleanup test files
+rm -f "$TEST_GLOBAL_SCRIPT" "${SLURM_JOB_SPOOL}/test_global_${GLOBAL_JOB_ID}."*
+echo
+
 echo "=== All tests passed! ==="
 exit 0