feat: job arrays

CHANGELOG.md

@@ -52,7 +52,7 @@
 
 ### Performance Improvements
 
-* performanc tweaks ([#417](https://github.com/snakemake/snakemake-executor-plugin-slurm/issues/417)) ([a3f6abf](https://github.com/snakemake/snakemake-executor-plugin-slurm/commit/a3f6abf47a1d3baa51b987fc0fcdb1972fc2bdd6))
+* performance tweaks ([#417](https://github.com/snakemake/snakemake-executor-plugin-slurm/issues/417)) ([a3f6abf](https://github.com/snakemake/snakemake-executor-plugin-slurm/commit/a3f6abf47a1d3baa51b987fc0fcdb1972fc2bdd6))
 
 ## [2.3.1](https://github.com/snakemake/snakemake-executor-plugin-slurm/compare/v2.3.0...v2.3.1) (2026-02-20)
 

docs/further.md

@@ -332,6 +332,19 @@ This node tracking works regardless of whether the `--slurm-requeue` flag is ena
 - **With `--slurm-requeue`**: SLURM will automatically requeue failed jobs, and they will be retried on different nodes
 - **Without `--slurm-requeue`**: Failed jobs will be reported as errors, but future retries (via `--retries` or other retry mechanisms) will avoid the problematic nodes
 
+#### SLURM Job Arrays
+
+Using `--slurm-array-jobs` SLURM job arrays can be submitted. `--slurm-array-jobs=rule1,rule2,...` lets you select specific rules by name to be selected as an array job. Alternatively, `--slurm-array-jobs=all` will submit all eligible rules as array jobs.
+
+Note: group jobs cannot be array jobs.
+
+.. note:: Using array jobs does impose a synchronization overhead (all jobs of a particular rule need to be ready for execution).
+
+When submitting array jobs, the `--slurm-array-limit` flag defines the
+maximum number of array tasks to be submitted in one job submission.
+If the number of tasks exceeds this limit, multiple array job submissions will be performed. This is useful to avoid hitting cluster limits on the maximum number of array tasks per job. Please obey your cluster limits and set this flag accordingly.
+
+
 #### MPI-specific Resources
 
 Snakemake's SLURM executor plugin supports the execution of MPI ([Message Passing Interface](https://en.wikipedia.org/wiki/Message_Passing_Interface)) jobs.

snakemake_executor_plugin_slurm/job_cancellation.py

@@ -26,7 +26,19 @@ def cancel_slurm_jobs(
     """
     if active_jobs:
         # TODO chunk jobids in order to avoid too long command lines
-        jobids = " ".join([job_info.external_jobid for job_info in active_jobs])
+        # Filter out None values in case some jobs haven't been assigned
+        # external IDs yet
+        jobids = " ".join(
+            [
+                job_info.external_jobid
+                for job_info in active_jobs
+                if job_info.external_jobid is not None
+            ]
+        )
+
+        if not jobids:
+            # No valid job IDs to cancel
+            return
 
         try:
             # timeout set to 60, because a scheduler cycle usually is
@@ -68,5 +80,5 @@ def cancel_slurm_jobs(
                     "HPC administrator."
                 )
             raise WorkflowError(
-                "Unable to cancel jobs with scancel " f"(exit code {e.returncode}){msg}"
+                f"Unable to cancel jobs with scancel (exit code {e.returncode}){msg}"
             ) from e

snakemake_executor_plugin_slurm/submit_string.py

@@ -16,13 +16,15 @@ def safe_quote(value):
     return shlex.quote(str_value)
 
 
-def get_submit_command(job, params):
+def get_submit_command(job, params, settings=None, failed_nodes=None) -> str:
     """
     Return the submit command for the job.
     """
     # Convert params dict to a SimpleNamespace for attribute-style access
     params = SimpleNamespace(**params)
 
+    failed_nodes = failed_nodes or set()
+
     call = (
         "sbatch "
         "--parsable "
@@ -76,6 +78,20 @@ def get_submit_command(job, params):
     if job.resources.get("nodes", False):
         call += f" --nodes={job.resources.get('nodes', 1)}"
 
+    if settings and settings.requeue:
+        call += " --requeue"
+
+    if settings and settings.qos:
+        call += f" --qos={safe_quote(settings.qos)}"
+
+    if settings and settings.reservation:
+        call += f" --reservation={safe_quote(settings.reservation)}"
+
+    # we exclude failed nodes from further job submissions, to avoid
+    # repeated failures.
+    if failed_nodes:
+        call += f" --exclude={','.join(failed_nodes)}"
+
     gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
     if gpu_job:
         # fixes #316 - allow unsetting of tasks per gpu

snakemake_executor_plugin_slurm/utils.py

@@ -1,17 +1,82 @@
 # utility functions for the SLURM executor plugin
 
+from collections import Counter
 import math
 import os
+import shlex
+import subprocess
 import re
 from pathlib import Path
 from typing import Union
 
+from snakemake_interface_executor_plugins.dag import DAGExecutorInterface
 from snakemake_interface_executor_plugins.jobs import (
     JobExecutorInterface,
 )
 from snakemake_interface_common.exceptions import WorkflowError
 
 
+def get_max_array_size() -> int:
+    """
+    Function to get the maximum array size for SLURM job arrays. This is used
+    to determine how many jobs can be submitted in a single array job.
+
+    Returns:
+        The maximum array size for SLURM job arrays, as an integer.
+        Defaults to 1000 if the SLURM_ARRAY_MAX environment variable is not set
+        or cannot be parsed as an integer.
+    """
+    max_array_size_str = None
+    scontrol_cmd = "scontrol show config"
+    try:
+        res = subprocess.run(
+            shlex.split(scontrol_cmd),
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        out = (res.stdout or "") + (res.stderr or "")
+        m = re.search(r"MaxArraySize\s*=?\s*(\d+)", out, re.IGNORECASE)
+        if m:
+            max_array_size_str = m.group(1)
+    except (subprocess.SubprocessError, OSError):
+        max_array_size_str = None
+
+    try:
+        max_array_size = int(max_array_size_str)
+    except (ValueError, TypeError):
+        max_array_size = 1000
+    # The SLURM_ARRAY_MAX limits to its value -1
+    return max_array_size - 1
+
+
+def get_job_wildcards(job: JobExecutorInterface) -> str:
+    """
+    Function to get the wildcards of a job as a string. This is used to
+    create the job name for the SLURM job submission.
+
+    Args:
+        job: The JobExecutorInterface instance representing the job
+    Returns:
+        A string representation of the job's wildcards, with slashes replaced
+        by underscores.
+    """
+    try:
+        wildcard_str = (
+            "_".join(job.wildcards).replace("/", "_") if job.wildcards else ""
+        )
+    except AttributeError:
+        wildcard_str = ""
+
+    return wildcard_str
+
+
+def pending_jobs_for_rule(dag: DAGExecutorInterface, rule_name: str) -> int:
+    """Count pending jobs for a given rule in the DAG."""
+    counts = Counter(job.rule.name for job in dag.needrun_jobs())
+    return counts.get(rule_name, 0)
+
+
 def round_half_up(n):
     return int(math.floor(n + 0.5))