__init__.py

__author__ = "David Lähnemann, Johannes Köster, Christian Meesters"
__copyright__ = "Copyright 2023, David Lähnemann, Johannes Köster, Christian Meesters"
__email__ = "johannes.koester@uni-due.de"
__license__ = "MIT"

import atexit
import asyncio
import base64
import errno
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import re
import shlex
import shutil
import subprocess
import time
from dataclasses import dataclass, field
from typing import List, Generator, Optional
import uuid
import zlib

from snakemake_interface_executor_plugins.executors.base import (
    SubmittedJobInfo,
)
from snakemake_interface_executor_plugins.executors.remote import (
    RemoteExecutor,
)
from snakemake_interface_executor_plugins.settings import (
    ExecutorSettingsBase,
    CommonSettings,
)
from snakemake_interface_executor_plugins.jobs import (
    JobExecutorInterface,
)
from snakemake_interface_common.exceptions import WorkflowError

from .accounts import (
    test_account,
    get_account,
)

from .utils import (
    get_max_array_size,
    get_job_wildcards,
    pending_jobs_for_rule,
    delete_slurm_environment,
    delete_empty_dirs,
    set_gres_string,
)
from .job_status_query import (
    get_min_job_age,
    is_query_tool_available,
    should_recommend_squeue_status_command,
    query_job_status_squeue,
    query_job_status_sacct,
    query_job_status,
)
from .job_cancellation import cancel_slurm_jobs
from .efficiency_report import create_efficiency_report
from .submit_string import get_submit_command
from .partitions import (
    get_default_partition,
    read_partition_file,
    get_best_partition,
)
from .validation import (
    validate_or_get_slurm_job_id,
    validate_slurm_extra,
    validate_executor_settings,
    validate_status_command_settings,
)


def _get_status_command_default():
    """Get smart default for status_command based on cluster configuration."""
    sacct_available = is_query_tool_available("sacct")
    squeue_available = is_query_tool_available("squeue")
    # squeue is assumed to always be available on SLURM clusters

    is_slurm_available = shutil.which("sinfo") is not None
    if not is_slurm_available:
        return None

    if not squeue_available and not sacct_available:
        raise WorkflowError(
            "Neither 'sacct' nor 'squeue' commands are available on this "
            "system. At least one of these commands is required for job "
            "status queries."
        )
    if sacct_available:
        return "sacct"
    else:
        return "squeue"


def _get_status_command_help():
    """Get help text with computed default."""
    default_cmd = _get_status_command_default()

    # if SLURM is not available (should not occur, only
    # in 3rd party CI tests)
    if default_cmd is None:
        return (
            "Command to query job status. Options: 'sacct', 'squeue'. "
            "SLURM not detected on this system, so no status command can be used."
        )

    sacct_available = is_query_tool_available("sacct")
    squeue_recommended = should_recommend_squeue_status_command()

    base_help = "Command to query job status. Options: 'sacct', 'squeue'. "

    if default_cmd == "sacct":
        if sacct_available and not squeue_recommended:
            info = (
                "'sacct' detected and will be used "
                "(MinJobAge may be too low for reliable 'squeue' usage)"
            )
        else:
            info = "'sacct' detected and will be used"
    else:  # default_cmd == "squeue"
        if squeue_recommended:
            # cumbersome, due to black and the need to stay below 80 chars
            msg_part1 = "'squeue' recommended (MinJobAge is sufficient )"
            msg_part2 = " for reliable usage"
            info = msg_part1 + msg_part2
        elif not sacct_available:
            info = (
                "'sacct' not available, falling back to 'squeue'. "
                "WARNING: 'squeue' may not work reliably if MinJobAge is "
                "too low"
            )
        else:
            info = (
                "'squeue' will be used. "
                "WARNING: MinJobAge may be too low for reliable 'squeue' usage"
            )

    return (
        f"{base_help}Default: '{default_cmd}' ({info}). "
        f"Set explicitly to override auto-detection."
    )


def _status_lookup_ids(external_jobid: str) -> List[str]:
    """Return candidate IDs for status lookup.

    For array jobs, Snakemake tracks task IDs as ``<jobid>_<taskid>``.
    Depending on SLURM command/options (e.g. ``sacct -X``), status output may
    only contain the parent array ID ``<jobid>``. This helper returns IDs in
    lookup order so callers can transparently fall back from task ID to parent
    ID.
    """
    candidates = [external_jobid]
    if "_" in external_jobid:
        parent_id, task_id = external_jobid.split("_", 1)
        if parent_id.isdigit() and task_id.isdigit():
            candidates.append(parent_id)
    return candidates


@dataclass
class ExecutorSettings(ExecutorSettingsBase):
    """Settings for the SLURM executor plugin."""

    array_jobs: Optional[str] = field(
        default=None,
        metadata={
            "help": "Will submit jobs as SLURM job arrays, if possible. "
            "Use as: --slurm-array-jobs='rule1, rule2' to submit jobs of "
            "rule1 and rule2 as array jobs. If a DAG contains only one job for "
            "a rule, it cannot be submitted as an array job. Selecting "
            "--slurm-array-jobs=all will submit all eligiblejobs as array jobs. "
            "Note: When choosing array job submission, the required jobs are "
            "subject to a synchronization overhead.",
            "env_var": False,
            "required": False,
        },
    )

    array_limit: Optional[int] = field(
        default=1000,
        metadata={
            "help": "When submitting array jobs, this flag defines the maximum "
            "number of array tasks to be submitted in one sbatch call. If the "
            "number of tasks exceeds this limit, multiple array job submissions "
            "will be performed. This is useful to avoid hitting cluster limits on "
            "the maximum number of array tasks per job. "
            "Please obey your cluster limits and set this flag accordingly.",
            "env_var": False,
            "required": False,
        },
    )

    logdir: Optional[Path] = field(
        default=None,
        metadata={
            "help": "Per default the SLURM log directory is relative to "
            "the working directory. "
            "This flag allows to set an alternative directory.",
            "env_var": False,
            "required": False,
        },
    )

    keep_successful_logs: bool = field(
        default=False,
        metadata={
            "help": "Per default SLURM log files will be deleted upon "
            "successful completion of a job. Whenever a SLURM job fails, "
            "its log file will be preserved. "
            "This flag allows to keep all SLURM log files, even those "
            "of successful jobs.",
            "env_var": False,
            "required": False,
        },
    )

    delete_logfiles_older_than: Optional[int] = field(
        default=10,
        metadata={
            "help": "Per default SLURM log files in the SLURM log directory "
            "of a workflow will be deleted after 10 days. For this, "
            "best leave the default log directory unaltered. "
            "Setting this flag allows to change this behaviour. "
            "If set to <=0, no old files will be deleted.",
        },
    )

    init_seconds_before_status_checks: Optional[int] = field(
        default=40,
        metadata={
            "help": "Defines the time in seconds before the first status "
            "check is performed on submitted jobs. Must be a positive "
            "integer",
        },
    )

    requeue: bool = field(
        default=False,
        metadata={
            "help": "Requeue jobs if they fail with exit code != 0, "
            "if no cluster default. Results in "
            "`sbatch ... --requeue ...` "
            "This flag has no effect, if not set.",
            "env_var": False,
            "required": False,
        },
    )

    exclude_failed_nodes: Optional[str] = field(
        default=None,
        metadata={
            "help": "Comma-separated list of nodes to exclude from job "
            "submission. This is useful to exclude known problematic nodes. "
            "This flag has no effect, if not set.",
            "env_var": False,
            "required": False,
        },
    )

    no_account: bool = field(
        default=False,
        metadata={
            "help": "Do not use any account for submission. "
            "This flag has no effect, if not set.",
            "env_var": False,
            "required": False,
        },
    )
    partition_config: Optional[Path] = field(
        default=None,
        metadata={
            "help": "Path to YAML file defining partition limits for dynamic "
            "partition selection. When provided, jobs will be dynamically "
            "assigned to the best-fitting partition based on their resource "
            "requirements. See documentation for complete list of available limits. "
            "Alternatively, the environment variable SNAKEMAKE_SLURM_PARTITIONS "
            "can be set to point to such a file. "
            "If both are set, this flag takes precedence.",
            "env_var": False,
            "required": False,
        },
    )
    efficiency_report: bool = field(
        default=False,
        metadata={
            "help": (
                "Generate an efficiency report at the end of the workflow. "
                "This flag has no effect, if not set."
            ),
            "env_var": False,
            "required": False,
        },
    )

    efficiency_report_path: Optional[Path] = field(
        default=None,
        metadata={
            "help": "Path to the efficiency report file. "
            "If not set, the report will be written to "
            "the current working directory with the name "
            "'efficiency_report_<run_uuid>.csv'. "
            "This flag has no effect, if not set.",
            "env_var": False,
            "required": False,
        },
    )

    efficiency_threshold: Optional[float] = field(
        default=0.8,
        metadata={
            "help": "Threshold for efficiency report. "
            "Jobs with efficiency below this threshold will be reported.",
            "env_var": False,
            "required": False,
        },
    )
    jobname_prefix: Optional[str] = field(
        default="",
        metadata={
            "help": "Prefix that is added to the job names. "
            "Must contain only alphanumeric characters, "
            "underscores or hyphens. Maximum length should "
            "not exceed 50 characters.",
            "env_var": False,
            "required": False,
        },
    )

    status_command: Optional[str] = field(
        default_factory=_get_status_command_default,
        metadata={
            "help": _get_status_command_help(),
            "env_var": False,
            "required": False,
        },
    )

    status_attempts: Optional[int] = field(
        default=5,
        metadata={
            "help": "Defines the number of attempts to query the status of "
            "all active jobs. If the status query fails, the next attempt "
            "will be performed after the next status check interval. "
            "The default is 5 status attempts before giving up. The maximum "
            "time between status checks is 180 seconds.",
            "env_var": False,
            "required": False,
        },
    )

    qos: Optional[str] = field(
        default=None,
        metadata={
            "help": "If set, the given QoS will be used for job submission.",
            "env_var": False,
            "required": False,
        },
    )

    reservation: Optional[str] = field(
        default=None,
        metadata={
            "help": ("If set, the given reservation will be used for job submission."),
            "env_var": False,
            "required": False,
        },
    )

    pass_command_as_script: bool = field(
        default=False,
        metadata={
            "help": (
                "Pass to sbatch and srun the command to be executed as a shell script"
                " (fed through stdin) instead of wrapping it in the command line "
                "call. Useful when a limit exists on SLURM command line length (ie. "
                "max_submit_line_size)."
            ),
            "env_var": False,
            "required": False,
        },
    )

    def __post_init__(self):
        """Validate settings after initialization."""
        validate_executor_settings(self)


# Required:
# Specify common settings shared by various executors.
common_settings = CommonSettings(
    # define whether your executor plugin executes locally
    # or remotely. In virtually all cases, it will be remote execution
    # (cluster, cloud, etc.). Only Snakemake's standard execution
    # plugins (snakemake-executor-plugin-dryrun,
    #          snakemake-executor-plugin-local)
    # are expected to specify False here.
    non_local_exec=True,
    # Define whether your executor plugin implies that there is no shared
    # filesystem (True) or not (False).
    # This is e.g. the case for cloud execution.
    implies_no_shared_fs=False,
    job_deploy_sources=False,
    pass_default_storage_provider_args=True,
    pass_default_resources_args=True,
    pass_envvar_declarations_to_cmd=False,
    auto_deploy_default_storage_provider=False,
)


def _select_logdir(workflow):
    """Selects where slurm_logdir should be created"""
    logdir = workflow.executor_settings.logdir
    # logdir is defined as absolute, keep "as is"
    if logdir and str(logdir).startswith("/"):
        return Path(logdir)
    # logdir is relative, we ensure it stays relative to the wokflow's workdir
    elif logdir:
        return Path(workflow.workdir_init) / workflow.executor_settings.logdir
    # logdir is unset
    else:
        return Path(".snakemake/slurm_logs").resolve()


# Required:
# Implementation of your executor
class Executor(RemoteExecutor):
    def __post_init__(self, test_mode: bool = False):
        # run check whether we are running in a SLURM job context
        self.warn_on_jobcontext()
        self.test_mode = test_mode

        self.run_uuid = str(uuid.uuid4())
        if self.workflow.executor_settings.exclude_failed_nodes:
            excluded_nodes = self.workflow.executor_settings.exclude_failed_nodes
            self._failed_nodes = set(
                node.strip() for node in excluded_nodes.split(",") if node.strip()
            )
        else:
            self._failed_nodes = set()
        # validate prefix: only allow alphanumeric, underscore, hyphen
        # cap length:
        if self.workflow.executor_settings.jobname_prefix:
            if not re.match(
                r"^[A-Za-z0-9_-]{1,50}$",
                self.workflow.executor_settings.jobname_prefix,
            ):
                raise WorkflowError(
                    "The jobname_prefix may only contain alphanumeric "
                    "characters, underscores or hyphens, and must not "
                    "exceed 50 characters in length."
                )
            self.run_uuid = "_".join(
                [self.workflow.executor_settings.jobname_prefix, self.run_uuid]
            )
        self.logger.info(f"SLURM run ID: {self.run_uuid}")
        self._fallback_account_arg = None
        self._fallback_partition = None
        self._preemption_warning = False  # no preemption warning has been issued
        self._submitted_job_clusters = set()  # track clusters of submitted jobs
        self._job_submission_executor = ThreadPoolExecutor(
            max_workers=2, thread_name_prefix="slurm_job_submit"
        )
        self._main_event_loop = None
        self._status_query_calls = 0
        self._status_query_failures = 0
        self._status_query_total_seconds = 0.0
        self._status_query_min_seconds = None
        self._status_query_max_seconds = 0.0
        self._status_query_cycle_rows = []
        array_job_setting = self.workflow.executor_settings.array_jobs
        if array_job_setting:
            normalized_setting = array_job_setting.replace(";", ",")
            self.array_jobs = {
                rule.strip() for rule in normalized_setting.split(",") if rule.strip()
            }
        else:
            self.array_jobs = set()
        self.max_array_size = min(
            get_max_array_size(), int(self.workflow.executor_settings.array_limit)
        )
        if self.max_array_size <= 10:
            self.logger.warning(
                "Array limit is set to "
                f"{self.max_array_size}, "
                "which is very low and may lead to excessive numbers of array "
                "job submissions. Please consider increasing this limit if your "
                "cluster allows it."
            )
        if self.max_array_size < 2:
            self.logger.error(
                "Array job submission is effectively disabled due to "
                f"max_array_size={self.max_array_size}. Consider increasing "
                "the array_limit setting to enable array job submission."
            )
            raise WorkflowError(
                "Array job submission is effectively disabled due to "
                "low array_limit."
            )
        self.slurm_logdir = _select_logdir(self.workflow)
        # Check the environment variable "SNAKEMAKE_SLURM_PARTITIONS",
        # if set, read the partitions from the given file. Let the CLI
        # option override this behavior.
        if (
            os.getenv("SNAKEMAKE_SLURM_PARTITIONS")
            and not self.workflow.executor_settings.partition_config
        ):
            partition_file = Path(os.getenv("SNAKEMAKE_SLURM_PARTITIONS"))
            self.logger.info(
                f"Reading SLURM partition configuration from "
                f"environment variable file: {partition_file}"
            )
            self._partitions = read_partition_file(partition_file)
        else:
            self._partitions = (
                read_partition_file(self.workflow.executor_settings.partition_config)
                if self.workflow.executor_settings.partition_config
                else None
            )
        atexit.register(self.clean_old_logs)
        # moved validation to validation.py
        validate_status_command_settings(self.workflow.executor_settings, self.logger)

    def get_status_command(self):
        """Get the status command to use, with fallback logic."""
        if hasattr(self.workflow.executor_settings, "status_command"):
            return self.workflow.executor_settings.status_command
        else:
            # Fallback: determine the best command based on
            # cluster configuration
            return _get_status_command_default()

    def shutdown(self) -> None:
        """
        Shutdown the executor.
        This method is overloaded, to include the cleaning of old log files
        and to optionally create an efficiency report.
        """
        # Ensure background submission tasks are finished before shutting down.
        self._job_submission_executor.shutdown(wait=True)

        # First, we invoke the original shutdown method
        super().shutdown()

        # Next, clean up old log files, unconditionally.
        self.clean_old_logs()
        # If the efficiency report is enabled, create it.
        if self.workflow.executor_settings.efficiency_report:
            threshold = self.workflow.executor_settings.efficiency_threshold
            report_path = self.workflow.executor_settings.efficiency_report_path
            create_efficiency_report(
                e_threshold=threshold,
                run_uuid=self.run_uuid,
                e_report_path=report_path,
                logger=self.logger,
            )

        # Finally, create a summary of status query timings and report it.
        # (only in debug mode).
        cumulative_avg_duration = (
            self._status_query_total_seconds / self._status_query_calls
            if self._status_query_calls
            else 0.0
        )
        min_duration = (
            f"{self._status_query_min_seconds:.3f}s"
            if self._status_query_min_seconds is not None
            else "n/a"
        )
        self.logger.debug(
            "Status query timing summary at shutdown: "
            f"calls={self._status_query_calls}, "
            f"failures={self._status_query_failures}, "
            f"min={min_duration}, "
            f"avg={cumulative_avg_duration:.3f}s, "
            f"max={self._status_query_max_seconds:.3f}s, "
            f"total={self._status_query_total_seconds:.3f}s"
        )

        # Report any failed nodes that were tracked during execution
        if self._failed_nodes:
            failed_nodes_str = ", ".join(sorted(self._failed_nodes))
            self.logger.warning(
                f"The following nodes failed during job execution and were "
                f"excluded from subsequent submissions: {failed_nodes_str}"
            )

    def clean_old_logs(self) -> None:
        """
        Delete files older than specified age from the SLURM log directory.
        """
        # shorthands:
        age_cutoff = self.workflow.executor_settings.delete_logfiles_older_than
        keep_all = self.workflow.executor_settings.keep_successful_logs
        if age_cutoff <= 0 or keep_all:
            return
        cutoff_secs = age_cutoff * 86400
        current_time = time.time()
        self.logger.info(f"Cleaning up SLURM log files older than {age_cutoff} day(s).")

        for path in self.slurm_logdir.rglob("*.log"):
            if path.is_file():
                try:
                    file_age = current_time - path.stat().st_mtime
                    if file_age > cutoff_secs:
                        path.unlink()
                except (OSError, FileNotFoundError) as e:
                    self.logger.error(f"Could not delete logfile {path}: {e}")
        # we need a 2nd iteration to remove putatively empty directories
        try:
            delete_empty_dirs(self.slurm_logdir)
        except (OSError, FileNotFoundError) as e:
            self.logger.error(
                f"Could not delete empty directories in {self.slurm_logdir}: {e}"
            )

    def warn_on_jobcontext(self, done=None):
        if not done:
            if "SLURM_JOB_ID" in os.environ:
                self.logger.warning(
                    "You are running snakemake in a SLURM job context. "
                    "This is not recommended, as it may lead to unexpected "
                    "behavior. "
                    "If possible, please run Snakemake directly on the "
                    "login node."
                )
                time.sleep(5)
                delete_slurm_environment()
        done = True

    def additional_general_args(self):
        """
        This function defines additional arguments to be
        passed to `exec_job`.
        """
        general_args = "--executor slurm-jobstep --jobs 1"
        # need to pass
        if self.workflow.executor_settings.pass_command_as_script:
            general_args += " --slurm-jobstep-pass-command-as-script"
        return general_args

    def run_jobs(self, jobs: List[JobExecutorInterface]):
        """
        This is a meta rule to delegate the job execution to either
        - `run_job` for individual job submission, or
        - `run_array_jobs` for array job submission, or
        - `run_pool_jobs` for pool job submission (to be implemented in the future).
        """
        if self._main_event_loop is None:
            try:
                self._main_event_loop = asyncio.get_running_loop()
            except RuntimeError:
                self._main_event_loop = None

        ready_jobs_by_rule = {}

        # check whether any other job is a group job, as these cannot be
        # submitted as array jobs and require special handling
        for job in jobs:
            if job.is_group():
                if job.name in self.array_jobs or "all" in self.array_jobs:
                    self.logger.warning(
                        f"Job '{job.name}' is a group job and cannot be "
                        "submitted as an array job. "
                        "Submitting it as a regular job instead."
                    )
                self._job_submission_executor.submit(self.run_job, job)
            else:
                ready_jobs_by_rule.setdefault(job.rule.name, []).append(job)

        for rule_name, same_rule_jobs in ready_jobs_by_rule.items():
            array_selected_for_rule = (
                "all" in self.array_jobs or rule_name in self.array_jobs
            )
            # TODO: use more sensible logging information, once finished
            self.logger.info(
                f"Running jobs for rule: {rule_name}, " f"{same_rule_jobs}"
            )
            self.logger.info("Current array job settings: " f"{self.array_jobs}")

            if array_selected_for_rule:
                dag = getattr(self.workflow, "dag", None)
                if dag is not None:
                    eligible_jobs = pending_jobs_for_rule(dag, rule_name)
                else:
                    eligible_jobs = len(same_rule_jobs)
                    self.logger.debug(
                        "workflow.dag unavailable in run_jobs(); "
                        "falling back to ready-job count for eligibility "
                        f"({rule_name}: {eligible_jobs})."
                    )

                # Keep synchronization against DAG eligibility, but do not block
                # once at least one full array chunk is ready.
                chunk_size = self.max_array_size

                if len(same_rule_jobs) == 1:
                    if eligible_jobs <= 1:
                        self.logger.debug(
                            f"Array submission requested for rule {rule_name}, "
                            "but only one pending job is available; submitting "
                            "as a regular job."
                        )
                        self._job_submission_executor.submit(
                            self.run_job, same_rule_jobs[0]
                        )
                    else:
                        self.logger.debug(
                            "Array job collection incomplete for rule "
                            f"{rule_name}: 1/{eligible_jobs} arrived. Waiting "
                            "for at least one full chunk."
                        )
                else:
                    if (
                        len(same_rule_jobs) < eligible_jobs
                        and len(same_rule_jobs) < chunk_size
                    ):
                        self.logger.debug(
                            "Array job collection incomplete for rule "
                            f"{rule_name}: {len(same_rule_jobs)}/{eligible_jobs} "
                            "arrived (< chunk size), waiting for more jobs."
                        )
                        continue

                    self.logger.debug(
                        "Submitting array-selected jobs for rule "
                        f"{rule_name}: {len(same_rule_jobs)} ready, "
                        f"{eligible_jobs} eligible, chunk_size={chunk_size}."
                    )
                    self._job_submission_executor.submit(
                        self.run_array_jobs, same_rule_jobs
                    )
                continue
            # Non-array mode: submit all ready jobs individually.
            elif len(same_rule_jobs) == 1:
                self.logger.debug(
                    f"Submitting single job for rule {rule_name} as "
                    "array mode is disabled."
                )
                self._job_submission_executor.submit(self.run_job, same_rule_jobs[0])
                continue
            else:
                self.logger.debug(
                    f"Submitting {len(same_rule_jobs)} ready jobs for rule "
                    f"{rule_name} individually (array mode disabled)."
                )
                for job in same_rule_jobs:
                    self._job_submission_executor.submit(self.run_job, job)

    def _report_job_submission_threadsafe(self, job_info: SubmittedJobInfo):
        if self._main_event_loop is not None:
            self._main_event_loop.call_soon_threadsafe(
                self.report_job_submission,
                job_info,
            )
        else:
            self.report_job_submission(job_info)

    def _report_job_error_threadsafe(self, job_info: SubmittedJobInfo, msg: str):
        if self._main_event_loop is not None:
            self._main_event_loop.call_soon_threadsafe(
                self.report_job_error,
                job_info,
                msg,
            )
        else:
            self.report_job_error(job_info, msg=msg)

    def run_array_jobs(self, jobs: List[JobExecutorInterface]):
        try:
            self.logger.debug(
                f"Preparing to submit array job for rule {jobs[0].rule.name} "
                f"with {len(jobs)} tasks."
            )
            group_or_rule = (
                f"group_{jobs[0].name}"
                if jobs[0].is_group()
                else f"rule_{jobs[0].name}"
            )

            # in an array job `sbatch --output` gets a single value
            # hence, we can only consider the first wildcard string
            # to create the SLURM log file path.
            wildcard_strs = [get_job_wildcards(job) for job in jobs]
            wildcard_str = wildcard_strs[0]
            # the wildcard string shall be ignored for the SLURM log
            # file path.
            slurm_logfile = self.slurm_logdir / group_or_rule / r"%A_%a.log"
            slurm_logfile.parent.mkdir(parents=True, exist_ok=True)

            # this behavior has been fixed in slurm 23.02, but there might be
            # plenty of older versions around, hence we should rather be
            # conservative here.
            assert "%A" not in str(self.slurm_logdir), (
                "bug: jobid placeholder in parent dir of logfile. This does not "
                "work as we have to create that dir before submission in order to "
                "make sbatch happy. Otherwise we get silent fails without "
                "logfiles being created."
            )
            assert r"%a" not in str(self.slurm_logdir), (
                "bug: jobid placeholder in parent dir of logfile. This does not "
                "work as we have to create that dir before submission in order to "
                "make sbatch happy. Otherwise we get silent fails without "
                "logfiles being created."
            )

            # generic part of a submission string:
            # we use a run_uuid as the job-name, to allow `--name`-based
            # filtering in the job status checks (`sacct --name` and
            # `squeue --name`)
            if wildcard_str == "":
                comment_str = f"rule_{jobs[0].name}"
            else:
                self.logger.warning(
                    "Array job submission does not allow for multiple different "
                    "wildcard combinations in the comment string. Only the first "
                    "one will be used."
                )
                comment_str = f"rule_{jobs[0].name}_wildcards_{wildcard_strs[0]}"

            for job in jobs:
                # check whether the 'slurm_extra' parameter is used correctly
                # prior to putatively setting in the sbatch call
                validate_slurm_extra(job)

            self.logger.debug("Building job params for array job")
            # Note: all jobs have the same resource requirement.
            #       Thus, we can simply take the first job to extract
            #       the relevant parameters for the sbatch call.
            job_params = {
                "run_uuid": self.run_uuid,
                "slurm_logfile": slurm_logfile,
                "comment_str": comment_str,
                "account": next(self.get_account_arg(jobs[0])),
                "partition": self.get_partition_arg(jobs[0]),
                "workdir": self.workflow.workdir_init,
            }

            call = get_submit_command(
                jobs[0],
                job_params,
                settings=self.workflow.executor_settings,
                failed_nodes=self._failed_nodes,
            )
            if self._failed_nodes:
                self.logger.debug(
                    "Excluding failed nodes from array job submission: "
                    f"{','.join(self._failed_nodes)}"
                )
            call += set_gres_string(jobs[0])

            if not jobs[0].resources.get("runtime"):
                self.logger.warning(
                    "No wall time information given. This might or might not "
                    "work on your cluster. "
                    "If not, specify the resource runtime in your rule or as "
                    "a reasonable default via --default-resources."
                )

            if not jobs[0].resources.get("mem_mb_per_cpu") and not jobs[
                0
            ].resources.get("mem_mb"):
                self.logger.warning(
                    "No job memory information ('mem_mb' or 'mem_mb_per_cpu') is "
                    "given - submitting without. This might or might not work on "
                    "your cluster."
                )
            # Build a compressed map of array task id -> full execution string
            # for all jobs.
            array_execs = {
                index: zlib.compress(
                    self.format_job_exec(job).encode("utf-8"), level=9
                ).hex()
                for index, job in enumerate(jobs, start=1)
            }

            # the actual array job call:

            # we need to cycle over all jobs and submit up to `array_limit` jobs per
            # submission, to avoid hitting cluster limits or oversaturating the
            # command line limits
            array_limit = min(self.max_array_size, len(jobs))
            for start_index in range(1, len(jobs) + 1, array_limit):
                end_index = min(start_index + array_limit - 1, len(jobs))
                # The first task of each chunk runs via the plain base command.
                # Remaining tasks are dispatched from --slurm-jobstep-array-execs.
                exec_job = self.format_job_exec(jobs[start_index - 1])
                sub_array_execs = {
                    str(i): array_execs[i]
                    for i in range(start_index + 1, end_index + 1)
                }
                array_execs_payload = base64.b64encode(
                    json.dumps(sub_array_execs).encode("utf-8")
                ).decode()

                use_script_submission = (
                    self.workflow.executor_settings.pass_command_as_script
                )
                submission_failed = False
                while True:
                    call_with_array = call + f" --array={start_index}-{end_index}"

                    if not use_script_submission:
                        # Use --wrap for the base execution command.
                        call_with_array += (
                            f' --wrap="{exec_job}'
                            f" --slurm-jobstep-array-execs="
                            f"{shlex.quote(array_execs_payload)}"
                            '"'
                        )
                        subprocess_stdin = None
                        self.logger.debug(f"call with array: {call_with_array}")
                    else:
                        # Use /dev/stdin to pass the base execution command as a script.
                        sbatch_script = "\n".join(
                            [
                                "#!/bin/sh",
                                f"{exec_job}",
                                "--slurm-jobstep-array-execs",
                                shlex.quote(array_execs_payload),
                            ]
                        )
                        call_with_array += " /dev/stdin"
                        subprocess_stdin = sbatch_script

                    self.logger.debug(
                        f"Submitting array job with sbatch call: {call_with_array}"
                    )
                    try:
                        process = subprocess.Popen(
                            call_with_array,
                            shell=True,
                            text=True,
                            stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                        )
                        out, err = process.communicate(input=subprocess_stdin)
                        if process.returncode != 0:
                            raise subprocess.CalledProcessError(
                                process.returncode, call_with_array, output=err
                            )
                        break
                    except OSError as e:
                        if e.errno == errno.E2BIG and not use_script_submission:
                            self.logger.warning(
                                "Array sbatch command exceeds argument-length "
                                "limits; retrying via /dev/stdin script mode "
                                f"for tasks {start_index}-{end_index}."
                            )
                            use_script_submission = True
                            continue
                        raise
                    except subprocess.CalledProcessError as e:
                        error_msg = (
                            "SLURM sbatch failed for array job submission "
                            f"(tasks {start_index}-{end_index}). "
                            f"The error message was '{e.output.strip()}'.\n"
                            f"    sbatch call:\n        {call_with_array}\n"
                        )
                        self.logger.error(error_msg)
                        for job in jobs[start_index - 1 : end_index]:
                            self._report_job_error_threadsafe(
                                SubmittedJobInfo(job),
                                (
                                    f"Part of failed array sbatch submission "
                                    f"(tasks {start_index}-{end_index}); "
                                    "see log for details."
                                ),
                            )
                        submission_failed = True
                        break

                if submission_failed:
                    continue

                # To extract the job id we split by semicolon and take the first
                # element (this also works if no cluster name was provided)
                slurm_jobid = out.strip().split(";")[0]
                # this slurm_jobid might be wrong: some cluster admin give convoluted
                # sbatch outputs. So we need to validate it properly (and replace it
                # if necessary).
                slurm_jobid = validate_or_get_slurm_job_id(slurm_jobid, out)
                # here, however we are dealing with array jobs and the job id is of
                # the form <jobid>_<array_task_id>, so we need to add the task ids
                job_ids = list()  # Snakemake interal ids
                for index in range(start_index, end_index + 1):
                    # Calculate the actual logfile path for this array task
                    job = jobs[index - 1]
                    job_ids.append(job.jobid)
                    job_wildcard_str = get_job_wildcards(job)
                    job_logfile = (
                        self.slurm_logdir
                        / group_or_rule
                        / job_wildcard_str
                        / f"{slurm_jobid}_{index}.log"
                    )

                    job_info = SubmittedJobInfo(
                        job,
                        external_jobid=f"{slurm_jobid}_{index}",
                        aux={"slurm_logfile": job_logfile},
                    )
                    self._report_job_submission_threadsafe(job_info)
                    self.logger.debug(
                        f"Registered array job task: "
                        f"external_jobid={slurm_jobid}_{index}, "
                        f"snakemake_jobid={job.jobid}"
                    )

                job_ids_str = ",".join(map(str, job_ids))
                self.logger.info(
                    f"Submitted array job with Snakemake IDs {job_ids_str} and "
                    f"SLURM job ID {slurm_jobid}. The individual task IDs are "
                    f"{start_index}-{end_index}."
                )

            # Track cluster specification for later use in cancel_jobs
            cluster_val = (
                jobs[0].resources.get("cluster")
                or jobs[0].resources.get("clusters")
                or jobs[0].resources.get("slurm_cluster")
            )
            if cluster_val:
                self._submitted_job_clusters.add(cluster_val)
        except Exception as e:
            self.logger.error(
                f"Exception in run_array_jobs for rule {jobs[0].rule.name}: {e}",
                exc_info=True,
            )
            for job in jobs:
                self._report_job_error_threadsafe(
                    SubmittedJobInfo(job),
                    f"Array job submission failed with exception: {e}",
                )

    def run_job(self, job: JobExecutorInterface):
        group_or_rule = f"group_{job.name}" if job.is_group() else f"rule_{job.name}"

        wildcard_str = get_job_wildcards(job)

        self.slurm_logdir.mkdir(parents=True, exist_ok=True)
        slurm_logfile = self.slurm_logdir / group_or_rule / wildcard_str / "%j.log"
        slurm_logfile.parent.mkdir(parents=True, exist_ok=True)
        # this behavior has been fixed in slurm 23.02, but there might be
        # plenty of older versions around, hence we should rather be
        # conservative here.
        assert "%j" not in str(self.slurm_logdir), (
            "bug: jobid placeholder in parent dir of logfile. This does not "
            "work as we have to create that dir before submission in order to "
            "make sbatch happy. Otherwise we get silent fails without "
            "logfiles being created."
        )

        # generic part of a submission string:
        # we use a run_uuid as the job-name, to allow `--name`-based
        # filtering in the job status checks (`sacct --name` and
        # `squeue --name`)
        if wildcard_str == "":
            comment_str = f"rule_{job.name}"
        else:
            comment_str = f"rule_{job.name}_wildcards_{wildcard_str}"
        # check whether the 'slurm_extra' parameter is used correctly
        # prior to putatively setting in the sbatch call
        validate_slurm_extra(job)

        # NOTE removed partition from below, such that partition
        # selection can benefit from resource checking as the call is built up.
        job_params = {
            "run_uuid": self.run_uuid,
            "slurm_logfile": slurm_logfile,
            "comment_str": comment_str,
            "account": next(self.get_account_arg(job)),
            "partition": self.get_partition_arg(job),
            "workdir": self.workflow.workdir_init,
        }

        call = get_submit_command(
            job,
            job_params,
            settings=self.workflow.executor_settings,
            failed_nodes=self._failed_nodes,
        )

        if self._failed_nodes:
            self.logger.debug(
                "Excluding failed nodes from job submission: "
                f"{','.join(self._failed_nodes)}"
            )

        call += set_gres_string(job)

        if not job.resources.get("runtime"):
            self.logger.warning(
                "No wall time information given. This might or might not "
                "work on your cluster. "
                "If not, specify the resource runtime in your rule or as "
                "a reasonable default via --default-resources."
            )

        if not job.resources.get("mem_mb_per_cpu") and not job.resources.get("mem_mb"):
            self.logger.warning(
                "No job memory information ('mem_mb' or 'mem_mb_per_cpu') is "
                "given - submitting without. This might or might not work on "
                "your cluster."
            )

        exec_job = self.format_job_exec(job)

        if not self.workflow.executor_settings.pass_command_as_script:
            # Escape potential double quotes in wrapped command.
            # Otherwise sbatch throws:
            # sbatch: error: script arguments not permitted with --wrap option
            # See:
            # https://github.com/snakemake/snakemake-executor-plugin-slurm/issues/29
            exec_job_escaped = exec_job.replace('"', '\\"')

            # and finally wrap the job to execute with all the snakemake parameters
            call += f' --wrap="{exec_job_escaped}"'
            subprocess_stdin = None
        else:
            # format the job to execute with all the snakemake parameters into a script
            sbatch_script = "\n".join(["#!/bin/sh", exec_job])
            self.logger.debug(f"sbatch script:\n{sbatch_script}")
            # feed the shell script to sbatch via stdin
            call += " /dev/stdin"
            subprocess_stdin = sbatch_script

        self.logger.debug(f"sbatch call: {call}")
        time.sleep(5)
        try:
            process = subprocess.Popen(
                call,
                shell=True,
                text=True,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            out, err = process.communicate(
                input=subprocess_stdin  # feed the sbatch shell script through stdin
            )
            if process.returncode != 0:
                raise subprocess.CalledProcessError(
                    process.returncode, call, output=err
                )
        except subprocess.CalledProcessError as e:
            self._report_job_error_threadsafe(
                SubmittedJobInfo(job),
                (
                    "SLURM sbatch failed. "
                    f"The error message was '{e.output.strip()}'.\n"
                    f"    sbatch call:\n        {call}\n"
                    + (
                        f"    sbatch script:\n{sbatch_script}\n"
                        if subprocess_stdin is not None
                        else ""
                    )
                ),
            )
            return
        # any other error message indicating failure?
        if "submission failed" in err:
            raise WorkflowError(
                f"SLURM job submission failed. The error message was {err}"
            )

        # multicluster submissions yield submission infos like
        # "Submitted batch job <id> on cluster <name>" by default, but with the
        # --parsable option it simply yields "<id>;<name>".
        # To extract the job id we split by semicolon and take the first
        # element (this also works if no cluster name was provided)
        slurm_jobid = out.strip().split(";")[0]
        # this slurm_jobid might be wrong: some cluster admin give convoluted
        # sbatch outputs. So we need to validate it properly (and replace it
        # if necessary).
        slurm_jobid = validate_or_get_slurm_job_id(slurm_jobid, out)
        slurm_logfile = slurm_logfile.with_name(
            slurm_logfile.name.replace("%j", slurm_jobid)
        )
        self.logger.info(
            f"Job {job.jobid} has been submitted with SLURM jobid "
            f"{slurm_jobid} (log: {slurm_logfile})."
        )
        # Track cluster specification for later use in cancel_jobs
        cluster_val = (
            job.resources.get("cluster")
            or job.resources.get("clusters")
            or job.resources.get("slurm_cluster")
        )
        if cluster_val:
            self._submitted_job_clusters.add(cluster_val)
        self._report_job_submission_threadsafe(
            SubmittedJobInfo(
                job,
                external_jobid=slurm_jobid,
                aux={"slurm_logfile": slurm_logfile},
            )
        )

    async def check_active_jobs(
        self, active_jobs: List[SubmittedJobInfo]
    ) -> Generator[SubmittedJobInfo, None, None]:
        # Check the status of active jobs.
        # You have to iterate over the given list active_jobs.
        # For jobs that have finished successfully, you have to call
        # self.report_job_success(job).
        # For jobs that have errored, you have to call
        # self.report_job_error(job).
        # Jobs that are still running have to be yielded.
        #
        # For queries to the remote middleware, please use
        # self.status_rate_limiter like this:
        #
        # async with self.status_rate_limiter:
        #    # query remote middleware here
        fail_stati = (
            "BOOT_FAIL",
            "CANCELLED",
            "DEADLINE",
            "FAILED",
            "NODE_FAIL",
            "OUT_OF_MEMORY",
            "TIMEOUT",
            "ERROR",
        )
        # Cap sleeping time between querying the status of all active jobs:
        # If `AccountingStorageType`` for `sacct` is set to
        # `accounting_storage/none`, `sacct` will query `slurmctld` (instead
        # of `slurmdbd`) and this in turn can rely on default config,
        # see: https://stackoverflow.com/a/46667605
        # This config defaults to `MinJobAge=300`, which implies that jobs will
        # be removed from `slurmctld` within 6 minutes of finishing. So we're
        # conservative here, with half that time.
        max_sleep_time = 180

        sacct_query_durations = []

        initial_interval = getattr(
            self.workflow.executor_settings,
            "init_seconds_before_status_checks",
            40,
        )
        # Fast path: if there are no active jobs, skip querying
        if not active_jobs:
            self.next_seconds_between_status_checks = initial_interval
            self.logger.debug("No active jobs; skipping status query.")
            return

        status_attempts = self.workflow.executor_settings.status_attempts
        self.logger.debug(
            f"Checking the status of {len(active_jobs)} active jobs "
            f"with {status_attempts} attempts."
        )

        active_jobs_ids = {job_info.external_jobid for job_info in active_jobs}
        active_jobs_seen_by_sacct = set()
        missing_sacct_status = set()

        # decide which status command to use
        status_command_name = self.get_status_command()
        min_job_age = get_min_job_age()
        dynamic_check_threshold = 3 * initial_interval
        if status_command_name == "squeue":
            if (
                min_job_age is None or min_job_age < dynamic_check_threshold
            ) and is_query_tool_available("sacct"):
                self.logger.info(
                    "Falling back to 'sacct' for status queries "
                    f"(MinJobAge={min_job_age}; threshold={dynamic_check_threshold}s)."
                )
                status_command_name = "sacct"
        if status_command_name == "sacct" and not is_query_tool_available("sacct"):
            self.logger.info("'sacct' unavailable, using 'squeue' for status queries.")
            status_command_name = "squeue"
        if status_command_name == "sacct":
            status_command = query_job_status_sacct(self.run_uuid)
        else:
            status_command = query_job_status_squeue(self.run_uuid)

        # this code is inspired by the snakemake profile:
        # https://github.com/Snakemake-Profiles/slurm
        cycle_failed_attempts = 0
        # In the typical case, status polling completes in a single attempt.
        # Additional attempts are mainly a resilience fallback for transient
        # slurmdbd/accounting hiccups where job states are briefly incomplete.
        for i in range(status_attempts):
            async with self.status_rate_limiter:
                (status_of_jobs, sacct_query_duration) = await query_job_status(
                    status_command, self.logger
                )
                if status_of_jobs is None and sacct_query_duration is None:
                    cycle_failed_attempts += 1
                    self._status_query_failures += 1
                    is_final_attempt = i + 1 == status_attempts
                    self.logger.debug(
                        f"could not check status of job {self.run_uuid}"
                        + (" (final attempt)" if is_final_attempt else "")
                    )
                    continue
                sacct_query_durations.append(sacct_query_duration)
                self._status_query_calls += 1
                self._status_query_total_seconds += sacct_query_duration
                if self._status_query_min_seconds is None:
                    self._status_query_min_seconds = sacct_query_duration
                else:
                    self._status_query_min_seconds = min(
                        self._status_query_min_seconds,
                        sacct_query_duration,
                    )
                self._status_query_max_seconds = max(
                    self._status_query_max_seconds,
                    sacct_query_duration,
                )
                # only take jobs that are still active; for array tasks fall
                # back from <jobid>_<taskid> to parent <jobid> if needed.
                status_keys = set(status_of_jobs.keys())
                active_jobs_ids_with_current_sacct_status = {
                    external_jobid
                    for external_jobid in active_jobs_ids
                    if any(
                        candidate in status_keys
                        for candidate in _status_lookup_ids(external_jobid)
                    )
                }
                missing_sacct_status = (
                    active_jobs_seen_by_sacct
                    - active_jobs_ids_with_current_sacct_status
                )
                active_jobs_seen_by_sacct = (
                    active_jobs_seen_by_sacct
                    | active_jobs_ids_with_current_sacct_status
                )
                # Only log detailed debug info on final attempt or when complete.
                # This avoids log flooding in verbose mode.
                if not missing_sacct_status or i + 1 == status_attempts:
                    self.logger.debug(
                        f"active_jobs_ids_with_current_sacct_status are: "
                        f"{active_jobs_ids_with_current_sacct_status}"
                    )
                    self.logger.debug(
                        f"active_jobs_seen_by_sacct are: {active_jobs_seen_by_sacct}"
                    )
                    self.logger.debug(
                        f"missing_sacct_status are: {missing_sacct_status}"
                    )
                if not missing_sacct_status:
                    break

        if sacct_query_durations:
            cycle_avg_duration = sum(sacct_query_durations) / len(sacct_query_durations)
            self._status_query_cycle_rows.append(
                {
                    "cycle_index": len(self._status_query_cycle_rows) + 1,
                    "command": status_command_name,
                    "successful_attempts": len(sacct_query_durations),
                    "failed_attempts": cycle_failed_attempts,
                    "min_seconds": f"{min(sacct_query_durations):.6f}",
                    "avg_seconds": f"{cycle_avg_duration:.6f}",
                    "max_seconds": f"{max(sacct_query_durations):.6f}",
                }
            )
            self.logger.debug(
                "Status query timing (cycle): "
                f"command={status_command_name}, "
                f"attempts={len(sacct_query_durations)}, "
                f"failed_attempts={cycle_failed_attempts}, "
                f"min={min(sacct_query_durations):.3f}s, "
                f"avg={cycle_avg_duration:.3f}s, "
                f"max={max(sacct_query_durations):.3f}s"
            )

        if self._status_query_calls and self._status_query_calls % 25 == 0:
            cumulative_avg_duration = (
                self._status_query_total_seconds / self._status_query_calls
            )
            self.logger.info(
                "Status query timing (cumulative): "
                f"calls={self._status_query_calls}, "
                f"failures={self._status_query_failures}, "
                f"min={self._status_query_min_seconds:.3f}s, "
                f"avg={cumulative_avg_duration:.3f}s, "
                f"max={self._status_query_max_seconds:.3f}s"
            )

        if missing_sacct_status:
            self.logger.warning(
                "Unable to get the status of all active jobs that should be "
                f"in slurmdbd, even after {status_attempts} attempts.\n"
                "The jobs with the following slurm job ids were previously "
                " seen by sacct, but sacct doesn't report them any more:\n"
                f"{missing_sacct_status}\n"
                "Please double-check with your slurm cluster administrator, "
                "that slurmdbd job accounting is properly set up.\n"
            )

        if status_of_jobs is not None:
            any_finished = False
            self.logger.debug(
                f"Status query returned {len(status_of_jobs)} job IDs: "
                f"{list(status_of_jobs.keys())[:10]}..."  # Show first 10
            )
            self.logger.debug(
                f"Checking {len(active_jobs)} active jobs with external IDs: "
                f"{[j.external_jobid for j in active_jobs][:10]}..."  # Show first 10
            )
            for j in active_jobs:
                slurm_logfile = j.aux.get("slurm_logfile")
                slurm_logfile_str = (
                    str(slurm_logfile) if slurm_logfile is not None else None
                )
                status_lookup_id = next(
                    (
                        candidate
                        for candidate in _status_lookup_ids(j.external_jobid)
                        if candidate in status_of_jobs
                    ),
                    None,
                )
                # the job probably didn't make it into slurmdbd yet, so
                # `sacct` doesn't return it
                if status_lookup_id is None:
                    # but the job should still be queueing or running and
                    # appear in slurmdbd (and thus `sacct` output) later
                    yield j
                    continue
                status = status_of_jobs[status_lookup_id]
                if status == "COMPLETED":
                    self.report_job_success(j)
                    any_finished = True
                    active_jobs_seen_by_sacct.discard(j.external_jobid)
                    if not self.workflow.executor_settings.keep_successful_logs:
                        self.logger.debug(
                            "removing SLURM log for successful job "
                            f"with SLURM ID '{j.external_jobid}'"
                        )
                        try:
                            if j.aux["slurm_logfile"].exists():
                                j.aux["slurm_logfile"].unlink()
                        except (OSError, FileNotFoundError) as e:
                            self.logger.warning(
                                "Could not remove SLURM log file"
                                f" {j.aux['slurm_logfile']}: {e}"
                            )
                elif status == "PREEMPTED" and not self._preemption_warning:
                    self._preemption_warning = True
                    self.logger.warning(
                        """
===== A Job preemption  occured! =====
Leave Snakemake running, if possible. Otherwise Snakemake
needs to restart this job upon a Snakemake restart.

We leave it to SLURM to resume your job(s)"""
                    )
                    yield j
                elif status == "UNKNOWN":
                    # the job probably does not exist anymore, but 'sacct' did not work
                    # so we assume it is finished
                    self.report_job_success(j)
                    any_finished = True
                    active_jobs_seen_by_sacct.discard(j.external_jobid)
                elif status == "NODE_FAIL":
                    # this is a special case: the job failed, but due to a node failure.
                    # Always track the failed node so future submissions exclude it,
                    # regardless of whether requeue is enabled.
                    if is_query_tool_available("sacct"):
                        try:
                            sacct_output = subprocess.check_output(
                                f"sacct -j {j.external_jobid} -n -X -o nodelist%-256",
                                shell=True,
                                text=True,
                                stderr=subprocess.PIPE,
                            )
                            node = sacct_output.strip()
                            if node:
                                self._failed_nodes.add(node)
                        except subprocess.CalledProcessError as e:
                            self.logger.warning(
                                f"Could not retrieve node information for job "
                                f"{j.external_jobid}: {e.stderr}"
                            )
                    else:
                        self.logger.debug(
                            "sacct not available; cannot track failed node"
                        )

                    if self.workflow.executor_settings.requeue:
                        self.logger.warning(
                            f"Job '{j.external_jobid}' failed with status "
                            f"'NODE_FAIL', but requeue is enabled. "
                            "Leaving it to SLURM to requeue the job. "
                            "Failed node will be excluded from future submissions."
                        )
                        yield j
                    else:
                        msg = (
                            f"SLURM-job '{j.external_jobid}' failed due to a "
                            "node failure, SLURM status is: "
                            f"'{status}'. "
                            "Failed nodes will be excluded from future job "
                            "submissions. Consider enabling requeueing for "
                            "such cases by setting the 'requeue' flag in the "
                            "executor settings."
                        )
                        self.report_job_error(j, msg=msg, aux_logs=[slurm_logfile_str])
                        active_jobs_seen_by_sacct.discard(j.external_jobid)
                elif status in fail_stati:
                    # we can only check for the fail status, if `sacct` is available
                    if status_command_name != "sacct":
                        msg = (
                            f"SLURM-job '{j.external_jobid}' failed, "
                            f"SLURM status is: '{status}'. "
                            "Detailed failure reason unavailable "
                            "(status command is not 'sacct')."
                        )
                        self.report_job_error(j, msg=msg, aux_logs=[slurm_logfile_str])
                        active_jobs_seen_by_sacct.discard(j.external_jobid)
                        continue
                    reasons = []
                    for step in range(10):  # Iterate over up to 10 job steps
                        reason_command = (
                            f"sacct -j {j.external_jobid}.{step} "
                            "--format=Reason --noheader"
                        )
                        try:
                            reason_output = subprocess.check_output(
                                reason_command,
                                shell=True,
                                text=True,
                                stderr=subprocess.PIPE,
                            ).strip()
                            if reason_output:
                                reasons.append(f"Step {step}: {reason_output}")
                        except subprocess.CalledProcessError as e:
                            self.logger.warning(
                                f"Failed to retrieve jobstep reason for SLURM job "
                                f"'{j.external_jobid}.{step}': {e.stderr.strip()}"
                            )
                            reasons.append(f"Step {step}: Unable to retrieve reason")

                    if not reasons:
                        reasons.append("Unknown")

                    if len(reasons) == 1:
                        msg = (
                            f"SLURM-job '{j.external_jobid}' failed, "
                            f"SLURM status is: '{status}'. "
                            f"Reason: {reasons[0]}."
                        )
                    else:
                        msg = (
                            f"SLURM-job '{j.external_jobid}' failed, "
                            f"SLURM status is: '{status}'. "
                            f"Reasons: {', '.join(reasons)}."
                        )
                    self.report_job_error(j, msg=msg, aux_logs=[slurm_logfile_str])
                    active_jobs_seen_by_sacct.discard(j.external_jobid)
                else:  # still running?
                    yield j

            if not any_finished:
                self.next_seconds_between_status_checks = min(
                    self.next_seconds_between_status_checks + 10,
                    max_sleep_time,
                )
            else:
                self.next_seconds_between_status_checks = initial_interval

    def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]):
        """Cancel all active jobs.

        This method is called when Snakemake is interrupted.
        """
        cancel_slurm_jobs(active_jobs, self._submitted_job_clusters, self.logger)

    def get_account_arg(self, job: JobExecutorInterface):
        """
        checks whether the desired account is valid,
        returns a default account, if applicable
        else raises an error - implicetly.
        """
        if job.resources.get("slurm_account"):
            # split the account upon ',' and whitespace, to allow
            # multiple accounts being given
            accounts = [
                a for a in re.split(r"[,\s]+", job.resources.slurm_account) if a
            ]
            for account in accounts:
                # here, we check whether the given or guessed account is valid
                # if not, a WorkflowError is raised
                test_account(account, self.logger)
            # sbatch only allows one account per submission
            # yield one after the other, if multiple were given
            # we have to quote the account, because it might
            # contain build-in shell commands - see issue #354
            for account in accounts:
                test_account(account, self.logger)
                yield f" -A {shlex.quote(account)}"
        else:
            if self._fallback_account_arg is None:
                self.logger.warning("No SLURM account given, trying to guess.")
                account = get_account(self.logger)
                if account:
                    self.logger.warning(f"Guessed SLURM account: {account}")
                    test_account(f"{account}", self.logger)
                    self._fallback_account_arg = f" -A {shlex.quote(account)}"
                else:
                    self.logger.warning(
                        "Unable to guess SLURM account. Trying to proceed without."
                    )
                    self._fallback_account_arg = (
                        ""  # no account specific args for sbatch
                    )
            yield self._fallback_account_arg

    def get_partition_arg(self, job: JobExecutorInterface):
        """
        checks whether the desired partition is valid,
        returns a default partition, if applicable
        else raises an error - implicetly.
        """
        partition = None

        # Check if a specific partition is requested
        if job.resources.get("slurm_partition"):
            # But also check if there's a cluster requirement that might override it
            job_cluster = (
                job.resources.get("slurm_cluster")
                or job.resources.get("cluster")
                or job.resources.get("clusters")
            )

            if job_cluster and self._partitions:
                # If a cluster is specified, verify the partition exists and matches
                # Otherwise, use auto-selection to find a partition for that cluster
                partition_obj = next(
                    (
                        p
                        for p in self._partitions
                        if p.name == job.resources.slurm_partition
                    ),
                    None,
                )
                if (
                    partition_obj
                    and partition_obj.partition_cluster
                    and partition_obj.partition_cluster != job_cluster
                ):
                    # Partition exists but is for a different cluster:
                    # use auto-selection
                    partition = get_best_partition(self._partitions, job, self.logger)
                else:
                    partition = job.resources.slurm_partition
            else:
                partition = job.resources.slurm_partition

        # If no partition was selected yet, try auto-selection
        if not partition and self._partitions:
            partition = get_best_partition(self._partitions, job, self.logger)

        # we didnt get a partition yet so try fallback.
        if not partition:
            if self._fallback_partition is None:
                self._fallback_partition = get_default_partition(job, self.logger)
            partition = self._fallback_partition
        if partition:
            # we have to quote the partition, because it might
            # contain build-in shell commands
            # string conversion needed for partition if partition is an integer
            return f" -p {shlex.quote(str(partition))}"
        else:
            return ""