Merge branch 'master' into benc-stdstreams-bugs

Author: benclifford

.github/workflows/parsl+slurm.yaml

@@ -0,0 +1,35 @@
+name: Test Slurm Scheduler
+on:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: read
+    strategy:
+      fail-fast: false
+      matrix:
+        container: ["ghcr.io/tylern4/slurm-standalone:slurm-24-05-0-1"]
+    timeout-minutes: 30
+
+    container:
+      image: ${{ matrix.container }}
+      options: "--platform=linux/amd64 --rm -h node01"
+
+    name: ${{ matrix.container }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Dependencies and Parsl
+        run: |
+          CC=/usr/lib64/openmpi/bin/mpicc pip3 install . -r test-requirements.txt
+
+      - name: Verify Parsl Installation
+        run: |
+          pytest parsl/tests/ -k "not cleannet and not unix_filesystem_permissions_required" --config parsl/tests/configs/local_threads.py --random-order --durations 10
+
+      - name: Test Parsl with Slurm Config
+        run: |
+          ./parsl/tests/slurm-entrypoint.sh  pytest parsl/tests/ -k "not cleannet and not unix_filesystem_permissions_required" --config parsl/tests/configs/slurm_local.py --random-order --durations 10

parsl/executors/taskvine/executor.py

@@ -3,6 +3,7 @@
 high-throughput system for delegating Parsl tasks to thousands of remote machines
 """
 
+import getpass
 import hashlib
 import inspect
 import itertools
@@ -18,6 +19,7 @@
 import threading
 import uuid
 from concurrent.futures import Future
+from datetime import datetime
 from typing import List, Literal, Optional, Union
 
 # Import other libraries
@@ -215,9 +217,9 @@ def __create_data_and_logging_dirs(self):
 
         # Create directories for data and results
         log_dir = os.path.join(run_dir, self.label)
-        self._function_data_dir = os.path.join(run_dir, self.label, "function_data")
         os.makedirs(log_dir)
-        os.makedirs(self._function_data_dir)
+        tmp_prefix = f'{self.label}-{getpass.getuser()}-{datetime.now().strftime("%Y%m%d%H%M%S%f")}-'
+        self._function_data_dir = tempfile.TemporaryDirectory(prefix=tmp_prefix)
 
         # put TaskVine logs outside of a Parsl run as TaskVine caches between runs while
         # Parsl does not.
@@ -227,7 +229,7 @@ def __create_data_and_logging_dirs(self):
 
         # factory logs go with manager logs regardless
         self.factory_config.scratch_dir = self.manager_config.vine_log_dir
-        logger.debug(f"Function data directory: {self._function_data_dir}, log directory: {log_dir}")
+        logger.debug(f"Function data directory: {self._function_data_dir.name}, log directory: {log_dir}")
         logger.debug(
             f"TaskVine manager log directory: {self.manager_config.vine_log_dir}, "
             f"factory log directory: {self.factory_config.scratch_dir}")
@@ -293,7 +295,7 @@ def _path_in_task(self, executor_task_id, *path_components):
             'map': Pickled file with a dict between local parsl names, and remote taskvine names.
         """
         task_dir = "{:04d}".format(executor_task_id)
-        return os.path.join(self._function_data_dir, task_dir, *path_components)
+        return os.path.join(self._function_data_dir.name, task_dir, *path_components)
 
     def submit(self, func, resource_specification, *args, **kwargs):
         """Processes the Parsl app by its arguments and submits the function

parsl/tests/configs/slurm_local.py

@@ -0,0 +1,26 @@
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
+
+
+def fresh_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label="docker_slurm",
+                encrypted=True,
+                provider=SlurmProvider(
+                    cmd_timeout=60,     # Add extra time for slow scheduler responses
+                    channel=LocalChannel(),
+                    nodes_per_block=1,
+                    init_blocks=1,
+                    min_blocks=1,
+                    max_blocks=1,
+                    walltime='00:10:00',
+                    launcher=SrunLauncher(),
+                ),
+            )
+        ],
+    )

parsl/tests/slurm-entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+echo "---> Starting the MUNGE Authentication service (munged) ..."
+gosu munge /usr/sbin/munged
+
+echo "---> Starting the slurmctld ..."
+exec gosu slurm /usr/sbin/slurmctld -i -Dvvv &
+
+echo "---> Waiting for slurmctld to become active before starting slurmd..."
+
+echo "---> Starting the Slurm Node Daemon (slurmd) ..."
+exec /usr/sbin/slurmd -Dvvv &
+
+echo "---> Running user command '${@}'"
+exec "$@"