Skip to content

Commit dc7911f

Browse files
TaskVineExecutor: write function data to temp directory (#3592)
This moves the working directory for TaskVine executor function data to a directory in /tmp These files were previously written adjacent to the logging in the working directory. They may be written in excess when running a large number of tasks.
1 parent a3ba87a commit dc7911f

File tree

1 file changed

+6
-4
lines changed

1 file changed

+6
-4
lines changed

parsl/executors/taskvine/executor.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
high-throughput system for delegating Parsl tasks to thousands of remote machines
44
"""
55

6+
import getpass
67
import hashlib
78
import inspect
89
import itertools
@@ -18,6 +19,7 @@
1819
import threading
1920
import uuid
2021
from concurrent.futures import Future
22+
from datetime import datetime
2123
from typing import List, Literal, Optional, Union
2224

2325
# Import other libraries
@@ -215,9 +217,9 @@ def __create_data_and_logging_dirs(self):
215217

216218
# Create directories for data and results
217219
log_dir = os.path.join(run_dir, self.label)
218-
self._function_data_dir = os.path.join(run_dir, self.label, "function_data")
219220
os.makedirs(log_dir)
220-
os.makedirs(self._function_data_dir)
221+
tmp_prefix = f'{self.label}-{getpass.getuser()}-{datetime.now().strftime("%Y%m%d%H%M%S%f")}-'
222+
self._function_data_dir = tempfile.TemporaryDirectory(prefix=tmp_prefix)
221223

222224
# put TaskVine logs outside of a Parsl run as TaskVine caches between runs while
223225
# Parsl does not.
@@ -227,7 +229,7 @@ def __create_data_and_logging_dirs(self):
227229

228230
# factory logs go with manager logs regardless
229231
self.factory_config.scratch_dir = self.manager_config.vine_log_dir
230-
logger.debug(f"Function data directory: {self._function_data_dir}, log directory: {log_dir}")
232+
logger.debug(f"Function data directory: {self._function_data_dir.name}, log directory: {log_dir}")
231233
logger.debug(
232234
f"TaskVine manager log directory: {self.manager_config.vine_log_dir}, "
233235
f"factory log directory: {self.factory_config.scratch_dir}")
@@ -293,7 +295,7 @@ def _path_in_task(self, executor_task_id, *path_components):
293295
'map': Pickled file with a dict between local parsl names, and remote taskvine names.
294296
"""
295297
task_dir = "{:04d}".format(executor_task_id)
296-
return os.path.join(self._function_data_dir, task_dir, *path_components)
298+
return os.path.join(self._function_data_dir.name, task_dir, *path_components)
297299

298300
def submit(self, func, resource_specification, *args, **kwargs):
299301
"""Processes the Parsl app by its arguments and submits the function

0 commit comments

Comments
 (0)