From 266a3854b441415ef4754327b249e066f1397962 Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Mon, 1 Feb 2021 15:42:28 +0100 Subject: [PATCH 1/2] use weakref in PROCESS_STACK A weak reference to the process is used in order to avoid that the 'process stack' context variable itself can keep the process in memory. Asynchronous function executions scheduled by asyncio's `call_soon`, `call_later` or `call_at` get individual copies of the context, which stay in memory as long as the corresponding handler stays in memory (even if the execution was cancelled). --- plumpy/processes.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plumpy/processes.py b/plumpy/processes.py index 782d9512..b8d7b83b 100644 --- a/plumpy/processes.py +++ b/plumpy/processes.py @@ -9,6 +9,7 @@ import time import sys import uuid +import weakref import asyncio from types import TracebackType from typing import ( @@ -519,9 +520,15 @@ def _process_scope(self) -> Generator[None, None, None]: This context manager function is used to make sure the process stack is correct meaning that globally someone can ask for Process.current() to get the last process that is on the call stack. + + Note: A weak reference to the process is used in order to avoid that the 'process stack' context variable + itself can keep the process in memory. + Asynchronous function executions scheduled by asyncio's `call_soon`, `call_later` or `call_at` get individual + copies of the context, which stay in memory as long as the corresponding handler stays in memory + (even if the execution was cancelled). See https://www.python.org/dev/peps/pep-0567/#asyncio . """ stack_copy = PROCESS_STACK.get().copy() - stack_copy.append(self) + stack_copy.append(weakref.ref(self)) PROCESS_STACK.set(stack_copy) try: yield None From bdab86d194ef6e8c178c21295910a4a48fc24bba Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Mon, 1 Feb 2021 17:09:16 +0100 Subject: [PATCH 2/2] dereference weakref --- plumpy/processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plumpy/processes.py b/plumpy/processes.py index b8d7b83b..d8691e78 100644 --- a/plumpy/processes.py +++ b/plumpy/processes.py @@ -145,7 +145,7 @@ def current(cls) -> Optional['Process']: """ if PROCESS_STACK.get(): - return PROCESS_STACK.get()[-1] + return PROCESS_STACK.get()[-1]() return None