Skip to content

Commit 20664f0

Browse files
authored
fix: Add Filelock to cache (#262)
## Description Issues were caused when running tests in parallel as a rename was triggered on a missing file. As can be seen [here](https://github.com/ecmwf/anemoi-plugins-ecmwf/actions/runs/21211982935/job/61022658486?pr=69) This adds a filelock to the cache to prevent that
1 parent 942936b commit 20664f0

File tree

2 files changed

+23
-17
lines changed

2 files changed

+23
-17
lines changed

pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ dependencies = [
4040
"aniso8601",
4141
"deprecation",
4242
"entrypoints",
43+
"filelock",
4344
"multiurl",
4445
"numpy",
4546
"pydantic>=2.9",

src/anemoi/utils/caching.py

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from typing import Any
1818

1919
import numpy as np
20+
from filelock import FileLock
2021

2122
LOCK = Lock()
2223
CACHE = {}
@@ -41,6 +42,8 @@ def _get_cache_path(collection: str) -> str:
4142
def clean_cache(collection: str = "default") -> None:
4243
"""Clean the cache for a collection.
4344
45+
This removes all cached data files and their associated lock files.
46+
4447
Parameters
4548
----------
4649
collection : str, optional
@@ -121,25 +124,27 @@ def cache(self, key: tuple, proc: Callable) -> Any:
121124
return CACHE[m]
122125

123126
path = _get_cache_path(self.collection)
124-
125-
filename = os.path.join(path, m) + self.ext
126-
if os.path.exists(filename):
127-
data = self.load(filename)
128-
if self.expires is None or data["expires"] > time.time():
129-
if data["key"] == key:
130-
return data["value"]
131-
132-
value = proc()
133-
data = {"key": key, "value": value}
134-
if self.expires is not None:
135-
data["expires"] = time.time() + self.expires
136-
137127
os.makedirs(path, exist_ok=True)
138-
temp_filename = self.save(filename, data)
139-
os.rename(temp_filename, filename)
140128

141-
CACHE[m] = value
142-
return value
129+
lock_file = os.path.join(path, f"{m}.lock")
130+
with FileLock(lock_file):
131+
filename = os.path.join(path, m) + self.ext
132+
if os.path.exists(filename):
133+
data = self.load(filename)
134+
if self.expires is None or data["expires"] > time.time():
135+
if data["key"] == key:
136+
return data["value"]
137+
138+
value = proc()
139+
data = {"key": key, "value": value}
140+
if self.expires is not None:
141+
data["expires"] = time.time() + self.expires
142+
143+
temp_filename = self.save(filename, data)
144+
os.rename(temp_filename, filename)
145+
146+
CACHE[m] = value
147+
return value
143148

144149

145150
class JsonCacher(Cacher):

0 commit comments

Comments
 (0)