Skip to content

Commit 8f33393

Browse files
authored
chore: Allow opening pre-existing unnamed storage (#1414)
### Description - Allow opening pre-existing unnamed storage ### Issues - Relates to: #1175 ### Testing - Added unit tests
1 parent 61c516d commit 8f33393

File tree

2 files changed

+21
-4
lines changed

2 files changed

+21
-4
lines changed

src/crawlee/storages/_storage_instance_manager.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -158,18 +158,22 @@ async def open_storage_instance(
158158
instance = cls(client, metadata.id, metadata.name) # type: ignore[call-arg]
159159
instance_name = getattr(instance, 'name', None)
160160

161-
# Cache the instance. Always cache by id and cache named or unnamed (alias).
161+
# Cache the instance.
162+
# Always cache by id.
162163
self._cache_by_storage_client[storage_client_type].by_id[cls][instance.id][additional_cache_key] = instance
164+
165+
# Cache named storage.
163166
if instance_name is not None:
164167
self._cache_by_storage_client[storage_client_type].by_name[cls][instance_name][additional_cache_key] = (
165168
instance
166169
)
167-
elif alias is not None:
170+
171+
# Cache unnamed storage.
172+
if alias is not None:
168173
self._cache_by_storage_client[storage_client_type].by_alias[cls][alias][additional_cache_key] = instance
169-
else:
170-
raise RuntimeError('Storage instance must have either a name or an alias.')
171174

172175
return instance
176+
173177
finally:
174178
# Make sure the client opener is closed.
175179
# If it was awaited, then closing is no operation, if it was not awaited, this is the cleanup.

tests/unit/storages/test_storage_instance_manager.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,3 +128,16 @@ async def test_identical_storage_remove_from_cache(storage_type: type[Storage])
128128
service_locator.storage_instance_manager.remove_from_cache(storage_1)
129129
storage_2 = await storage_type.open()
130130
assert storage_1 is not storage_2
131+
132+
133+
async def test_preexisting_unnamed_storage_open_by_id(storage_type: type[Storage]) -> None:
134+
"""Test that persisted pre-existing unnamed storage can be opened by ID."""
135+
storage_client = FileSystemStorageClient()
136+
storage_1 = await storage_type.open(alias='custom_name', storage_client=storage_client)
137+
138+
# Make service_locator unaware of this storage
139+
service_locator.storage_instance_manager.clear_cache()
140+
141+
storage_1_again = await storage_type.open(id=storage_1.id, storage_client=storage_client)
142+
143+
assert storage_1.id == storage_1_again.id

0 commit comments

Comments
 (0)