ApifyFileSystemStorageClient will hvae identical cache key as FileSystemStorageClient

Author: Pijukatel

src/apify/storage_clients/_file_system/_storage_client.py

@@ -10,6 +10,8 @@
 from ._key_value_store_client import ApifyFileSystemKeyValueStoreClient
 
 if TYPE_CHECKING:
+    from collections.abc import Hashable
+
     from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient
 
 
@@ -21,6 +23,13 @@ class ApifyFileSystemStorageClient(FileSystemStorageClient):
     except for the metadata file and the `INPUT.json` file.
     """
 
+    @override
+    def get_storage_client_cache_key(self, configuration: Configuration) -> Hashable:
+        # Ensure same cache key as the `FileSystemStorageClient` to prevent potential purging of the path twice.
+        # If `FileSystemStorageClient` opens the storage first, it will be used even in successive open calls by
+        # `ApifyFileSystemStorageClient` and vice versa.
+        return FileSystemStorageClient().get_storage_client_cache_key(configuration)
+
     @override
     async def create_kvs_client(
         self,

tests/unit/test_apify_storages.py

@@ -4,12 +4,15 @@
 
 import pytest
 
+from crawlee.storage_clients import FileSystemStorageClient
+from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient
 from crawlee.storage_clients.models import StorageMetadata
 from crawlee.storages._base import Storage
 
-from apify import Configuration
+from apify import Actor, Configuration
 from apify.storage_clients import ApifyStorageClient
 from apify.storage_clients._apify import ApifyDatasetClient, ApifyKeyValueStoreClient, ApifyRequestQueueClient
+from apify.storage_clients._file_system import ApifyFileSystemKeyValueStoreClient, ApifyFileSystemStorageClient
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 
@@ -61,3 +64,31 @@ def create_metadata(id: str) -> StorageMetadata:
         # Equivalent configuration results in same storage clients.
         assert storage_1 is storage_4
         assert storage_3 is storage_5
+
+
+async def test_no_double_purge_for_filesystem_storage_client() -> None:
+    expected_value = 'some value'
+    expected_key = 'some key'
+
+    async with Actor():
+        await Actor.set_value(expected_key, expected_value)
+        # RQ uses KVS under the hood for persistence, so it will try to open same default KVS as it was already opened,
+        # but based on different client - FileSystemStorageClient.
+        await Actor.open_request_queue()
+        assert expected_value == await Actor.get_value(expected_key)
+
+
+async def test_first_filesystem_storage_client_wins() -> None:
+    """Test that when two different FileSystemStorageClient variants are used to open the same storage, they both use
+    the same client that was used to open the storage first"""
+    kvs_1 = await KeyValueStore.open(storage_client=ApifyFileSystemStorageClient())
+    kvs_2 = await KeyValueStore.open(storage_client=FileSystemStorageClient())
+
+    kvs_3 = await KeyValueStore.open(name='a', storage_client=FileSystemStorageClient())
+    kvs_4 = await KeyValueStore.open(name='a', storage_client=ApifyFileSystemStorageClient())
+
+    assert kvs_1 is kvs_2
+    assert type(kvs_2._client) is ApifyFileSystemKeyValueStoreClient
+
+    assert kvs_3 is kvs_4
+    assert type(kvs_4._client) is FileSystemKeyValueStoreClient