up tests

Author: Mantisus

src/crawlee/request_loaders/_sitemap_request_loader.py

@@ -80,7 +80,10 @@ def __init__(
             exclude: List of glob or regex patterns to exclude URLs.
             max_buffer_size: Maximum number of URLs to buffer in memory.
             http_client: the instance of `HttpClient` to use for fetching sitemaps.
-            persist_state_key: Key in key-value store for persisting.
+            persist_state_key: Key in key-value store for persisting state. Each SitemapRequestLoader
+                instance must use a unique key to avoid conflicts when multiple loaders run
+                concurrently. Sharing the same key between instances will cause state corruption
+                and unpredictable behavior.
             persist_enabled: Flag to enable/disable persistence.
         """
         self._http_client = http_client

tests/unit/request_loaders/test_sitemap_request_loader.py

@@ -114,7 +114,7 @@ async def test_abort_sitemap_loading(server_url: URL, http_client: HttpClient) -
 
 async def test_create_persist_state_for_sitemap_loading(server_url: URL, http_client: HttpClient) -> None:
     sitemap_url = (server_url / 'sitemap.xml').with_query(base64=encode_base64(BASIC_SITEMAP.encode()))
-    persist_key = f'test_sitemap_loader_{id(http_client)}'
+    persist_key = f'create_persist_state_{id(http_client)}'
     sitemap_loader = SitemapRequestLoader(
         [str(sitemap_url)], http_client=http_client, persist_state_key=persist_key, persist_enabled=True
     )
@@ -132,7 +132,7 @@ async def test_create_persist_state_for_sitemap_loading(server_url: URL, http_cl
 
 async def test_data_persistence_for_sitemap_loading(server_url: URL, http_client: HttpClient) -> None:
     sitemap_url = (server_url / 'sitemap.xml').with_query(base64=encode_base64(BASIC_SITEMAP.encode()))
-    persist_key = f'test_sitemap_loader_{id(http_client)}'
+    persist_key = f'data_persist_state_{id(http_client)}'
     sitemap_loader = SitemapRequestLoader(
         [str(sitemap_url)], http_client=http_client, persist_state_key=persist_key, persist_enabled=True
     )
@@ -155,7 +155,7 @@ async def test_recovery_data_persistence_for_sitemap_loading(
     http_client: HttpClient,
 ) -> None:
     sitemap_url = (server_url / 'sitemap.xml').with_query(base64=encode_base64(BASIC_SITEMAP.encode()))
-    persist_key = f'test_sitemap_loader_{id(http_client)}'
+    persist_key = f'recovery_persist_state_{id(http_client)}'
     sitemap_loader = SitemapRequestLoader(
         [str(sitemap_url)], http_client=http_client, persist_state_key=persist_key, persist_enabled=True
     )