polishment

Author: vdusek

src/crawlee/storage_clients/_memory/_request_queue_client.py

@@ -35,9 +35,6 @@ class MemoryRequestQueueClient(RequestQueueClient):
     does not support data sharing across different processes.
     """
 
-    # Class variable to store request queue instances by ID
-    _instances: ClassVar[dict[str, MemoryRequestQueueClient]] = {}
-
     def __init__(
         self,
         *,

src/crawlee/storages/_base.py

@@ -58,17 +58,3 @@ async def purge(self) -> None:
         This method does not remove the storage itself, e.g. don't remove the metadata,
         but clears all items within it.
         """
-
-    @classmethod
-    def compute_cache_key(
-        cls,
-        id: str | None = None,
-        name: str | None = None,
-        configuration: Configuration | None = None,
-        storage_client: StorageClient | None = None,
-    ) -> str:
-        """Compute the cache key for the storage.
-
-        The cache key computed based on the storage ID, name, configuration fields, and storage client class.
-        """
-        return f'{id}|{name}|{configuration}|{storage_client.__class__}'

src/crawlee/storages/_key_value_store.py

@@ -73,13 +73,13 @@ class KeyValueStore(Storage):
     _default_instance: ClassVar[KeyValueStore | None] = None
     """Cache for the default key-value store instance."""
 
-    # Cache for recoverable (auto-saved) values
     _autosaved_values: ClassVar[
         dict[
             str,
             dict[str, RecoverableState[AutosavedValue]],
         ]
     ] = {}
+    """Cache for recoverable (auto-saved) values."""
 
     def __init__(self, client: KeyValueStoreClient) -> None:
         """Initialize a new instance.
@@ -155,17 +155,12 @@ async def open(
 
     @override
     async def drop(self) -> None:
-        # Remove from cache before dropping
         if self.id in self._cache_by_id:
             del self._cache_by_id[self.id]
-
         if self.name is not None and self.name in self._cache_by_name:
             del self._cache_by_name[self.name]
 
-        # Clear cache with persistent values
-        await self._clear_cache()
-
-        # Drop the key-value store client
+        await self._clear_cache()  # Clear cache with persistent values.
         await self._client.drop()
 
     @override

tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py

@@ -500,10 +500,10 @@ async def test_adaptive_playwright_crawler_timeout_in_sub_crawler(test_urls: lis
     """Tests that timeout in static sub crawler forces fall back to browser sub crawler.
 
     Create situation where static sub crawler blocks(should time out), such error should start browser sub
-    crawler."""
-
+    crawler.
+    """
     static_only_predictor_no_detection = _SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0]))
-    request_handler_timeout = timedelta(seconds=0.1)
+    request_handler_timeout = timedelta(seconds=1)
 
     crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
         max_request_retries=1,
@@ -522,9 +522,9 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
         except AdaptiveContextError:
             mocked_static_handler()
             # Relax timeout for the fallback browser request to avoid flakiness in test
-            crawler._request_handler_timeout = timedelta(seconds=5)
+            crawler._request_handler_timeout = timedelta(seconds=10)
             # Sleep for time obviously larger than top crawler timeout.
-            await asyncio.sleep(request_handler_timeout.total_seconds() * 2)
+            await asyncio.sleep(request_handler_timeout.total_seconds() * 3)
 
     await crawler.run(test_urls[:1])
 

tests/unit/storages/test_request_queue.py

@@ -259,57 +259,6 @@ async def test_add_requests_batch_with_forefront(rq: RequestQueue) -> None:
     assert next_request6 is None
 
 
-async def test_add_requests_mixed_forefront(rq: RequestQueue) -> None:
-    """Test the ordering when adding requests with mixed forefront values."""
-    # Add normal requests
-    await rq.add_request('https://example.com/normal1')
-    await rq.add_request('https://example.com/normal2')
-
-    # Add a batch with forefront=True
-    await rq.add_requests(
-        ['https://example.com/priority1', 'https://example.com/priority2'],
-        forefront=True,
-    )
-
-    # Add another normal request
-    await rq.add_request('https://example.com/normal3')
-
-    # Add another priority request
-    await rq.add_request('https://example.com/priority3', forefront=True)
-
-    # Wait for background tasks
-    await asyncio.sleep(0.1)
-
-    # The expected order should be:
-    # 1. priority3 (most recent forefront)
-    # 2. priority1 (from batch, forefront)
-    # 3. priority2 (from batch, forefront)
-    # 4. normal1 (oldest normal)
-    # 5. normal2
-    # 6. normal3 (newest normal)
-
-    requests = []
-    while True:
-        req = await rq.fetch_next_request()
-        if req is None:
-            break
-        requests.append(req)
-        await rq.mark_request_as_handled(req)
-
-    assert len(requests) == 6
-    assert requests[0].url == 'https://example.com/priority3'
-
-    # The next two should be from the forefront batch (exact order within batch may vary)
-    batch_urls = {requests[1].url, requests[2].url}
-    assert 'https://example.com/priority1' in batch_urls
-    assert 'https://example.com/priority2' in batch_urls
-
-    # Then the normal requests in order
-    assert requests[3].url == 'https://example.com/normal1'
-    assert requests[4].url == 'https://example.com/normal2'
-    assert requests[5].url == 'https://example.com/normal3'
-
-
 async def test_add_requests_with_forefront(rq: RequestQueue) -> None:
     """Test adding requests to the front of the queue."""
     # Add some initial requests