chore(deps): change custom LRUCache to cachetools.LRUCache (apify#988)

### Description

- change custom `LRUCache` to `cachetools.LRUCache`. In my opinion,
`functools.lru_cache's` logic isn't well-suited for this use case.
Therefore, if we want to modify our caching approach, using `cachetools`
appears to be a better option.

### Issues

- Closes: apify#86

Author: Mantisus

poetry.lock

@@ -44,6 +44,7 @@ keywords = [
 python = "^3.9"
 beautifulsoup4 = { version = ">=4.12.0", optional = true }
 browserforge = { version = ">=1.2.3", optional = true }
+cachetools = ">=5.5.1"
 colorama = ">=0.4.0"
 cookiecutter = ">=2.6.0"
 curl-cffi = { version = ">=0.7.2", optional = true }
@@ -93,6 +94,7 @@ ruff = "~0.8.0"
 setuptools = "~75.8.0"                    # setuptools are used by pytest, but not explicitly required
 sortedcontainers-stubs = "~2.4.0"
 types-beautifulsoup4 = "~4.12.0.20240229"
+types-cachetools = "~=5.5.0.20240820"
 types-colorama = "~0.4.15.20240106"
 types-psutil = "~5.9.5.20240205"
 types-python-dateutil = "~2.9.0.20240316"

pyproject.toml

@@ -7,12 +7,12 @@
 from logging import getLogger
 from typing import TYPE_CHECKING, Any, Generic, TypedDict, TypeVar
 
+from cachetools import LRUCache
 from typing_extensions import override
 
 from crawlee import service_locator
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.docs import docs_group
-from crawlee._utils.lru_cache import LRUCache
 from crawlee._utils.requests import unique_key_to_request_id
 from crawlee._utils.wait import wait_for_all_tasks_for_finish
 from crawlee.events import Event
@@ -135,7 +135,7 @@ def __init__(self, id: str, name: str | None, storage_client: StorageClient) ->
         self._in_progress: set[str] = set()
         self._last_activity = datetime.now(timezone.utc)
         self._recently_handled: BoundedSet[str] = BoundedSet(max_length=self._RECENTLY_HANDLED_CACHE_SIZE)
-        self._requests_cache: LRUCache[CachedRequest] = LRUCache(max_length=self._MAX_CACHED_REQUESTS)
+        self._requests_cache: LRUCache[str, CachedRequest] = LRUCache(maxsize=self._MAX_CACHED_REQUESTS)
 
     @property
     @override

src/crawlee/_utils/lru_cache.py

@@ -246,3 +246,22 @@ async def test_deduplication_of_requests_with_valid_custom_unique_key() -> None:
     assert await rq.get_total_count() == 1
 
     assert await rq.fetch_next_request() == request_1
+
+
+async def test_cache_requests(request_queue: RequestQueue) -> None:
+    request_1 = Request.from_url('https://apify.com')
+    request_2 = Request.from_url('https://crawlee.dev')
+
+    await request_queue.add_request(request_1)
+    await request_queue.add_request(request_2)
+
+    assert request_queue._requests_cache.currsize == 2
+
+    fetched_request = await request_queue.fetch_next_request()
+
+    assert fetched_request is not None
+    assert fetched_request.id == request_1.id
+
+    # After calling fetch_next_request request_1 moved to the end of the cache store.
+    cached_items = [request_queue._requests_cache.popitem()[0] for _ in range(2)]
+    assert cached_items == [request_2.id, request_1.id]