feat: Handle request list user input (#326)

Add helper function to handle request list inputs.

Closes: #310

---------

Co-authored-by: Jan Buchar <[email protected]>

Author: Pijukatel

pyproject.toml

@@ -141,6 +141,9 @@ indent-style = "space"
 docstring-quotes = "double"
 inline-quotes = "single"
 
+[tool.ruff.lint.flake8-type-checking]
+runtime-evaluated-base-classes = ["pydantic.BaseModel", "crawlee.configuration.Configuration"]
+
 [tool.ruff.lint.flake8-builtins]
 builtins-ignorelist = ["id"]
 

src/apify/_actor.py

@@ -8,7 +8,6 @@
 
 from lazy_object_proxy import Proxy
 from pydantic import AliasChoices
-from typing_extensions import Self
 
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
@@ -31,6 +30,8 @@
     import logging
     from types import TracebackType
 
+    from typing_extensions import Self
+
     from crawlee.proxy_configuration import _NewUrlFunction
 
     from apify._models import Webhook

src/apify/_configuration.py

@@ -1,4 +1,3 @@
-# ruff: noqa: TCH001 TCH002 TCH003 (so that pydantic annotations work)
 from __future__ import annotations
 
 from datetime import datetime, timedelta

src/apify/_models.py

@@ -1,4 +1,3 @@
-# ruff: noqa: TCH001 TCH002 TCH003 (Pydantic)
 from __future__ import annotations
 
 from datetime import datetime, timedelta

src/apify/_platform_event_manager.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import asyncio
-from datetime import datetime  # noqa: TCH003
+from datetime import datetime
 from typing import TYPE_CHECKING, Annotated, Any, Literal, Union
 
 import websockets.client

src/apify/scrapy/middlewares/apify_proxy.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
 from urllib.parse import ParseResult, urlparse
 
 try:
-    from scrapy import Request, Spider  # noqa: TCH002
+    if TYPE_CHECKING:
+        from scrapy import Request, Spider
+        from scrapy.crawler import Crawler
     from scrapy.core.downloader.handlers.http11 import TunnelError
-    from scrapy.crawler import Crawler  # noqa: TCH002
     from scrapy.exceptions import NotConfigured
 except ImportError as exc:
     raise ImportError(

src/apify/scrapy/pipelines/actor_dataset_push.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from itemadapter.adapter import ItemAdapter
 
 try:
-    from scrapy import Item, Spider  # noqa: TCH002
+    if TYPE_CHECKING:
+        from scrapy import Item, Spider
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',

src/apify/scrapy/scheduler.py

@@ -1,15 +1,18 @@
 from __future__ import annotations
 
 import traceback
+from typing import TYPE_CHECKING
 
 from apify._configuration import Configuration
 from apify.apify_storage_client import ApifyStorageClient
 
 try:
     from scrapy import Spider
     from scrapy.core.scheduler import BaseScheduler
-    from scrapy.http.request import Request  # noqa: TCH002
     from scrapy.utils.reactor import is_asyncio_reactor_installed
+
+    if TYPE_CHECKING:
+        from scrapy.http.request import Request
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',

src/apify/scrapy/utils.py

@@ -2,14 +2,17 @@
 
 import asyncio
 from base64 import b64encode
+from typing import TYPE_CHECKING
 from urllib.parse import unquote
 
 from apify_shared.utils import ignore_docs
 
 try:
-    from scrapy.settings import Settings  # noqa: TCH002
     from scrapy.utils.project import get_project_settings
     from scrapy.utils.python import to_bytes
+
+    if TYPE_CHECKING:
+        from scrapy.settings import Settings
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. For example, if you use pip, run '

src/apify/storages/__init__.py

@@ -1,3 +1,5 @@
 from crawlee.storages import Dataset, KeyValueStore, RequestQueue
 
-__all__ = ['Dataset', 'KeyValueStore', 'RequestQueue']
+from ._request_list import RequestList
+
+__all__ = ['Dataset', 'KeyValueStore', 'RequestQueue', 'RequestList']