Review comments.

Update TCH001, TCH002, TCH003 uses.

Author: Pijukatel

pyproject.toml

@@ -142,7 +142,7 @@ docstring-quotes = "double"
 inline-quotes = "single"
 
 [tool.ruff.lint.flake8-type-checking]
-runtime-evaluated-base-classes = ["pydantic.BaseModel"]
+runtime-evaluated-base-classes = ["pydantic.BaseModel", "crawlee.configuration.Configuration", "ApifyHttpProxyMiddleware"]
 
 [tool.ruff.lint.flake8-builtins]
 builtins-ignorelist = ["id"]

src/apify/_configuration.py

@@ -1,4 +1,3 @@
-# ruff: noqa: TCH001 TCH002 TCH003 (so that pydantic annotations work)
 from __future__ import annotations
 
 from datetime import datetime, timedelta

src/apify/_models.py

@@ -1,4 +1,3 @@
-# ruff: noqa: TCH001 TCH002 TCH003 (Pydantic)
 from __future__ import annotations
 
 from datetime import datetime, timedelta

src/apify/scrapy/middlewares/apify_proxy.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
 from urllib.parse import ParseResult, urlparse
 
 try:
-    from scrapy import Request, Spider  # noqa: TCH002
+    if TYPE_CHECKING:
+        from scrapy import Request, Spider
+        from scrapy.crawler import Crawler
     from scrapy.core.downloader.handlers.http11 import TunnelError
-    from scrapy.crawler import Crawler  # noqa: TCH002
     from scrapy.exceptions import NotConfigured
 except ImportError as exc:
     raise ImportError(

src/apify/scrapy/pipelines/actor_dataset_push.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from itemadapter.adapter import ItemAdapter
 
 try:
-    from scrapy import Item, Spider  # noqa: TCH002
+    if TYPE_CHECKING:
+        from scrapy import Item, Spider
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',

src/apify/scrapy/scheduler.py

@@ -1,15 +1,18 @@
 from __future__ import annotations
 
 import traceback
+from typing import TYPE_CHECKING
 
 from apify._configuration import Configuration
 from apify.apify_storage_client import ApifyStorageClient
 
 try:
     from scrapy import Spider
     from scrapy.core.scheduler import BaseScheduler
-    from scrapy.http.request import Request  # noqa: TCH002
     from scrapy.utils.reactor import is_asyncio_reactor_installed
+
+    if TYPE_CHECKING:
+        from scrapy.http.request import Request
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',

src/apify/scrapy/utils.py

@@ -2,14 +2,17 @@
 
 import asyncio
 from base64 import b64encode
+from typing import TYPE_CHECKING
 from urllib.parse import unquote
 
 from apify_shared.utils import ignore_docs
 
 try:
-    from scrapy.settings import Settings  # noqa: TCH002
     from scrapy.utils.project import get_project_settings
     from scrapy.utils.python import to_bytes
+
+    if TYPE_CHECKING:
+        from scrapy.settings import Settings
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. For example, if you use pip, run '

src/apify/storages/__init__.py

@@ -1,5 +1,5 @@
 from crawlee.storages import Dataset, KeyValueStore, RequestQueue
 
-from ._actor_inputs import create_request_list
+from .request_list import RequestList
 
-__all__ = ['Dataset', 'KeyValueStore', 'RequestQueue', 'create_request_list']
+__all__ = ['Dataset', 'KeyValueStore', 'RequestQueue', 'RequestList']

src/apify/storages/_known_actor_input_keys.py

@@ -4,16 +4,14 @@
 import re
 from asyncio import Task
 from functools import partial
-from typing import Any
+from typing import Any, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, TypeAdapter
 
 from crawlee import Request
 from crawlee._types import HttpMethod
 from crawlee.http_clients import BaseHttpClient, HttpxHttpClient
-from crawlee.storages import RequestList
-
-from ._known_actor_input_keys import ActorInputKeys
+from crawlee.storages import RequestList as CrawleeRequestList
 
 URL_NO_COMMAS_REGEX = re.compile(
     r'https?:\/\/(www\.)?([^\W_]|[^\W_][-\w0-9@:%._+~#=]{0,254}[^\W_])\.[a-z]{2,63}(:\d{1,5})?(\/[-\w@:%+.~#?&/=()]*)?'
@@ -24,50 +22,63 @@ class _RequestDetails(BaseModel):
     method: HttpMethod = 'GET'
     payload: str = ''
     headers: dict[str, str] = Field(default_factory=dict)
-    user_data: dict[str, str] = Field(default_factory=dict, alias=ActorInputKeys.startUrls.userData)
+    user_data: dict[str, str] = Field(default_factory=dict, alias='userData')
 
 
 class _RequestsFromUrlInput(_RequestDetails):
-    requests_from_url: str = Field(alias=ActorInputKeys.startUrls.requestsFromUrl)
+    requests_from_url: str = Field(alias='requestsFromUrl')
 
 
 class _SimpleUrlInput(_RequestDetails):
     url: str
 
 
-async def create_request_list(
-    actor_start_urls_input: list[dict[str, Any]], http_client: BaseHttpClient | None = None
+url_input_adapter = TypeAdapter(list[Union[_RequestsFromUrlInput, _SimpleUrlInput]])
+
+
+class RequestList(CrawleeRequestList):
+    """Extends crawlee RequestList."""
+
+    @classmethod
+    async def open(
+        cls,
+        name: str | None = None,
+        actor_start_urls_input: list[dict[str, Any]] | None = None,
+        http_client: BaseHttpClient | None = None,
+    ) -> RequestList:
+        """Creates RequestList from Actor input requestListSources.
+
+        name is name of the returned RequestList
+        actor_start_urls_input  can contain list dicts with either url or requestsFromUrl key
+        http_client is client that will be used to send get request to url defined in requestsFromUrl
+
+        Example actor_start_urls_input:
+            [
+                # Gather urls from response body.
+                {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
+                # Directly include this url.
+                {'url': 'https://crawlee.dev', 'method': 'GET'}
+            ]
+        """
+        actor_start_urls_input = actor_start_urls_input or []
+        return await _create_request_list(name, actor_start_urls_input, http_client)
+
+
+async def _create_request_list(
+    name: str | None, actor_start_urls_input: list[dict[str, Any]], http_client: BaseHttpClient | None
 ) -> RequestList:
-    """Creates RequestList from Actor input requestListSources.
-
-    actor_start_urls_input  can contain list dicts with either url or requestsFromUrl key
-    http_client is client that will be used to send get request to url defined in requestsFromUrl
-
-    Example:
-        actor_start_urls_input = [
-            # Gather urls from response body.
-            {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
-            # Directly include this url.
-            {'url': 'https://crawlee.dev', 'method': 'GET'}
-        ]
-    """
     if not http_client:
         http_client = HttpxHttpClient()
-    simple_url_requests_inputs = [
-        _SimpleUrlInput(**request_input)
-        for request_input in actor_start_urls_input
-        if ActorInputKeys.startUrls.url in request_input
-    ]
-    remote_url_requests_inputs = [
-        _RequestsFromUrlInput(**request_input)
-        for request_input in actor_start_urls_input
-        if ActorInputKeys.startUrls.requestsFromUrl in request_input
-    ]
 
-    simple_url_requests = _create_requests_from_input(simple_url_requests_inputs)
-    remote_url_requests = await _create_requests_from_url(remote_url_requests_inputs, http_client=http_client)
+    ulr_inputs = url_input_adapter.validate_python(actor_start_urls_input)  # instance of list[Union[...]]
+
+    simple_url_inputs = [url_input for url_input in ulr_inputs if type(url_input) is _SimpleUrlInput]
+    remote_url_inputs = [url_input for url_input in ulr_inputs if type(url_input) is _RequestsFromUrlInput]
+
+    simple_url_requests = _create_requests_from_input(simple_url_inputs)
+    remote_url_requests = await _create_requests_from_url(remote_url_inputs, http_client=http_client)
 
-    return RequestList(requests=simple_url_requests + remote_url_requests)
+    return RequestList(name=name, requests=simple_url_requests + remote_url_requests)
 
 
 def _create_requests_from_input(simple_url_inputs: list[_SimpleUrlInput]) -> list[Request]: