chore: Use runtime-evaluated-base-classes for ruff (#711)

This allows to remove ruff exceptions, without breaking the ruff check.
Also some exceptions were not necessary in the first place, so fix them.

Author: Pijukatel

pyproject.toml

@@ -168,6 +168,9 @@ indent-style = "space"
 docstring-quotes = "double"
 inline-quotes = "single"
 
+[tool.ruff.lint.flake8-type-checking]
+runtime-evaluated-base-classes = ["pydantic.BaseModel", "pydantic_settings.BaseSettings"]
+
 [tool.ruff.lint.flake8-builtins]
 builtins-ignorelist = ["id"]
 

src/crawlee/_request.py

@@ -1,22 +1,22 @@
-# ruff: noqa: TCH001, TCH002, TCH003 (because of Pydantic)
-
 from __future__ import annotations
 
 from collections.abc import Iterator, MutableMapping
 from datetime import datetime
 from decimal import Decimal
 from enum import IntEnum
-from typing import Annotated, Any, cast
+from typing import TYPE_CHECKING, Annotated, Any, cast
 
 from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, PlainSerializer, PlainValidator, TypeAdapter
-from typing_extensions import Self
 
 from crawlee._types import EnqueueStrategy, HttpHeaders, HttpMethod, HttpPayload, JsonSerializable
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.docs import docs_group
 from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
 from crawlee._utils.urls import extract_query_params, validate_http_url
 
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
 
 class RequestState(IntEnum):
     """Crawlee-specific request handling state."""

src/crawlee/_utils/system.py

@@ -1,5 +1,3 @@
-# ruff: noqa: TCH001, TCH002, TCH003 (because of Pydantic)
-
 from __future__ import annotations
 
 import os

src/crawlee/base_storage_client/_models.py

@@ -1,5 +1,3 @@
-# ruff: noqa: TCH001, TCH002, TCH003 (because of Pydantic)
-
 from __future__ import annotations
 
 from datetime import datetime

src/crawlee/beautifulsoup_crawler/_beautifulsoup_crawler.py

@@ -6,7 +6,6 @@
 
 from bs4 import BeautifulSoup, Tag
 from pydantic import ValidationError
-from typing_extensions import Unpack
 
 from crawlee import EnqueueStrategy
 from crawlee._request import BaseRequestData
@@ -20,6 +19,8 @@
 from crawlee.http_crawler import HttpCrawlingContext
 
 if TYPE_CHECKING:
+    from typing_extensions import Unpack
+
     from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs
 
 BeautifulSoupParser = Literal['html.parser', 'lxml', 'xml', 'html5lib']

src/crawlee/configuration.py

@@ -1,17 +1,17 @@
-# ruff: noqa: TCH003 TCH002 TCH001
-
 from __future__ import annotations
 
 from datetime import timedelta
-from typing import Annotated, Literal
+from typing import TYPE_CHECKING, Annotated, Literal
 
 from pydantic import AliasChoices, BeforeValidator, Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
-from typing_extensions import Self
 
 from crawlee._utils.docs import docs_group
 from crawlee._utils.models import timedelta_ms
 
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
 __all__ = ['Configuration']
 
 

src/crawlee/events/_event_manager.py

@@ -10,7 +10,6 @@
 from typing import TYPE_CHECKING, TypedDict
 
 from pyee.asyncio import AsyncIOEventEmitter
-from typing_extensions import NotRequired
 
 from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
@@ -20,6 +19,8 @@
 if TYPE_CHECKING:
     from types import TracebackType
 
+    from typing_extensions import NotRequired
+
     from crawlee.events._types import EventData, Listener, WrappedListener
 
 logger = getLogger(__name__)

src/crawlee/events/_local_event_manager.py

@@ -7,8 +7,6 @@
 from logging import getLogger
 from typing import TYPE_CHECKING
 
-from typing_extensions import Unpack
-
 from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
 from crawlee._utils.system import get_cpu_info, get_memory_info
@@ -18,6 +16,8 @@
 if TYPE_CHECKING:
     from types import TracebackType
 
+    from typing_extensions import Unpack
+
 logger = getLogger(__name__)
 
 

src/crawlee/events/_types.py

@@ -1,4 +1,3 @@
-# ruff: noqa: TCH001 TCH002
 from __future__ import annotations
 
 from collections.abc import Callable, Coroutine

src/crawlee/http_crawler/_http_crawler.py

@@ -3,15 +3,15 @@
 import logging
 from typing import TYPE_CHECKING, AsyncGenerator, Iterable
 
-from typing_extensions import Unpack
-
 from crawlee._utils.docs import docs_group
 from crawlee.basic_crawler import BasicCrawler, BasicCrawlerOptions, ContextPipeline
 from crawlee.errors import SessionError
 from crawlee.http_clients import HttpxHttpClient
 from crawlee.http_crawler._http_crawling_context import HttpCrawlingContext
 
 if TYPE_CHECKING:
+    from typing_extensions import Unpack
+
     from crawlee._types import BasicCrawlingContext