Skip to content

Commit 6606de4

Browse files
vdusekvdusek
authored
chore: Migrate to ty type checker (#1625)
1 parent 6dbfcb5 commit 6606de4

File tree

61 files changed

+353
-396
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+353
-396
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Cache
22
__pycache__
3-
.mypy_cache
43
.pytest_cache
54
.ruff_cache
5+
.ty_cache
66
.uv-cache
77

88
# Virtual envs

‎CONTRIBUTING.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ make format
4646

4747
### Type checking
4848

49-
Type checking is handled by [mypy](https://mypy.readthedocs.io/), verifying code against type annotations. Configuration settings can be found in `pyproject.toml`.
49+
Type checking is handled by [ty](https://docs.astral.sh/ty/), verifying code against type annotations. Configuration settings can be found in `pyproject.toml`.
5050

5151
To run type checking:
5252

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
E2E_TESTS_CONCURRENCY = 1
66

77
clean:
8-
rm -rf .mypy_cache .pytest_cache .ruff_cache .uv-cache build dist htmlcov .coverage
8+
rm -rf .uv_cache .pytest_cache .ruff_cache .uv-cache build dist htmlcov .coverage
99

1010
install-sync:
1111
uv sync --all-extras
@@ -27,7 +27,7 @@ lint:
2727
uv run ruff check
2828

2929
type-check:
30-
uv run mypy
30+
uv run ty check
3131

3232
unit-tests:
3333
uv run pytest \

‎docs/deployment/code_examples/google/cloud_run_example.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# mypy: disable-error-code="misc"
21
import json
32
import os
43

@@ -9,7 +8,7 @@
98
from crawlee.storage_clients import MemoryStorageClient
109

1110

12-
@get('/') # type: ignore[untyped-decorator]
11+
@get('/')
1312
async def main() -> str:
1413
"""The crawler entry point that will be called when the HTTP endpoint is accessed."""
1514
# highlight-start

‎docs/deployment/code_examples/google/google_example.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# mypy: disable-error-code="misc"
21
import asyncio
32
import json
43
from datetime import timedelta
@@ -48,7 +47,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
4847
# highlight-end
4948

5049

51-
@functions_framework.http # type: ignore[untyped-decorator]
50+
@functions_framework.http
5251
def crawlee_run(request: Request) -> Response:
5352
# You can pass data to your crawler using `request`
5453
function_id = request.headers['Function-Execution-Id']

‎docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
async def main() -> None:
1111
fingerprint_generator = DefaultFingerprintGenerator(
12-
header_options=HeaderGeneratorOptions(browsers=['chromium']),
12+
header_options=HeaderGeneratorOptions(browsers=['chrome']),
1313
screen_options=ScreenOptions(min_width=400),
1414
)
1515

‎docs/guides/code_examples/http_crawlers/selectolax_parser.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ async def parse(self, response: HttpResponse) -> LexborHTMLParser:
2222
"""Parse HTTP response body into a document object."""
2323
response_body = await response.read()
2424
# Run parsing in a thread to avoid blocking the event loop.
25-
return await asyncio.to_thread(lambda: LexborHTMLParser(response_body))
25+
return await asyncio.to_thread(LexborHTMLParser, response_body)
2626

2727
@override
2828
async def parse_text(self, text: str) -> LexborHTMLParser:

‎docs/guides/code_examples/running_in_web_server/server.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
app = FastAPI(lifespan=lifespan, title='Crawler app')
1515

1616

17-
@app.get('/', response_class=HTMLResponse) # type: ignore[untyped-decorator]
17+
@app.get('/', response_class=HTMLResponse)
1818
def index() -> str:
1919
return """
2020
<!DOCTYPE html>
@@ -32,7 +32,7 @@ def index() -> str:
3232
"""
3333

3434

35-
@app.get('/scrape') # type: ignore[untyped-decorator]
35+
@app.get('/scrape')
3636
async def scrape_url(request: Request, url: str | None = None) -> dict:
3737
if not url:
3838
return {'url': 'missing', 'scrape result': 'no results'}

‎pyproject.toml‎

Lines changed: 12 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,6 @@ dev = [
102102
"build<2.0.0", # For e2e tests.
103103
"dycw-pytest-only<3.0.0",
104104
"fakeredis[probabilistic,json,lua]<3.0.0",
105-
"mypy~=1.19.0",
106105
"pre-commit<5.0.0",
107106
"proxy-py<3.0.0",
108107
"pydoc-markdown<5.0.0",
@@ -113,6 +112,7 @@ dev = [
113112
"pytest<9.0.0",
114113
"ruff~=0.14.0",
115114
"setuptools", # setuptools are used by pytest, but not explicitly required
115+
"ty~=0.0.0",
116116
"types-beautifulsoup4<5.0.0",
117117
"types-cachetools<7.0.0",
118118
"types-colorama<1.0.0",
@@ -230,62 +230,24 @@ filterwarnings = [
230230
"ignore:websockets.server.WebSocketServerProtocol is deprecated:DeprecationWarning",
231231
]
232232

233-
[tool.mypy]
234-
python_version = "3.10"
235-
plugins = ["pydantic.mypy"]
233+
[tool.ty.environment]
234+
python-version = "3.10"
235+
236+
[tool.ty.src]
237+
include = ["src", "tests", "scripts", "docs", "website"]
236238
exclude = [
237239
"src/crawlee/project_template",
238240
"docs/guides/code_examples/storage_clients/custom_storage_client_example.py",
239241
]
240-
files = ["src", "tests", "docs", "website"]
241-
check_untyped_defs = true
242-
disallow_incomplete_defs = true
243-
disallow_untyped_calls = true
244-
disallow_untyped_decorators = true
245-
disallow_untyped_defs = true
246-
no_implicit_optional = true
247-
warn_redundant_casts = true
248-
warn_return_any = true
249-
warn_unreachable = true
250-
warn_unused_ignores = true
251-
252-
[[tool.mypy.overrides]]
253-
# Example codes are sometimes showing integration of crawlee with external tool, which is not dependency of crawlee.
254-
module = [
255-
"apify", # Example code shows integration of apify and crawlee.
256-
"apify_fingerprint_datapoints", # Untyped and stubs not available
257-
"camoufox", # Example code shows integration of camoufox and crawlee.
258-
"fastapi", # Example code shows running in webserver.
259-
"saxonche", # Example code shows HttpCrawler with custom parser.
260-
"scrapling.*", # Example code shows HttpCrawler with custom parser.
261-
"selectolax.*", # Example code shows HttpCrawler with custom parser.
262-
"stagehand.*", # Example code shows integration of Stagehand and crawlee.
263-
"starlette.*", # Example code shows running in webserver.
264-
"flask", # Example code shows deploy on Google Cloud.
265-
"functions_framework", # Example code shows deploy on Google Cloud.
266-
"jaro", # Untyped and stubs not available
267-
"litestar", # Example code shows deploy on Google Cloud Run.
268-
"loguru", # Example code shows integration of loguru and crawlee for JSON logging.
269-
"lxml.*", # Example code shows HttpCrawler with custom parser.
270-
"sklearn.linear_model", # Untyped and stubs not available
271-
"cookiecutter.*", # Untyped and stubs not available
272-
"inquirer.*", # Untyped and stubs not available
273-
"pyquery", # Example code shows HttpCrawler with custom parser.
274-
"warcio.*", # Example code shows WARC files creation.
275-
"wrapt" # Untyped and stubs not available
276-
]
277-
ignore_missing_imports = true
278242

279-
[[tool.mypy.overrides]]
280-
module = [
281-
"running_in_web_server.*", # False positive when fastapi not available
243+
[[tool.ty.overrides]]
244+
include = [
245+
"docs/**/*.py",
246+
"website/**/*.py",
282247
]
283-
disable_error_code = ["misc"]
284248

285-
[tool.basedpyright]
286-
pythonVersion = "3.10"
287-
typeCheckingMode = "standard"
288-
include = ["src", "tests", "docs", "website"]
249+
[tool.ty.overrides.rules]
250+
unresolved-import = "ignore"
289251

290252
[tool.coverage.report]
291253
exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"]

‎src/crawlee/_browserforge_workaround.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def patch_browserforge() -> None:
2020
def DownloadIfNotExists(**flags: bool) -> None:
2121
pass
2222

23-
download.DownloadIfNotExists = DownloadIfNotExists
23+
download.DownloadIfNotExists = DownloadIfNotExists # ty: ignore[invalid-assignment]
2424

2525
import browserforge.bayesian_network
2626

0 commit comments

Comments
 (0)