Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,13 @@ timeout = 300
markers = [
"run_alone: marks tests that must run in isolation",
]
# Ignore DeprecationWarnings coming from Uvicorn's internal imports. Uvicorn relies on deprecated
# modules from `websockets`, which triggers warnings during tests. These are safe to ignore until
# Uvicorn updates its internals.
filterwarnings = [
"ignore:websockets.legacy is deprecated:DeprecationWarning",
"ignore:websockets.server.WebSocketServerProtocol is deprecated:DeprecationWarning",
]

[tool.mypy]
python_version = "3.10"
Expand Down
4 changes: 2 additions & 2 deletions src/crawlee/_utils/context.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

import asyncio
import inspect
from collections.abc import Callable
from functools import wraps
from typing import Any, TypeVar
Expand Down Expand Up @@ -44,4 +44,4 @@ async def async_wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:

return await method(self, *args, **kwargs)

return async_wrapper if asyncio.iscoroutinefunction(method) else sync_wrapper # type: ignore[return-value]
return async_wrapper if inspect.iscoroutinefunction(method) else sync_wrapper # type: ignore[return-value]
3 changes: 2 additions & 1 deletion src/crawlee/_utils/recurring_task.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import asyncio
import inspect
from logging import getLogger
from typing import TYPE_CHECKING

Expand Down Expand Up @@ -49,7 +50,7 @@ async def _wrapper(self) -> None:
"""
sleep_time_secs = self.delay.total_seconds()
while True:
await self.func() if asyncio.iscoroutinefunction(self.func) else self.func()
await self.func() if inspect.iscoroutinefunction(self.func) else self.func()
await asyncio.sleep(sleep_time_secs)

def start(self) -> None:
Expand Down
4 changes: 1 addition & 3 deletions src/crawlee/events/_event_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,11 +174,9 @@ async def listener_wrapper(event_data: EventData) -> None:
# to avoid blocking the event loop
coro = (
listener(*bound_args.args, **bound_args.kwargs)
if asyncio.iscoroutinefunction(listener)
if inspect.iscoroutinefunction(listener)
else asyncio.to_thread(cast('Callable[..., None]', listener), *bound_args.args, **bound_args.kwargs)
)
# Note: use `asyncio.iscoroutinefunction` rather then `inspect.iscoroutinefunction` since it works with
# unittests.mock.AsyncMock. See https://github.com/python/cpython/issues/84753.

listener_task = asyncio.create_task(coro, name=f'Task-{event.value}-{listener.__name__}')
self._listener_tasks.add(listener_task)
Expand Down
6 changes: 5 additions & 1 deletion tests/unit/browsers/test_playwright_browser_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from unittest.mock import AsyncMock

import pytest
from playwright.async_api import Browser, Playwright, async_playwright
from playwright.async_api import Browser, BrowserContext, Page, Playwright, async_playwright

from crawlee.browsers import PlaywrightBrowserController, PlaywrightPersistentBrowser

Expand Down Expand Up @@ -115,6 +115,10 @@ async def test_memory_leak_on_concurrent_context_creation() -> None:
# Prepare mocked browser with relevant methods and attributes
mocked_browser = AsyncMock()
mocked_context_launcher = AsyncMock()
mocked_context = AsyncMock(spec=BrowserContext)

mocked_context_launcher.return_value = mocked_context
mocked_context.new_page.return_value = AsyncMock(spec=Page)

async def delayed_launch_persistent_context(*args: Any, **kwargs: Any) -> Any:
"""Ensure that both calls to create context overlap in time."""
Expand Down
1 change: 1 addition & 0 deletions tests/unit/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ def redirect_http_server(unused_tcp_port_factory: Callable[[], int]) -> Iterator
timeout_graceful_shutdown=10,
log_level='error',
access_log=False,
ws='websockets-sansio',
)
server = TestServer(config=config)
yield from serve_in_thread(server)
Expand Down
21 changes: 12 additions & 9 deletions tests/unit/storages/test_key_value_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -1095,25 +1095,28 @@ async def test_validate_name(storage_client: StorageClient, name: str, *, is_val


@pytest.mark.parametrize(
'tested_storage_client',
'tested_storage_client_class',
[
pytest.param(MemoryStorageClient(), id='tested=MemoryStorageClient'),
pytest.param(FileSystemStorageClient(), id='tested=FileSystemStorageClient'),
pytest.param(SqlStorageClient(), id='tested=SqlStorageClient'),
pytest.param(MemoryStorageClient, id='tested=MemoryStorageClient'),
pytest.param(FileSystemStorageClient, id='tested=FileSystemStorageClient'),
pytest.param(SqlStorageClient, id='tested=SqlStorageClient'),
],
)
@pytest.mark.parametrize(
'global_storage_client',
'global_storage_client_class',
[
pytest.param(MemoryStorageClient(), id='global=MemoryStorageClient'),
pytest.param(FileSystemStorageClient(), id='global=FileSystemStorageClient'),
pytest.param(SqlStorageClient(), id='global=SqlStorageClient'),
pytest.param(MemoryStorageClient, id='global=MemoryStorageClient'),
pytest.param(FileSystemStorageClient, id='global=FileSystemStorageClient'),
pytest.param(SqlStorageClient, id='global=SqlStorageClient'),
],
)
async def test_get_auto_saved_value_various_global_clients(
tmp_path: Path, tested_storage_client: StorageClient, global_storage_client: StorageClient
tmp_path: Path, tested_storage_client_class: type[StorageClient], global_storage_client_class: type[StorageClient]
) -> None:
"""Ensure that persistence is working for all clients regardless of what is set in service locator."""
tested_storage_client = tested_storage_client_class()
global_storage_client = global_storage_client_class()

service_locator.set_configuration(
Configuration(
storage_dir=str(tmp_path),
Expand Down
Loading