Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,10 @@ timeout = 300
markers = [
"run_alone: marks tests that must run in isolation",
]
filterwarnings = [
"ignore:websockets.legacy is deprecated:DeprecationWarning",
"ignore:websockets.server.WebSocketServerProtocol is deprecated:DeprecationWarning",
]

[tool.mypy]
python_version = "3.10"
Expand Down
4 changes: 2 additions & 2 deletions src/crawlee/_utils/context.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

import asyncio
import inspect
from collections.abc import Callable
from functools import wraps
from typing import Any, TypeVar
Expand Down Expand Up @@ -44,4 +44,4 @@ async def async_wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:

return await method(self, *args, **kwargs)

return async_wrapper if asyncio.iscoroutinefunction(method) else sync_wrapper # type: ignore[return-value]
return async_wrapper if inspect.iscoroutinefunction(method) else sync_wrapper # type: ignore[return-value]
3 changes: 2 additions & 1 deletion src/crawlee/_utils/recurring_task.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import asyncio
import inspect
from logging import getLogger
from typing import TYPE_CHECKING

Expand Down Expand Up @@ -49,7 +50,7 @@ async def _wrapper(self) -> None:
"""
sleep_time_secs = self.delay.total_seconds()
while True:
await self.func() if asyncio.iscoroutinefunction(self.func) else self.func()
await self.func() if inspect.iscoroutinefunction(self.func) else self.func()
await asyncio.sleep(sleep_time_secs)

def start(self) -> None:
Expand Down
4 changes: 1 addition & 3 deletions src/crawlee/events/_event_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,11 +174,9 @@ async def listener_wrapper(event_data: EventData) -> None:
# to avoid blocking the event loop
coro = (
listener(*bound_args.args, **bound_args.kwargs)
if asyncio.iscoroutinefunction(listener)
if inspect.iscoroutinefunction(listener)
else asyncio.to_thread(cast('Callable[..., None]', listener), *bound_args.args, **bound_args.kwargs)
)
# Note: use `asyncio.iscoroutinefunction` rather then `inspect.iscoroutinefunction` since it works with
# unittests.mock.AsyncMock. See https://github.com/python/cpython/issues/84753.

listener_task = asyncio.create_task(coro, name=f'Task-{event.value}-{listener.__name__}')
self._listener_tasks.add(listener_task)
Expand Down
6 changes: 5 additions & 1 deletion tests/unit/browsers/test_playwright_browser_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from unittest.mock import AsyncMock

import pytest
from playwright.async_api import Browser, Playwright, async_playwright
from playwright.async_api import Browser, BrowserContext, Page, Playwright, async_playwright

from crawlee.browsers import PlaywrightBrowserController, PlaywrightPersistentBrowser

Expand Down Expand Up @@ -115,6 +115,10 @@ async def test_memory_leak_on_concurrent_context_creation() -> None:
# Prepare mocked browser with relevant methods and attributes
mocked_browser = AsyncMock()
mocked_context_launcher = AsyncMock()
mocked_context = AsyncMock(spec=BrowserContext)

mocked_context_launcher.return_value = mocked_context
mocked_context.new_page.return_value = AsyncMock(spec=Page)

async def delayed_launch_persistent_context(*args: Any, **kwargs: Any) -> Any:
"""Ensure that both calls to create context overlap in time."""
Expand Down
1 change: 1 addition & 0 deletions tests/unit/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ def redirect_http_server(unused_tcp_port_factory: Callable[[], int]) -> Iterator
timeout_graceful_shutdown=10,
log_level='error',
access_log=False,
ws='websockets-sansio',
)
server = TestServer(config=config)
yield from serve_in_thread(server)
Expand Down
21 changes: 12 additions & 9 deletions tests/unit/storages/test_key_value_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -1095,25 +1095,28 @@ async def test_validate_name(storage_client: StorageClient, name: str, *, is_val


@pytest.mark.parametrize(
'tested_storage_client',
'tested_storage_client_class',
[
pytest.param(MemoryStorageClient(), id='tested=MemoryStorageClient'),
pytest.param(FileSystemStorageClient(), id='tested=FileSystemStorageClient'),
pytest.param(SqlStorageClient(), id='tested=SqlStorageClient'),
pytest.param(MemoryStorageClient, id='tested=MemoryStorageClient'),
pytest.param(FileSystemStorageClient, id='tested=FileSystemStorageClient'),
pytest.param(SqlStorageClient, id='tested=SqlStorageClient'),
],
)
@pytest.mark.parametrize(
'global_storage_client',
'global_storage_client_class',
[
pytest.param(MemoryStorageClient(), id='global=MemoryStorageClient'),
pytest.param(FileSystemStorageClient(), id='global=FileSystemStorageClient'),
pytest.param(SqlStorageClient(), id='global=SqlStorageClient'),
pytest.param(MemoryStorageClient, id='global=MemoryStorageClient'),
pytest.param(FileSystemStorageClient, id='global=FileSystemStorageClient'),
pytest.param(SqlStorageClient, id='global=SqlStorageClient'),
],
)
async def test_get_auto_saved_value_various_global_clients(
tmp_path: Path, tested_storage_client: StorageClient, global_storage_client: StorageClient
tmp_path: Path, tested_storage_client_class: type[StorageClient], global_storage_client_class: type[StorageClient]
) -> None:
"""Ensure that persistence is working for all clients regardless of what is set in service locator."""
tested_storage_client = tested_storage_client_class()
global_storage_client = global_storage_client_class()

service_locator.set_configuration(
Configuration(
storage_dir=str(tmp_path),
Expand Down
Loading