diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml index 91edf890..82164516 100644 --- a/.github/workflows/run_release.yaml +++ b/.github/workflows/run_release.yaml @@ -64,12 +64,11 @@ jobs: needs: [should_release] uses: ./.github/workflows/_version_conflict_check.yaml - # tmp disabled due to instability - # integration_tests: - # name: Integration tests - # needs: [should_release] - # uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main - # secrets: inherit + integration_tests: + name: Integration tests + needs: [should_release] + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit publish_to_pypi: name: Publish to PyPI @@ -81,7 +80,7 @@ jobs: unit_tests, changelog_entry_check, version_conflict_check, - # integration_tests, # tmp disabled due to instability + integration_tests, ] runs-on: ubuntu-latest permissions: diff --git a/src/apify/scrapy/middlewares/apify_proxy.py b/src/apify/scrapy/middlewares/apify_proxy.py index 94069d57..3a7f7b75 100644 --- a/src/apify/scrapy/middlewares/apify_proxy.py +++ b/src/apify/scrapy/middlewares/apify_proxy.py @@ -27,7 +27,7 @@ class ApifyHttpProxyMiddleware: proxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []} """ - def __init__(self: ApifyHttpProxyMiddleware, proxy_settings: dict) -> None: + def __init__(self, proxy_settings: dict) -> None: """Create a new instance. Args: @@ -66,7 +66,7 @@ def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> Apify return cls(proxy_settings) - async def process_request(self: ApifyHttpProxyMiddleware, request: Request, spider: Spider) -> None: + async def process_request(self, request: Request, spider: Spider) -> None: """Process a Scrapy request by assigning a new proxy. Args: @@ -89,7 +89,7 @@ async def process_request(self: ApifyHttpProxyMiddleware, request: Request, spid Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: updated request.meta={request.meta}') def process_exception( - self: ApifyHttpProxyMiddleware, + self, request: Request, exception: Exception, spider: Spider, @@ -116,7 +116,7 @@ def process_exception( 'reason="{exception}", skipping...' ) - async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult: + async def _get_new_proxy_url(self) -> ParseResult: """Get a new proxy URL. Raises: diff --git a/src/apify/scrapy/pipelines/actor_dataset_push.py b/src/apify/scrapy/pipelines/actor_dataset_push.py index 8f371788..15026475 100644 --- a/src/apify/scrapy/pipelines/actor_dataset_push.py +++ b/src/apify/scrapy/pipelines/actor_dataset_push.py @@ -19,7 +19,7 @@ class ActorDatasetPushPipeline: """ async def process_item( - self: ActorDatasetPushPipeline, + self, item: Item, spider: Spider, ) -> Item: diff --git a/src/apify/scrapy/scheduler.py b/src/apify/scrapy/scheduler.py index 36d149c9..849e5376 100644 --- a/src/apify/scrapy/scheduler.py +++ b/src/apify/scrapy/scheduler.py @@ -29,7 +29,7 @@ class ApifyScheduler(BaseScheduler): This scheduler requires the asyncio Twisted reactor to be installed. """ - def __init__(self: ApifyScheduler) -> None: + def __init__(self) -> None: """Create a new instance.""" if not is_asyncio_reactor_installed(): raise ValueError( @@ -40,7 +40,7 @@ def __init__(self: ApifyScheduler) -> None: self._rq: RequestQueue | None = None self.spider: Spider | None = None - def open(self: ApifyScheduler, spider: Spider) -> None: # this has to be named "open" + def open(self, spider: Spider) -> None: # this has to be named "open" """Open the scheduler. Args: @@ -58,7 +58,7 @@ async def open_queue() -> RequestQueue: traceback.print_exc() raise - def has_pending_requests(self: ApifyScheduler) -> bool: + def has_pending_requests(self) -> bool: """Check if the scheduler has any pending requests. Returns: @@ -75,7 +75,7 @@ def has_pending_requests(self: ApifyScheduler) -> bool: return not is_finished - def enqueue_request(self: ApifyScheduler, request: Request) -> bool: + def enqueue_request(self, request: Request) -> bool: """Add a request to the scheduler. This could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...). @@ -111,7 +111,7 @@ def enqueue_request(self: ApifyScheduler, request: Request) -> bool: Actor.log.debug(f'[{call_id}]: rq.add_request.result={result}...') return bool(result.was_already_present) - def next_request(self: ApifyScheduler) -> Request | None: + def next_request(self) -> Request | None: """Fetch the next request from the scheduler. Returns: diff --git a/tests/integration/README.md b/tests/integration/README.md index 331acad1..a3b2dbcf 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,22 +1,16 @@ -Integration tests -================= +# Integration tests -We have integration tests which build and run Actors using the Python SDK on the Apify Platform. -To run these tests, you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to use for the tests, -and then start them with `make integration-tests`. +We have integration tests which build and run Actors using the Python SDK on the Apify Platform. To run these tests, you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to use for the tests, and then start them with `make integration-tests`. -If you want to run the integration tests on a different environment than the main Apify Platform, -you need to set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to the Apify API you want to use. +If you want to run the integration tests on a different environment than the main Apify Platform, you need to set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to the Apify API you want to use. -How to write tests ------------------- +## How to write tests There are two fixtures which you can use to write tests: ### `apify_client_async` -This fixture just gives you an instance of `ApifyClientAsync` configured with the right token and API URL, -so you don't have to do that yourself. +This fixture just gives you an instance of `ApifyClientAsync` configured with the right token and API URL, so you don't have to do that yourself. ```python async def test_something(apify_client_async: ApifyClientAsync) -> None: @@ -27,40 +21,38 @@ async def test_something(apify_client_async: ApifyClientAsync) -> None: This fixture returns a factory function for creating Actors on the Apify Platform. -For the Actor source, the fixture takes the files from `tests/integration/actor_source_base`, -builds the Apify SDK wheel from the current codebase, -and adds the Actor source you passed to the fixture as an argument. -You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. +For the Actor source, the fixture takes the files from `tests/integration/actor_source_base`, builds the Apify SDK wheel from the current codebase, and adds the Actor source you passed to the fixture as an argument. You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. -The created Actor will be uploaded to the platform, built there, and after the test finishes, it will be automatically deleted. -If the Actor build fails, it will not be deleted, so that you can check why the build failed. +The created Actor will be uploaded to the platform, built there, and after the test finishes, it will be automatically deleted. If the Actor build fails, it will not be deleted, so that you can check why the build failed. ### Creating test Actor straight from a Python function -You can create Actors straight from a Python function. -This is great because you can have the test Actor source code checked with the linter. +You can create Actors straight from a Python function. This is great because you can have the test Actor source code checked with the linter. ```python -async def test_something(self, make_actor: ActorFactory) -> None: +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: print('Hello!') - actor = await make_actor('something', main_func=main) + actor = await make_actor(label='something', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' ``` -These Actors will have the `src/main.py` file set to the `main` function definition, -prepended with `import asyncio` and `from apify import Actor`, for your convenience. +These Actors will have the `src/main.py` file set to the `main` function definition, prepended with `import asyncio` and `from apify import Actor`, for your convenience. You can also pass extra imports directly to the main function: ```python -async def test_something(self, make_actor: ActorFactory) -> None: +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main(): import os from apify_shared.consts import ActorEventTypes, ActorEnvVars @@ -68,23 +60,23 @@ async def test_something(self, make_actor: ActorFactory) -> None: print('The Actor is running with ' + os.getenv(ActorEnvVars.MEMORY_MBYTES) + 'MB of memory') await Actor.on(ActorEventTypes.SYSTEM_INFO, lambda event_data: print(event_data)) - actor = await make_actor('something', main_func=main) - - run_result = await actor.call() + actor = await make_actor(label='something', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' ``` ### Creating Actor from source files -You can also pass the source files directly if you need something more complex -(e.g. pass some fixed value to the Actor source code or use multiple source files). +You can also pass the source files directly if you need something more complex (e.g. pass some fixed value to the Actor source code or use multiple source files). To pass the source code of the `src/main.py` file directly, use the `main_py` argument to `make_actor`: ```python -async def test_something(self, make_actor: ActorFactory) -> None: +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' main_py_source = f""" import asyncio @@ -96,9 +88,8 @@ async def test_something(self, make_actor: ActorFactory) -> None: await Actor.set_value('OUTPUT', '{expected_output}') """ - actor = await make_actor('something', main_py=main_py_source) - - await actor.call() + actor = await make_actor(label='something', main_py=main_py_source) + await run_actor(actor) output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None @@ -106,11 +97,13 @@ async def test_something(self, make_actor: ActorFactory) -> None: ``` -Or you can pass multiple source files with the `source_files` argument, -if you need something really complex: +Or you can pass multiple source files with the `source_files` argument, if you need something really complex: ```python -async def test_something(self, make_actor: ActorFactory) -> None: +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: actor_source_files = { 'src/utils.py': """ from datetime import datetime, timezone @@ -129,9 +122,8 @@ async def test_something(self, make_actor: ActorFactory) -> None: print('Hello! It is ' + current_datetime.time()) """, } - actor = await make_actor('something', source_files=actor_source_files) + actor = await make_actor(label='something', source_files=actor_source_files) + actor_run = await run_actor(actor) - actor_run = await actor.call() - assert actor_run is not None - assert actor_run['status'] == 'SUCCEEDED' + assert actor_run.status == 'SUCCEEDED' ``` diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 913cad32..2ff45a68 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,7 +7,7 @@ import sys import textwrap from pathlib import Path -from typing import TYPE_CHECKING, Callable, Protocol, cast +from typing import TYPE_CHECKING, Any, Callable, Coroutine, Protocol, cast import pytest from filelock import FileLock @@ -17,21 +17,25 @@ import apify._actor from ._utils import generate_unique_resource_name +from apify._models import ActorRun if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Mapping from apify_client.clients.resource_clients import ActorClientAsync -TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' -API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' -SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() +_TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' +_API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' +_SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() -# To isolate the tests, we need to reset the used singletons before each test case -# We also patch the default storage client with a tmp_path @pytest.fixture(autouse=True) def _reset_and_patch_default_instances() -> None: + """Reset the used singletons and patch the default storage client with a temporary directory. + + To isolate the tests, we need to reset the used singletons before each test case. We also patch the default + storage client with a tmp_path. + """ from crawlee import service_container cast(dict, service_container._services).clear() @@ -40,37 +44,44 @@ def _reset_and_patch_default_instances() -> None: # TODO: StorageClientManager local storage client purge # noqa: TD003 -# This fixture can't be session-scoped, -# because then you start getting `RuntimeError: Event loop is closed` errors, -# because `httpx.AsyncClient` in `ApifyClientAsync` tries to reuse the same event loop across requests, -# but `pytest-asyncio` closes the event loop after each test, -# and uses a new one for the next test. @pytest.fixture def apify_client_async() -> ApifyClientAsync: - api_token = os.getenv(TOKEN_ENV_VAR) - api_url = os.getenv(API_URL_ENV_VAR) + """Create an instance of the ApifyClientAsync. + + This fixture can't be session-scoped, because then you start getting `RuntimeError: Event loop is closed` errors, + because `httpx.AsyncClient` in `ApifyClientAsync` tries to reuse the same event loop across requests, + but `pytest-asyncio` closes the event loop after each test, and uses a new one for the next test. + """ + api_token = os.getenv(_TOKEN_ENV_VAR) + api_url = os.getenv(_API_URL_ENV_VAR) if not api_token: - raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') + raise RuntimeError(f'{_TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') return ApifyClientAsync(api_token, api_url=api_url) -# Build the package wheel if it hasn't been built yet, and return the path to the wheel @pytest.fixture(scope='session') def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) -> Path: + """Build the package wheel if it hasn't been built yet, and return the path to the wheel.""" # Make sure the wheel is not being built concurrently across all the pytest-xdist runners, - # through locking the building process with a temp file + # through locking the building process with a temp file. with FileLock(tmp_path_factory.getbasetemp().parent / 'sdk_wheel_build.lock'): # Make sure the wheel is built exactly once across across all the pytest-xdist runners, - # through an indicator file saying that the wheel was already built + # through an indicator file saying that the wheel was already built. was_wheel_built_this_test_run_file = tmp_path_factory.getbasetemp() / f'wheel_was_built_in_run_{testrun_uid}' if not was_wheel_built_this_test_run_file.exists(): - subprocess.run('python -m build', cwd=SDK_ROOT_PATH, shell=True, check=True, capture_output=True) # noqa: S602, S607 + subprocess.run( + args='python -m build', + cwd=_SDK_ROOT_PATH, + shell=True, + check=True, + capture_output=True, + ) was_wheel_built_this_test_run_file.touch() - # Read the current package version, necessary for getting the right wheel filename - pyproject_toml_file = (SDK_ROOT_PATH / 'pyproject.toml').read_text(encoding='utf-8') + # Read the current package version, necessary for getting the right wheel filename. + pyproject_toml_file = (_SDK_ROOT_PATH / 'pyproject.toml').read_text(encoding='utf-8') for line in pyproject_toml_file.splitlines(): if line.startswith('version = '): delim = '"' if '"' in line else "'" @@ -79,9 +90,9 @@ def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) - else: raise RuntimeError('Unable to find version string.') - wheel_path = SDK_ROOT_PATH / 'dist' / f'apify-{sdk_version}-py3-none-any.whl' + wheel_path = _SDK_ROOT_PATH / 'dist' / f'apify-{sdk_version}-py3-none-any.whl' - # Just to be sure + # Just to be sure. assert wheel_path.exists() return wheel_path @@ -127,34 +138,18 @@ def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: return source_files -# Just a type for the make_actor result, so that we can import it in tests -class ActorFactory(Protocol): - def __call__( - self: ActorFactory, - actor_label: str, - *, - main_func: Callable | None = None, - main_py: str | None = None, - source_files: Mapping[str, str | bytes] | None = None, - ) -> Awaitable[ActorClientAsync]: ... - - -@pytest.fixture -async def make_actor( - actor_base_source_files: dict[str, str | bytes], - apify_client_async: ApifyClientAsync, -) -> AsyncIterator[ActorFactory]: - """A fixture for returning a temporary Actor factory.""" - actor_clients_for_cleanup: list[ActorClientAsync] = [] +class MakeActorFunction(Protocol): + """A type for the `make_actor` fixture.""" - async def _make_actor( - actor_label: str, + def __call__( + self, + label: str, *, main_func: Callable | None = None, main_py: str | None = None, source_files: Mapping[str, str | bytes] | None = None, - ) -> ActorClientAsync: - """Create a temporary Actor from the given main function or source file(s). + ) -> Awaitable[ActorClientAsync]: + """Create a temporary Actor from the given main function or source files. The Actor will be uploaded to the Apify Platform, built there, and after the test finishes, it will be automatically deleted. @@ -162,7 +157,7 @@ async def _make_actor( You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. Args: - actor_label: The label which will be a part of the generated Actor name + label: The label which will be a part of the generated Actor name. main_func: The main function of the Actor. main_py: The `src/main.py` file of the Actor. source_files: A dictionary of the source files of the Actor. @@ -170,13 +165,34 @@ async def _make_actor( Returns: A resource client for the created Actor. """ + + +@pytest.fixture +async def make_actor( + actor_base_source_files: dict[str, str | bytes], + apify_client_async: ApifyClientAsync, +) -> AsyncIterator[MakeActorFunction]: + """Fixture for creating temporary Actors for testing purposes. + + This returns a function that creates a temporary Actor from the given main function or source files. The Actor + will be uploaded to the Apify Platform, built there, and after the test finishes, it will be automatically deleted. + """ + actor_clients_for_cleanup: list[ActorClientAsync] = [] + + async def _make_actor( + label: str, + *, + main_func: Callable | None = None, + main_py: str | None = None, + source_files: Mapping[str, str | bytes] | None = None, + ) -> ActorClientAsync: if not (main_func or main_py or source_files): raise TypeError('One of `main_func`, `main_py` or `source_files` arguments must be specified') if (main_func and main_py) or (main_func and source_files) or (main_py and source_files): raise TypeError('Cannot specify more than one of `main_func`, `main_py` and `source_files` arguments') - actor_name = generate_unique_resource_name(actor_label) + actor_name = generate_unique_resource_name(label) # Get the source of main_func and convert it into a reasonable main_py file. if main_func: @@ -204,7 +220,7 @@ async def _make_actor( actor_source_files = actor_base_source_files.copy() actor_source_files.update(source_files) - # Reformat the source files in a format that the Apify API understands + # Reformat the source files in a format that the Apify API understands. source_files_for_api = [] for file_name, file_contents in actor_source_files.items(): if isinstance(file_contents, str): @@ -242,19 +258,63 @@ async def _make_actor( actor_client = apify_client_async.actor(created_actor['id']) print(f'Building Actor {actor_name}...') - build = await actor_client.build(version_number='0.0', wait_for_finish=300) + build_result = await actor_client.build(version_number='0.0') + build_client = apify_client_async.build(build_result['id']) + build_client_result = await build_client.wait_for_finish(wait_secs=600) - assert build['status'] == ActorJobStatus.SUCCEEDED + assert build_client_result is not None + assert build_client_result['status'] == ActorJobStatus.SUCCEEDED - # We only mark the client for cleanup if the build succeeded, - # so that if something goes wrong here, - # you have a chance to check the error + # We only mark the client for cleanup if the build succeeded, so that if something goes wrong here, + # you have a chance to check the error. actor_clients_for_cleanup.append(actor_client) return actor_client yield _make_actor - # Delete all the generated actors + # Delete all the generated Actors. for actor_client in actor_clients_for_cleanup: await actor_client.delete() + + +class RunActorFunction(Protocol): + """A type for the `run_actor` fixture.""" + + def __call__( + self, + actor: ActorClientAsync, + *, + run_input: Any = None, + ) -> Coroutine[None, None, ActorRun]: + """Initiate an Actor run and wait for its completion. + + Args: + actor: Actor async client, in testing context usually created by `make_actor` fixture. + run_input: Optional input for the Actor run. + + Returns: + Actor run result. + """ + + +@pytest.fixture +async def run_actor(apify_client_async: ApifyClientAsync) -> RunActorFunction: + """Fixture for calling an Actor run and waiting for its completion. + + This fixture returns a function that initiates an Actor run with optional run input, waits for its completion, + and retrieves the final result. It uses the `wait_for_finish` method with a timeout of 10 minutes. + """ + + async def _run_actor(actor: ActorClientAsync, *, run_input: Any = None) -> ActorRun: + call_result = await actor.call(run_input=run_input) + + assert isinstance(call_result, dict), 'The result of ActorClientAsync.call() is not a dictionary.' + assert 'id' in call_result, 'The result of ActorClientAsync.call() does not contain an ID.' + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=600) + + return ActorRun.model_validate(run_result) + + return _run_actor diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index 7e88251a..5327af9c 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -8,27 +8,32 @@ from ._utils import generate_unique_resource_name from apify import Actor +from apify._models import ActorRun if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_actor_reports_running_on_platform(make_actor: ActorFactory) -> None: +async def test_actor_reports_running_on_platform( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: assert Actor.is_at_home() is True - actor = await make_actor('is-at-home', main_func=main) + actor = await make_actor(label='is-at-home', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() + assert run_result.status == 'SUCCEEDED' - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' - -async def test_actor_retrieves_env_vars(make_actor: ActorFactory) -> None: +async def test_actor_retrieves_env_vars( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: env_dict = Actor.get_env() @@ -45,15 +50,16 @@ async def main() -> None: assert len(env_dict.get('default_key_value_store_id', '')) == 17 assert len(env_dict.get('default_request_queue_id', '')) == 17 - actor = await make_actor('get-env', main_func=main) - - run_result = await actor.call() + actor = await make_actor(label='get-env', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_actor_creates_new_client_instance(make_actor: ActorFactory) -> None: +async def test_actor_creates_new_client_instance( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: import os @@ -68,42 +74,43 @@ async def main() -> None: kv_store_client = new_client.key_value_store(default_key_value_store_id) await kv_store_client.set_record('OUTPUT', 'TESTING-OUTPUT') - actor = await make_actor('new-client', main_func=main) + actor = await make_actor(label='new-client', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None assert output_record['value'] == 'TESTING-OUTPUT' -async def test_actor_sets_status_message(make_actor: ActorFactory) -> None: +async def test_actor_sets_status_message( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: actor_input = await Actor.get_input() or {} await Actor.set_status_message('testing-status-message', **actor_input) - actor = await make_actor('set-status-message', main_func=main) - - run_result = await actor.call() + actor = await make_actor(label='set-status-message', main_func=main) + run_result_1 = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' - assert run_result['statusMessage'] == 'testing-status-message' - assert run_result['isStatusMessageTerminal'] is None + assert run_result_1.status == 'SUCCEEDED' + assert run_result_1.status_message == 'testing-status-message' + assert run_result_1.is_status_message_terminal is None - run_result = await actor.call(run_input={'is_terminal': True}) + run_result_2 = await run_actor(actor, run_input={'is_terminal': True}) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' - assert run_result['statusMessage'] == 'testing-status-message' - assert run_result['isStatusMessageTerminal'] is True + assert run_result_2.status == 'SUCCEEDED' + assert run_result_2.status_message == 'testing-status-message' + assert run_result_2.is_status_message_terminal is True -async def test_actor_starts_another_actor_instance(make_actor: ActorFactory) -> None: +async def test_actor_starts_another_actor_instance( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main_inner() -> None: async with Actor: await asyncio.sleep(5) @@ -125,25 +132,30 @@ async def main_outer() -> None: assert inner_run_status is not None assert inner_run_status.get('status') in ['READY', 'RUNNING'] - inner_actor = await make_actor('start-inner', main_func=main_inner) - outer_actor = await make_actor('start-outer', main_func=main_outer) + inner_actor = await make_actor(label='start-inner', main_func=main_inner) + outer_actor = await make_actor(label='start-outer', main_func=main_outer) inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_run_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + run_result_outer = await run_actor( + outer_actor, + run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, + ) - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish() + await inner_actor.last_run().wait_for_finish(wait_secs=600) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None assert inner_output_record['value'] == f'{test_value}_XXX_{test_value}' -async def test_actor_calls_another_actor(make_actor: ActorFactory) -> None: +async def test_actor_calls_another_actor( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main_inner() -> None: async with Actor: await asyncio.sleep(5) @@ -165,25 +177,31 @@ async def main_outer() -> None: assert inner_run_status is not None assert inner_run_status.get('status') == 'SUCCEEDED' - inner_actor = await make_actor('call-inner', main_func=main_inner) - outer_actor = await make_actor('call-outer', main_func=main_outer) + inner_actor = await make_actor(label='call-inner', main_func=main_inner) + outer_actor = await make_actor(label='call-outer', main_func=main_outer) inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_run_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + run_result_outer = await run_actor( + outer_actor, + run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, + ) - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish() + await inner_actor.last_run().wait_for_finish(wait_secs=600) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None assert inner_output_record['value'] == f'{test_value}_XXX_{test_value}' -async def test_actor_calls_task(make_actor: ActorFactory, apify_client_async: ApifyClientAsync) -> None: +async def test_actor_calls_task( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, + apify_client_async: ApifyClientAsync, +) -> None: async def main_inner() -> None: async with Actor: await asyncio.sleep(5) @@ -204,8 +222,8 @@ async def main_outer() -> None: assert inner_run_status is not None assert inner_run_status.get('status') == 'SUCCEEDED' - inner_actor = await make_actor('call-task-inner', main_func=main_inner) - outer_actor = await make_actor('call-task-outer', main_func=main_outer) + inner_actor = await make_actor(label='call-task-inner', main_func=main_inner) + outer_actor = await make_actor(label='call-task-outer', main_func=main_outer) inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() @@ -216,12 +234,14 @@ async def main_outer() -> None: task_input={'test_value': test_value}, ) - outer_run_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_task_id': task['id']}) + run_result_outer = await run_actor( + outer_actor, + run_input={'test_value': test_value, 'inner_task_id': task['id']}, + ) - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish() + await inner_actor.last_run().wait_for_finish(wait_secs=600) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None @@ -230,7 +250,10 @@ async def main_outer() -> None: await apify_client_async.task(task['id']).delete() -async def test_actor_aborts_another_actor_run(make_actor: ActorFactory) -> None: +async def test_actor_aborts_another_actor_run( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main_inner() -> None: async with Actor: await asyncio.sleep(180) @@ -246,26 +269,32 @@ async def main_outer() -> None: await Actor.abort(inner_run_id) - inner_actor = await make_actor('abort-inner', main_func=main_inner) - outer_actor = await make_actor('abort-outer', main_func=main_outer) + inner_actor = await make_actor(label='abort-inner', main_func=main_inner) + outer_actor = await make_actor(label='abort-outer', main_func=main_outer) inner_run_id = (await inner_actor.start())['id'] - outer_run_result = await outer_actor.call(run_input={'inner_run_id': inner_run_id}) + run_result_outer = await run_actor( + outer_actor, + run_input={'inner_run_id': inner_run_id}, + ) + + assert run_result_outer.status == 'SUCCEEDED' - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + await inner_actor.last_run().wait_for_finish(wait_secs=600) + inner_actor_last_run_dict = await inner_actor.last_run().get() + inner_actor_last_run = ActorRun.model_validate(inner_actor_last_run_dict) - await inner_actor.last_run().wait_for_finish() - inner_actor_last_run = await inner_actor.last_run().get() - assert inner_actor_last_run is not None - assert inner_actor_last_run['status'] == 'ABORTED' + assert inner_actor_last_run.status == 'ABORTED' inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is None -async def test_actor_metamorphs_into_another_actor(make_actor: ActorFactory) -> None: +async def test_actor_metamorphs_into_another_actor( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main_inner() -> None: import os @@ -297,16 +326,18 @@ async def main_outer() -> None: await Actor.set_value('RECORD_AFTER_METAMORPH_CALL', 'dummy') raise AssertionError('The Actor should have been metamorphed by now') - inner_actor = await make_actor('metamorph-inner', main_func=main_inner) - outer_actor = await make_actor('metamorph-outer', main_func=main_outer) + inner_actor = await make_actor(label='metamorph-inner', main_func=main_inner) + outer_actor = await make_actor(label='metamorph-outer', main_func=main_outer) inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_run_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + run_result_outer = await run_actor( + outer_actor, + run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, + ) - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' outer_run_key_value_store = outer_actor.last_run().key_value_store() @@ -320,7 +351,10 @@ async def main_outer() -> None: assert await inner_actor.last_run().get() is None -async def test_actor_reboots_successfully(make_actor: ActorFactory) -> None: +async def test_actor_reboots_successfully( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: print('Starting...') @@ -334,11 +368,14 @@ async def main() -> None: print('Finishing...') - actor = await make_actor('actor_rebooter', main_func=main) - run_result = await actor.call(run_input={'counter_key': 'reboot_counter'}) + actor = await make_actor(label='actor_rebooter', main_func=main) + + run_result = await run_actor( + actor, + run_input={'counter_key': 'reboot_counter'}, + ) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' not_written_value = await actor.last_run().key_value_store().get_record('THIS_KEY_SHOULD_NOT_BE_WRITTEN') assert not_written_value is None @@ -348,7 +385,10 @@ async def main() -> None: assert reboot_counter['value'] == 2 -async def test_actor_adds_webhook_and_receives_event(make_actor: ActorFactory) -> None: +async def test_actor_adds_webhook_and_receives_event( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main_server() -> None: import os from http.server import BaseHTTPRequestHandler, HTTPServer @@ -369,9 +409,7 @@ def do_POST(self) -> None: # noqa: N802 nonlocal webhook_body content_length = self.headers.get('content-length') length = int(content_length) if content_length else 0 - webhook_body = self.rfile.read(length).decode('utf-8') - self.send_response(200) self.end_headers() self.wfile.write(bytes('Hello, world!', encoding='utf-8')) @@ -399,33 +437,34 @@ async def main_client() -> None: ) server_actor, client_actor = await asyncio.gather( - make_actor('add-webhook-server', main_func=main_server), - make_actor('add-webhook-client', main_func=main_client), + make_actor(label='add-webhook-server', main_func=main_server), + make_actor(label='add-webhook-client', main_func=main_client), ) server_actor_run = await server_actor.start() server_actor_container_url = server_actor_run['containerUrl'] - # Give the server actor some time to start running server_actor_initialized = await server_actor.last_run().key_value_store().get_record('INITIALIZED') while not server_actor_initialized: server_actor_initialized = await server_actor.last_run().key_value_store().get_record('INITIALIZED') await asyncio.sleep(1) - client_actor_run_result = await client_actor.call( - run_input={'server_actor_container_url': server_actor_container_url} + ac_run_result = await run_actor( + client_actor, + run_input={'server_actor_container_url': server_actor_container_url}, ) - assert client_actor_run_result is not None - assert client_actor_run_result['status'] == 'SUCCEEDED' - server_actor_run_result = await server_actor.last_run().wait_for_finish() - assert server_actor_run_result is not None - assert server_actor_run_result['status'] == 'SUCCEEDED' + assert ac_run_result.status == 'SUCCEEDED' + + sa_run_result_dict = await server_actor.last_run().wait_for_finish(wait_secs=600) + sa_run_result = ActorRun.model_validate(sa_run_result_dict) + + assert sa_run_result.status == 'SUCCEEDED' webhook_body_record = await server_actor.last_run().key_value_store().get_record('WEBHOOK_BODY') assert webhook_body_record is not None assert webhook_body_record['value'] != '' parsed_webhook_body = json.loads(webhook_body_record['value']) - assert parsed_webhook_body['eventData']['actorId'] == client_actor_run_result['actId'] - assert parsed_webhook_body['eventData']['actorRunId'] == client_actor_run_result['id'] + assert parsed_webhook_body['eventData']['actorId'] == ac_run_result.act_id + assert parsed_webhook_body['eventData']['actorRunId'] == ac_run_result.id diff --git a/tests/integration/test_actor_create_proxy_configuration.py b/tests/integration/test_actor_create_proxy_configuration.py index f9737a7a..5861d43e 100644 --- a/tests/integration/test_actor_create_proxy_configuration.py +++ b/tests/integration/test_actor_create_proxy_configuration.py @@ -5,10 +5,13 @@ from apify import Actor if TYPE_CHECKING: - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_create_basic_proxy_configuration(make_actor: ActorFactory) -> None: +async def test_create_basic_proxy_configuration( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: groups = ['SHADER'] country_code = 'US' @@ -24,26 +27,26 @@ async def main() -> None: assert proxy_configuration._password is not None assert proxy_configuration._country_code == country_code - actor = await make_actor('proxy-configuration', main_func=main) + actor = await make_actor(label='proxy-configuration', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_create_proxy_configuration_with_groups_and_country(make_actor: ActorFactory) -> None: +async def test_create_proxy_configuration_with_groups_and_country( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: await Actor.init() proxy_url_suffix = f'{Actor.config.proxy_password}@{Actor.config.proxy_hostname}:{Actor.config.proxy_port}' + proxy_configuration = await Actor.create_proxy_configuration(actor_proxy_input={'useApifyProxy': True}) - proxy_configuration = await Actor.create_proxy_configuration( - actor_proxy_input={ - 'useApifyProxy': True, - } - ) assert proxy_configuration is not None - assert await proxy_configuration.new_url() == f'http://auto:{proxy_url_suffix}' + + new_url = await proxy_configuration.new_url() + assert new_url == f'http://auto:{proxy_url_suffix}' groups = ['SHADER', 'BUYPROXIES94952'] country_code = 'US' @@ -55,15 +58,13 @@ async def main() -> None: } ) assert proxy_configuration is not None - assert ( - await proxy_configuration.new_url() - == f'http://groups-{"+".join(groups)},country-{country_code}:{proxy_url_suffix}' - ) + + new_url = await proxy_configuration.new_url() + assert new_url == f'http://groups-{"+".join(groups)},country-{country_code}:{proxy_url_suffix}' await Actor.exit() - actor = await make_actor('proxy-configuration', main_func=main) + actor = await make_actor(label='proxy-configuration', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' diff --git a/tests/integration/test_actor_dataset.py b/tests/integration/test_actor_dataset.py index d4eeb0fa..c1c69d72 100644 --- a/tests/integration/test_actor_dataset.py +++ b/tests/integration/test_actor_dataset.py @@ -12,10 +12,13 @@ from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_push_and_verify_data_in_default_dataset(make_actor: ActorFactory) -> None: +async def test_push_and_verify_data_in_default_dataset( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: desired_item_count = 100 # Also change inside main() if you're changing this async def main() -> None: @@ -23,48 +26,54 @@ async def main() -> None: async with Actor: await Actor.push_data([{'id': i} for i in range(desired_item_count)]) - actor = await make_actor('push-data', main_func=main) + actor = await make_actor(label='push-data', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() + assert run_result.status == 'SUCCEEDED' - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' list_page = await actor.last_run().dataset().list_items() assert list_page.items[0]['id'] == 0 assert list_page.items[-1]['id'] == desired_item_count - 1 assert len(list_page.items) == list_page.count == desired_item_count -async def test_push_large_data_chunks_over_9mb(make_actor: ActorFactory) -> None: +async def test_push_large_data_chunks_over_9mb( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: await Actor.push_data([{'str': 'x' * 10000} for _ in range(5000)]) # ~50MB - actor = await make_actor('push-data-over-9mb', main_func=main) + actor = await make_actor(label='push-data-over-9mb', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() + assert run_result.status == 'SUCCEEDED' - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' async for item in actor.last_run().dataset().iterate_items(): assert item['str'] == 'x' * 10000 -async def test_same_references_in_default_dataset(make_actor: ActorFactory) -> None: +async def test_same_references_in_default_dataset( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: dataset1 = await Actor.open_dataset() dataset2 = await Actor.open_dataset() assert dataset1 is dataset2 - actor = await make_actor('dataset-same-ref-default', main_func=main) + actor = await make_actor(label='dataset-same-ref-default', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_same_references_in_named_dataset(make_actor: ActorFactory) -> None: +async def test_same_references_in_named_dataset( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: dataset_name = generate_unique_resource_name('dataset') async def main() -> None: @@ -82,14 +91,16 @@ async def main() -> None: await dataset_by_name_1.drop() - actor = await make_actor('dataset-same-ref-named', main_func=main) + actor = await make_actor(label='dataset-same-ref-named', main_func=main) + run_result = await run_actor(actor, run_input={'datasetName': dataset_name}) - run_result = await actor.call(run_input={'datasetName': dataset_name}) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_force_cloud( + apify_client_async: ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, +) -> None: assert apify_client_async.token is not None monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_client_async.token) diff --git a/tests/integration/test_actor_events.py b/tests/integration/test_actor_events.py index 7509a71d..31890dd6 100644 --- a/tests/integration/test_actor_events.py +++ b/tests/integration/test_actor_events.py @@ -8,10 +8,13 @@ from apify import Actor if TYPE_CHECKING: - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_emit_and_capture_interval_events(make_actor: ActorFactory) -> None: +async def test_emit_and_capture_interval_events( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: import os from datetime import datetime @@ -42,22 +45,21 @@ async def log_event(data: Any) -> None: Actor.on(Event.PERSIST_STATE, on_event(ActorEventTypes.PERSIST_STATE)) await asyncio.sleep(3) - # The SYSTEM_INFO event sometimes takes a while to appear, let's wait for it for a while longer + # The SYSTEM_INFO event sometimes takes a while to appear, let's wait for it for a while longer. for _ in range(20): if was_system_info_emitted: break await asyncio.sleep(1) - # Check that parsing datetimes works correctly - # Check `createdAt` is a datetime (so it's the same locally and on platform) + # Check that parsing datetimes works correctly. + # Check `createdAt` is a datetime (so it's the same locally and on platform). assert isinstance(system_infos[0].cpu_info.created_at, datetime) - actor = await make_actor('actor-interval-events', main_func=main) + actor = await make_actor(label='actor-interval-events', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() + assert run_result.status == 'SUCCEEDED' - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' dataset_items_page = await actor.last_run().dataset().list_items() persist_state_events = [ item for item in dataset_items_page.items if item['event_type'] == ActorEventTypes.PERSIST_STATE @@ -69,7 +71,10 @@ async def log_event(data: Any) -> None: assert len(system_info_events) > 0 -async def test_event_listener_can_be_removed_successfully(make_actor: ActorFactory) -> None: +async def test_event_listener_can_be_removed_successfully( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: import os @@ -94,9 +99,7 @@ def count_event(data): # type: ignore # noqa: ANN202, ANN001 await asyncio.sleep(0.5) assert counter == last_count - actor = await make_actor('actor-off-event', main_func=main) + actor = await make_actor(label='actor-off-event', main_func=main) + run_result = await run_actor(actor) - run = await actor.call() - - assert run is not None - assert run['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' diff --git a/tests/integration/test_actor_key_value_store.py b/tests/integration/test_actor_key_value_store.py index 98cac610..6ed64123 100644 --- a/tests/integration/test_actor_key_value_store.py +++ b/tests/integration/test_actor_key_value_store.py @@ -12,24 +12,29 @@ from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_same_references_in_default_kvs(make_actor: ActorFactory) -> None: +async def test_same_references_in_default_kvs( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: kvs1 = await Actor.open_key_value_store() kvs2 = await Actor.open_key_value_store() assert kvs1 is kvs2 - actor = await make_actor('kvs-same-ref-default', main_func=main) + actor = await make_actor(label='kvs-same-ref-default', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_same_references_in_named_kvs(make_actor: ActorFactory) -> None: +async def test_same_references_in_named_kvs( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: kvs_name = generate_unique_resource_name('key-value-store') async def main() -> None: @@ -47,14 +52,16 @@ async def main() -> None: await kvs_by_name_1.drop() - actor = await make_actor('kvs-same-ref-named', main_func=main) + actor = await make_actor(label='kvs-same-ref-named', main_func=main) + run_result = await run_actor(actor, run_input={'kvsName': kvs_name}) - run_result = await actor.call(run_input={'kvsName': kvs_name}) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_force_cloud( + apify_client_async: ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, +) -> None: assert apify_client_async.token is not None monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_client_async.token) @@ -80,7 +87,10 @@ async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: py await key_value_store_client.delete() -async def test_set_and_get_value_in_same_run(make_actor: ActorFactory) -> None: +async def test_set_and_get_value_in_same_run( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: await Actor.set_value('test', {'number': 123, 'string': 'a string', 'nested': {'test': 1}}) @@ -89,23 +99,25 @@ async def main() -> None: assert value['string'] == 'a string' assert value['nested']['test'] == 1 - actor = await make_actor('actor-get-set-value', main_func=main) + actor = await make_actor(label='actor-get-set-value', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_set_value_in_one_run_and_get_value_in_another(make_actor: ActorFactory) -> None: +async def test_set_value_in_one_run_and_get_value_in_another( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main_set() -> None: async with Actor: await Actor.set_value('test', {'number': 123, 'string': 'a string', 'nested': {'test': 1}}) - actor_set = await make_actor('actor-set-value', main_func=main_set) + actor_set = await make_actor(label='actor-set-value', main_func=main_set) + run_result_set = await run_actor(actor_set) + + assert run_result_set.status == 'SUCCEEDED' - run_result_set = await actor_set.call() - assert run_result_set is not None - assert run_result_set['status'] == 'SUCCEEDED' # Externally check if the value is present in key-value store test_record = await actor_set.last_run().key_value_store().get_record('test') assert test_record is not None @@ -124,16 +136,19 @@ async def main_get() -> None: assert value['string'] == 'a string' assert value['nested']['test'] == 1 - actor_get = await make_actor('actor-get-value', main_func=main_get) + actor_get = await make_actor(label='actor-get-value', main_func=main_get) default_kvs_info = await actor_set.last_run().key_value_store().get() assert default_kvs_info is not None - run_result_get = await actor_get.call(run_input={'kvs-id': default_kvs_info['id']}) - assert run_result_get is not None - assert run_result_get['status'] == 'SUCCEEDED' + run_result_get = await run_actor(actor_get, run_input={'kvs-id': default_kvs_info['id']}) + + assert run_result_get.status == 'SUCCEEDED' -async def test_actor_get_input_from_run(make_actor: ActorFactory) -> None: +async def test_actor_get_input_from_run( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: actor_source_files = { 'INPUT_SCHEMA.json': """ { @@ -166,21 +181,25 @@ async def main(): assert input_object['password'] == 'very secret' """, } - actor = await make_actor('actor-get-input', source_files=actor_source_files) + actor = await make_actor(label='actor-get-input', source_files=actor_source_files) - run_result = await actor.call( + run_result = await run_actor( + actor, run_input={ 'number': 123, 'string': 'a string', 'nested': {'test': 1}, 'password': 'very secret', - } + }, ) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + + assert run_result.status == 'SUCCEEDED' -async def test_generate_public_url_for_kvs_record(make_actor: ActorFactory) -> None: +async def test_generate_public_url_for_kvs_record( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: from typing import cast @@ -196,8 +215,7 @@ async def main() -> None: assert record_url == f'{public_api_url}/v2/key-value-stores/{default_store_id}/records/dummy' - actor = await make_actor('kvs-get-public-url', main_func=main) + actor = await make_actor(label='kvs-get-public-url', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' diff --git a/tests/integration/test_actor_lifecycle.py b/tests/integration/test_actor_lifecycle.py index b2f756f9..7a975c99 100644 --- a/tests/integration/test_actor_lifecycle.py +++ b/tests/integration/test_actor_lifecycle.py @@ -5,10 +5,13 @@ from apify import Actor if TYPE_CHECKING: - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_actor_init_and_double_init_prevention(make_actor: ActorFactory) -> None: +async def test_actor_init_and_double_init_prevention( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: my_actor = Actor await my_actor.init() @@ -32,15 +35,16 @@ async def main() -> None: assert double_init is False assert my_actor._is_initialized is False - actor = await make_actor('actor-init', main_func=main) + actor = await make_actor(label='actor-init', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() + assert run_result.status == 'SUCCEEDED' - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' - -async def test_actor_init_correctly_in_async_with_block(make_actor: ActorFactory) -> None: +async def test_actor_init_correctly_in_async_with_block( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: import apify._actor @@ -48,62 +52,68 @@ async def main() -> None: assert apify._actor.Actor._is_initialized assert apify._actor.Actor._is_initialized is False - actor = await make_actor('with-actor-init', main_func=main) - - run_result = await actor.call() + actor = await make_actor(label='with-actor-init', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_actor_exit_with_different_exit_codes(make_actor: ActorFactory) -> None: +async def test_actor_exit_with_different_exit_codes( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: input = await Actor.get_input() # noqa: A001 await Actor.exit(**input) - actor = await make_actor('actor-exit', main_func=main) + actor = await make_actor(label='actor-exit', main_func=main) for exit_code in [0, 1, 101]: - run_result = await actor.call(run_input={'exit_code': exit_code}) - assert run_result is not None - assert run_result['exitCode'] == exit_code - assert run_result['status'] == 'FAILED' if exit_code > 0 else 'SUCCEEDED' + run_result = await run_actor(actor, run_input={'exit_code': exit_code}) + + assert run_result.exit_code == exit_code + assert run_result.status == 'FAILED' if exit_code > 0 else 'SUCCEEDED' -async def test_actor_fail_with_custom_exit_codes_and_status_messages(make_actor: ActorFactory) -> None: +async def test_actor_fail_with_custom_exit_codes_and_status_messages( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: input = await Actor.get_input() # noqa: A001 await Actor.fail(**input) if input else await Actor.fail() - actor = await make_actor('actor-fail', main_func=main) + actor = await make_actor(label='actor-fail', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['exitCode'] == 1 - assert run_result['status'] == 'FAILED' + assert run_result.exit_code == 1 + assert run_result.status == 'FAILED' for exit_code in [1, 10, 100]: - run_result = await actor.call(run_input={'exit_code': exit_code}) - assert run_result is not None - assert run_result['exitCode'] == exit_code - assert run_result['status'] == 'FAILED' + run_result = await run_actor(actor, run_input={'exit_code': exit_code}) - # fail with status message - run_result = await actor.call(run_input={'status_message': 'This is a test message'}) - assert run_result is not None - assert run_result['status'] == 'FAILED' - assert run_result.get('statusMessage') == 'This is a test message' + assert run_result.exit_code == exit_code + assert run_result.status == 'FAILED' + # Fail with a status message. + run_result = await run_actor(actor, run_input={'status_message': 'This is a test message'}) -async def test_actor_fails_correctly_with_exception(make_actor: ActorFactory) -> None: + assert run_result.status == 'FAILED' + assert run_result.status_message == 'This is a test message' + + +async def test_actor_fails_correctly_with_exception( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: raise Exception('This is a test exception') # noqa: TRY002 - actor = await make_actor('with-actor-fail', main_func=main) - run_result = await actor.call() - assert run_result is not None - assert run_result['exitCode'] == 91 - assert run_result['status'] == 'FAILED' + actor = await make_actor(label='with-actor-fail', main_func=main) + run_result = await run_actor(actor) + + assert run_result.exit_code == 91 + assert run_result.status == 'FAILED' diff --git a/tests/integration/test_actor_log.py b/tests/integration/test_actor_log.py index 103e4794..7ec1b24d 100644 --- a/tests/integration/test_actor_log.py +++ b/tests/integration/test_actor_log.py @@ -5,10 +5,13 @@ from apify import Actor, __version__ if TYPE_CHECKING: - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_actor_logging(make_actor: ActorFactory) -> None: +async def test_actor_logging( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: import logging @@ -37,12 +40,10 @@ async def main() -> None: # Test that exception in Actor.main is logged with the traceback raise RuntimeError('Dummy RuntimeError') - actor = await make_actor('actor-log', main_func=main) + actor = await make_actor(label='actor-log', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - - assert run_result is not None - assert run_result['status'] == 'FAILED' + assert run_result.status == 'FAILED' run_log = await actor.last_run().log().get() assert run_log is not None diff --git a/tests/integration/test_actor_request_queue.py b/tests/integration/test_actor_request_queue.py index fc1902a3..06e8529e 100644 --- a/tests/integration/test_actor_request_queue.py +++ b/tests/integration/test_actor_request_queue.py @@ -13,24 +13,29 @@ from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_same_references_in_default_rq(make_actor: ActorFactory) -> None: +async def test_same_references_in_default_rq( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: rq1 = await Actor.open_request_queue() rq2 = await Actor.open_request_queue() assert rq1 is rq2 - actor = await make_actor('rq-same-ref-default', main_func=main) + actor = await make_actor(label='rq-same-ref-default', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_same_references_in_named_rq(make_actor: ActorFactory) -> None: +async def test_same_references_in_named_rq( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: rq_name = generate_unique_resource_name('request-queue') async def main() -> None: @@ -48,14 +53,16 @@ async def main() -> None: await rq_by_name_1.drop() - actor = await make_actor('rq-same-ref-named', main_func=main) + actor = await make_actor(label='rq-same-ref-named', main_func=main) + run_result = await run_actor(actor, run_input={'rqName': rq_name}) - run_result = await actor.call(run_input={'rqName': rq_name}) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_force_cloud( + apify_client_async: ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, +) -> None: assert apify_client_async.token is not None monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_client_async.token) diff --git a/tests/integration/test_fixtures.py b/tests/integration/test_fixtures.py index 049a3d73..865effa9 100644 --- a/tests/integration/test_fixtures.py +++ b/tests/integration/test_fixtures.py @@ -10,10 +10,13 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_actor_from_main_func(make_actor: ActorFactory) -> None: +async def test_actor_from_main_func( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: import os @@ -22,19 +25,21 @@ async def main() -> None: async with Actor: await Actor.set_value('OUTPUT', os.getenv(ActorEnvVars.ID)) - actor = await make_actor('make-actor-main-func', main_func=main) + actor = await make_actor(label='make-actor-main-func', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') + assert output_record is not None - assert run_result['actId'] == output_record['value'] + assert run_result.act_id == output_record['value'] -async def test_actor_from_main_py(make_actor: ActorFactory) -> None: +async def test_actor_from_main_py( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' main_py_source = f""" import asyncio @@ -44,18 +49,21 @@ async def main(): await Actor.set_value('OUTPUT', '{expected_output}') """ - actor = await make_actor('make-actor-main-py', main_py=main_py_source) + actor = await make_actor(label='make-actor-main-py', main_py=main_py_source) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') + assert output_record is not None assert output_record['value'] == expected_output -async def test_actor_from_source_files(make_actor: ActorFactory) -> None: +async def test_actor_from_source_files( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: test_started_at = datetime.now(timezone.utc) actor_source_files = { 'src/utils.py': """ @@ -75,11 +83,10 @@ async def main(): await Actor.set_value('OUTPUT', current_datetime) """, } - actor = await make_actor('make-actor-source-files', source_files=actor_source_files) + actor = await make_actor(label='make-actor-source-files', source_files=actor_source_files) + run_result = await run_actor(actor) - actor_run = await actor.call() - assert actor_run is not None - assert actor_run['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index 19251ad0..4bce884a 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -5,10 +5,13 @@ from apify import Actor if TYPE_CHECKING: - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction -async def test_add_and_fetch_requests(make_actor: ActorFactory) -> None: +async def test_add_and_fetch_requests( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: desired_request_count = 100 @@ -33,14 +36,16 @@ async def main() -> None: is_finished = await rq.is_finished() assert is_finished is True - actor = await make_actor('rq-simple-test', main_func=main) + actor = await make_actor(label='rq-simple-test', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_add_requests_in_batches(make_actor: ActorFactory) -> None: +async def test_add_requests_in_batches( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: desired_request_count = 100 @@ -63,14 +68,16 @@ async def main() -> None: is_finished = await rq.is_finished() assert is_finished is True - actor = await make_actor('rq-batch-test', main_func=main) + actor = await make_actor(label='rq-batch-test', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' -async def test_add_non_unique_requests_in_batch(make_actor: ActorFactory) -> None: +async def test_add_non_unique_requests_in_batch( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: from crawlee import Request @@ -100,8 +107,7 @@ async def main() -> None: is_finished = await rq.is_finished() assert is_finished is True - actor = await make_actor('rq-batch-test', main_func=main) + actor = await make_actor(label='rq-batch-test', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index fd74d99e..40d27c54 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -64,12 +64,12 @@ def _reset_and_patch_default_instances( # This class is used to patch the ApifyClientAsync methods to return a fixed value or be replaced with another method. class ApifyClientAsyncPatcher: - def __init__(self: ApifyClientAsyncPatcher, monkeypatch: pytest.MonkeyPatch) -> None: + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: self.monkeypatch = monkeypatch self.calls: dict[str, dict[str, list[tuple[Any, Any]]]] = defaultdict(lambda: defaultdict(list)) def patch( - self: ApifyClientAsyncPatcher, + self, method: str, submethod: str, *,