From 6bf520c4d9f8ff9d4006f6d018dbf784f214d7de Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 29 Oct 2024 18:35:40 +0100 Subject: [PATCH 01/12] ci: make building Actors in intgrt tests more stable relates: #301 --- tests/integration/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 913cad32..2bf615a0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -242,9 +242,12 @@ async def _make_actor( actor_client = apify_client_async.actor(created_actor['id']) print(f'Building Actor {actor_name}...') - build = await actor_client.build(version_number='0.0', wait_for_finish=300) + build_result = await actor_client.build(version_number='0.0', wait_for_finish=60) + build_client = apify_client_async.build(build_result['id']) + build_client_result = await build_client.wait_for_finish(wait_secs=600) - assert build['status'] == ActorJobStatus.SUCCEEDED + assert build_client_result is not None + assert build_client_result['status'] == ActorJobStatus.SUCCEEDED # We only mark the client for cleanup if the build succeeded, # so that if something goes wrong here, From df521e239dc9b174e51eab64a4d8fbb8ea382f86 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 29 Oct 2024 18:44:04 +0100 Subject: [PATCH 02/12] try to improve run actor as well --- tests/integration/conftest.py | 4 +-- tests/integration/test_actor_api_helpers.py | 35 ++++++++++++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2bf615a0..734af969 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -242,9 +242,9 @@ async def _make_actor( actor_client = apify_client_async.actor(created_actor['id']) print(f'Building Actor {actor_name}...') - build_result = await actor_client.build(version_number='0.0', wait_for_finish=60) + build_result = await actor_client.build(version_number='0.0') build_client = apify_client_async.build(build_result['id']) - build_client_result = await build_client.wait_for_finish(wait_secs=600) + build_client_result = await build_client.wait_for_finish(wait_secs=300) assert build_client_result is not None assert build_client_result['status'] == ActorJobStatus.SUCCEEDED diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index 7e88251a..fb6580a7 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -15,7 +15,10 @@ from .conftest import ActorFactory -async def test_actor_reports_running_on_platform(make_actor: ActorFactory) -> None: +async def test_actor_reports_running_on_platform( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: assert Actor.is_at_home() is True @@ -23,12 +26,19 @@ async def main() -> None: actor = await make_actor('is-at-home', main_func=main) run_result = await actor.call() - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + run_client = apify_client_async.run(run_result['id']) + run_client_result = await run_client.wait_for_finish(wait_secs=300) + + assert run_client_result is not None + assert run_client_result['status'] == 'SUCCEEDED' -async def test_actor_retrieves_env_vars(make_actor: ActorFactory) -> None: + +async def test_actor_retrieves_env_vars( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: env_dict = Actor.get_env() @@ -48,12 +58,19 @@ async def main() -> None: actor = await make_actor('get-env', main_func=main) run_result = await actor.call() - assert run_result is not None + + run_client = apify_client_async.run(run_result['id']) + run_client_result = await run_client.wait_for_finish(wait_secs=300) + + assert run_client_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_actor_creates_new_client_instance(make_actor: ActorFactory) -> None: +async def test_actor_creates_new_client_instance( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: import os @@ -71,8 +88,12 @@ async def main() -> None: actor = await make_actor('new-client', main_func=main) run_result = await actor.call() - assert run_result is not None + + run_client = apify_client_async.run(run_result['id']) + run_client_result = await run_client.wait_for_finish(wait_secs=300) + + assert run_client_result is not None assert run_result['status'] == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') From a9fdb846e7cdf171761208a953df017b862d2755 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 29 Oct 2024 19:08:16 +0100 Subject: [PATCH 03/12] more --- tests/integration/test_actor_api_helpers.py | 72 +++++++++++++-------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index fb6580a7..b5fdc3e3 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -25,14 +25,14 @@ async def main() -> None: actor = await make_actor('is-at-home', main_func=main) - run_result = await actor.call() - assert run_result is not None + call_result = await actor.call() + assert call_result is not None - run_client = apify_client_async.run(run_result['id']) - run_client_result = await run_client.wait_for_finish(wait_secs=300) + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) - assert run_client_result is not None - assert run_client_result['status'] == 'SUCCEEDED' + assert run_result is not None + assert run_result['status'] == 'SUCCEEDED' async def test_actor_retrieves_env_vars( @@ -57,13 +57,13 @@ async def main() -> None: actor = await make_actor('get-env', main_func=main) - run_result = await actor.call() - assert run_result is not None + call_result = await actor.call() + assert call_result is not None - run_client = apify_client_async.run(run_result['id']) - run_client_result = await run_client.wait_for_finish(wait_secs=300) + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) - assert run_client_result is not None + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -87,13 +87,13 @@ async def main() -> None: actor = await make_actor('new-client', main_func=main) - run_result = await actor.call() - assert run_result is not None + call_result = await actor.call() + assert call_result is not None - run_client = apify_client_async.run(run_result['id']) - run_client_result = await run_client.wait_for_finish(wait_secs=300) + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) - assert run_client_result is not None + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') @@ -101,7 +101,10 @@ async def main() -> None: assert output_record['value'] == 'TESTING-OUTPUT' -async def test_actor_sets_status_message(make_actor: ActorFactory) -> None: +async def test_actor_sets_status_message( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: actor_input = await Actor.get_input() or {} @@ -109,22 +112,33 @@ async def main() -> None: actor = await make_actor('set-status-message', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' assert run_result['statusMessage'] == 'testing-status-message' assert run_result['isStatusMessageTerminal'] is None - run_result = await actor.call(run_input={'is_terminal': True}) + call_result_2 = await actor.call(run_input={'is_terminal': True}) + assert call_result_2 is not None - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' - assert run_result['statusMessage'] == 'testing-status-message' - assert run_result['isStatusMessageTerminal'] is True + run_client_2 = apify_client_async.run(call_result_2['id']) + run_result_2 = await run_client_2.wait_for_finish(wait_secs=300) + assert run_result_2 is not None + assert run_result_2['status'] == 'SUCCEEDED' + assert run_result_2['statusMessage'] == 'testing-status-message' + assert run_result_2['isStatusMessageTerminal'] is True -async def test_actor_starts_another_actor_instance(make_actor: ActorFactory) -> None: + +async def test_actor_starts_another_actor_instance( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main_inner() -> None: async with Actor: await asyncio.sleep(5) @@ -152,10 +166,14 @@ async def main_outer() -> None: inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_run_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + outer_call_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + assert outer_call_result is not None - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + run_client_outer = apify_client_async.run(outer_call_result['id']) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) + + assert run_result_outer is not None + assert run_result_outer['status'] == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish() From 346d53cd3dc0d43e29c6d3f353f417c1d97e1827 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 29 Oct 2024 19:39:45 +0100 Subject: [PATCH 04/12] api helpers and create proxy confs are done --- tests/integration/test_actor_api_helpers.py | 112 ++++++++++++------ .../test_actor_create_proxy_configuration.py | 26 +++- 2 files changed, 98 insertions(+), 40 deletions(-) diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index b5fdc3e3..bb9ed43d 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -175,14 +175,17 @@ async def main_outer() -> None: assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish() + await inner_actor.last_run().wait_for_finish(wait_secs=300) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None assert inner_output_record['value'] == f'{test_value}_XXX_{test_value}' -async def test_actor_calls_another_actor(make_actor: ActorFactory) -> None: +async def test_actor_calls_another_actor( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main_inner() -> None: async with Actor: await asyncio.sleep(5) @@ -210,19 +213,26 @@ async def main_outer() -> None: inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_run_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + outer_call_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + assert outer_call_result is not None + + run_client_outer = apify_client_async.run(outer_call_result['id']) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + assert run_result_outer is not None + assert run_result_outer['status'] == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish() + await inner_actor.last_run().wait_for_finish(wait_secs=300) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None assert inner_output_record['value'] == f'{test_value}_XXX_{test_value}' -async def test_actor_calls_task(make_actor: ActorFactory, apify_client_async: ApifyClientAsync) -> None: +async def test_actor_calls_task( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main_inner() -> None: async with Actor: await asyncio.sleep(5) @@ -255,12 +265,16 @@ async def main_outer() -> None: task_input={'test_value': test_value}, ) - outer_run_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_task_id': task['id']}) + outer_call_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_task_id': task['id']}) + assert outer_call_result is not None + + run_client_outer = apify_client_async.run(outer_call_result['id']) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + assert run_result_outer is not None + assert run_result_outer['status'] == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish() + await inner_actor.last_run().wait_for_finish(wait_secs=300) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None @@ -269,7 +283,10 @@ async def main_outer() -> None: await apify_client_async.task(task['id']).delete() -async def test_actor_aborts_another_actor_run(make_actor: ActorFactory) -> None: +async def test_actor_aborts_another_actor_run( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main_inner() -> None: async with Actor: await asyncio.sleep(180) @@ -290,12 +307,16 @@ async def main_outer() -> None: inner_run_id = (await inner_actor.start())['id'] - outer_run_result = await outer_actor.call(run_input={'inner_run_id': inner_run_id}) + outer_call_result = await outer_actor.call(run_input={'inner_run_id': inner_run_id}) + assert outer_call_result is not None - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + run_client_outer = apify_client_async.run(outer_call_result['id']) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) + + assert run_result_outer is not None + assert run_result_outer['status'] == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish() + await inner_actor.last_run().wait_for_finish(wait_secs=300) inner_actor_last_run = await inner_actor.last_run().get() assert inner_actor_last_run is not None assert inner_actor_last_run['status'] == 'ABORTED' @@ -304,7 +325,10 @@ async def main_outer() -> None: assert inner_output_record is None -async def test_actor_metamorphs_into_another_actor(make_actor: ActorFactory) -> None: +async def test_actor_metamorphs_into_another_actor( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main_inner() -> None: import os @@ -342,10 +366,14 @@ async def main_outer() -> None: inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_run_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + outer_call_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) + assert outer_call_result is not None + + run_client_outer = apify_client_async.run(outer_call_result['id']) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) - assert outer_run_result is not None - assert outer_run_result['status'] == 'SUCCEEDED' + assert run_result_outer is not None + assert run_result_outer['status'] == 'SUCCEEDED' outer_run_key_value_store = outer_actor.last_run().key_value_store() @@ -359,7 +387,10 @@ async def main_outer() -> None: assert await inner_actor.last_run().get() is None -async def test_actor_reboots_successfully(make_actor: ActorFactory) -> None: +async def test_actor_reboots_successfully( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: print('Starting...') @@ -374,7 +405,12 @@ async def main() -> None: print('Finishing...') actor = await make_actor('actor_rebooter', main_func=main) - run_result = await actor.call(run_input={'counter_key': 'reboot_counter'}) + + call_result = await actor.call(run_input={'counter_key': 'reboot_counter'}) + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -387,7 +423,10 @@ async def main() -> None: assert reboot_counter['value'] == 2 -async def test_actor_adds_webhook_and_receives_event(make_actor: ActorFactory) -> None: +async def test_actor_adds_webhook_and_receives_event( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main_server() -> None: import os from http.server import BaseHTTPRequestHandler, HTTPServer @@ -408,9 +447,7 @@ def do_POST(self) -> None: # noqa: N802 nonlocal webhook_body content_length = self.headers.get('content-length') length = int(content_length) if content_length else 0 - webhook_body = self.rfile.read(length).decode('utf-8') - self.send_response(200) self.end_headers() self.wfile.write(bytes('Hello, world!', encoding='utf-8')) @@ -445,26 +482,29 @@ async def main_client() -> None: server_actor_run = await server_actor.start() server_actor_container_url = server_actor_run['containerUrl'] - # Give the server actor some time to start running server_actor_initialized = await server_actor.last_run().key_value_store().get_record('INITIALIZED') while not server_actor_initialized: server_actor_initialized = await server_actor.last_run().key_value_store().get_record('INITIALIZED') await asyncio.sleep(1) - client_actor_run_result = await client_actor.call( - run_input={'server_actor_container_url': server_actor_container_url} - ) - assert client_actor_run_result is not None - assert client_actor_run_result['status'] == 'SUCCEEDED' + ac_call_result = await client_actor.call(run_input={'server_actor_container_url': server_actor_container_url}) + assert ac_call_result is not None + + ac_run_client = apify_client_async.run(ac_call_result['id']) + ac_run_result = await ac_run_client.wait_for_finish(wait_secs=300) + + assert ac_run_result is not None + assert ac_run_result['status'] == 'SUCCEEDED' + + sa_run_result = await server_actor.last_run().wait_for_finish(wait_secs=300) - server_actor_run_result = await server_actor.last_run().wait_for_finish() - assert server_actor_run_result is not None - assert server_actor_run_result['status'] == 'SUCCEEDED' + assert sa_run_result is not None + assert sa_run_result['status'] == 'SUCCEEDED' webhook_body_record = await server_actor.last_run().key_value_store().get_record('WEBHOOK_BODY') assert webhook_body_record is not None assert webhook_body_record['value'] != '' parsed_webhook_body = json.loads(webhook_body_record['value']) - assert parsed_webhook_body['eventData']['actorId'] == client_actor_run_result['actId'] - assert parsed_webhook_body['eventData']['actorRunId'] == client_actor_run_result['id'] + assert parsed_webhook_body['eventData']['actorId'] == ac_run_result['actId'] + assert parsed_webhook_body['eventData']['actorRunId'] == ac_run_result['id'] diff --git a/tests/integration/test_actor_create_proxy_configuration.py b/tests/integration/test_actor_create_proxy_configuration.py index f9737a7a..4ace7a01 100644 --- a/tests/integration/test_actor_create_proxy_configuration.py +++ b/tests/integration/test_actor_create_proxy_configuration.py @@ -5,10 +5,15 @@ from apify import Actor if TYPE_CHECKING: + from apify_client import ApifyClientAsync + from .conftest import ActorFactory -async def test_create_basic_proxy_configuration(make_actor: ActorFactory) -> None: +async def test_create_basic_proxy_configuration( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: groups = ['SHADER'] country_code = 'US' @@ -26,12 +31,20 @@ async def main() -> None: actor = await make_actor('proxy-configuration', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_create_proxy_configuration_with_groups_and_country(make_actor: ActorFactory) -> None: +async def test_create_proxy_configuration_with_groups_and_country( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: await Actor.init() @@ -64,6 +77,11 @@ async def main() -> None: actor = await make_actor('proxy-configuration', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' From 6ee7e8e2df1a866a3f08a9272ecc2744885261bf Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 29 Oct 2024 20:13:31 +0100 Subject: [PATCH 05/12] all integration tests should be more stable --- tests/integration/test_actor_dataset.py | 53 ++++++++++-- tests/integration/test_actor_events.py | 29 +++++-- .../integration/test_actor_key_value_store.py | 85 ++++++++++++++++--- tests/integration/test_actor_lifecycle.py | 75 +++++++++++++--- tests/integration/test_actor_log.py | 13 ++- tests/integration/test_actor_request_queue.py | 29 +++++-- tests/integration/test_fixtures.py | 39 +++++++-- tests/integration/test_request_queue.py | 38 +++++++-- 8 files changed, 299 insertions(+), 62 deletions(-) diff --git a/tests/integration/test_actor_dataset.py b/tests/integration/test_actor_dataset.py index d4eeb0fa..dfcf35f5 100644 --- a/tests/integration/test_actor_dataset.py +++ b/tests/integration/test_actor_dataset.py @@ -15,7 +15,10 @@ from .conftest import ActorFactory -async def test_push_and_verify_data_in_default_dataset(make_actor: ActorFactory) -> None: +async def test_push_and_verify_data_in_default_dataset( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: desired_item_count = 100 # Also change inside main() if you're changing this async def main() -> None: @@ -25,32 +28,48 @@ async def main() -> None: actor = await make_actor('push-data', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' + list_page = await actor.last_run().dataset().list_items() assert list_page.items[0]['id'] == 0 assert list_page.items[-1]['id'] == desired_item_count - 1 assert len(list_page.items) == list_page.count == desired_item_count -async def test_push_large_data_chunks_over_9mb(make_actor: ActorFactory) -> None: +async def test_push_large_data_chunks_over_9mb( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: await Actor.push_data([{'str': 'x' * 10000} for _ in range(5000)]) # ~50MB actor = await make_actor('push-data-over-9mb', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' + async for item in actor.last_run().dataset().iterate_items(): assert item['str'] == 'x' * 10000 -async def test_same_references_in_default_dataset(make_actor: ActorFactory) -> None: +async def test_same_references_in_default_dataset( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: dataset1 = await Actor.open_dataset() @@ -59,12 +78,20 @@ async def main() -> None: actor = await make_actor('dataset-same-ref-default', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_same_references_in_named_dataset(make_actor: ActorFactory) -> None: +async def test_same_references_in_named_dataset( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: dataset_name = generate_unique_resource_name('dataset') async def main() -> None: @@ -84,12 +111,20 @@ async def main() -> None: actor = await make_actor('dataset-same-ref-named', main_func=main) - run_result = await actor.call(run_input={'datasetName': dataset_name}) + call_result = await actor.call(run_input={'datasetName': dataset_name}) + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_force_cloud( + apify_client_async: ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, +) -> None: assert apify_client_async.token is not None monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_client_async.token) diff --git a/tests/integration/test_actor_events.py b/tests/integration/test_actor_events.py index 7509a71d..84c2ffce 100644 --- a/tests/integration/test_actor_events.py +++ b/tests/integration/test_actor_events.py @@ -8,10 +8,15 @@ from apify import Actor if TYPE_CHECKING: + from apify_client import ApifyClientAsync + from .conftest import ActorFactory -async def test_emit_and_capture_interval_events(make_actor: ActorFactory) -> None: +async def test_emit_and_capture_interval_events( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: import os from datetime import datetime @@ -54,10 +59,15 @@ async def log_event(data: Any) -> None: actor = await make_actor('actor-interval-events', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' + dataset_items_page = await actor.last_run().dataset().list_items() persist_state_events = [ item for item in dataset_items_page.items if item['event_type'] == ActorEventTypes.PERSIST_STATE @@ -69,7 +79,10 @@ async def log_event(data: Any) -> None: assert len(system_info_events) > 0 -async def test_event_listener_can_be_removed_successfully(make_actor: ActorFactory) -> None: +async def test_event_listener_can_be_removed_successfully( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: import os @@ -96,7 +109,11 @@ def count_event(data): # type: ignore # noqa: ANN202, ANN001 actor = await make_actor('actor-off-event', main_func=main) - run = await actor.call() + call_result = await actor.call() + assert call_result is not None - assert run is not None - assert run['status'] == 'SUCCEEDED' + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + + assert run_result is not None + assert run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_actor_key_value_store.py b/tests/integration/test_actor_key_value_store.py index 98cac610..61290886 100644 --- a/tests/integration/test_actor_key_value_store.py +++ b/tests/integration/test_actor_key_value_store.py @@ -15,7 +15,10 @@ from .conftest import ActorFactory -async def test_same_references_in_default_kvs(make_actor: ActorFactory) -> None: +async def test_same_references_in_default_kvs( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: kvs1 = await Actor.open_key_value_store() @@ -24,12 +27,20 @@ async def main() -> None: actor = await make_actor('kvs-same-ref-default', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_same_references_in_named_kvs(make_actor: ActorFactory) -> None: +async def test_same_references_in_named_kvs( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: kvs_name = generate_unique_resource_name('key-value-store') async def main() -> None: @@ -49,12 +60,20 @@ async def main() -> None: actor = await make_actor('kvs-same-ref-named', main_func=main) - run_result = await actor.call(run_input={'kvsName': kvs_name}) + call_result = await actor.call(run_input={'kvsName': kvs_name}) + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_force_cloud( + apify_client_async: ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, +) -> None: assert apify_client_async.token is not None monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_client_async.token) @@ -80,7 +99,10 @@ async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: py await key_value_store_client.delete() -async def test_set_and_get_value_in_same_run(make_actor: ActorFactory) -> None: +async def test_set_and_get_value_in_same_run( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: await Actor.set_value('test', {'number': 123, 'string': 'a string', 'nested': {'test': 1}}) @@ -91,21 +113,35 @@ async def main() -> None: actor = await make_actor('actor-get-set-value', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_set_value_in_one_run_and_get_value_in_another(make_actor: ActorFactory) -> None: +async def test_set_value_in_one_run_and_get_value_in_another( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main_set() -> None: async with Actor: await Actor.set_value('test', {'number': 123, 'string': 'a string', 'nested': {'test': 1}}) actor_set = await make_actor('actor-set-value', main_func=main_set) - run_result_set = await actor_set.call() + call_result_set = await actor_set.call() + assert call_result_set is not None + + run_client_set = apify_client_async.run(call_result_set['id']) + run_result_set = await run_client_set.wait_for_finish(wait_secs=300) + assert run_result_set is not None assert run_result_set['status'] == 'SUCCEEDED' + # Externally check if the value is present in key-value store test_record = await actor_set.last_run().key_value_store().get_record('test') assert test_record is not None @@ -128,12 +164,20 @@ async def main_get() -> None: default_kvs_info = await actor_set.last_run().key_value_store().get() assert default_kvs_info is not None - run_result_get = await actor_get.call(run_input={'kvs-id': default_kvs_info['id']}) + call_result_get = await actor_get.call(run_input={'kvs-id': default_kvs_info['id']}) + assert call_result_get is not None + + run_client_get = apify_client_async.run(call_result_get['id']) + run_result_get = await run_client_get.wait_for_finish(wait_secs=300) + assert run_result_get is not None assert run_result_get['status'] == 'SUCCEEDED' -async def test_actor_get_input_from_run(make_actor: ActorFactory) -> None: +async def test_actor_get_input_from_run( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: actor_source_files = { 'INPUT_SCHEMA.json': """ { @@ -168,7 +212,7 @@ async def main(): } actor = await make_actor('actor-get-input', source_files=actor_source_files) - run_result = await actor.call( + call_result = await actor.call( run_input={ 'number': 123, 'string': 'a string', @@ -176,11 +220,19 @@ async def main(): 'password': 'very secret', } ) + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_generate_public_url_for_kvs_record(make_actor: ActorFactory) -> None: +async def test_generate_public_url_for_kvs_record( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: from typing import cast @@ -198,6 +250,11 @@ async def main() -> None: actor = await make_actor('kvs-get-public-url', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_actor_lifecycle.py b/tests/integration/test_actor_lifecycle.py index b2f756f9..9239405e 100644 --- a/tests/integration/test_actor_lifecycle.py +++ b/tests/integration/test_actor_lifecycle.py @@ -5,10 +5,15 @@ from apify import Actor if TYPE_CHECKING: + from apify_client import ApifyClientAsync + from .conftest import ActorFactory -async def test_actor_init_and_double_init_prevention(make_actor: ActorFactory) -> None: +async def test_actor_init_and_double_init_prevention( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: my_actor = Actor await my_actor.init() @@ -34,13 +39,20 @@ async def main() -> None: actor = await make_actor('actor-init', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_actor_init_correctly_in_async_with_block(make_actor: ActorFactory) -> None: +async def test_actor_init_correctly_in_async_with_block( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: import apify._actor @@ -50,13 +62,20 @@ async def main() -> None: actor = await make_actor('with-actor-init', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_actor_exit_with_different_exit_codes(make_actor: ActorFactory) -> None: +async def test_actor_exit_with_different_exit_codes( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: input = await Actor.get_input() # noqa: A001 @@ -65,13 +84,21 @@ async def main() -> None: actor = await make_actor('actor-exit', main_func=main) for exit_code in [0, 1, 101]: - run_result = await actor.call(run_input={'exit_code': exit_code}) + call_result = await actor.call(run_input={'exit_code': exit_code}) + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['exitCode'] == exit_code assert run_result['status'] == 'FAILED' if exit_code > 0 else 'SUCCEEDED' -async def test_actor_fail_with_custom_exit_codes_and_status_messages(make_actor: ActorFactory) -> None: +async def test_actor_fail_with_custom_exit_codes_and_status_messages( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: input = await Actor.get_input() # noqa: A001 @@ -79,31 +106,55 @@ async def main() -> None: actor = await make_actor('actor-fail', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['exitCode'] == 1 assert run_result['status'] == 'FAILED' for exit_code in [1, 10, 100]: - run_result = await actor.call(run_input={'exit_code': exit_code}) + call_result = await actor.call(run_input={'exit_code': exit_code}) + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['exitCode'] == exit_code assert run_result['status'] == 'FAILED' # fail with status message - run_result = await actor.call(run_input={'status_message': 'This is a test message'}) + call_result = await actor.call(run_input={'status_message': 'This is a test message'}) + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'FAILED' assert run_result.get('statusMessage') == 'This is a test message' -async def test_actor_fails_correctly_with_exception(make_actor: ActorFactory) -> None: +async def test_actor_fails_correctly_with_exception( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: raise Exception('This is a test exception') # noqa: TRY002 actor = await make_actor('with-actor-fail', main_func=main) - run_result = await actor.call() + + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['exitCode'] == 91 assert run_result['status'] == 'FAILED' diff --git a/tests/integration/test_actor_log.py b/tests/integration/test_actor_log.py index 103e4794..404cad79 100644 --- a/tests/integration/test_actor_log.py +++ b/tests/integration/test_actor_log.py @@ -5,10 +5,15 @@ from apify import Actor, __version__ if TYPE_CHECKING: + from apify_client import ApifyClientAsync + from .conftest import ActorFactory -async def test_actor_logging(make_actor: ActorFactory) -> None: +async def test_actor_logging( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: import logging @@ -39,7 +44,11 @@ async def main() -> None: actor = await make_actor('actor-log', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'FAILED' diff --git a/tests/integration/test_actor_request_queue.py b/tests/integration/test_actor_request_queue.py index fc1902a3..8f27324b 100644 --- a/tests/integration/test_actor_request_queue.py +++ b/tests/integration/test_actor_request_queue.py @@ -16,7 +16,10 @@ from .conftest import ActorFactory -async def test_same_references_in_default_rq(make_actor: ActorFactory) -> None: +async def test_same_references_in_default_rq( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: rq1 = await Actor.open_request_queue() @@ -25,12 +28,20 @@ async def main() -> None: actor = await make_actor('rq-same-ref-default', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_same_references_in_named_rq(make_actor: ActorFactory) -> None: +async def test_same_references_in_named_rq( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: rq_name = generate_unique_resource_name('request-queue') async def main() -> None: @@ -50,12 +61,20 @@ async def main() -> None: actor = await make_actor('rq-same-ref-named', main_func=main) - run_result = await actor.call(run_input={'rqName': rq_name}) + call_result = await actor.call(run_input={'rqName': rq_name}) + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_force_cloud( + apify_client_async: ApifyClientAsync, + monkeypatch: pytest.MonkeyPatch, +) -> None: assert apify_client_async.token is not None monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_client_async.token) diff --git a/tests/integration/test_fixtures.py b/tests/integration/test_fixtures.py index 049a3d73..80f6ed1f 100644 --- a/tests/integration/test_fixtures.py +++ b/tests/integration/test_fixtures.py @@ -13,7 +13,10 @@ from .conftest import ActorFactory -async def test_actor_from_main_func(make_actor: ActorFactory) -> None: +async def test_actor_from_main_func( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: import os @@ -24,7 +27,11 @@ async def main() -> None: actor = await make_actor('make-actor-main-func', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -34,7 +41,10 @@ async def main() -> None: assert run_result['actId'] == output_record['value'] -async def test_actor_from_main_py(make_actor: ActorFactory) -> None: +async def test_actor_from_main_py( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' main_py_source = f""" import asyncio @@ -46,7 +56,12 @@ async def main(): actor = await make_actor('make-actor-main-py', main_py=main_py_source) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -55,7 +70,10 @@ async def main(): assert output_record['value'] == expected_output -async def test_actor_from_source_files(make_actor: ActorFactory) -> None: +async def test_actor_from_source_files( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: test_started_at = datetime.now(timezone.utc) actor_source_files = { 'src/utils.py': """ @@ -77,9 +95,14 @@ async def main(): } actor = await make_actor('make-actor-source-files', source_files=actor_source_files) - actor_run = await actor.call() - assert actor_run is not None - assert actor_run['status'] == 'SUCCEEDED' + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + + assert run_result is not None + assert run_result['status'] == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index 19251ad0..182a78da 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -5,10 +5,15 @@ from apify import Actor if TYPE_CHECKING: + from apify_client import ApifyClientAsync + from .conftest import ActorFactory -async def test_add_and_fetch_requests(make_actor: ActorFactory) -> None: +async def test_add_and_fetch_requests( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: desired_request_count = 100 @@ -35,12 +40,20 @@ async def main() -> None: actor = await make_actor('rq-simple-test', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_add_requests_in_batches(make_actor: ActorFactory) -> None: +async def test_add_requests_in_batches( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: async with Actor: desired_request_count = 100 @@ -65,12 +78,20 @@ async def main() -> None: actor = await make_actor('rq-batch-test', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' -async def test_add_non_unique_requests_in_batch(make_actor: ActorFactory) -> None: +async def test_add_non_unique_requests_in_batch( + apify_client_async: ApifyClientAsync, + make_actor: ActorFactory, +) -> None: async def main() -> None: from crawlee import Request @@ -102,6 +123,11 @@ async def main() -> None: actor = await make_actor('rq-batch-test', main_func=main) - run_result = await actor.call() + call_result = await actor.call() + assert call_result is not None + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=300) + assert run_result is not None assert run_result['status'] == 'SUCCEEDED' From 8f8da5d1b52718b9f39c8a954173d37ca875425e Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 29 Oct 2024 20:14:52 +0100 Subject: [PATCH 06/12] uncomment integration test workflow --- .github/workflows/run_release.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml index 91edf890..82164516 100644 --- a/.github/workflows/run_release.yaml +++ b/.github/workflows/run_release.yaml @@ -64,12 +64,11 @@ jobs: needs: [should_release] uses: ./.github/workflows/_version_conflict_check.yaml - # tmp disabled due to instability - # integration_tests: - # name: Integration tests - # needs: [should_release] - # uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main - # secrets: inherit + integration_tests: + name: Integration tests + needs: [should_release] + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit publish_to_pypi: name: Publish to PyPI @@ -81,7 +80,7 @@ jobs: unit_tests, changelog_entry_check, version_conflict_check, - # integration_tests, # tmp disabled due to instability + integration_tests, ] runs-on: ubuntu-latest permissions: From 267dc21900cdb017a3ae483981f0ae02d05e98e3 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 29 Oct 2024 20:42:36 +0100 Subject: [PATCH 07/12] wait 10 minutes --- tests/integration/conftest.py | 2 +- tests/integration/test_actor_api_helpers.py | 34 +++++++++---------- .../test_actor_create_proxy_configuration.py | 4 +-- tests/integration/test_actor_dataset.py | 8 ++--- tests/integration/test_actor_events.py | 4 +-- .../integration/test_actor_key_value_store.py | 14 ++++---- tests/integration/test_actor_lifecycle.py | 14 ++++---- tests/integration/test_actor_log.py | 2 +- tests/integration/test_actor_request_queue.py | 4 +-- tests/integration/test_fixtures.py | 6 ++-- tests/integration/test_request_queue.py | 6 ++-- 11 files changed, 49 insertions(+), 49 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 734af969..e660d557 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -244,7 +244,7 @@ async def _make_actor( print(f'Building Actor {actor_name}...') build_result = await actor_client.build(version_number='0.0') build_client = apify_client_async.build(build_result['id']) - build_client_result = await build_client.wait_for_finish(wait_secs=300) + build_client_result = await build_client.wait_for_finish(wait_secs=600) assert build_client_result is not None assert build_client_result['status'] == ActorJobStatus.SUCCEEDED diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index bb9ed43d..3d8ca822 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -29,7 +29,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -61,7 +61,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -91,7 +91,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -116,7 +116,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -127,7 +127,7 @@ async def main() -> None: assert call_result_2 is not None run_client_2 = apify_client_async.run(call_result_2['id']) - run_result_2 = await run_client_2.wait_for_finish(wait_secs=300) + run_result_2 = await run_client_2.wait_for_finish(wait_secs=600) assert run_result_2 is not None assert run_result_2['status'] == 'SUCCEEDED' @@ -170,12 +170,12 @@ async def main_outer() -> None: assert outer_call_result is not None run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish(wait_secs=300) + await inner_actor.last_run().wait_for_finish(wait_secs=600) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None @@ -217,12 +217,12 @@ async def main_outer() -> None: assert outer_call_result is not None run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish(wait_secs=300) + await inner_actor.last_run().wait_for_finish(wait_secs=600) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None @@ -269,12 +269,12 @@ async def main_outer() -> None: assert outer_call_result is not None run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish(wait_secs=300) + await inner_actor.last_run().wait_for_finish(wait_secs=600) inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is not None @@ -311,12 +311,12 @@ async def main_outer() -> None: assert outer_call_result is not None run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish(wait_secs=300) + await inner_actor.last_run().wait_for_finish(wait_secs=600) inner_actor_last_run = await inner_actor.last_run().get() assert inner_actor_last_run is not None assert inner_actor_last_run['status'] == 'ABORTED' @@ -370,7 +370,7 @@ async def main_outer() -> None: assert outer_call_result is not None run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=300) + run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' @@ -410,7 +410,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -491,12 +491,12 @@ async def main_client() -> None: assert ac_call_result is not None ac_run_client = apify_client_async.run(ac_call_result['id']) - ac_run_result = await ac_run_client.wait_for_finish(wait_secs=300) + ac_run_result = await ac_run_client.wait_for_finish(wait_secs=600) assert ac_run_result is not None assert ac_run_result['status'] == 'SUCCEEDED' - sa_run_result = await server_actor.last_run().wait_for_finish(wait_secs=300) + sa_run_result = await server_actor.last_run().wait_for_finish(wait_secs=600) assert sa_run_result is not None assert sa_run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_actor_create_proxy_configuration.py b/tests/integration/test_actor_create_proxy_configuration.py index 4ace7a01..e963dbff 100644 --- a/tests/integration/test_actor_create_proxy_configuration.py +++ b/tests/integration/test_actor_create_proxy_configuration.py @@ -35,7 +35,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -81,7 +81,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_actor_dataset.py b/tests/integration/test_actor_dataset.py index dfcf35f5..1b0656d5 100644 --- a/tests/integration/test_actor_dataset.py +++ b/tests/integration/test_actor_dataset.py @@ -32,7 +32,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -57,7 +57,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -82,7 +82,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -115,7 +115,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_actor_events.py b/tests/integration/test_actor_events.py index 84c2ffce..f64b5260 100644 --- a/tests/integration/test_actor_events.py +++ b/tests/integration/test_actor_events.py @@ -63,7 +63,7 @@ async def log_event(data: Any) -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -113,7 +113,7 @@ def count_event(data): # type: ignore # noqa: ANN202, ANN001 assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_actor_key_value_store.py b/tests/integration/test_actor_key_value_store.py index 61290886..988444c8 100644 --- a/tests/integration/test_actor_key_value_store.py +++ b/tests/integration/test_actor_key_value_store.py @@ -31,7 +31,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -64,7 +64,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -117,7 +117,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -137,7 +137,7 @@ async def main_set() -> None: assert call_result_set is not None run_client_set = apify_client_async.run(call_result_set['id']) - run_result_set = await run_client_set.wait_for_finish(wait_secs=300) + run_result_set = await run_client_set.wait_for_finish(wait_secs=600) assert run_result_set is not None assert run_result_set['status'] == 'SUCCEEDED' @@ -168,7 +168,7 @@ async def main_get() -> None: assert call_result_get is not None run_client_get = apify_client_async.run(call_result_get['id']) - run_result_get = await run_client_get.wait_for_finish(wait_secs=300) + run_result_get = await run_client_get.wait_for_finish(wait_secs=600) assert run_result_get is not None assert run_result_get['status'] == 'SUCCEEDED' @@ -223,7 +223,7 @@ async def main(): assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -254,7 +254,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_actor_lifecycle.py b/tests/integration/test_actor_lifecycle.py index 9239405e..5ebe6308 100644 --- a/tests/integration/test_actor_lifecycle.py +++ b/tests/integration/test_actor_lifecycle.py @@ -43,7 +43,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -66,7 +66,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -88,7 +88,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['exitCode'] == exit_code @@ -110,7 +110,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['exitCode'] == 1 @@ -121,7 +121,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['exitCode'] == exit_code @@ -132,7 +132,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'FAILED' @@ -153,7 +153,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['exitCode'] == 91 diff --git a/tests/integration/test_actor_log.py b/tests/integration/test_actor_log.py index 404cad79..2e2e1d69 100644 --- a/tests/integration/test_actor_log.py +++ b/tests/integration/test_actor_log.py @@ -48,7 +48,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'FAILED' diff --git a/tests/integration/test_actor_request_queue.py b/tests/integration/test_actor_request_queue.py index 8f27324b..987e5185 100644 --- a/tests/integration/test_actor_request_queue.py +++ b/tests/integration/test_actor_request_queue.py @@ -32,7 +32,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -65,7 +65,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_fixtures.py b/tests/integration/test_fixtures.py index 80f6ed1f..555c472a 100644 --- a/tests/integration/test_fixtures.py +++ b/tests/integration/test_fixtures.py @@ -31,7 +31,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -60,7 +60,7 @@ async def main(): assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -99,7 +99,7 @@ async def main(): assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index 182a78da..050946c7 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -44,7 +44,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -82,7 +82,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' @@ -127,7 +127,7 @@ async def main() -> None: assert call_result is not None run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=300) + run_result = await run_client.wait_for_finish(wait_secs=600) assert run_result is not None assert run_result['status'] == 'SUCCEEDED' From 0ddc7606224de5a4c7e9470fc9048fa61a84d614 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 30 Oct 2024 11:24:43 +0100 Subject: [PATCH 08/12] do it better --- tests/integration/conftest.py | 51 +++++++- tests/integration/test_actor_api_helpers.py | 138 +++++++------------- tests/unit/conftest.py | 4 +- 3 files changed, 99 insertions(+), 94 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e660d557..a5c8fd9f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,7 +7,7 @@ import sys import textwrap from pathlib import Path -from typing import TYPE_CHECKING, Callable, Protocol, cast +from typing import TYPE_CHECKING, Any, Callable, Coroutine, Protocol, cast import pytest from filelock import FileLock @@ -56,6 +56,53 @@ def apify_client_async() -> ApifyClientAsync: return ApifyClientAsync(api_token, api_url=api_url) +class RunActorFunction(Protocol): + """A type for the `run_actor` fixture.""" + + def __call__( + self, + actor: ActorClientAsync, + *, + run_input: Any = None, + ) -> Coroutine[None, None, dict]: + """Initiate an Actor run and wait for its completion. + + Args: + actor: Actor async client, in testing context usually created by `make_actor` fixture. + run_input: Optional input for the Actor run. + + Returns: + Actor run result. + """ + + +@pytest.fixture +async def run_actor(apify_client_async: ApifyClientAsync) -> RunActorFunction: + """Fixture for calling an Actor run and waiting for its completion. + + This fixture returns a function that initiates an Actor run with optional run input, waits for its completion, + and retrieves the final result. It uses the `wait_for_finish` method with a timeout of 10 minutes. + + Returns: + A coroutine that initiates an Actor run and waits for its completion. + """ + + async def _run_actor(actor: ActorClientAsync, *, run_input: Any = None) -> dict: + call_result = await actor.call(run_input=run_input) + + assert isinstance(call_result, dict), 'The result of ActorClientAsync.call() is not a dictionary.' + assert 'id' in call_result, 'The result of ActorClientAsync.call() does not contain an ID.' + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=600) + + assert isinstance(run_result, dict), 'The result of RunClientAsync.wait_for_finish() is not a dictionary.' + assert 'status' in run_result, 'The result of RunClientAsync.wait_for_finish() does not contain a status.' + return run_result + + return _run_actor + + # Build the package wheel if it hasn't been built yet, and return the path to the wheel @pytest.fixture(scope='session') def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) -> Path: @@ -130,7 +177,7 @@ def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: # Just a type for the make_actor result, so that we can import it in tests class ActorFactory(Protocol): def __call__( - self: ActorFactory, + self, actor_label: str, *, main_func: Callable | None = None, diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index 3d8ca822..cb239806 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -12,11 +12,11 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import ActorFactory, RunActorFunction async def test_actor_reports_running_on_platform( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main() -> None: @@ -24,19 +24,13 @@ async def main() -> None: assert Actor.is_at_home() is True actor = await make_actor('is-at-home', main_func=main) + run_result = await run_actor(actor) - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None assert run_result['status'] == 'SUCCEEDED' async def test_actor_retrieves_env_vars( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main() -> None: @@ -56,19 +50,13 @@ async def main() -> None: assert len(env_dict.get('default_request_queue_id', '')) == 17 actor = await make_actor('get-env', main_func=main) + run_result = await run_actor(actor) - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None assert run_result['status'] == 'SUCCEEDED' async def test_actor_creates_new_client_instance( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main() -> None: @@ -86,14 +74,8 @@ async def main() -> None: await kv_store_client.set_record('OUTPUT', 'TESTING-OUTPUT') actor = await make_actor('new-client', main_func=main) + run_result = await run_actor(actor) - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None assert run_result['status'] == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') @@ -102,7 +84,7 @@ async def main() -> None: async def test_actor_sets_status_message( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main() -> None: @@ -111,32 +93,21 @@ async def main() -> None: await Actor.set_status_message('testing-status-message', **actor_input) actor = await make_actor('set-status-message', main_func=main) + run_result_1 = await run_actor(actor) - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' - assert run_result['statusMessage'] == 'testing-status-message' - assert run_result['isStatusMessageTerminal'] is None - - call_result_2 = await actor.call(run_input={'is_terminal': True}) - assert call_result_2 is not None + assert run_result_1['status'] == 'SUCCEEDED' + assert run_result_1['statusMessage'] == 'testing-status-message' + assert run_result_1['isStatusMessageTerminal'] is None - run_client_2 = apify_client_async.run(call_result_2['id']) - run_result_2 = await run_client_2.wait_for_finish(wait_secs=600) + run_result_2 = await run_actor(actor, run_input={'is_terminal': True}) - assert run_result_2 is not None assert run_result_2['status'] == 'SUCCEEDED' assert run_result_2['statusMessage'] == 'testing-status-message' assert run_result_2['isStatusMessageTerminal'] is True async def test_actor_starts_another_actor_instance( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main_inner() -> None: @@ -166,13 +137,11 @@ async def main_outer() -> None: inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_call_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) - assert outer_call_result is not None - - run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) + run_result_outer = await run_actor( + outer_actor, + run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, + ) - assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -183,7 +152,7 @@ async def main_outer() -> None: async def test_actor_calls_another_actor( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main_inner() -> None: @@ -213,13 +182,11 @@ async def main_outer() -> None: inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_call_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) - assert outer_call_result is not None - - run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) + run_result_outer = await run_actor( + outer_actor, + run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, + ) - assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -231,6 +198,7 @@ async def main_outer() -> None: async def test_actor_calls_task( apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main_inner() -> None: @@ -265,13 +233,11 @@ async def main_outer() -> None: task_input={'test_value': test_value}, ) - outer_call_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_task_id': task['id']}) - assert outer_call_result is not None - - run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) + run_result_outer = await run_actor( + outer_actor, + run_input={'test_value': test_value, 'inner_task_id': task['id']}, + ) - assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -284,7 +250,7 @@ async def main_outer() -> None: async def test_actor_aborts_another_actor_run( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main_inner() -> None: @@ -307,13 +273,11 @@ async def main_outer() -> None: inner_run_id = (await inner_actor.start())['id'] - outer_call_result = await outer_actor.call(run_input={'inner_run_id': inner_run_id}) - assert outer_call_result is not None - - run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) + run_result_outer = await run_actor( + outer_actor, + run_input={'inner_run_id': inner_run_id}, + ) - assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -326,7 +290,7 @@ async def main_outer() -> None: async def test_actor_metamorphs_into_another_actor( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main_inner() -> None: @@ -366,13 +330,11 @@ async def main_outer() -> None: inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() - outer_call_result = await outer_actor.call(run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}) - assert outer_call_result is not None - - run_client_outer = apify_client_async.run(outer_call_result['id']) - run_result_outer = await run_client_outer.wait_for_finish(wait_secs=600) + run_result_outer = await run_actor( + outer_actor, + run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, + ) - assert run_result_outer is not None assert run_result_outer['status'] == 'SUCCEEDED' outer_run_key_value_store = outer_actor.last_run().key_value_store() @@ -388,7 +350,7 @@ async def main_outer() -> None: async def test_actor_reboots_successfully( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main() -> None: @@ -406,13 +368,11 @@ async def main() -> None: actor = await make_actor('actor_rebooter', main_func=main) - call_result = await actor.call(run_input={'counter_key': 'reboot_counter'}) - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + run_result = await run_actor( + actor, + run_input={'counter_key': 'reboot_counter'}, + ) - assert run_result is not None assert run_result['status'] == 'SUCCEEDED' not_written_value = await actor.last_run().key_value_store().get_record('THIS_KEY_SHOULD_NOT_BE_WRITTEN') @@ -424,7 +384,7 @@ async def main() -> None: async def test_actor_adds_webhook_and_receives_event( - apify_client_async: ApifyClientAsync, + run_actor: RunActorFunction, make_actor: ActorFactory, ) -> None: async def main_server() -> None: @@ -487,13 +447,11 @@ async def main_client() -> None: server_actor_initialized = await server_actor.last_run().key_value_store().get_record('INITIALIZED') await asyncio.sleep(1) - ac_call_result = await client_actor.call(run_input={'server_actor_container_url': server_actor_container_url}) - assert ac_call_result is not None - - ac_run_client = apify_client_async.run(ac_call_result['id']) - ac_run_result = await ac_run_client.wait_for_finish(wait_secs=600) + ac_run_result = await run_actor( + client_actor, + run_input={'server_actor_container_url': server_actor_container_url}, + ) - assert ac_run_result is not None assert ac_run_result['status'] == 'SUCCEEDED' sa_run_result = await server_actor.last_run().wait_for_finish(wait_secs=600) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index fd74d99e..40d27c54 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -64,12 +64,12 @@ def _reset_and_patch_default_instances( # This class is used to patch the ApifyClientAsync methods to return a fixed value or be replaced with another method. class ApifyClientAsyncPatcher: - def __init__(self: ApifyClientAsyncPatcher, monkeypatch: pytest.MonkeyPatch) -> None: + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: self.monkeypatch = monkeypatch self.calls: dict[str, dict[str, list[tuple[Any, Any]]]] = defaultdict(lambda: defaultdict(list)) def patch( - self: ApifyClientAsyncPatcher, + self, method: str, submethod: str, *, From 7cea0b3d9382a08d196d08665629b174c7b22a85 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 30 Oct 2024 11:46:12 +0100 Subject: [PATCH 09/12] try to use model validate --- tests/integration/conftest.py | 9 ++-- tests/integration/test_actor_api_helpers.py | 50 +++++++++++---------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a5c8fd9f..d5883c84 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -17,6 +17,7 @@ import apify._actor from ._utils import generate_unique_resource_name +from apify._models import ActorRun if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Mapping @@ -64,7 +65,7 @@ def __call__( actor: ActorClientAsync, *, run_input: Any = None, - ) -> Coroutine[None, None, dict]: + ) -> Coroutine[None, None, ActorRun]: """Initiate an Actor run and wait for its completion. Args: @@ -87,7 +88,7 @@ async def run_actor(apify_client_async: ApifyClientAsync) -> RunActorFunction: A coroutine that initiates an Actor run and waits for its completion. """ - async def _run_actor(actor: ActorClientAsync, *, run_input: Any = None) -> dict: + async def _run_actor(actor: ActorClientAsync, *, run_input: Any = None) -> ActorRun: call_result = await actor.call(run_input=run_input) assert isinstance(call_result, dict), 'The result of ActorClientAsync.call() is not a dictionary.' @@ -96,9 +97,7 @@ async def _run_actor(actor: ActorClientAsync, *, run_input: Any = None) -> dict: run_client = apify_client_async.run(call_result['id']) run_result = await run_client.wait_for_finish(wait_secs=600) - assert isinstance(run_result, dict), 'The result of RunClientAsync.wait_for_finish() is not a dictionary.' - assert 'status' in run_result, 'The result of RunClientAsync.wait_for_finish() does not contain a status.' - return run_result + return ActorRun.model_validate(run_result) return _run_actor diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index cb239806..22ac68b1 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -8,6 +8,7 @@ from ._utils import generate_unique_resource_name from apify import Actor +from apify._models import ActorRun if TYPE_CHECKING: from apify_client import ApifyClientAsync @@ -26,7 +27,7 @@ async def main() -> None: actor = await make_actor('is-at-home', main_func=main) run_result = await run_actor(actor) - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_actor_retrieves_env_vars( @@ -52,7 +53,7 @@ async def main() -> None: actor = await make_actor('get-env', main_func=main) run_result = await run_actor(actor) - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_actor_creates_new_client_instance( @@ -76,7 +77,7 @@ async def main() -> None: actor = await make_actor('new-client', main_func=main) run_result = await run_actor(actor) - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None @@ -95,15 +96,15 @@ async def main() -> None: actor = await make_actor('set-status-message', main_func=main) run_result_1 = await run_actor(actor) - assert run_result_1['status'] == 'SUCCEEDED' - assert run_result_1['statusMessage'] == 'testing-status-message' - assert run_result_1['isStatusMessageTerminal'] is None + assert run_result_1.status == 'SUCCEEDED' + assert run_result_1.status_message == 'testing-status-message' + assert run_result_1.is_status_message_terminal is None run_result_2 = await run_actor(actor, run_input={'is_terminal': True}) - assert run_result_2['status'] == 'SUCCEEDED' - assert run_result_2['statusMessage'] == 'testing-status-message' - assert run_result_2['isStatusMessageTerminal'] is True + assert run_result_2.status == 'SUCCEEDED' + assert run_result_2.status_message == 'testing-status-message' + assert run_result_2.is_status_message_terminal is True async def test_actor_starts_another_actor_instance( @@ -142,7 +143,7 @@ async def main_outer() -> None: run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, ) - assert run_result_outer['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -187,7 +188,7 @@ async def main_outer() -> None: run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, ) - assert run_result_outer['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -238,7 +239,7 @@ async def main_outer() -> None: run_input={'test_value': test_value, 'inner_task_id': task['id']}, ) - assert run_result_outer['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -278,12 +279,13 @@ async def main_outer() -> None: run_input={'inner_run_id': inner_run_id}, ) - assert run_result_outer['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) - inner_actor_last_run = await inner_actor.last_run().get() - assert inner_actor_last_run is not None - assert inner_actor_last_run['status'] == 'ABORTED' + inner_actor_last_run_dict = await inner_actor.last_run().get() + inner_actor_last_run = ActorRun.model_validate(inner_actor_last_run_dict) + + assert inner_actor_last_run.status == 'ABORTED' inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is None @@ -335,7 +337,7 @@ async def main_outer() -> None: run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, ) - assert run_result_outer['status'] == 'SUCCEEDED' + assert run_result_outer.status == 'SUCCEEDED' outer_run_key_value_store = outer_actor.last_run().key_value_store() @@ -373,7 +375,7 @@ async def main() -> None: run_input={'counter_key': 'reboot_counter'}, ) - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' not_written_value = await actor.last_run().key_value_store().get_record('THIS_KEY_SHOULD_NOT_BE_WRITTEN') assert not_written_value is None @@ -452,17 +454,17 @@ async def main_client() -> None: run_input={'server_actor_container_url': server_actor_container_url}, ) - assert ac_run_result['status'] == 'SUCCEEDED' + assert ac_run_result.status == 'SUCCEEDED' - sa_run_result = await server_actor.last_run().wait_for_finish(wait_secs=600) + sa_run_result_dict = await server_actor.last_run().wait_for_finish(wait_secs=600) + sa_run_result = ActorRun.model_validate(sa_run_result_dict) - assert sa_run_result is not None - assert sa_run_result['status'] == 'SUCCEEDED' + assert sa_run_result.status == 'SUCCEEDED' webhook_body_record = await server_actor.last_run().key_value_store().get_record('WEBHOOK_BODY') assert webhook_body_record is not None assert webhook_body_record['value'] != '' parsed_webhook_body = json.loads(webhook_body_record['value']) - assert parsed_webhook_body['eventData']['actorId'] == ac_run_result['actId'] - assert parsed_webhook_body['eventData']['actorRunId'] == ac_run_result['id'] + assert parsed_webhook_body['eventData']['actorId'] == ac_run_result.act_id + assert parsed_webhook_body['eventData']['actorRunId'] == ac_run_result.id From 2865b3e47d693c9b76cabe19a32bae0410adaf2d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 30 Oct 2024 12:06:28 +0100 Subject: [PATCH 10/12] api helpers and proxy config are ready --- tests/integration/conftest.py | 185 ++++++++++-------- tests/integration/test_actor_api_helpers.py | 60 +++--- .../test_actor_create_proxy_configuration.py | 53 ++--- tests/integration/test_actor_dataset.py | 10 +- tests/integration/test_actor_events.py | 6 +- .../integration/test_actor_key_value_store.py | 14 +- tests/integration/test_actor_lifecycle.py | 12 +- tests/integration/test_actor_log.py | 4 +- tests/integration/test_actor_request_queue.py | 6 +- tests/integration/test_fixtures.py | 8 +- tests/integration/test_request_queue.py | 8 +- 11 files changed, 180 insertions(+), 186 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d5883c84..339fe5ce 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -29,10 +29,13 @@ SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() -# To isolate the tests, we need to reset the used singletons before each test case -# We also patch the default storage client with a tmp_path @pytest.fixture(autouse=True) def _reset_and_patch_default_instances() -> None: + """Reset the used singletons and patch the default storage client with a temporary directory. + + To isolate the tests, we need to reset the used singletons before each test case. We also patch the default + storage client with a tmp_path. + """ from crawlee import service_container cast(dict, service_container._services).clear() @@ -41,13 +44,14 @@ def _reset_and_patch_default_instances() -> None: # TODO: StorageClientManager local storage client purge # noqa: TD003 -# This fixture can't be session-scoped, -# because then you start getting `RuntimeError: Event loop is closed` errors, -# because `httpx.AsyncClient` in `ApifyClientAsync` tries to reuse the same event loop across requests, -# but `pytest-asyncio` closes the event loop after each test, -# and uses a new one for the next test. @pytest.fixture def apify_client_async() -> ApifyClientAsync: + """Create an instance of the ApifyClientAsync. + + This fixture can't be session-scoped, because then you start getting `RuntimeError: Event loop is closed` errors, + because `httpx.AsyncClient` in `ApifyClientAsync` tries to reuse the same event loop across requests, + but `pytest-asyncio` closes the event loop after each test, and uses a new one for the next test. + """ api_token = os.getenv(TOKEN_ENV_VAR) api_url = os.getenv(API_URL_ENV_VAR) @@ -57,65 +61,26 @@ def apify_client_async() -> ApifyClientAsync: return ApifyClientAsync(api_token, api_url=api_url) -class RunActorFunction(Protocol): - """A type for the `run_actor` fixture.""" - - def __call__( - self, - actor: ActorClientAsync, - *, - run_input: Any = None, - ) -> Coroutine[None, None, ActorRun]: - """Initiate an Actor run and wait for its completion. - - Args: - actor: Actor async client, in testing context usually created by `make_actor` fixture. - run_input: Optional input for the Actor run. - - Returns: - Actor run result. - """ - - -@pytest.fixture -async def run_actor(apify_client_async: ApifyClientAsync) -> RunActorFunction: - """Fixture for calling an Actor run and waiting for its completion. - - This fixture returns a function that initiates an Actor run with optional run input, waits for its completion, - and retrieves the final result. It uses the `wait_for_finish` method with a timeout of 10 minutes. - - Returns: - A coroutine that initiates an Actor run and waits for its completion. - """ - - async def _run_actor(actor: ActorClientAsync, *, run_input: Any = None) -> ActorRun: - call_result = await actor.call(run_input=run_input) - - assert isinstance(call_result, dict), 'The result of ActorClientAsync.call() is not a dictionary.' - assert 'id' in call_result, 'The result of ActorClientAsync.call() does not contain an ID.' - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - return ActorRun.model_validate(run_result) - - return _run_actor - - -# Build the package wheel if it hasn't been built yet, and return the path to the wheel @pytest.fixture(scope='session') def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) -> Path: + """Build the package wheel if it hasn't been built yet, and return the path to the wheel.""" # Make sure the wheel is not being built concurrently across all the pytest-xdist runners, - # through locking the building process with a temp file + # through locking the building process with a temp file. with FileLock(tmp_path_factory.getbasetemp().parent / 'sdk_wheel_build.lock'): # Make sure the wheel is built exactly once across across all the pytest-xdist runners, - # through an indicator file saying that the wheel was already built + # through an indicator file saying that the wheel was already built. was_wheel_built_this_test_run_file = tmp_path_factory.getbasetemp() / f'wheel_was_built_in_run_{testrun_uid}' if not was_wheel_built_this_test_run_file.exists(): - subprocess.run('python -m build', cwd=SDK_ROOT_PATH, shell=True, check=True, capture_output=True) # noqa: S602, S607 + subprocess.run( + args='python -m build', + cwd=SDK_ROOT_PATH, + shell=True, + check=True, + capture_output=True, + ) was_wheel_built_this_test_run_file.touch() - # Read the current package version, necessary for getting the right wheel filename + # Read the current package version, necessary for getting the right wheel filename. pyproject_toml_file = (SDK_ROOT_PATH / 'pyproject.toml').read_text(encoding='utf-8') for line in pyproject_toml_file.splitlines(): if line.startswith('version = '): @@ -127,7 +92,7 @@ def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) - wheel_path = SDK_ROOT_PATH / 'dist' / f'apify-{sdk_version}-py3-none-any.whl' - # Just to be sure + # Just to be sure. assert wheel_path.exists() return wheel_path @@ -173,34 +138,18 @@ def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: return source_files -# Just a type for the make_actor result, so that we can import it in tests -class ActorFactory(Protocol): +class MakeActorFunction(Protocol): + """A type for the `make_actor` fixture.""" + def __call__( self, - actor_label: str, + label: str, *, main_func: Callable | None = None, main_py: str | None = None, source_files: Mapping[str, str | bytes] | None = None, - ) -> Awaitable[ActorClientAsync]: ... - - -@pytest.fixture -async def make_actor( - actor_base_source_files: dict[str, str | bytes], - apify_client_async: ApifyClientAsync, -) -> AsyncIterator[ActorFactory]: - """A fixture for returning a temporary Actor factory.""" - actor_clients_for_cleanup: list[ActorClientAsync] = [] - - async def _make_actor( - actor_label: str, - *, - main_func: Callable | None = None, - main_py: str | None = None, - source_files: Mapping[str, str | bytes] | None = None, - ) -> ActorClientAsync: - """Create a temporary Actor from the given main function or source file(s). + ) -> Awaitable[ActorClientAsync]: + """Create a temporary Actor from the given main function or source files. The Actor will be uploaded to the Apify Platform, built there, and after the test finishes, it will be automatically deleted. @@ -208,7 +157,7 @@ async def _make_actor( You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. Args: - actor_label: The label which will be a part of the generated Actor name + label: The label which will be a part of the generated Actor name. main_func: The main function of the Actor. main_py: The `src/main.py` file of the Actor. source_files: A dictionary of the source files of the Actor. @@ -216,13 +165,34 @@ async def _make_actor( Returns: A resource client for the created Actor. """ + + +@pytest.fixture +async def make_actor( + actor_base_source_files: dict[str, str | bytes], + apify_client_async: ApifyClientAsync, +) -> AsyncIterator[MakeActorFunction]: + """Fixture for creating temporary Actors for testing purposes. + + This returns a function that creates a temporary Actor from the given main function or source files. The Actor + will be uploaded to the Apify Platform, built there, and after the test finishes, it will be automatically deleted. + """ + actor_clients_for_cleanup: list[ActorClientAsync] = [] + + async def _make_actor( + label: str, + *, + main_func: Callable | None = None, + main_py: str | None = None, + source_files: Mapping[str, str | bytes] | None = None, + ) -> ActorClientAsync: if not (main_func or main_py or source_files): raise TypeError('One of `main_func`, `main_py` or `source_files` arguments must be specified') if (main_func and main_py) or (main_func and source_files) or (main_py and source_files): raise TypeError('Cannot specify more than one of `main_func`, `main_py` and `source_files` arguments') - actor_name = generate_unique_resource_name(actor_label) + actor_name = generate_unique_resource_name(label) # Get the source of main_func and convert it into a reasonable main_py file. if main_func: @@ -250,7 +220,7 @@ async def _make_actor( actor_source_files = actor_base_source_files.copy() actor_source_files.update(source_files) - # Reformat the source files in a format that the Apify API understands + # Reformat the source files in a format that the Apify API understands. source_files_for_api = [] for file_name, file_contents in actor_source_files.items(): if isinstance(file_contents, str): @@ -295,15 +265,56 @@ async def _make_actor( assert build_client_result is not None assert build_client_result['status'] == ActorJobStatus.SUCCEEDED - # We only mark the client for cleanup if the build succeeded, - # so that if something goes wrong here, - # you have a chance to check the error + # We only mark the client for cleanup if the build succeeded, so that if something goes wrong here, + # you have a chance to check the error. actor_clients_for_cleanup.append(actor_client) return actor_client yield _make_actor - # Delete all the generated actors + # Delete all the generated Actors. for actor_client in actor_clients_for_cleanup: await actor_client.delete() + + +class RunActorFunction(Protocol): + """A type for the `run_actor` fixture.""" + + def __call__( + self, + actor: ActorClientAsync, + *, + run_input: Any = None, + ) -> Coroutine[None, None, ActorRun]: + """Initiate an Actor run and wait for its completion. + + Args: + actor: Actor async client, in testing context usually created by `make_actor` fixture. + run_input: Optional input for the Actor run. + + Returns: + Actor run result. + """ + + +@pytest.fixture +async def run_actor(apify_client_async: ApifyClientAsync) -> RunActorFunction: + """Fixture for calling an Actor run and waiting for its completion. + + This fixture returns a function that initiates an Actor run with optional run input, waits for its completion, + and retrieves the final result. It uses the `wait_for_finish` method with a timeout of 10 minutes. + """ + + async def _run_actor(actor: ActorClientAsync, *, run_input: Any = None) -> ActorRun: + call_result = await actor.call(run_input=run_input) + + assert isinstance(call_result, dict), 'The result of ActorClientAsync.call() is not a dictionary.' + assert 'id' in call_result, 'The result of ActorClientAsync.call() does not contain an ID.' + + run_client = apify_client_async.run(call_result['id']) + run_result = await run_client.wait_for_finish(wait_secs=600) + + return ActorRun.model_validate(run_result) + + return _run_actor diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index 22ac68b1..5327af9c 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -13,26 +13,26 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory, RunActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_actor_reports_running_on_platform( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main() -> None: async with Actor: assert Actor.is_at_home() is True - actor = await make_actor('is-at-home', main_func=main) + actor = await make_actor(label='is-at-home', main_func=main) run_result = await run_actor(actor) assert run_result.status == 'SUCCEEDED' async def test_actor_retrieves_env_vars( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main() -> None: async with Actor: @@ -50,15 +50,15 @@ async def main() -> None: assert len(env_dict.get('default_key_value_store_id', '')) == 17 assert len(env_dict.get('default_request_queue_id', '')) == 17 - actor = await make_actor('get-env', main_func=main) + actor = await make_actor(label='get-env', main_func=main) run_result = await run_actor(actor) assert run_result.status == 'SUCCEEDED' async def test_actor_creates_new_client_instance( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main() -> None: import os @@ -74,7 +74,7 @@ async def main() -> None: kv_store_client = new_client.key_value_store(default_key_value_store_id) await kv_store_client.set_record('OUTPUT', 'TESTING-OUTPUT') - actor = await make_actor('new-client', main_func=main) + actor = await make_actor(label='new-client', main_func=main) run_result = await run_actor(actor) assert run_result.status == 'SUCCEEDED' @@ -85,15 +85,15 @@ async def main() -> None: async def test_actor_sets_status_message( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main() -> None: async with Actor: actor_input = await Actor.get_input() or {} await Actor.set_status_message('testing-status-message', **actor_input) - actor = await make_actor('set-status-message', main_func=main) + actor = await make_actor(label='set-status-message', main_func=main) run_result_1 = await run_actor(actor) assert run_result_1.status == 'SUCCEEDED' @@ -108,8 +108,8 @@ async def main() -> None: async def test_actor_starts_another_actor_instance( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main_inner() -> None: async with Actor: @@ -132,8 +132,8 @@ async def main_outer() -> None: assert inner_run_status is not None assert inner_run_status.get('status') in ['READY', 'RUNNING'] - inner_actor = await make_actor('start-inner', main_func=main_inner) - outer_actor = await make_actor('start-outer', main_func=main_outer) + inner_actor = await make_actor(label='start-inner', main_func=main_inner) + outer_actor = await make_actor(label='start-outer', main_func=main_outer) inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() @@ -153,8 +153,8 @@ async def main_outer() -> None: async def test_actor_calls_another_actor( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main_inner() -> None: async with Actor: @@ -177,8 +177,8 @@ async def main_outer() -> None: assert inner_run_status is not None assert inner_run_status.get('status') == 'SUCCEEDED' - inner_actor = await make_actor('call-inner', main_func=main_inner) - outer_actor = await make_actor('call-outer', main_func=main_outer) + inner_actor = await make_actor(label='call-inner', main_func=main_inner) + outer_actor = await make_actor(label='call-outer', main_func=main_outer) inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() @@ -198,9 +198,9 @@ async def main_outer() -> None: async def test_actor_calls_task( - apify_client_async: ApifyClientAsync, + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, + apify_client_async: ApifyClientAsync, ) -> None: async def main_inner() -> None: async with Actor: @@ -222,8 +222,8 @@ async def main_outer() -> None: assert inner_run_status is not None assert inner_run_status.get('status') == 'SUCCEEDED' - inner_actor = await make_actor('call-task-inner', main_func=main_inner) - outer_actor = await make_actor('call-task-outer', main_func=main_outer) + inner_actor = await make_actor(label='call-task-inner', main_func=main_inner) + outer_actor = await make_actor(label='call-task-outer', main_func=main_outer) inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() @@ -251,8 +251,8 @@ async def main_outer() -> None: async def test_actor_aborts_another_actor_run( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main_inner() -> None: async with Actor: @@ -269,8 +269,8 @@ async def main_outer() -> None: await Actor.abort(inner_run_id) - inner_actor = await make_actor('abort-inner', main_func=main_inner) - outer_actor = await make_actor('abort-outer', main_func=main_outer) + inner_actor = await make_actor(label='abort-inner', main_func=main_inner) + outer_actor = await make_actor(label='abort-outer', main_func=main_outer) inner_run_id = (await inner_actor.start())['id'] @@ -292,8 +292,8 @@ async def main_outer() -> None: async def test_actor_metamorphs_into_another_actor( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main_inner() -> None: import os @@ -326,8 +326,8 @@ async def main_outer() -> None: await Actor.set_value('RECORD_AFTER_METAMORPH_CALL', 'dummy') raise AssertionError('The Actor should have been metamorphed by now') - inner_actor = await make_actor('metamorph-inner', main_func=main_inner) - outer_actor = await make_actor('metamorph-outer', main_func=main_outer) + inner_actor = await make_actor(label='metamorph-inner', main_func=main_inner) + outer_actor = await make_actor(label='metamorph-outer', main_func=main_outer) inner_actor_id = (await inner_actor.get() or {})['id'] test_value = crypto_random_object_id() @@ -352,8 +352,8 @@ async def main_outer() -> None: async def test_actor_reboots_successfully( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main() -> None: async with Actor: @@ -368,7 +368,7 @@ async def main() -> None: print('Finishing...') - actor = await make_actor('actor_rebooter', main_func=main) + actor = await make_actor(label='actor_rebooter', main_func=main) run_result = await run_actor( actor, @@ -386,8 +386,8 @@ async def main() -> None: async def test_actor_adds_webhook_and_receives_event( + make_actor: MakeActorFunction, run_actor: RunActorFunction, - make_actor: ActorFactory, ) -> None: async def main_server() -> None: import os @@ -437,8 +437,8 @@ async def main_client() -> None: ) server_actor, client_actor = await asyncio.gather( - make_actor('add-webhook-server', main_func=main_server), - make_actor('add-webhook-client', main_func=main_client), + make_actor(label='add-webhook-server', main_func=main_server), + make_actor(label='add-webhook-client', main_func=main_client), ) server_actor_run = await server_actor.start() diff --git a/tests/integration/test_actor_create_proxy_configuration.py b/tests/integration/test_actor_create_proxy_configuration.py index e963dbff..ba2e361f 100644 --- a/tests/integration/test_actor_create_proxy_configuration.py +++ b/tests/integration/test_actor_create_proxy_configuration.py @@ -5,14 +5,12 @@ from apify import Actor if TYPE_CHECKING: - from apify_client import ApifyClientAsync - - from .conftest import ActorFactory + from .conftest import MakeActorFunction, RunActorFunction async def test_create_basic_proxy_configuration( - apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: groups = ['SHADER'] @@ -29,34 +27,26 @@ async def main() -> None: assert proxy_configuration._password is not None assert proxy_configuration._country_code == country_code - actor = await make_actor('proxy-configuration', main_func=main) - - call_result = await actor.call() - assert call_result is not None + actor = await make_actor(label='proxy-configuration', main_func=main) + run_result = await run_actor(actor=actor) - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_create_proxy_configuration_with_groups_and_country( - apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: await Actor.init() proxy_url_suffix = f'{Actor.config.proxy_password}@{Actor.config.proxy_hostname}:{Actor.config.proxy_port}' + proxy_configuration = await Actor.create_proxy_configuration(actor_proxy_input={'useApifyProxy': True}) - proxy_configuration = await Actor.create_proxy_configuration( - actor_proxy_input={ - 'useApifyProxy': True, - } - ) assert proxy_configuration is not None - assert await proxy_configuration.new_url() == f'http://auto:{proxy_url_suffix}' + + new_url = await proxy_configuration.new_url() + assert new_url == f'http://auto:{proxy_url_suffix}' groups = ['SHADER', 'BUYPROXIES94952'] country_code = 'US' @@ -68,20 +58,13 @@ async def main() -> None: } ) assert proxy_configuration is not None - assert ( - await proxy_configuration.new_url() - == f'http://groups-{"+".join(groups)},country-{country_code}:{proxy_url_suffix}' - ) - await Actor.exit() - - actor = await make_actor('proxy-configuration', main_func=main) + new_url = await proxy_configuration.new_url() + assert new_url == f'http://groups-{"+".join(groups)},country-{country_code}:{proxy_url_suffix}' - call_result = await actor.call() - assert call_result is not None + await Actor.exit() - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='proxy-configuration', main_func=main) + run_result = await run_actor(actor=actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' diff --git a/tests/integration/test_actor_dataset.py b/tests/integration/test_actor_dataset.py index 1b0656d5..cbf931d2 100644 --- a/tests/integration/test_actor_dataset.py +++ b/tests/integration/test_actor_dataset.py @@ -12,12 +12,12 @@ from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction async def test_push_and_verify_data_in_default_dataset( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: desired_item_count = 100 # Also change inside main() if you're changing this @@ -45,7 +45,7 @@ async def main() -> None: async def test_push_large_data_chunks_over_9mb( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -68,7 +68,7 @@ async def main() -> None: async def test_same_references_in_default_dataset( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -90,7 +90,7 @@ async def main() -> None: async def test_same_references_in_named_dataset( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: dataset_name = generate_unique_resource_name('dataset') diff --git a/tests/integration/test_actor_events.py b/tests/integration/test_actor_events.py index f64b5260..0fbbd9b6 100644 --- a/tests/integration/test_actor_events.py +++ b/tests/integration/test_actor_events.py @@ -10,12 +10,12 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction async def test_emit_and_capture_interval_events( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: import os @@ -81,7 +81,7 @@ async def log_event(data: Any) -> None: async def test_event_listener_can_be_removed_successfully( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: import os diff --git a/tests/integration/test_actor_key_value_store.py b/tests/integration/test_actor_key_value_store.py index 988444c8..aff90951 100644 --- a/tests/integration/test_actor_key_value_store.py +++ b/tests/integration/test_actor_key_value_store.py @@ -12,12 +12,12 @@ from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction async def test_same_references_in_default_kvs( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -39,7 +39,7 @@ async def main() -> None: async def test_same_references_in_named_kvs( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: kvs_name = generate_unique_resource_name('key-value-store') @@ -101,7 +101,7 @@ async def test_force_cloud( async def test_set_and_get_value_in_same_run( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -125,7 +125,7 @@ async def main() -> None: async def test_set_value_in_one_run_and_get_value_in_another( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main_set() -> None: async with Actor: @@ -176,7 +176,7 @@ async def main_get() -> None: async def test_actor_get_input_from_run( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: actor_source_files = { 'INPUT_SCHEMA.json': """ @@ -231,7 +231,7 @@ async def main(): async def test_generate_public_url_for_kvs_record( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: from typing import cast diff --git a/tests/integration/test_actor_lifecycle.py b/tests/integration/test_actor_lifecycle.py index 5ebe6308..17faa915 100644 --- a/tests/integration/test_actor_lifecycle.py +++ b/tests/integration/test_actor_lifecycle.py @@ -7,12 +7,12 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction async def test_actor_init_and_double_init_prevention( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: my_actor = Actor @@ -51,7 +51,7 @@ async def main() -> None: async def test_actor_init_correctly_in_async_with_block( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: import apify._actor @@ -74,7 +74,7 @@ async def main() -> None: async def test_actor_exit_with_different_exit_codes( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -97,7 +97,7 @@ async def main() -> None: async def test_actor_fail_with_custom_exit_codes_and_status_messages( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -141,7 +141,7 @@ async def main() -> None: async def test_actor_fails_correctly_with_exception( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: diff --git a/tests/integration/test_actor_log.py b/tests/integration/test_actor_log.py index 2e2e1d69..8d37b9e2 100644 --- a/tests/integration/test_actor_log.py +++ b/tests/integration/test_actor_log.py @@ -7,12 +7,12 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction async def test_actor_logging( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: import logging diff --git a/tests/integration/test_actor_request_queue.py b/tests/integration/test_actor_request_queue.py index 987e5185..2a54fd5d 100644 --- a/tests/integration/test_actor_request_queue.py +++ b/tests/integration/test_actor_request_queue.py @@ -13,12 +13,12 @@ from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction async def test_same_references_in_default_rq( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -40,7 +40,7 @@ async def main() -> None: async def test_same_references_in_named_rq( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: rq_name = generate_unique_resource_name('request-queue') diff --git a/tests/integration/test_fixtures.py b/tests/integration/test_fixtures.py index 555c472a..fd3e44b4 100644 --- a/tests/integration/test_fixtures.py +++ b/tests/integration/test_fixtures.py @@ -10,12 +10,12 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction async def test_actor_from_main_func( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: import os @@ -43,7 +43,7 @@ async def main() -> None: async def test_actor_from_main_py( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' main_py_source = f""" @@ -72,7 +72,7 @@ async def main(): async def test_actor_from_source_files( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: test_started_at = datetime.now(timezone.utc) actor_source_files = { diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index 050946c7..dfe16daa 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -7,12 +7,12 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import ActorFactory + from .conftest import MakeActorFunction async def test_add_and_fetch_requests( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -52,7 +52,7 @@ async def main() -> None: async def test_add_requests_in_batches( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -90,7 +90,7 @@ async def main() -> None: async def test_add_non_unique_requests_in_batch( apify_client_async: ApifyClientAsync, - make_actor: ActorFactory, + make_actor: MakeActorFunction, ) -> None: async def main() -> None: from crawlee import Request From 2de09d7ef9a5cfdf2a0df02ed22251efd1f406dd Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 30 Oct 2024 12:06:52 +0100 Subject: [PATCH 11/12] just rm self: Type --- src/apify/scrapy/middlewares/apify_proxy.py | 8 ++++---- src/apify/scrapy/pipelines/actor_dataset_push.py | 2 +- src/apify/scrapy/scheduler.py | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/apify/scrapy/middlewares/apify_proxy.py b/src/apify/scrapy/middlewares/apify_proxy.py index 94069d57..3a7f7b75 100644 --- a/src/apify/scrapy/middlewares/apify_proxy.py +++ b/src/apify/scrapy/middlewares/apify_proxy.py @@ -27,7 +27,7 @@ class ApifyHttpProxyMiddleware: proxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []} """ - def __init__(self: ApifyHttpProxyMiddleware, proxy_settings: dict) -> None: + def __init__(self, proxy_settings: dict) -> None: """Create a new instance. Args: @@ -66,7 +66,7 @@ def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> Apify return cls(proxy_settings) - async def process_request(self: ApifyHttpProxyMiddleware, request: Request, spider: Spider) -> None: + async def process_request(self, request: Request, spider: Spider) -> None: """Process a Scrapy request by assigning a new proxy. Args: @@ -89,7 +89,7 @@ async def process_request(self: ApifyHttpProxyMiddleware, request: Request, spid Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: updated request.meta={request.meta}') def process_exception( - self: ApifyHttpProxyMiddleware, + self, request: Request, exception: Exception, spider: Spider, @@ -116,7 +116,7 @@ def process_exception( 'reason="{exception}", skipping...' ) - async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult: + async def _get_new_proxy_url(self) -> ParseResult: """Get a new proxy URL. Raises: diff --git a/src/apify/scrapy/pipelines/actor_dataset_push.py b/src/apify/scrapy/pipelines/actor_dataset_push.py index 8f371788..15026475 100644 --- a/src/apify/scrapy/pipelines/actor_dataset_push.py +++ b/src/apify/scrapy/pipelines/actor_dataset_push.py @@ -19,7 +19,7 @@ class ActorDatasetPushPipeline: """ async def process_item( - self: ActorDatasetPushPipeline, + self, item: Item, spider: Spider, ) -> Item: diff --git a/src/apify/scrapy/scheduler.py b/src/apify/scrapy/scheduler.py index 36d149c9..849e5376 100644 --- a/src/apify/scrapy/scheduler.py +++ b/src/apify/scrapy/scheduler.py @@ -29,7 +29,7 @@ class ApifyScheduler(BaseScheduler): This scheduler requires the asyncio Twisted reactor to be installed. """ - def __init__(self: ApifyScheduler) -> None: + def __init__(self) -> None: """Create a new instance.""" if not is_asyncio_reactor_installed(): raise ValueError( @@ -40,7 +40,7 @@ def __init__(self: ApifyScheduler) -> None: self._rq: RequestQueue | None = None self.spider: Spider | None = None - def open(self: ApifyScheduler, spider: Spider) -> None: # this has to be named "open" + def open(self, spider: Spider) -> None: # this has to be named "open" """Open the scheduler. Args: @@ -58,7 +58,7 @@ async def open_queue() -> RequestQueue: traceback.print_exc() raise - def has_pending_requests(self: ApifyScheduler) -> bool: + def has_pending_requests(self) -> bool: """Check if the scheduler has any pending requests. Returns: @@ -75,7 +75,7 @@ def has_pending_requests(self: ApifyScheduler) -> bool: return not is_finished - def enqueue_request(self: ApifyScheduler, request: Request) -> bool: + def enqueue_request(self, request: Request) -> bool: """Add a request to the scheduler. This could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...). @@ -111,7 +111,7 @@ def enqueue_request(self: ApifyScheduler, request: Request) -> bool: Actor.log.debug(f'[{call_id}]: rq.add_request.result={result}...') return bool(result.was_already_present) - def next_request(self: ApifyScheduler) -> Request | None: + def next_request(self) -> Request | None: """Fetch the next request from the scheduler. Returns: From 0b6f2b721ef651e406b9cdce8306e750fff9975d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 30 Oct 2024 12:38:00 +0100 Subject: [PATCH 12/12] done --- tests/integration/README.md | 84 +++++++------- tests/integration/conftest.py | 18 +-- .../test_actor_create_proxy_configuration.py | 4 +- tests/integration/test_actor_dataset.py | 58 +++------- tests/integration/test_actor_events.py | 38 ++----- .../integration/test_actor_key_value_store.py | 99 +++++------------ tests/integration/test_actor_lifecycle.py | 103 ++++++------------ tests/integration/test_actor_log.py | 18 +-- tests/integration/test_actor_request_queue.py | 30 ++--- tests/integration/test_fixtures.py | 48 +++----- tests/integration/test_request_queue.py | 46 +++----- 11 files changed, 182 insertions(+), 364 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index 331acad1..a3b2dbcf 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,22 +1,16 @@ -Integration tests -================= +# Integration tests -We have integration tests which build and run Actors using the Python SDK on the Apify Platform. -To run these tests, you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to use for the tests, -and then start them with `make integration-tests`. +We have integration tests which build and run Actors using the Python SDK on the Apify Platform. To run these tests, you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to use for the tests, and then start them with `make integration-tests`. -If you want to run the integration tests on a different environment than the main Apify Platform, -you need to set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to the Apify API you want to use. +If you want to run the integration tests on a different environment than the main Apify Platform, you need to set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to the Apify API you want to use. -How to write tests ------------------- +## How to write tests There are two fixtures which you can use to write tests: ### `apify_client_async` -This fixture just gives you an instance of `ApifyClientAsync` configured with the right token and API URL, -so you don't have to do that yourself. +This fixture just gives you an instance of `ApifyClientAsync` configured with the right token and API URL, so you don't have to do that yourself. ```python async def test_something(apify_client_async: ApifyClientAsync) -> None: @@ -27,40 +21,38 @@ async def test_something(apify_client_async: ApifyClientAsync) -> None: This fixture returns a factory function for creating Actors on the Apify Platform. -For the Actor source, the fixture takes the files from `tests/integration/actor_source_base`, -builds the Apify SDK wheel from the current codebase, -and adds the Actor source you passed to the fixture as an argument. -You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. +For the Actor source, the fixture takes the files from `tests/integration/actor_source_base`, builds the Apify SDK wheel from the current codebase, and adds the Actor source you passed to the fixture as an argument. You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. -The created Actor will be uploaded to the platform, built there, and after the test finishes, it will be automatically deleted. -If the Actor build fails, it will not be deleted, so that you can check why the build failed. +The created Actor will be uploaded to the platform, built there, and after the test finishes, it will be automatically deleted. If the Actor build fails, it will not be deleted, so that you can check why the build failed. ### Creating test Actor straight from a Python function -You can create Actors straight from a Python function. -This is great because you can have the test Actor source code checked with the linter. +You can create Actors straight from a Python function. This is great because you can have the test Actor source code checked with the linter. ```python -async def test_something(self, make_actor: ActorFactory) -> None: +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main() -> None: async with Actor: print('Hello!') - actor = await make_actor('something', main_func=main) + actor = await make_actor(label='something', main_func=main) + run_result = await run_actor(actor) - run_result = await actor.call() - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' ``` -These Actors will have the `src/main.py` file set to the `main` function definition, -prepended with `import asyncio` and `from apify import Actor`, for your convenience. +These Actors will have the `src/main.py` file set to the `main` function definition, prepended with `import asyncio` and `from apify import Actor`, for your convenience. You can also pass extra imports directly to the main function: ```python -async def test_something(self, make_actor: ActorFactory) -> None: +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: async def main(): import os from apify_shared.consts import ActorEventTypes, ActorEnvVars @@ -68,23 +60,23 @@ async def test_something(self, make_actor: ActorFactory) -> None: print('The Actor is running with ' + os.getenv(ActorEnvVars.MEMORY_MBYTES) + 'MB of memory') await Actor.on(ActorEventTypes.SYSTEM_INFO, lambda event_data: print(event_data)) - actor = await make_actor('something', main_func=main) - - run_result = await actor.call() + actor = await make_actor(label='something', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' ``` ### Creating Actor from source files -You can also pass the source files directly if you need something more complex -(e.g. pass some fixed value to the Actor source code or use multiple source files). +You can also pass the source files directly if you need something more complex (e.g. pass some fixed value to the Actor source code or use multiple source files). To pass the source code of the `src/main.py` file directly, use the `main_py` argument to `make_actor`: ```python -async def test_something(self, make_actor: ActorFactory) -> None: +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' main_py_source = f""" import asyncio @@ -96,9 +88,8 @@ async def test_something(self, make_actor: ActorFactory) -> None: await Actor.set_value('OUTPUT', '{expected_output}') """ - actor = await make_actor('something', main_py=main_py_source) - - await actor.call() + actor = await make_actor(label='something', main_py=main_py_source) + await run_actor(actor) output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None @@ -106,11 +97,13 @@ async def test_something(self, make_actor: ActorFactory) -> None: ``` -Or you can pass multiple source files with the `source_files` argument, -if you need something really complex: +Or you can pass multiple source files with the `source_files` argument, if you need something really complex: ```python -async def test_something(self, make_actor: ActorFactory) -> None: +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: actor_source_files = { 'src/utils.py': """ from datetime import datetime, timezone @@ -129,9 +122,8 @@ async def test_something(self, make_actor: ActorFactory) -> None: print('Hello! It is ' + current_datetime.time()) """, } - actor = await make_actor('something', source_files=actor_source_files) + actor = await make_actor(label='something', source_files=actor_source_files) + actor_run = await run_actor(actor) - actor_run = await actor.call() - assert actor_run is not None - assert actor_run['status'] == 'SUCCEEDED' + assert actor_run.status == 'SUCCEEDED' ``` diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 339fe5ce..2ff45a68 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -24,9 +24,9 @@ from apify_client.clients.resource_clients import ActorClientAsync -TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' -API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' -SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() +_TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' +_API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' +_SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() @pytest.fixture(autouse=True) @@ -52,11 +52,11 @@ def apify_client_async() -> ApifyClientAsync: because `httpx.AsyncClient` in `ApifyClientAsync` tries to reuse the same event loop across requests, but `pytest-asyncio` closes the event loop after each test, and uses a new one for the next test. """ - api_token = os.getenv(TOKEN_ENV_VAR) - api_url = os.getenv(API_URL_ENV_VAR) + api_token = os.getenv(_TOKEN_ENV_VAR) + api_url = os.getenv(_API_URL_ENV_VAR) if not api_token: - raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') + raise RuntimeError(f'{_TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') return ApifyClientAsync(api_token, api_url=api_url) @@ -73,7 +73,7 @@ def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) - if not was_wheel_built_this_test_run_file.exists(): subprocess.run( args='python -m build', - cwd=SDK_ROOT_PATH, + cwd=_SDK_ROOT_PATH, shell=True, check=True, capture_output=True, @@ -81,7 +81,7 @@ def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) - was_wheel_built_this_test_run_file.touch() # Read the current package version, necessary for getting the right wheel filename. - pyproject_toml_file = (SDK_ROOT_PATH / 'pyproject.toml').read_text(encoding='utf-8') + pyproject_toml_file = (_SDK_ROOT_PATH / 'pyproject.toml').read_text(encoding='utf-8') for line in pyproject_toml_file.splitlines(): if line.startswith('version = '): delim = '"' if '"' in line else "'" @@ -90,7 +90,7 @@ def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) - else: raise RuntimeError('Unable to find version string.') - wheel_path = SDK_ROOT_PATH / 'dist' / f'apify-{sdk_version}-py3-none-any.whl' + wheel_path = _SDK_ROOT_PATH / 'dist' / f'apify-{sdk_version}-py3-none-any.whl' # Just to be sure. assert wheel_path.exists() diff --git a/tests/integration/test_actor_create_proxy_configuration.py b/tests/integration/test_actor_create_proxy_configuration.py index ba2e361f..5861d43e 100644 --- a/tests/integration/test_actor_create_proxy_configuration.py +++ b/tests/integration/test_actor_create_proxy_configuration.py @@ -28,7 +28,7 @@ async def main() -> None: assert proxy_configuration._country_code == country_code actor = await make_actor(label='proxy-configuration', main_func=main) - run_result = await run_actor(actor=actor) + run_result = await run_actor(actor) assert run_result.status == 'SUCCEEDED' @@ -65,6 +65,6 @@ async def main() -> None: await Actor.exit() actor = await make_actor(label='proxy-configuration', main_func=main) - run_result = await run_actor(actor=actor) + run_result = await run_actor(actor) assert run_result.status == 'SUCCEEDED' diff --git a/tests/integration/test_actor_dataset.py b/tests/integration/test_actor_dataset.py index cbf931d2..c1c69d72 100644 --- a/tests/integration/test_actor_dataset.py +++ b/tests/integration/test_actor_dataset.py @@ -12,12 +12,12 @@ from apify_client import ApifyClientAsync - from .conftest import MakeActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_push_and_verify_data_in_default_dataset( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: desired_item_count = 100 # Also change inside main() if you're changing this @@ -26,16 +26,10 @@ async def main() -> None: async with Actor: await Actor.push_data([{'id': i} for i in range(desired_item_count)]) - actor = await make_actor('push-data', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='push-data', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' list_page = await actor.last_run().dataset().list_items() assert list_page.items[0]['id'] == 0 @@ -44,31 +38,25 @@ async def main() -> None: async def test_push_large_data_chunks_over_9mb( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: await Actor.push_data([{'str': 'x' * 10000} for _ in range(5000)]) # ~50MB - actor = await make_actor('push-data-over-9mb', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='push-data-over-9mb', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async for item in actor.last_run().dataset().iterate_items(): assert item['str'] == 'x' * 10000 async def test_same_references_in_default_dataset( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -76,21 +64,15 @@ async def main() -> None: dataset2 = await Actor.open_dataset() assert dataset1 is dataset2 - actor = await make_actor('dataset-same-ref-default', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='dataset-same-ref-default', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_same_references_in_named_dataset( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: dataset_name = generate_unique_resource_name('dataset') @@ -109,16 +91,10 @@ async def main() -> None: await dataset_by_name_1.drop() - actor = await make_actor('dataset-same-ref-named', main_func=main) - - call_result = await actor.call(run_input={'datasetName': dataset_name}) - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='dataset-same-ref-named', main_func=main) + run_result = await run_actor(actor, run_input={'datasetName': dataset_name}) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_force_cloud( diff --git a/tests/integration/test_actor_events.py b/tests/integration/test_actor_events.py index 0fbbd9b6..31890dd6 100644 --- a/tests/integration/test_actor_events.py +++ b/tests/integration/test_actor_events.py @@ -8,14 +8,12 @@ from apify import Actor if TYPE_CHECKING: - from apify_client import ApifyClientAsync - - from .conftest import MakeActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_emit_and_capture_interval_events( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: import os @@ -47,26 +45,20 @@ async def log_event(data: Any) -> None: Actor.on(Event.PERSIST_STATE, on_event(ActorEventTypes.PERSIST_STATE)) await asyncio.sleep(3) - # The SYSTEM_INFO event sometimes takes a while to appear, let's wait for it for a while longer + # The SYSTEM_INFO event sometimes takes a while to appear, let's wait for it for a while longer. for _ in range(20): if was_system_info_emitted: break await asyncio.sleep(1) - # Check that parsing datetimes works correctly - # Check `createdAt` is a datetime (so it's the same locally and on platform) + # Check that parsing datetimes works correctly. + # Check `createdAt` is a datetime (so it's the same locally and on platform). assert isinstance(system_infos[0].cpu_info.created_at, datetime) - actor = await make_actor('actor-interval-events', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='actor-interval-events', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' dataset_items_page = await actor.last_run().dataset().list_items() persist_state_events = [ @@ -80,8 +72,8 @@ async def log_event(data: Any) -> None: async def test_event_listener_can_be_removed_successfully( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: import os @@ -107,13 +99,7 @@ def count_event(data): # type: ignore # noqa: ANN202, ANN001 await asyncio.sleep(0.5) assert counter == last_count - actor = await make_actor('actor-off-event', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='actor-off-event', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' diff --git a/tests/integration/test_actor_key_value_store.py b/tests/integration/test_actor_key_value_store.py index aff90951..6ed64123 100644 --- a/tests/integration/test_actor_key_value_store.py +++ b/tests/integration/test_actor_key_value_store.py @@ -12,12 +12,12 @@ from apify_client import ApifyClientAsync - from .conftest import MakeActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_same_references_in_default_kvs( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -25,21 +25,15 @@ async def main() -> None: kvs2 = await Actor.open_key_value_store() assert kvs1 is kvs2 - actor = await make_actor('kvs-same-ref-default', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='kvs-same-ref-default', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_same_references_in_named_kvs( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: kvs_name = generate_unique_resource_name('key-value-store') @@ -58,16 +52,10 @@ async def main() -> None: await kvs_by_name_1.drop() - actor = await make_actor('kvs-same-ref-named', main_func=main) - - call_result = await actor.call(run_input={'kvsName': kvs_name}) - assert call_result is not None + actor = await make_actor(label='kvs-same-ref-named', main_func=main) + run_result = await run_actor(actor, run_input={'kvsName': kvs_name}) - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_force_cloud( @@ -100,8 +88,8 @@ async def test_force_cloud( async def test_set_and_get_value_in_same_run( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -111,36 +99,24 @@ async def main() -> None: assert value['string'] == 'a string' assert value['nested']['test'] == 1 - actor = await make_actor('actor-get-set-value', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='actor-get-set-value', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_set_value_in_one_run_and_get_value_in_another( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main_set() -> None: async with Actor: await Actor.set_value('test', {'number': 123, 'string': 'a string', 'nested': {'test': 1}}) - actor_set = await make_actor('actor-set-value', main_func=main_set) - - call_result_set = await actor_set.call() - assert call_result_set is not None - - run_client_set = apify_client_async.run(call_result_set['id']) - run_result_set = await run_client_set.wait_for_finish(wait_secs=600) + actor_set = await make_actor(label='actor-set-value', main_func=main_set) + run_result_set = await run_actor(actor_set) - assert run_result_set is not None - assert run_result_set['status'] == 'SUCCEEDED' + assert run_result_set.status == 'SUCCEEDED' # Externally check if the value is present in key-value store test_record = await actor_set.last_run().key_value_store().get_record('test') @@ -160,23 +136,18 @@ async def main_get() -> None: assert value['string'] == 'a string' assert value['nested']['test'] == 1 - actor_get = await make_actor('actor-get-value', main_func=main_get) + actor_get = await make_actor(label='actor-get-value', main_func=main_get) default_kvs_info = await actor_set.last_run().key_value_store().get() assert default_kvs_info is not None - call_result_get = await actor_get.call(run_input={'kvs-id': default_kvs_info['id']}) - assert call_result_get is not None + run_result_get = await run_actor(actor_get, run_input={'kvs-id': default_kvs_info['id']}) - run_client_get = apify_client_async.run(call_result_get['id']) - run_result_get = await run_client_get.wait_for_finish(wait_secs=600) - - assert run_result_get is not None - assert run_result_get['status'] == 'SUCCEEDED' + assert run_result_get.status == 'SUCCEEDED' async def test_actor_get_input_from_run( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: actor_source_files = { 'INPUT_SCHEMA.json': """ @@ -210,28 +181,24 @@ async def main(): assert input_object['password'] == 'very secret' """, } - actor = await make_actor('actor-get-input', source_files=actor_source_files) + actor = await make_actor(label='actor-get-input', source_files=actor_source_files) - call_result = await actor.call( + run_result = await run_actor( + actor, run_input={ 'number': 123, 'string': 'a string', 'nested': {'test': 1}, 'password': 'very secret', - } + }, ) - assert call_result is not None - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_generate_public_url_for_kvs_record( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: from typing import cast @@ -248,13 +215,7 @@ async def main() -> None: assert record_url == f'{public_api_url}/v2/key-value-stores/{default_store_id}/records/dummy' - actor = await make_actor('kvs-get-public-url', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='kvs-get-public-url', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' diff --git a/tests/integration/test_actor_lifecycle.py b/tests/integration/test_actor_lifecycle.py index 17faa915..7a975c99 100644 --- a/tests/integration/test_actor_lifecycle.py +++ b/tests/integration/test_actor_lifecycle.py @@ -5,14 +5,12 @@ from apify import Actor if TYPE_CHECKING: - from apify_client import ApifyClientAsync - - from .conftest import MakeActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_actor_init_and_double_init_prevention( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: my_actor = Actor @@ -37,21 +35,15 @@ async def main() -> None: assert double_init is False assert my_actor._is_initialized is False - actor = await make_actor('actor-init', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='actor-init', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_actor_init_correctly_in_async_with_block( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: import apify._actor @@ -60,101 +52,68 @@ async def main() -> None: assert apify._actor.Actor._is_initialized assert apify._actor.Actor._is_initialized is False - actor = await make_actor('with-actor-init', main_func=main) + actor = await make_actor(label='with-actor-init', main_func=main) + run_result = await run_actor(actor) - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_actor_exit_with_different_exit_codes( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: input = await Actor.get_input() # noqa: A001 await Actor.exit(**input) - actor = await make_actor('actor-exit', main_func=main) + actor = await make_actor(label='actor-exit', main_func=main) for exit_code in [0, 1, 101]: - call_result = await actor.call(run_input={'exit_code': exit_code}) - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + run_result = await run_actor(actor, run_input={'exit_code': exit_code}) - assert run_result is not None - assert run_result['exitCode'] == exit_code - assert run_result['status'] == 'FAILED' if exit_code > 0 else 'SUCCEEDED' + assert run_result.exit_code == exit_code + assert run_result.status == 'FAILED' if exit_code > 0 else 'SUCCEEDED' async def test_actor_fail_with_custom_exit_codes_and_status_messages( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: input = await Actor.get_input() # noqa: A001 await Actor.fail(**input) if input else await Actor.fail() - actor = await make_actor('actor-fail', main_func=main) + actor = await make_actor(label='actor-fail', main_func=main) + run_result = await run_actor(actor) - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['exitCode'] == 1 - assert run_result['status'] == 'FAILED' + assert run_result.exit_code == 1 + assert run_result.status == 'FAILED' for exit_code in [1, 10, 100]: - call_result = await actor.call(run_input={'exit_code': exit_code}) - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + run_result = await run_actor(actor, run_input={'exit_code': exit_code}) - assert run_result is not None - assert run_result['exitCode'] == exit_code - assert run_result['status'] == 'FAILED' + assert run_result.exit_code == exit_code + assert run_result.status == 'FAILED' - # fail with status message - call_result = await actor.call(run_input={'status_message': 'This is a test message'}) - assert call_result is not None + # Fail with a status message. + run_result = await run_actor(actor, run_input={'status_message': 'This is a test message'}) - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'FAILED' - assert run_result.get('statusMessage') == 'This is a test message' + assert run_result.status == 'FAILED' + assert run_result.status_message == 'This is a test message' async def test_actor_fails_correctly_with_exception( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: raise Exception('This is a test exception') # noqa: TRY002 - actor = await make_actor('with-actor-fail', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='with-actor-fail', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['exitCode'] == 91 - assert run_result['status'] == 'FAILED' + assert run_result.exit_code == 91 + assert run_result.status == 'FAILED' diff --git a/tests/integration/test_actor_log.py b/tests/integration/test_actor_log.py index 8d37b9e2..7ec1b24d 100644 --- a/tests/integration/test_actor_log.py +++ b/tests/integration/test_actor_log.py @@ -5,14 +5,12 @@ from apify import Actor, __version__ if TYPE_CHECKING: - from apify_client import ApifyClientAsync - - from .conftest import MakeActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_actor_logging( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: import logging @@ -42,16 +40,10 @@ async def main() -> None: # Test that exception in Actor.main is logged with the traceback raise RuntimeError('Dummy RuntimeError') - actor = await make_actor('actor-log', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='actor-log', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'FAILED' + assert run_result.status == 'FAILED' run_log = await actor.last_run().log().get() assert run_log is not None diff --git a/tests/integration/test_actor_request_queue.py b/tests/integration/test_actor_request_queue.py index 2a54fd5d..06e8529e 100644 --- a/tests/integration/test_actor_request_queue.py +++ b/tests/integration/test_actor_request_queue.py @@ -13,12 +13,12 @@ from apify_client import ApifyClientAsync - from .conftest import MakeActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_same_references_in_default_rq( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -26,21 +26,15 @@ async def main() -> None: rq2 = await Actor.open_request_queue() assert rq1 is rq2 - actor = await make_actor('rq-same-ref-default', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='rq-same-ref-default', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_same_references_in_named_rq( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: rq_name = generate_unique_resource_name('request-queue') @@ -59,16 +53,10 @@ async def main() -> None: await rq_by_name_1.drop() - actor = await make_actor('rq-same-ref-named', main_func=main) - - call_result = await actor.call(run_input={'rqName': rq_name}) - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='rq-same-ref-named', main_func=main) + run_result = await run_actor(actor, run_input={'rqName': rq_name}) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_force_cloud( diff --git a/tests/integration/test_fixtures.py b/tests/integration/test_fixtures.py index fd3e44b4..865effa9 100644 --- a/tests/integration/test_fixtures.py +++ b/tests/integration/test_fixtures.py @@ -10,12 +10,12 @@ if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .conftest import MakeActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_actor_from_main_func( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: import os @@ -25,25 +25,20 @@ async def main() -> None: async with Actor: await Actor.set_value('OUTPUT', os.getenv(ActorEnvVars.ID)) - actor = await make_actor('make-actor-main-func', main_func=main) + actor = await make_actor(label='make-actor-main-func', main_func=main) + run_result = await run_actor(actor) - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') + assert output_record is not None - assert run_result['actId'] == output_record['value'] + assert run_result.act_id == output_record['value'] async def test_actor_from_main_py( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' main_py_source = f""" @@ -54,25 +49,20 @@ async def main(): await Actor.set_value('OUTPUT', '{expected_output}') """ - actor = await make_actor('make-actor-main-py', main_py=main_py_source) - - call_result = await actor.call() - assert call_result is not None + actor = await make_actor(label='make-actor-main-py', main_py=main_py_source) + run_result = await run_actor(actor) - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') + assert output_record is not None assert output_record['value'] == expected_output async def test_actor_from_source_files( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: test_started_at = datetime.now(timezone.utc) actor_source_files = { @@ -93,16 +83,10 @@ async def main(): await Actor.set_value('OUTPUT', current_datetime) """, } - actor = await make_actor('make-actor-source-files', source_files=actor_source_files) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='make-actor-source-files', source_files=actor_source_files) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index dfe16daa..4bce884a 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -5,14 +5,12 @@ from apify import Actor if TYPE_CHECKING: - from apify_client import ApifyClientAsync - - from .conftest import MakeActorFunction + from .conftest import MakeActorFunction, RunActorFunction async def test_add_and_fetch_requests( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -38,21 +36,15 @@ async def main() -> None: is_finished = await rq.is_finished() assert is_finished is True - actor = await make_actor('rq-simple-test', main_func=main) - - call_result = await actor.call() - assert call_result is not None + actor = await make_actor(label='rq-simple-test', main_func=main) + run_result = await run_actor(actor) - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_add_requests_in_batches( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: async with Actor: @@ -76,21 +68,15 @@ async def main() -> None: is_finished = await rq.is_finished() assert is_finished is True - actor = await make_actor('rq-batch-test', main_func=main) - - call_result = await actor.call() - assert call_result is not None + actor = await make_actor(label='rq-batch-test', main_func=main) + run_result = await run_actor(actor) - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) - - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED' async def test_add_non_unique_requests_in_batch( - apify_client_async: ApifyClientAsync, make_actor: MakeActorFunction, + run_actor: RunActorFunction, ) -> None: async def main() -> None: from crawlee import Request @@ -121,13 +107,7 @@ async def main() -> None: is_finished = await rq.is_finished() assert is_finished is True - actor = await make_actor('rq-batch-test', main_func=main) - - call_result = await actor.call() - assert call_result is not None - - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + actor = await make_actor(label='rq-batch-test', main_func=main) + run_result = await run_actor(actor) - assert run_result is not None - assert run_result['status'] == 'SUCCEEDED' + assert run_result.status == 'SUCCEEDED'