Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
609 changes: 294 additions & 315 deletions tests/integration/test_actor_api_helpers.py

Large diffs are not rendered by default.

114 changes: 54 additions & 60 deletions tests/integration/test_actor_create_proxy_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,68 +8,62 @@
from .conftest import ActorFactory


class TestActorCreateProxyConfiguration:
async def test_create_proxy_configuration_basic(
self: TestActorCreateProxyConfiguration,
make_actor: ActorFactory,
) -> None:
async def main() -> None:
groups = ['SHADER']
country_code = 'US'

async with Actor:
proxy_configuration = await Actor.create_proxy_configuration(
groups=groups,
country_code=country_code,
)

assert proxy_configuration is not None
assert proxy_configuration._groups == groups
assert proxy_configuration._password is not None
assert proxy_configuration._country_code == country_code

actor = await make_actor('proxy-configuration', main_func=main)

run_result = await actor.call()
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'

async def test_create_proxy_configuration_complex(
self: TestActorCreateProxyConfiguration,
make_actor: ActorFactory,
) -> None:
async def main() -> None:
await Actor.init()

proxy_url_suffix = f'{Actor.config.proxy_password}@{Actor.config.proxy_hostname}:{Actor.config.proxy_port}'
async def test_create_basic_proxy_configuration(make_actor: ActorFactory) -> None:
async def main() -> None:
groups = ['SHADER']
country_code = 'US'

async with Actor:
proxy_configuration = await Actor.create_proxy_configuration(
actor_proxy_input={
'useApifyProxy': True,
}
groups=groups,
country_code=country_code,
)
assert proxy_configuration is not None
assert await proxy_configuration.new_url() == f'http://auto:{proxy_url_suffix}'

groups = ['SHADER', 'BUYPROXIES94952']
country_code = 'US'
proxy_configuration = await Actor.create_proxy_configuration(
actor_proxy_input={
'useApifyProxy': True,
'apifyProxyGroups': groups,
'apifyProxyCountry': country_code,
}
)
assert proxy_configuration is not None
assert (
await proxy_configuration.new_url()
== f'http://groups-{"+".join(groups)},country-{country_code}:{proxy_url_suffix}'
)

await Actor.exit()

actor = await make_actor('proxy-configuration', main_func=main)

run_result = await actor.call()
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'
assert proxy_configuration._groups == groups
assert proxy_configuration._password is not None
assert proxy_configuration._country_code == country_code

actor = await make_actor('proxy-configuration', main_func=main)

run_result = await actor.call()
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'


async def test_create_proxy_configuration_with_groups_and_country(make_actor: ActorFactory) -> None:
async def main() -> None:
await Actor.init()

proxy_url_suffix = f'{Actor.config.proxy_password}@{Actor.config.proxy_hostname}:{Actor.config.proxy_port}'

proxy_configuration = await Actor.create_proxy_configuration(
actor_proxy_input={
'useApifyProxy': True,
}
)
assert proxy_configuration is not None
assert await proxy_configuration.new_url() == f'http://auto:{proxy_url_suffix}'

groups = ['SHADER', 'BUYPROXIES94952']
country_code = 'US'
proxy_configuration = await Actor.create_proxy_configuration(
actor_proxy_input={
'useApifyProxy': True,
'apifyProxyGroups': groups,
'apifyProxyCountry': country_code,
}
)
assert proxy_configuration is not None
assert (
await proxy_configuration.new_url()
== f'http://groups-{"+".join(groups)},country-{country_code}:{proxy_url_suffix}'
)

await Actor.exit()

actor = await make_actor('proxy-configuration', main_func=main)

run_result = await actor.call()
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'
149 changes: 73 additions & 76 deletions tests/integration/test_actor_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,104 +15,101 @@
from .conftest import ActorFactory


class TestActorPushData:
async def test_push_data(self: TestActorPushData, make_actor: ActorFactory) -> None:
desired_item_count = 100 # Also change inside main() if you're changing this
async def test_push_and_verify_data_in_default_dataset(make_actor: ActorFactory) -> None:
desired_item_count = 100 # Also change inside main() if you're changing this

async def main() -> None:
desired_item_count = 100
async with Actor:
await Actor.push_data([{'id': i} for i in range(desired_item_count)])
async def main() -> None:
desired_item_count = 100
async with Actor:
await Actor.push_data([{'id': i} for i in range(desired_item_count)])

actor = await make_actor('push-data', main_func=main)

run_result = await actor.call()

assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'
list_page = await actor.last_run().dataset().list_items()
assert list_page.items[0]['id'] == 0
assert list_page.items[-1]['id'] == desired_item_count - 1
assert len(list_page.items) == list_page.count == desired_item_count

actor = await make_actor('push-data', main_func=main)

run_result = await actor.call()
async def test_push_large_data_chunks_over_9mb(make_actor: ActorFactory) -> None:
async def main() -> None:
async with Actor:
await Actor.push_data([{'str': 'x' * 10000} for _ in range(5000)]) # ~50MB

assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'
list_page = await actor.last_run().dataset().list_items()
assert list_page.items[0]['id'] == 0
assert list_page.items[-1]['id'] == desired_item_count - 1
assert len(list_page.items) == list_page.count == desired_item_count
actor = await make_actor('push-data-over-9mb', main_func=main)

async def test_push_data_over_9mb(self: TestActorPushData, make_actor: ActorFactory) -> None:
async def main() -> None:
async with Actor:
await Actor.push_data([{'str': 'x' * 10000} for _ in range(5000)]) # ~50MB
run_result = await actor.call()

actor = await make_actor('push-data-over-9mb', main_func=main)
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'
async for item in actor.last_run().dataset().iterate_items():
assert item['str'] == 'x' * 10000

run_result = await actor.call()

assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'
async for item in actor.last_run().dataset().iterate_items():
assert item['str'] == 'x' * 10000
async def test_same_references_in_default_dataset(make_actor: ActorFactory) -> None:
async def main() -> None:
async with Actor:
dataset1 = await Actor.open_dataset()
dataset2 = await Actor.open_dataset()
assert dataset1 is dataset2

actor = await make_actor('dataset-same-ref-default', main_func=main)

class TestActorOpenDataset:
async def test_same_references_default(self: TestActorOpenDataset, make_actor: ActorFactory) -> None:
async def main() -> None:
async with Actor:
dataset1 = await Actor.open_dataset()
dataset2 = await Actor.open_dataset()
assert dataset1 is dataset2
run_result = await actor.call()
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'

actor = await make_actor('dataset-same-ref-default', main_func=main)

run_result = await actor.call()
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'
async def test_same_references_in_named_dataset(make_actor: ActorFactory) -> None:
dataset_name = generate_unique_resource_name('dataset')

async def test_same_references_named(self: TestActorOpenDataset, make_actor: ActorFactory) -> None:
dataset_name = generate_unique_resource_name('dataset')
async def main() -> None:
async with Actor:
input_object = await Actor.get_input()
dataset_name = input_object['datasetName']
dataset_by_name_1 = await Actor.open_dataset(name=dataset_name)
dataset_by_name_2 = await Actor.open_dataset(name=dataset_name)
assert dataset_by_name_1 is dataset_by_name_2

async def main() -> None:
async with Actor:
input_object = await Actor.get_input()
dataset_name = input_object['datasetName']
dataset_by_name_1 = await Actor.open_dataset(name=dataset_name)
dataset_by_name_2 = await Actor.open_dataset(name=dataset_name)
assert dataset_by_name_1 is dataset_by_name_2
dataset_by_id_1 = await Actor.open_dataset(id=dataset_by_name_1._id)
dataset_by_id_2 = await Actor.open_dataset(id=dataset_by_name_1._id)
assert dataset_by_id_1 is dataset_by_name_1
assert dataset_by_id_2 is dataset_by_id_1

dataset_by_id_1 = await Actor.open_dataset(id=dataset_by_name_1._id)
dataset_by_id_2 = await Actor.open_dataset(id=dataset_by_name_1._id)
assert dataset_by_id_1 is dataset_by_name_1
assert dataset_by_id_2 is dataset_by_id_1
await dataset_by_name_1.drop()

await dataset_by_name_1.drop()
actor = await make_actor('dataset-same-ref-named', main_func=main)

actor = await make_actor('dataset-same-ref-named', main_func=main)
run_result = await actor.call(run_input={'datasetName': dataset_name})
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'

run_result = await actor.call(run_input={'datasetName': dataset_name})
assert run_result is not None
assert run_result['status'] == 'SUCCEEDED'

async def test_force_cloud(
self: TestActorOpenDataset,
apify_client_async: ApifyClientAsync,
monkeypatch: pytest.MonkeyPatch,
) -> None:
assert apify_client_async.token is not None
monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_client_async.token)
async def test_force_cloud(apify_client_async: ApifyClientAsync, monkeypatch: pytest.MonkeyPatch) -> None:
assert apify_client_async.token is not None
monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_client_async.token)

dataset_name = generate_unique_resource_name('dataset')
dataset_item = {'foo': 'bar'}
dataset_name = generate_unique_resource_name('dataset')
dataset_item = {'foo': 'bar'}

async with Actor:
dataset = await Actor.open_dataset(name=dataset_name, force_cloud=True)
dataset_id = dataset._id
async with Actor:
dataset = await Actor.open_dataset(name=dataset_name, force_cloud=True)
dataset_id = dataset._id

await dataset.push_data(dataset_item)
await dataset.push_data(dataset_item)

dataset_client = apify_client_async.dataset(dataset_id)
dataset_client = apify_client_async.dataset(dataset_id)

try:
dataset_details = await dataset_client.get()
assert dataset_details is not None
assert dataset_details.get('name') == dataset_name
try:
dataset_details = await dataset_client.get()
assert dataset_details is not None
assert dataset_details.get('name') == dataset_name

dataset_items = await dataset_client.list_items()
assert dataset_items.items == [dataset_item]
finally:
await dataset_client.delete()
dataset_items = await dataset_client.list_items()
assert dataset_items.items == [dataset_item]
finally:
await dataset_client.delete()
Loading