Fix unit tests

Author: vdusek

Makefile

@@ -26,13 +26,13 @@ type-check:
 	uv run mypy
 
 unit-tests:
-	uv run pytest --numprocesses=auto --verbose --cov=src/apify tests/unit
+	uv run pytest --numprocesses=auto -vv --cov=src/apify tests/unit
 
 unit-tests-cov:
-	uv run pytest --numprocesses=auto --verbose --cov=src/apify --cov-report=html tests/unit
+	uv run pytest --numprocesses=auto -vv --cov=src/apify --cov-report=html tests/unit
 
 integration-tests:
-	uv run pytest --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) --verbose tests/integration
+	uv run pytest --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) -vv tests/integration
 
 format:
 	uv run ruff check --fix

pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
     "apify-client>=1.11.0",
     "apify-shared>=1.3.0",
     "cachetools>=5.5.0",
-    "crawlee@git+https://github.com/apify/crawlee-python.git@1cbf15e13af882c864b87f8ed48252bcb3747993",
+    "crawlee@git+https://github.com/apify/crawlee-python.git@new-storage-clients",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
     # TODO: ensure compatibility with the latest version of lazy-object-proxy

src/apify/_actor.py

@@ -682,7 +682,23 @@ def get_env(self) -> dict:
                 config[alias] = getattr(self._configuration, field_name)
 
         env_vars = {env_var.value.lower(): env_var.name.lower() for env_var in [*ActorEnvVars, *ApifyEnvVars]}
-        return {option_name: config[env_var] for env_var, option_name in env_vars.items() if env_var in config}
+        result = {option_name: config[env_var] for env_var, option_name in env_vars.items() if env_var in config}
+
+        # These environment variables are not part of the Configuration model,
+        # so we need to add them manually to the result dictionary.
+        result[ActorEnvVars.DEFAULT_DATASET_ID.name.lower()] = os.environ.get(
+            ActorEnvVars.DEFAULT_DATASET_ID.value
+        ) or os.environ.get(ApifyEnvVars.DEFAULT_DATASET_ID.value)
+
+        result[ActorEnvVars.DEFAULT_KEY_VALUE_STORE_ID.name.lower()] = os.environ.get(
+            ActorEnvVars.DEFAULT_KEY_VALUE_STORE_ID.value
+        ) or os.environ.get(ApifyEnvVars.DEFAULT_KEY_VALUE_STORE_ID.value)
+
+        result[ActorEnvVars.DEFAULT_REQUEST_QUEUE_ID.name.lower()] = os.environ.get(
+            ActorEnvVars.DEFAULT_REQUEST_QUEUE_ID.value
+        ) or os.environ.get(ApifyEnvVars.DEFAULT_REQUEST_QUEUE_ID.value)
+
+        return result
 
     async def start(
         self,

src/apify/storage_clients/_apify/_dataset_client.py

@@ -8,6 +8,7 @@
 from typing_extensions import override
 
 from apify_client import ApifyClientAsync
+from apify_shared.consts import ActorEnvVars, ApifyEnvVars
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
@@ -99,10 +100,10 @@ async def open(
         # If both id and name are None, try to get the default storage ID from environment variables.
         if id is None and name is None:
             id = os.environ.get(
-                'ACTOR_DEFAULT_DATASET_ID',
+                ActorEnvVars.DEFAULT_DATASET_ID.value,
                 None,
             ) or os.environ.get(
-                'APIFY_DEFAULT_DATASET_ID',
+                ApifyEnvVars.DEFAULT_DATASET_ID.value,
                 None,
             )
 

src/apify/storage_clients/_apify/_key_value_store_client.py

@@ -9,6 +9,7 @@
 from yarl import URL
 
 from apify_client import ApifyClientAsync
+from apify_shared.consts import ActorEnvVars, ApifyEnvVars
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
@@ -101,10 +102,10 @@ async def open(
         # If both id and name are None, try to get the default storage ID from environment variables.
         if id is None and name is None:
             id = os.environ.get(
-                'ACTOR_DEFAULT_KEY_VALUE_STORE_ID',
+                ActorEnvVars.DEFAULT_KEY_VALUE_STORE_ID.value,
                 None,
             ) or os.environ.get(
-                'APIFY_DEFAULT_KEY_VALUE_STORE_ID',
+                ApifyEnvVars.DEFAULT_KEY_VALUE_STORE_ID.value,
                 None,
             )
 

src/apify/storage_clients/_apify/_request_queue_client.py

@@ -11,6 +11,7 @@
 from typing_extensions import override
 
 from apify_client import ApifyClientAsync
+from apify_shared.consts import ActorEnvVars, ApifyEnvVars
 from crawlee._utils.requests import unique_key_to_request_id
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
@@ -131,10 +132,10 @@ async def open(
         # If both id and name are None, try to get the default storage ID from environment variables.
         if id is None and name is None:
             id = os.environ.get(
-                'ACTOR_DEFAULT_REQUEST_QUEUE_ID',
+                ActorEnvVars.DEFAULT_REQUEST_QUEUE_ID.value,
                 None,
             ) or os.environ.get(
-                'APIFY_DEFAULT_REQUEST_QUEUE_ID',
+                ApifyEnvVars.DEFAULT_REQUEST_QUEUE_ID.value,
                 None,
             )
 

tests/integration/test_actor_dataset.py

@@ -37,6 +37,10 @@ async def main() -> None:
     assert len(list_page.items) == list_page.count == desired_item_count
 
 
+import pytest
+
+
+@pytest.mark.only
 async def test_push_large_data_chunks_over_9mb(
     make_actor: MakeActorFunction,
     run_actor: RunActorFunction,

tests/unit/actor/test_actor_dataset.py

@@ -2,14 +2,8 @@
 
 import pytest
 
-from apify_shared.consts import ActorEnvVars
-from crawlee.storage_clients import FileSystemStorageClient
-
 from apify import Actor
 
-# NOTE: We only test the dataset methods available on Actor class/instance.
-# Actual tests for the implementations are in storages/.
-
 
 async def test_throws_error_without_actor_init() -> None:
     with pytest.raises(RuntimeError):
@@ -34,20 +28,6 @@ async def test_open_dataset_returns_same_references() -> None:
         assert dataset_by_id_2 is dataset_by_id_1
 
 
-@pytest.mark.skip(reason='TODO: fix this test')
-async def test_open_dataset_uses_env_var(monkeypatch: pytest.MonkeyPatch) -> None:
-    memory_storage_client = FileSystemStorageClient()
-
-    default_dataset_id = 'my-new-default-id'
-    monkeypatch.setenv(ActorEnvVars.DEFAULT_DATASET_ID, default_dataset_id)
-
-    async with Actor:
-        ddt = await Actor.open_dataset()
-        assert ddt.metadata.id == default_dataset_id
-        dataset = await memory_storage_client.create_dataset_client(id=ddt.metadata.id)
-        await dataset.drop()
-
-
 async def test_push_data_to_dataset() -> None:
     async with Actor as actor:
         dataset = await actor.open_dataset()

tests/unit/actor/test_actor_env_helpers.py

@@ -4,9 +4,8 @@
 import string
 from datetime import datetime, timedelta
 from decimal import Decimal
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
-import pytest
 from pydantic_core import TzInfo
 
 from apify_shared.consts import (
@@ -22,14 +21,16 @@
 
 from apify import Actor
 
+if TYPE_CHECKING:
+    import pytest
+
 
 async def test_actor_is_not_at_home_when_local() -> None:
     async with Actor as actor:
         is_at_home = actor.is_at_home()
         assert is_at_home is False
 
 
-@pytest.mark.skip(reason='TODO: fix this test')
 async def test_get_env_with_randomized_env_vars(monkeypatch: pytest.MonkeyPatch) -> None:  # noqa: PLR0912
     ignored_env_vars = {
         ApifyEnvVars.INPUT_KEY,
@@ -43,6 +44,7 @@ async def test_get_env_with_randomized_env_vars(monkeypatch: pytest.MonkeyPatch)
         ApifyEnvVars.LOG_FORMAT,
         ApifyEnvVars.LOG_LEVEL,
         ActorEnvVars.STANDBY_PORT,
+        ApifyEnvVars.PERSIST_STORAGE,
     }
 
     legacy_env_vars = {
@@ -58,7 +60,7 @@ async def test_get_env_with_randomized_env_vars(monkeypatch: pytest.MonkeyPatch)
     }
 
     # Set up random env vars
-    expected_get_env: dict[str, Any] = {}
+    expected_get_env = dict[str, Any]()
     expected_get_env[ApifyEnvVars.LOG_LEVEL.name.lower()] = 'INFO'
 
     for int_env_var in INTEGER_ENV_VARS:

tests/unit/actor/test_actor_key_value_store.py

@@ -10,8 +10,6 @@
 from apify._crypto import public_encrypt
 
 
-# NOTE: We only test the key-value store methods available on Actor class/instance.
-# Actual tests for the implementations are in storages/.
 async def test_open_returns_same_references() -> None:
     async with Actor:
         kvs1 = await Actor.open_key_value_store()