rm force flag

Author: vdusek

src/crawlee/service_container.py

@@ -95,21 +95,16 @@ def get_configuration() -> Configuration:
 
 
 @docs_group('Functions')
-def set_configuration(
-    configuration: Configuration,
-    *,
-    force: bool = False,
-) -> None:
+def set_configuration(configuration: Configuration) -> None:
     """Set the configuration.
 
     Args:
         configuration: The configuration to set.
-        force: If True, the configuration will be set even if it was already set.
 
     Raises:
         ServiceConflictError: If the configuration was already set.
     """
-    if _service_locator.configuration_was_set and not force:
+    if _service_locator.configuration_was_set:
         raise ServiceConflictError(Configuration, configuration, _service_locator.configuration)
 
     _service_locator.configuration = configuration
@@ -122,21 +117,16 @@ def get_event_manager() -> EventManager:
 
 
 @docs_group('Functions')
-def set_event_manager(
-    event_manager: EventManager,
-    *,
-    force: bool = False,
-) -> None:
+def set_event_manager(event_manager: EventManager) -> None:
     """Set the event manager.
 
     Args:
         event_manager: The event manager to set.
-        force: If True, the event manager will be set even if it was already set.
 
     Raises:
         ServiceConflictError: If the event manager was already set.
     """
-    if _service_locator.event_manager_was_set and not force:
+    if _service_locator.event_manager_was_set:
         raise ServiceConflictError(EventManager, event_manager, _service_locator.event_manager)
 
     _service_locator.event_manager = event_manager
@@ -149,21 +139,16 @@ def get_storage_client() -> BaseStorageClient:
 
 
 @docs_group('Functions')
-def set_storage_client(
-    storage_client: BaseStorageClient,
-    *,
-    force: bool = False,
-) -> None:
+def set_storage_client(storage_client: BaseStorageClient) -> None:
     """Set the storage client.
 
     Args:
         storage_client: The storage client to set.
-        force: If True, the storage client will be set even if it was already set.
 
     Raises:
         ServiceConflictError: If the storage client was already set.
     """
-    if _service_locator.storage_client_was_set and not force:
+    if _service_locator.storage_client_was_set:
         raise ServiceConflictError(BaseStorageClient, storage_client, _service_locator.storage_client)
 
     _service_locator.storage_client = storage_client

tests/unit/_memory_storage_client/test_memory_storage_client.py

@@ -224,15 +224,15 @@ async def test_not_implemented_method(tmp_path: Path) -> None:
 async def test_default_storage_path_used(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the default value to be used
     monkeypatch.delenv('CRAWLEE_STORAGE_DIR', raising=False)
-    service_container.set_configuration(Configuration(), force=True)
+    service_container.set_configuration(Configuration())
     ms = MemoryStorageClient()
     assert ms.storage_dir == './storage'
 
 
 async def test_storage_path_from_env_var_overrides_default(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the env var to override the default value
     monkeypatch.setenv('CRAWLEE_STORAGE_DIR', './env_var_storage_dir')
-    service_container.set_configuration(Configuration(), force=True)
+    service_container.set_configuration(Configuration())
     ms = MemoryStorageClient()
     assert ms.storage_dir == './env_var_storage_dir'
 

tests/unit/_memory_storage_client/test_memory_storage_e2e.py

@@ -14,7 +14,7 @@
 async def test_actor_memory_storage_client_key_value_store_e2e(
     monkeypatch: pytest.MonkeyPatch,
     purge_on_start: bool,  # noqa: FBT001
-    reset_globals: Callable[[], None],
+    prepare_test_env: Callable[[], None],
 ) -> None:
     """This test simulates two clean runs using memory storage.
     The second run attempts to access data created by the first one.
@@ -32,7 +32,7 @@ async def test_actor_memory_storage_client_key_value_store_e2e(
 
     # We simulate another clean run, we expect the memory storage to read from the local data directory
     # Default storages are purged based on purge_on_start parameter.
-    reset_globals()
+    prepare_test_env()
 
     # Check if we're using a different memory storage instance
     assert old_client is not service_container.get_storage_client()
@@ -54,7 +54,7 @@ async def test_actor_memory_storage_client_key_value_store_e2e(
 async def test_actor_memory_storage_client_request_queue_e2e(
     monkeypatch: pytest.MonkeyPatch,
     purge_on_start: bool,  # noqa: FBT001
-    reset_globals: Callable[[], None],
+    prepare_test_env: Callable[[], None],
 ) -> None:
     """This test simulates two clean runs using memory storage.
     The second run attempts to access data created by the first one.
@@ -82,7 +82,7 @@ async def test_actor_memory_storage_client_request_queue_e2e(
 
     # We simulate another clean run, we expect the memory storage to read from the local data directory
     # Default storages are purged based on purge_on_start parameter.
-    reset_globals()
+    prepare_test_env()
 
     # Add some more requests to the default queue
     default_queue = await RequestQueue.open()

tests/unit/basic_crawler/test_basic_crawler.py

@@ -798,7 +798,7 @@ async def handler(context: BasicCrawlingContext) -> None:
 
 async def test_respects_no_persist_storage() -> None:
     config = Configuration(persist_storage=False)
-    service_container.set_configuration(config, force=True)
+    service_container.set_configuration(config)
     crawler = BasicCrawler()
 
     @crawler.router.default_handler

tests/unit/conftest.py

@@ -18,47 +18,96 @@
 from crawlee.storages import _creation_management
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator
+    from collections.abc import AsyncGenerator, Generator
     from pathlib import Path
 
 
 @pytest.fixture
-def reset_globals(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Callable[[], None]:
-    def reset() -> None:
-        # Set the environment variable for the local storage directory to the temporary path
+def prepare_test_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Callable[[], None]:
+    """Prepare the testing environment by resetting the global state before each test.
+
+    This fixture ensures that the global state of the package is reset to a known baseline before each test runs.
+    It also configures a temporary storage directory for test isolation.
+
+    Args:
+        monkeypatch: Test utility provided by pytest for patching.
+        tmp_path: A unique temporary directory path provided by pytest for test isolation.
+
+    Returns:
+        A callable that prepares the test environment.
+    """
+
+    def _prepare_test_env() -> None:
+        # Set the environment variable for the local storage directory to the temporary path.
         monkeypatch.setenv('CRAWLEE_STORAGE_DIR', str(tmp_path))
 
-        # Reset services in crawlee.service_container
-        service_container.set_configuration(Configuration(), force=True)
-        service_container.set_storage_client(MemoryStorageClient(), force=True)
-        service_container.set_event_manager(LocalEventManager(), force=True)
+        # Initialize services in the service container with default values.
+        service_container.set_configuration(Configuration())
+        service_container.set_storage_client(MemoryStorageClient())
+        service_container.set_event_manager(LocalEventManager())
+
+        # Reset the global state flags in the service locator.
+        service_container._service_locator._configuration_was_set = False
+        service_container._service_locator._storage_client_was_set = False
+        service_container._service_locator._event_manager_was_set = False
 
-        # Clear creation-related caches to ensure no state is carried over between tests
+        # Clear creation-related caches to ensure no state is carried over between tests.
         monkeypatch.setattr(_creation_management, '_cache_dataset_by_id', {})
         monkeypatch.setattr(_creation_management, '_cache_dataset_by_name', {})
         monkeypatch.setattr(_creation_management, '_cache_kvs_by_id', {})
         monkeypatch.setattr(_creation_management, '_cache_kvs_by_name', {})
         monkeypatch.setattr(_creation_management, '_cache_rq_by_id', {})
         monkeypatch.setattr(_creation_management, '_cache_rq_by_name', {})
 
-        # Verify that the environment variable is set correctly
+        # Verify that the test environment was set up correctly.
         assert os.environ.get('CRAWLEE_STORAGE_DIR') == str(tmp_path)
+        assert service_container._service_locator.configuration_was_set is False
+        assert service_container._service_locator.storage_client_was_set is False
+        assert service_container._service_locator.event_manager_was_set is False
 
-    return reset
+    return _prepare_test_env
+
+
+@pytest.fixture
+def cleanup_test_env() -> Callable[[], None]:
+    """Clean up and reset global state after a test has completed.
+
+    This fixture ensures that any modifications to the global state during a test are undone
+    after the test finishes. Restoring the environment for the subsequent tests.
+
+    Returns:
+        A callable that cleans up the test environment.
+    """
+
+    def _cleanup_test_env() -> None:
+        # Reset the flags in the service locator to indicate no services are explicitly set.
+        service_container._service_locator._configuration_was_set = False
+        service_container._service_locator._storage_client_was_set = False
+        service_container._service_locator._event_manager_was_set = False
+
+    return _cleanup_test_env
 
 
 @pytest.fixture(autouse=True)
-def _isolate_test_environment(reset_globals: Callable[[], None]) -> None:
-    """Isolate tests by resetting the storage clients, clearing caches, and setting the environment variables.
+def _isolate_test_environment(
+    prepare_test_env: Callable[[], None],
+    cleanup_test_env: Callable[[], None],
+) -> Generator[None, None, None]:
+    """Isolate the testing environment by resetting global state before and after each test.
 
-    The fixture is applied automatically to all test cases.
+    This fixture ensures that each test starts with a clean slate and that any modifications during the test
+    do not affect subsequent tests. It runs automatically for all tests.
 
     Args:
-        monkeypatch: Test utility provided by pytest.
-        tmp_path: A unique temporary directory path provided by pytest for test isolation.
-    """
+        prepare_test_env: Fixture to prepare the environment before each test.
+        cleanup_test_env: Fixture to clean up the environment after each test.
 
-    reset_globals()
+    Yields:
+        None. This fixture works as a setup and teardown mechanism.
+    """
+    prepare_test_env()
+    yield
+    cleanup_test_env()
 
 
 @pytest.fixture

tests/unit/sessions/test_session_pool.py

@@ -114,7 +114,7 @@ async def test_create_session_function() -> None:
 
 async def test_session_pool_persist(event_manager: EventManager, kvs: KeyValueStore) -> None:
     """Test persistence of session pool state to KVS and validate stored data integrity."""
-    service_container.set_event_manager(event_manager, force=True)
+    service_container.set_event_manager(event_manager)
 
     async with SessionPool(
         max_pool_size=MAX_POOL_SIZE,
@@ -145,7 +145,7 @@ async def test_session_pool_persist(event_manager: EventManager, kvs: KeyValueSt
 
 async def test_session_pool_persist_and_restore(event_manager: EventManager, kvs: KeyValueStore) -> None:
     """Check session pool's ability to persist its state and then restore it accurately after reset."""
-    service_container.set_event_manager(event_manager, force=True)
+    service_container.set_event_manager(event_manager)
 
     async with SessionPool(
         max_pool_size=MAX_POOL_SIZE,

tests/unit/test_configuration.py

@@ -32,7 +32,7 @@ async def test_storage_not_persisted_when_disabled(tmp_path: Path, httpbin: URL)
         crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
     )
     storage_client = MemoryStorageClient(config)
-    set_storage_client(storage_client, force=True)
+    set_storage_client(storage_client)
 
     crawler = HttpCrawler()
 
@@ -54,7 +54,7 @@ async def test_storage_persisted_when_enabled(tmp_path: Path, httpbin: URL) -> N
         crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
     )
     storage_client = MemoryStorageClient(config)
-    set_storage_client(storage_client, force=True)
+    set_storage_client(storage_client)
 
     crawler = HttpCrawler()
 

tests/unit/test_service_container.py

@@ -22,38 +22,35 @@ def test_configuration() -> None:
     assert config == default_config
 
     custom_config = Configuration(default_browser_path='custom_path')
+    set_configuration(custom_config)
+    config = get_configuration()
+    assert config == custom_config
 
     with pytest.raises(ServiceConflictError, match='Configuration has already been set.'):
         set_configuration(custom_config)
 
-    set_configuration(custom_config, force=True)
-    config = get_configuration()
-    assert config == custom_config
-
 
 def test_event_manager() -> None:
     default_event_manager = get_event_manager()
     assert isinstance(default_event_manager, LocalEventManager)
 
     custom_event_manager = LocalEventManager()
+    set_event_manager(custom_event_manager)
+    event_manager = get_event_manager()
+    assert event_manager == custom_event_manager
 
     with pytest.raises(ServiceConflictError, match='EventManager has already been set.'):
         set_event_manager(custom_event_manager)
 
-    set_event_manager(custom_event_manager, force=True)
-    event_manager = get_event_manager()
-    assert event_manager == custom_event_manager
-
 
 def test_storage_client() -> None:
     default_storage_client = get_storage_client()
     assert isinstance(default_storage_client, MemoryStorageClient)
 
     custom_storage_client = MemoryStorageClient()
+    set_storage_client(custom_storage_client)
+    storage_client = get_storage_client()
+    assert storage_client == custom_storage_client
 
     with pytest.raises(ServiceConflictError, match='StorageClient has already been set.'):
         set_storage_client(custom_storage_client)
-
-    set_storage_client(custom_storage_client, force=True)
-    storage_client = get_storage_client()
-    assert storage_client == custom_storage_client