refactor: Add util function for checking kwarg count (#1423)

### Description

- Add a simple util function to limit the number of kwargs and reuse it
in the code.


### Testing

- Added unit tests.

Author: Pijukatel

src/crawlee/_utils/raise_if_too_many_kwargs.py

@@ -0,0 +1,12 @@
+from typing import Any
+
+
+def raise_if_too_many_kwargs(max_kwargs: int = 1, **kwargs: Any) -> None:
+    """Raise ValueError if there are more non-None kwargs then max_kwargs."""
+    none_kwargs_names = [f'"{kwarg_name}"' for kwarg_name, value in kwargs.items() if value is not None]
+    if len(none_kwargs_names) > max_kwargs:
+        all_kwargs_names = [f'"{kwarg_name}"' for kwarg_name in kwargs]
+        raise ValueError(
+            f'Only one of {", ".join(all_kwargs_names)} can be specified, but following arguments were '
+            f'specified: {", ".join(none_kwargs_names)}.'
+        )

src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -14,6 +14,7 @@
 from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import atomic_write, json_dumps
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
@@ -114,9 +115,7 @@ async def open(
                 or if both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         dataset_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 

src/crawlee/storage_clients/_file_system/_key_value_store_client.py

@@ -15,6 +15,7 @@
 from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import atomic_write, infer_mime_type, json_dumps
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
@@ -113,9 +114,7 @@ async def open(
                 or if both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         kvs_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 

src/crawlee/storage_clients/_file_system/_request_queue_client.py

@@ -17,6 +17,7 @@
 from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import atomic_write, json_dumps
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee._utils.recoverable_state import RecoverableState
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import (
@@ -164,9 +165,7 @@ async def open(
                 or if both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         rq_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 

src/crawlee/storage_clients/_memory/_dataset_client.py

@@ -7,6 +7,7 @@
 from typing_extensions import override
 
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
@@ -76,9 +77,7 @@ async def open(
             ValueError: If both name and alias are provided, or if neither id, name, nor alias is provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         # Create a new dataset
         dataset_id = id or crypto_random_object_id()

src/crawlee/storage_clients/_memory/_key_value_store_client.py

@@ -8,6 +8,7 @@
 
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import infer_mime_type
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
@@ -74,9 +75,7 @@ async def open(
             ValueError: If both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         # Create a new key-value store
         store_id = id or crypto_random_object_id()

src/crawlee/storage_clients/_memory/_request_queue_client.py

@@ -10,6 +10,7 @@
 
 from crawlee import Request
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
 
@@ -86,9 +87,7 @@ async def open(
             ValueError: If both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         # Create a new queue
         queue_id = id or crypto_random_object_id()

src/crawlee/storages/_storage_instance_manager.py

@@ -5,6 +5,7 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, TypeVar
 
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, RequestQueueClient
 
 if TYPE_CHECKING:
@@ -107,13 +108,11 @@ async def open_storage_instance(
                 )
 
             # Validate input parameters.
-            specified_params = sum(1 for param in [id, name, alias] if param is not None)
-            if specified_params > 1:
-                raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+            raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
             # Auto-set alias='default' when no parameters are specified.
             # Default unnamed storage is equal to alias=default unnamed storage.
-            if specified_params == 0:
+            if not any([name, alias, id]):
                 alias = self._DEFAULT_STORAGE_ALIAS
 
             # Check cache

tests/unit/_utils/test_raise_if_too_many_kwargs.py

@@ -0,0 +1,38 @@
+from contextlib import nullcontext
+from typing import Any
+
+import pytest
+
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
+
+
+@pytest.mark.parametrize(
+    ('kwargs', 'should_raise'),
+    [
+        ({'alias': 'alias', 'name': None, 'id': None}, False),
+        ({'alias': None, 'name': 'name', 'id': None}, False),
+        ({'alias': None, 'name': None, 'id': 'id'}, False),
+        ({'alias': 'alias', 'name': 'name', 'id': None}, True),
+        ({'alias': 'alias', 'name': None, 'id': 'id'}, True),
+        ({'alias': None, 'name': 'name', 'id': 'id'}, True),
+        ({'alias': 'alias', 'name': 'name', 'id': 'id'}, True),
+        ({'alias': None, 'name': None, 'id': None}, False),
+    ],
+)
+def test_limit_kwargs_default(kwargs: dict[str, Any], *, should_raise: bool) -> None:
+    context = pytest.raises(ValueError, match=r'^Only one of .*') if should_raise else nullcontext()
+    with context:
+        raise_if_too_many_kwargs(**kwargs)
+
+
+@pytest.mark.parametrize(
+    ('kwargs', 'should_raise'),
+    [
+        ({'alias': 'alias', 'name': 'name', 'id': 'id'}, True),
+        ({'alias': 'alias', 'name': 'name', 'id': None}, False),
+    ],
+)
+def test_limit_kwargs(kwargs: dict[str, Any], *, should_raise: bool) -> None:
+    context = pytest.raises(ValueError, match=r'^Only one of .*') if should_raise else nullcontext()
+    with context:
+        raise_if_too_many_kwargs(max_kwargs=2, **kwargs)

tests/unit/storages/test_dataset.py

@@ -150,7 +150,11 @@ async def test_open_with_id_and_name(
     storage_client: StorageClient,
 ) -> None:
     """Test that open() raises an error when both id and name are provided."""
-    with pytest.raises(ValueError, match=r'Only one of "id", "name", or "alias" can be specified, not multiple.'):
+    with pytest.raises(
+        ValueError,
+        match=r'Only one of "id", "name", "alias" can be specified, but following arguments '
+        r'were specified: "id", "name".',
+    ):
         await Dataset.open(
             id='some-id',
             name='some-name',
@@ -651,7 +655,11 @@ async def test_alias_with_id_error(
     storage_client: StorageClient,
 ) -> None:
     """Test that providing both alias and id raises error."""
-    with pytest.raises(ValueError, match=r'Only one of "id", "name", or "alias" can be specified, not multiple.'):
+    with pytest.raises(
+        ValueError,
+        match=r'Only one of "id", "name", "alias" can be specified, but following arguments '
+        r'were specified: "id", "alias".',
+    ):
         await Dataset.open(
             id='some-id',
             alias='some-alias',
@@ -663,7 +671,11 @@ async def test_alias_with_name_error(
     storage_client: StorageClient,
 ) -> None:
     """Test that providing both alias and name raises error."""
-    with pytest.raises(ValueError, match=r'Only one of "id", "name", or "alias" can be specified, not multiple.'):
+    with pytest.raises(
+        ValueError,
+        match=r'Only one of "id", "name", "alias" can be specified, but following arguments '
+        r'were specified: "name", "alias".',
+    ):
         await Dataset.open(
             name='some-name',
             alias='some-alias',
@@ -675,7 +687,11 @@ async def test_alias_with_all_parameters_error(
     storage_client: StorageClient,
 ) -> None:
     """Test that providing id, name, and alias raises error."""
-    with pytest.raises(ValueError, match=r'Only one of "id", "name", or "alias" can be specified, not multiple.'):
+    with pytest.raises(
+        ValueError,
+        match=r'Only one of "id", "name", "alias" can be specified, but following arguments '
+        r'were specified: "id", "name", "alias".',
+    ):
         await Dataset.open(
             id='some-id',
             name='some-name',