param utils and tests restructure

Author: salabi

digital_land/plugins/arcgis.py

@@ -1,17 +1,24 @@
 import json
 import logging
 import time
-from utils.validate_parameter_utils import validate_parameters
 from esridump.dumper import EsriDumper
+from utils.validate_parameter_utils import non_negative_int
+from utils.validate_parameter_utils import positive_int
+from utils.validate_parameter_utils import validate_plugin_parameters
 
 DEFAULT_TIMEOUT = 60
 DEFAULT_RETRIES = 2
 DEFAULT_RETRY_BACKOFF_SECONDS = 2
-ALLOWED_ARCGIS_PARAMETERS = {
-    "max_page_size",
-    "timeout",
-    "retries",
-    "retry_backoff_seconds",
+ARCGIS_PARAMETER_DEFAULTS = {
+    "timeout": DEFAULT_TIMEOUT,
+    "retries": DEFAULT_RETRIES,
+    "retry_backoff_seconds": DEFAULT_RETRY_BACKOFF_SECONDS,
+}
+ARCGIS_PARAMETER_VALIDATORS = {
+    "max_page_size": positive_int,
+    "timeout": positive_int,
+    "retries": non_negative_int,
+    "retry_backoff_seconds": positive_int,
 }
 
 
@@ -89,50 +96,10 @@ def get(collector, url, log={}, plugin="arcgis", parameters=None):
     return log, content
 
 
-# def validate_parameters(parameters):
-#     if parameters is None:
-#         parameters = {}
-
-#     if not isinstance(parameters, dict):
-#         raise ValueError("ArcGIS parameters must be a dictionary")
-
-#     unknown = set(parameters) - ALLOWED_ARCGIS_PARAMETERS
-#     if unknown:
-#         raise ValueError(
-#             f"Unsupported ArcGIS parameters: {sorted(unknown)}. "
-#             f"Allowed parameters: {sorted(ALLOWED_ARCGIS_PARAMETERS)}"
-#         )
-
-#     validated = {
-#         "timeout": DEFAULT_TIMEOUT,
-#         "retries": DEFAULT_RETRIES,
-#         "retry_backoff_seconds": DEFAULT_RETRY_BACKOFF_SECONDS,
-#     }
-
-#     if "max_page_size" in parameters:
-#         value = parameters["max_page_size"]
-#         if not isinstance(value, int) or value <= 0:
-#             raise ValueError("ArcGIS parameter 'max_page_size' must be a positive integer")
-#         validated["max_page_size"] = value
-
-#     if "timeout" in parameters:
-#         value = parameters["timeout"]
-#         if not isinstance(value, int) or value <= 0:
-#             raise ValueError("ArcGIS parameter 'timeout' must be a positive integer")
-#         validated["timeout"] = value
-
-#     if "retries" in parameters:
-#         value = parameters["retries"]
-#         if not isinstance(value, int) or value < 0:
-#             raise ValueError("ArcGIS parameter 'retries' must be a non-negative integer")
-#         validated["retries"] = value
-
-#     if "retry_backoff_seconds" in parameters:
-#         value = parameters["retry_backoff_seconds"]
-#         if not isinstance(value, int) or value <= 0:
-#             raise ValueError(
-#                 "ArcGIS parameter 'retry_backoff_seconds' must be a positive integer"
-#             )
-#         validated["retry_backoff_seconds"] = value
-
-#     return validated
+def validate_parameters(parameters):
+    return validate_plugin_parameters(
+        parameters=parameters,
+        plugin_name="ArcGIS",
+        defaults=ARCGIS_PARAMETER_DEFAULTS,
+        validators=ARCGIS_PARAMETER_VALIDATORS,
+    )

digital_land/utils/validate_parameter_utils.py

@@ -1,47 +1,40 @@
-def validate_parameters(parameters):
+def positive_int(value, plugin_name, parameter_name):
+    if not isinstance(value, int) or value <= 0:
+        raise ValueError(
+            f"{plugin_name} parameter '{parameter_name}' must be a positive integer"
+        )
+    return value
+
+
+def non_negative_int(value, plugin_name, parameter_name):
+    if not isinstance(value, int) or value < 0:
+        raise ValueError(
+            f"{plugin_name} parameter '{parameter_name}' must be a non-negative integer"
+        )
+    return value
+
+
+def validate_plugin_parameters(parameters, plugin_name, defaults, validators):
     if parameters is None:
         parameters = {}
 
     if not isinstance(parameters, dict):
-        raise ValueError("ArcGIS parameters must be a dictionary")
+        raise ValueError(f"{plugin_name} parameters must be a dictionary")
 
-    unknown = set(parameters) - ALLOWED_ARCGIS_PARAMETERS
+    allowed_parameters = set(validators)
+    unknown = set(parameters) - allowed_parameters
     if unknown:
         raise ValueError(
-            f"Unsupported ArcGIS parameters: {sorted(unknown)}. "
-            f"Allowed parameters: {sorted(ALLOWED_ARCGIS_PARAMETERS)}"
+            f"Unsupported {plugin_name} parameters: {sorted(unknown)}. "
+            f"Allowed parameters: {sorted(allowed_parameters)}"
         )
 
-    validated = {
-        "timeout": DEFAULT_TIMEOUT,
-        "retries": DEFAULT_RETRIES,
-        "retry_backoff_seconds": DEFAULT_RETRY_BACKOFF_SECONDS,
-    }
-
-    if "max_page_size" in parameters:
-        value = parameters["max_page_size"]
-        if not isinstance(value, int) or value <= 0:
-            raise ValueError("ArcGIS parameter 'max_page_size' must be a positive integer")
-        validated["max_page_size"] = value
-
-    if "timeout" in parameters:
-        value = parameters["timeout"]
-        if not isinstance(value, int) or value <= 0:
-            raise ValueError("ArcGIS parameter 'timeout' must be a positive integer")
-        validated["timeout"] = value
-
-    if "retries" in parameters:
-        value = parameters["retries"]
-        if not isinstance(value, int) or value < 0:
-            raise ValueError("ArcGIS parameter 'retries' must be a non-negative integer")
-        validated["retries"] = value
-
-    if "retry_backoff_seconds" in parameters:
-        value = parameters["retry_backoff_seconds"]
-        if not isinstance(value, int) or value <= 0:
-            raise ValueError(
-                "ArcGIS parameter 'retry_backoff_seconds' must be a positive integer"
+    validated = defaults.copy()
+
+    for parameter_name, validator in validators.items():
+        if parameter_name in parameters:
+            validated[parameter_name] = validator(
+                parameters[parameter_name], plugin_name, parameter_name
             )
-        validated["retry_backoff_seconds"] = value
 
     return validated

tests/unit/test_collect.py

@@ -1,5 +1,4 @@
 import hashlib
-import io
 import json
 import csv
 import os
@@ -14,6 +13,8 @@
 from digital_land.collect import Collector, FetchStatus
 from digital_land.plugins.arcgis import get as arcgis_get
 from digital_land.plugins.arcgis import validate_parameters as validate_arcgis_parameters
+from digital_land.utils.validate_parameter_utils import positive_int
+from digital_land.utils.validate_parameter_utils import validate_plugin_parameters
 
 
 @pytest.fixture
@@ -258,6 +259,16 @@ def test_arcgis_validate_parameters_defaults():
     }
 
 
+def test_validate_plugin_parameters_rejects_unknown_parameters():
+    with pytest.raises(ValueError, match="Unsupported Example parameters"):
+        validate_plugin_parameters(
+            parameters={"unknown": 1},
+            plugin_name="Example",
+            defaults={},
+            validators={"page_size": positive_int},
+        )
+
+
 def test_arcgis_get_retries_and_logs_failed_step(monkeypatch):
     attempts = {"count": 0}
     sleep_calls = []
@@ -299,7 +310,7 @@ def __iter__(self):
     assert log["arcgis-timeout-seconds"] == 90
     assert log["arcgis-retries"] == 1
     assert log["arcgis-retry-backoff-seconds"] == 3
-    assert "exception" not in log  
+    assert "exception" not in log
 
 def test_fetch_passes_parameters_to_arcgis_plugin(collector, monkeypatch):
     captured = {}