Merge pull request #43 from DACCS-Climate/keywords-to-types

replace keywords with types and relax restrictions on keywords

Author: mishaschwartz

doc/node_registry.example.json

@@ -41,6 +41,12 @@
           "service-wms",
           "service-wfs"
         ],
+        "types": [
+          "data",
+          "wps",
+          "wms",
+          "wfs"
+        ],
         "description": "GeoServer is a server that allows users to view and edit geospatial data.",
         "links": [
           {
@@ -60,6 +66,9 @@
         "keywords": [
           "service-ogcapi_processes"
         ],
+        "types": [
+          "ogcapi_processes"
+        ],
         "description": "An OGC-API flavored Execution Management Service",
         "links": [
           {

marble_node_registry/migrations.py

@@ -0,0 +1,46 @@
+# Migrations are used to ensure that data provided by nodes that are using an
+# older version of the schema are updated automatically to comply with the newest
+# version of the schema.
+#
+# If a backwards incompatible change is introduced in the schema, please make a 
+# new migration function here to ensure that older data is properly updated.
+#
+# Migration functions will be applied to each node's data in the order that they
+# appear in the MIGRATIONS variable at the bottom of this file.
+#
+# All migration function should take a single argument which contains the node's
+# data and modify that data in place.
+
+def convert_keywords_to_types(data: dict) -> None:
+    """
+    Add service types if they don't exist.
+
+    Since version 1.3.0 service "types" are now required. If they don't exist
+    then they can be derived from service "keywords" which were used in place
+    of "types" prior to this version.
+    """
+
+    keyword2type = {
+        "catalog": "catalog",
+        "data": "data",
+        "jupyterhub": "jupyterhub",
+        "other": "other",
+        "service-wps": "wps",
+        "service-wms": "wms",
+        "service-wfs": "wfs",
+        "service-wcs": "wcs",
+        "service-ogcapi_processes": "ogcapi_processes"
+    }
+    for service in data["services"]:
+        if "types" not in service:
+            service["types"] = []
+            for keyword in service["keywords"]:
+                if (type_ := keyword2type.get(keyword)):
+                    service["types"].append(type_)
+            if not service["types"]:
+                service["types"].append("other")
+
+
+MIGRATIONS = (
+    convert_keywords_to_types,
+)

marble_node_registry/update.py

@@ -6,6 +6,8 @@
 import datetime
 from copy import deepcopy
 
+from migrations import MIGRATIONS
+
 THIS_DIR = os.path.dirname(__file__)
 ROOT_DIR = os.path.dirname(THIS_DIR)
 SCHEMA_FILE = os.path.join(ROOT_DIR, "node_registry.schema.json")
@@ -89,6 +91,15 @@ def update_registry() -> None:
             )
             continue
 
+        try:
+            for migration in MIGRATIONS:
+                migration(data)
+        except Exception as e:
+            registry[name] = org_data
+            registry[name]["status"] = "invalid_configuration"
+            sys.stderr.write(f"unable to apply migrations for Node named {name}: {e}.")
+            continue
+
         try:
             jsonschema.validate(instance=registry, schema=schema)
         except jsonschema.exceptions.ValidationError as e:

node_registry.schema.json

@@ -1,5 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/refs/tags/1.3.0/node_registry.schema.json",
   "patternProperties": {
     "^[a-zA-Z0-9]+$": {
       "type": "object",
@@ -150,6 +151,7 @@
       "type": "object",
       "required": [
         "name",
+        "types",
         "keywords",
         "description",
         "links"
@@ -159,12 +161,36 @@
           "type": "string",
           "minLength": 1
         },
-        "keywords": {
+        "types": {
           "type": "array",
           "minItems": 1,
           "items": {
             "type": "string",
-            "pattern": "^catalog|data|jupyterhub|other|service-(wps|wms|wfs|wcs|ogcapi_processes)$"
+            "enum": [
+              "auth",
+              "management",
+              "catalog",
+              "stac",
+              "data",
+              "jupyterhub",
+              "other",
+              "wps",
+              "wms",
+              "wfs",
+              "wcs",
+              "ogcapi_processes",
+              "ogcapi_dggs",
+              "ogcapi_coverages",
+              "ogcapi_features",
+              "ogcapi_records"
+            ]
+          }
+        },
+        "keywords": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
           }
         },
         "description": {

pyproject.toml

@@ -2,4 +2,4 @@
 addopts = [
     "--import-mode=importlib",
 ]
-pythonpath = "."
+pythonpath = "./marble_node_registry"

tests/test_registry.py

@@ -35,6 +35,7 @@ def registry_content_with_services(registry_content):
         {
             "name": "test-service",
             "keywords": ["other"],
+            "types": ["other"],
             "description": "test service",
             "version": "1.2.3",
             "links": [

tests/test_update.py

@@ -5,12 +5,13 @@
 import jsonschema
 import pytest
 
-from marble_node_registry import update
+import update # type: ignore
 
 GOOD_SERVICES = {
     "services": [
         {
             "name": "geoserver",
+            "types": ["data", "wps", "wms", "wfs"],
             "keywords": ["data", "service-wps", "service-wms", "service-wfs"],
             "description": "GeoServer is a server that allows users to view and edit geospatial data.",
             "links": [
@@ -20,7 +21,8 @@
         },
         {
             "name": "weaver",
-            "keywords": ["service-ogcapi_processes"],
+            "types": ["ogcapi_processes"],
+            "keywords": ["service-ogcapi_processes", "some-other-keyword"],
             "description": "An OGC-API flavored Execution Management Service",
             "links": [
                 {"rel": "service", "type": "application/json", "href": "https://daccs-uoft.example.com/weaver/"},
@@ -265,7 +267,7 @@ def test_last_updated_updated(self, example_node_name, example_registry_content,
             example_node_name
         ].get("last_updated")
 
-    def test_final_registry_valid(self, updated_registry, node_registry_schema):
+    def test_final_registry_validity(self, updated_registry, node_registry_schema):
         jsonschema.validate(instance=updated_registry.call_args.args[0], schema=node_registry_schema)
 
 
@@ -300,7 +302,7 @@ def test_last_updated_no_change(self, example_node_name, example_registry_conten
             example_node_name
         ].get("last_updated")
 
-    def test_final_registry_valid(self, updated_registry, node_registry_schema):
+    def test_final_registry_validity(self, updated_registry, node_registry_schema):
         jsonschema.validate(instance=updated_registry.call_args.args[0], schema=node_registry_schema)
 
 
@@ -350,11 +352,30 @@ class TestOnlineNodeUpdateWithInvalidServices(InvalidResponseTests, NonInitialTe
     services = {"services": [{"bad_key": "some_value"}]}
 
 
-class TestOnlineNodeUpdateWithInvalidServiceKeywords(InvalidResponseTests, NonInitialTests):
-    """Test when updates have previously been run and the reported services keywords are not valid"""
+class TestOnlineNodeUpdateWithInvalidServiceTypes(InvalidResponseTests, NonInitialTests):
+    """Test when updates have previously been run and the reported services types are not valid"""
 
     services = deepcopy(GOOD_SERVICES)
 
     @pytest.fixture(scope="class", autouse=True)
-    def bad_keywords(self):
-        self.services["services"][0]["keywords"] = ["something-bad"]
+    def bad_types(self):
+        self.services["services"][0]["types"] = ["something-bad"]
+
+
+class TestOnlineNodeUpdateWithNoTypes(ValidResponseTests, NonInitialTests):
+    """
+    Test when updates have previously been run and there are no services types
+
+    This ensures that service types are updated as expected from the provided keywords
+    """
+
+    services = deepcopy(GOOD_SERVICES)
+
+    @pytest.fixture(scope="class", autouse=True)
+    def no_types(self):
+        for service in self.services["services"]:
+            service.pop("types")
+
+    def test_services_updated(self, example_node_name, updated_registry):
+        """Test that the services values are updated"""
+        assert updated_registry.call_args.args[0][example_node_name]["services"] == GOOD_SERVICES["services"]