split topic validation to be subscription or publication driven (#14)

tomkralidis · web-flow · commit bb5c5dc03cde · 2026-02-05T07:55:29.000-05:00
* split topic validation to be subscription or publication driven

* update README

* cache IANA TLDs
diff --git a/README.md b/README.md
@@ -49,20 +49,31 @@ pywis-topics --version
 pywis-topics bundle sync
 ```
 
-### Listing and validation
+### Listing and validating topics for subscription
 
 ```bash
 # validate a WIS2 topic hierarchy
-pywis-topics topic validate origin/a/wis2/ca-eccc-msc
+pywis-topics topic validate subscription origin/a/wis2/ca-eccc-msc
 
 # validate a WIS2 topic hierarchy in no-strict mode
-pywis-topics topic validate --no-strict origin/a/wis2/fake-centre-id/data/core
+pywis-topics topic validate subscription --no-strict origin/a/wis2/fake-centre-id/data/core
 
 # list children of a given WIS2 topic hierarchy level
 pywis-topics topic list wis2/a
 
 # validate a WIS2 topic hierarchy with wildcards (needs no-strict mode)
-pywis-topics topic validate origin/a/wis2/+/data/core --no-strict
+pywis-topics topic validate subscription origin/a/wis2/+/data/core --no-strict
+```
+
+### Validating topics for publication
+
+```bash
+# validate a WIS2 topic hierarchy
+pywis-topics topic validate publication origin/a/wis2/ca-eccc-msc
+pywis-topics topic validate publication origin/a/wis2/ca-eccc-msc/ocean
+pywis-topics topic validate publication origin/a/wis2/us-noaa-nws/data/core/weather/surface-based-observations/synop
+# validate a WIS2 topic hierarchy in no-strict mode
+pywis-topics topic validate publication --no-strict origin/a/wis2/fake-centre-id/data/core
 ```
 
 ### Centre identification validation
diff --git a/pywis_topics/bundle.py b/pywis_topics/bundle.py
@@ -78,7 +78,7 @@ def sync_bundle() -> None:
                 shutil.copyfileobj(src, dest)
 
     LOGGER.debug('Downloading IANA TLDs')
-    IANA_URL = 'https://data.iana.org/TLD/tlds-alpha-by-domain.txt'
+    IANA_URL = 'https://wmo-im.github.io/wis2-topic-hierarchy/iana-tlds/tlds-alpha-by-domain.txt'  # noqa
     iana_file = WIS2_TOPIC_HIERARCHY_DIR_TEMP / 'tlds-alpha-by-domain.txt'
     with iana_file.open('wb') as fh:
         fh.write(urlopen_(f'{IANA_URL}').read())
diff --git a/pywis_topics/pygeoapi_plugin.py b/pywis_topics/pygeoapi_plugin.py
@@ -138,6 +138,16 @@
     'links': WIS2_TOPIC_HIERARCHY_LINKS,
     'inputs': {
         'topic': WIS2_TOPIC_HIERARCHY_INPUT_TOPIC,
+        'strict': {
+            'type': 'boolean',
+            'default': False,
+            'description': 'Validate in strict mode (default true)'
+        },
+        'publication': {
+            'type': 'boolean',
+            'default': False,
+            'description': 'Topic is publication-based (default false)'
+        }
     },
     'outputs': {
         'result': {
@@ -223,6 +233,8 @@ def execute(self, data):
         response = None
         mimetype = 'application/json'
         topic = data.get('topic')
+        publication = data.get('publication', False)
+        strict = data.get('strict', True)
 
         if topic is None:
             msg = 'Missing topic'
@@ -231,8 +243,11 @@ def execute(self, data):
 
         LOGGER.debug('Querying topic')
         th = TopicHierarchy()
+        topic_is_valid = th.validate(
+            topic, strict=strict, publication=publication)
+
         response = {
-            'topic_is_valid': th.validate(topic)
+            'topic_is_valid': topic_is_valid
         }
         return mimetype, response
 
diff --git a/pywis_topics/topics.py b/pywis_topics/topics.py
@@ -125,13 +125,15 @@ def list_children(self, topic_hierarchy: str = None) -> List[str]:
         return matches
 
     def validate(self, topic_hierarchy: str = None,
-                 strict: bool = True) -> bool:
+                 strict: bool = True, publication: bool = False) -> bool:
         """
         Validates a topic hierarchy
 
         :param topic_hierarchy: `str` of topic hierarchy
         :param strict: `bool` of whether to perform strict validation,
                        including centre-id
+        :param publication: `bool` of whether to perform validation
+                            for publication
 
         :returns: `bool` of whether topic hierarchy is valid
         """
@@ -149,6 +151,27 @@ def validate(self, topic_hierarchy: str = None,
         core_tokens = all_tokens[:6]
         esd_subtopic = '/'.join(all_tokens[6:])
 
+        if publication:
+            if any(c in topic_hierarchy for c in ['#', '+']):
+                msg = 'Invalid characters for publication'
+                LOGGER.warning(msg)
+                return False
+
+            if len(core_tokens) < 5:
+                msg = 'Not enough tokens for publication'
+                LOGGER.warning(msg)
+                return False
+
+            if core_tokens[4] not in ['data', 'metadata']:
+                msg = 'Invalid token for publication (must be data or metadata)'  # noqa
+                LOGGER.warning(msg)
+                return False
+
+            if core_tokens[-2] == 'data':
+                if esd_subtopic in [None, '']:
+                    LOGGER.debug('Earth system discipline subtopic is empty')
+                    return False
+
         LOGGER.debug(f'Core tokens: {core_tokens}')
         LOGGER.debug(f'Earth system discipline subtopic: {esd_subtopic}')
         LOGGER.debug('Validating core tokens')
@@ -296,6 +319,12 @@ def topic():
     pass
 
 
+@click.group()
+def validate():
+    """Topic hierarchy validation utilities"""
+    pass
+
+
 @click.command('list')
 @click.pass_context
 @get_cli_common_options
@@ -322,8 +351,8 @@ def list_(ctx, topic_hierarchy, logfile, verbosity):
 @click.option('--strict/--no-strict', default=True,
               help='Validate in strict mode')
 @click.argument('topic-hierarchy')
-def validate(ctx, topic_hierarchy, logfile, verbosity, strict=True):
-    """Validate topic hierarchy"""
+def subscription(ctx, topic_hierarchy, logfile, verbosity, strict=True):
+    """Validate topic hierarchy for subscription"""
 
     setup_logger(verbosity, logfile)
 
@@ -335,5 +364,24 @@ def validate(ctx, topic_hierarchy, logfile, verbosity, strict=True):
         click.echo('Invalid')
 
 
+@click.command()
+@click.pass_context
+@get_cli_common_options
+@click.argument('topic-hierarchy')
+def publication(ctx, topic_hierarchy, logfile, verbosity):
+    """Validate topic hierarchy for publication"""
+
+    setup_logger(verbosity, logfile)
+
+    th = TopicHierarchy()
+
+    if th.validate(topic_hierarchy, publication=True):
+        click.echo('Valid')
+    else:
+        click.echo('Invalid')
+
+
 topic.add_command(list_)
 topic.add_command(validate)
+validate.add_command(subscription)
+validate.add_command(publication)
diff --git a/tests/run_tests.py b/tests/run_tests.py
@@ -75,11 +75,16 @@ def tearDown(self):
         """return to pristine state"""
         pass
 
-    def test_validate(self):
+    def test_validate_subscription(self):
         value = None
         with self.assertRaises(ValueError):
             _ = self.th.validate(value)
 
+        valid_topics = [
+            'cache/a/wis2',
+            'cache/a/wis2/ca-eccc-msc/data/core',
+        ]
+
         invalid_topics = [
             'invalid.topic.hierarchy',
             'ORIGIN/A/wis2',
@@ -88,17 +93,12 @@ def test_validate(self):
             'a/wis2'
         ]
 
-        valid_topics = [
-            'cache/a/wis2',
-            'cache/a/wis2/ca-eccc-msc/data/core',
-        ]
+        for valid_topic in valid_topics:
+            self.assertTrue(self.th.validate(valid_topic))
 
         for invalid_topic in invalid_topics:
             self.assertFalse(self.th.validate(invalid_topic))
 
-        for valid_topic in valid_topics:
-            self.assertTrue(self.th.validate(valid_topic))
-
         value = 'cache/a/wis2/fake-centre-id/data/core'
         self.assertTrue(self.th.validate(value, strict=False))
 
@@ -150,6 +150,71 @@ def test_validate(self):
         self.assertTrue(self.th.validate(value, strict=False))
         self.assertFalse(self.th.validate(value))
 
+    def test_validate_publication(self):
+        value = None
+        with self.assertRaises(ValueError):
+            _ = self.th.validate(value)
+
+        valid_topics = [
+            'origin/a/wis2/kz-kazhydromet/data/core/weather/surface-based-observations/synop',  # noqa
+            'origin/a/wis2/uk-metoffice/data/core/ocean/surface-based-observations/drifting-ocean-profilers',  # noqa
+            'origin/a/wis2/ca-eccc-msc/data/core/ocean/experimental'  # noqa
+        ]
+
+        invalid_topics = [
+            'invalid.topic.hierarchy',
+            'ORIGIN/A/wis2',
+            'origin/a/wis2/ca-é',
+            'invalid/topic/hierarchy',
+            'a/wis2',
+            'cache/a/wis2',
+            'origin/a/wis2/ca-eccc-msc/data/core',
+            'cache/a/wis2/ca-eccc-msc/data/core',
+            'cache/a/wis2/ca-eccc-msc/data/core/weather/surface-based-observations1'  # noqa
+        ]
+
+        for valid_topic in valid_topics:
+            self.assertTrue(self.th.validate(valid_topic, publication=True))
+
+        for invalid_topic in invalid_topics:
+            self.assertFalse(self.th.validate(invalid_topic, publication=True))
+
+        value = 'cache/a/wis2/fake-centre-id/data/core'
+        self.assertFalse(self.th.validate(value, strict=False,
+                         publication=True))
+
+        value = 'cache/a/+'
+        self.assertFalse(self.th.validate(value, strict=False,
+                         publication=True))
+
+        value = 'cache/a/#'
+        self.assertFalse(self.th.validate(value, strict=False,
+                         publication=True))
+
+        value = 'cache/a/wis2/+/data/core/#'
+        self.assertFalse(self.th.validate(value, strict=False,
+                         publication=True))
+
+        value = 'cache/a/wis2/+/data/core/weather/#'
+        self.assertFalse(self.th.validate(value, strict=False,
+                         publication=True))
+
+        value = 'cache/a/wis2/+/data/#/weather'
+        self.assertFalse(self.th.validate(value, publication=True))
+
+        value = 'cache/a/wis2/+/data/core/weather/surface-based-observations'
+        self.assertFalse(self.th.validate(value, strict=False,
+                         publication=True))
+
+        value = 'origin/a/wis2/ca-eccc-msc/data/core/ocean'
+        self.assertTrue(self.th.validate(value, publication=True))
+
+        value = 'cache/a/wis2/ca-eccc-msc/data/core/ocean'
+        self.assertTrue(self.th.validate(value, publication=True))
+
+        value = 'cache/a/wis2/io-wis2dev-11-test/data/core/ocean'
+        self.assertTrue(self.th.validate(value, publication=True))
+
     def test_list_children(self):
         value = None
         with self.assertRaises(ValueError):
