STAC model updates (#1183)

* STAC API: add created, updated, platform, instrument and gsd parsing, constrain STAC API queries to item search only
* fix broker/MQTT symbols
* update docs
* update OARec tests

Author: tomkralidis

docs/metadata-model-reference.rst

@@ -224,15 +224,15 @@ Model Crosswalk
      - ``gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date[gmd:dateType/gmd:CI_DateTypeCode/@codeListValue="revision"]/gmd:date/gco:Date``
      - 
      - ``properties.updated``
-     - ``properties.updated``
+     - ``created`` or ``properties.updated``
      - 
    * - ``date_creation``
      - ``pycsw:CreationDate``
      - ``apiso:CreationDate``
      - ``gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date[gmd:dateType/gmd:CI_DateTypeCode/@codeListValue="creation"]/gmd:date/gco:Date``
      - 
      - ``properties.created``
-     - ``properties.created``
+     - ``created`` or ``properties.created``
      - 
    * - ``date_publication``
      - ``pycsw:PublicationDate``
@@ -304,15 +304,15 @@ Model Crosswalk
      - ``gmd:identificationInfo/gmd:MD_DataIdentification/gmd:spatialResolution/gmd:MD_Resolution/gmd:distance/gco:Distance``
      - 
      - 
-     - 
+     - ``gsd`` or ``properties.gsd``
      - 
    * - ``distanceuom``
      - ``pycsw:DistanceUOM``
      - ``apiso:DistanceUOM``
      - ``gmd:identificationInfo/gmd:MD_DataIdentification/gmd:spatialResolution/gmd:MD_Resolution/gmd:distance/gco:Distance/@uom``
      - 
      - 
-     - 
+     - fixed to ``m``
      - 
    * - ``time_begin``
      - ``pycsw:TempExtent_begin``
@@ -504,15 +504,15 @@ Model Crosswalk
      - ``gmi:acquisitionInfo/gmi:MI_AcquisitionInformation/gmi:platform/gmi:MI_Platform/gmi:identifier``
      - 
      - 
-     - 
+     - ``platform`` or ``properties.platform``
      - 
    * - ``instrument``
      - ``pycsw:Instrument``
      - ``apiso:Instrument``
      - ``gmi:acquisitionInfo/gmi:MI_AcquisitionInformation/gmi:platform/gmi:MI_Platform/gmi:instrument/gmi:MI_Instrument/gmi:identifier``
      - 
      - 
-     - 
+     - ``instruments`` or ``properties.instruments``
      - 
    * - ``sensortype``
      - ``pycsw:SensorType``

pycsw/broker/mqtt.py

@@ -45,7 +45,7 @@ def __init__(self, broker_url):
 
         :param publisher_def: provider definition
 
-        :returns: pycsw.pubsub.mqtt.MQTTClient
+        :returns: pycsw.pubsub.mqtt.MQTTPubSubClient
         """
 
         super().__init__(broker_url)
@@ -79,25 +79,28 @@ def __init__(self, broker_url):
     def connect(self) -> None:
         """
         Connect to an MQTT broker
+
         :returns: None
         """
 
         self.conn.connect(self.broker_url.hostname, self.port)
         LOGGER.debug('Connected to broker')
 
-    def pub(self, topic: str, message: str, qos: int = 1) -> bool:
+    def pub(self, channel: str, message: str, qos: int = 1) -> bool:
         """
-        Publish a message to a broker/topic
-        :param topic: `str` of topic
+        Publish a message to a broker/channel
+
+        :param channel: `str` of channel
         :param message: `str` of message
+
         :returns: `bool` of publish result
         """
 
         LOGGER.debug(f'Publishing to broker {self.broker_safe_url}')
-        LOGGER.debug(f'Topic: {topic}')
+        LOGGER.debug(f'Channel: {channel}')
         LOGGER.debug(f'Message: {message}')
 
-        result = self.conn.publish(topic, message, qos)
+        result = self.conn.publish(channel, message, qos)
         LOGGER.debug(f'Result: {result}')
 
         # TODO: investigate implication

pycsw/core/metadata.py

@@ -1504,12 +1504,11 @@ def _parse_iso(context, repos, exml):
             _set(context, recobj, 'pycsw:DistanceValue', md_identification.distance[0])
         if len(md_identification.uom) > 0:
             _set(context, recobj, 'pycsw:DistanceUOM', md_identification.uom[0])
-
         if len(md_identification.classification) > 0:
             _set(context, recobj, 'pycsw:Classification', md_identification.classification[0])
         if len(md_identification.uselimitation) > 0:
             _set(context, recobj, 'pycsw:ConditionApplyingToAccessAndUse',
-            md_identification.uselimitation[0])
+                 md_identification.uselimitation[0])
 
     service_types = []
     from owslib.iso import SV_ServiceIdentification
@@ -1527,14 +1526,13 @@ def _parse_iso(context, repos, exml):
         _set(context, recobj, 'pycsw:SpecificationTitle', md.dataquality.specificationtitle)
         if hasattr(md.dataquality, 'specificationdate'):
             _set(context, recobj, 'pycsw:specificationDate',
-            md.dataquality.specificationdate[0].date)
+                 md.dataquality.specificationdate[0].date)
             _set(context, recobj, 'pycsw:SpecificationDateType',
-            md.dataquality.specificationdate[0].datetype)
+                 md.dataquality.specificationdate[0].datetype)
 
     if hasattr(md, 'contact') and len(md.contact) > 0:
         _set(context, recobj, 'pycsw:ResponsiblePartyRole', md.contact[0].role)
 
-
     if hasattr(md, 'contentinfo') and len(md.contentinfo) > 0:
         for ci in md.contentinfo:
             if isinstance(ci, MD_ImageDescription):
@@ -1545,9 +1543,9 @@ def _parse_iso(context, repos, exml):
                 if ci.processing_level is not None:
                     pl_keyword = 'eo:processingLevel:' + ci.processing_level
                     if keywords is None:
-                        keywords  = pl_keyword
+                        keywords = pl_keyword
                     else:
-                        keywords  += ',' + pl_keyword
+                        keywords += ',' + pl_keyword
 
                     _set(context, recobj, 'pycsw:Keywords', keywords)
 
@@ -1573,7 +1571,7 @@ def _parse_iso(context, repos, exml):
             _set(context, recobj, 'pycsw:Instrument', instrument.identifier)
             _set(context, recobj, 'pycsw:SensorType', instrument.type)
 
-    all_formats=[]
+    all_formats = []
     if hasattr(md.distribution, 'format') and md.distribution.format is not None:
         all_formats.append(md.distribution.format)
 
@@ -1616,11 +1614,11 @@ def _parse_iso(context, repos, exml):
                             'url': scpt.url
                         }
                         links.append(linkobj)
-    except Exception as err:  # srv: identification does not exist
+    except Exception:  # srv: identification does not exist
         LOGGER.exception('no srv:SV_ServiceIdentification links found')
 
     if hasattr(md_identification, 'graphicoverview'):
-        for thumb in  md_identification.graphicoverview:
+        for thumb in md_identification.graphicoverview:
             links.append({
                 'name': 'preview',
                 'description': 'Web image thumbnail (URL)',
@@ -1643,6 +1641,7 @@ def _parse_iso(context, repos, exml):
 
     return recobj
 
+
 def _parse_dc(context, repos, exml):
 
     from owslib.csw import CswRecord
@@ -1830,6 +1829,15 @@ def _parse_stac_resource(context, repos, record):
         stype = 'item'
         title = record['properties'].get('title')
         abstract = record['properties'].get('description')
+        _set(context, recobj, 'pycsw:CreationDate', record['properties'].get('created'))
+        _set(context, recobj, 'pycsw:Modified', record['properties'].get('updated'))
+        _set(context, recobj, 'pycsw:Platform', record['properties'].get('platform'))
+        instruments = record['properties'].get('instruments')
+        if instruments is not None:
+            _set(context, recobj, 'pycsw:Instrument', ','.join(instruments))
+        if record['properties'].get('gsd') is not None:
+            _set(context, recobj, 'pycsw:DistanceValue', record['properties']['gsd'])
+            _set(context, recobj, 'pycsw:DistanceUOM', 'm')
         if record.get('geometry') is not None:
             bbox_wkt = util.bbox2wktpolygon(util.geojson_geometry2bbox(record['geometry']))
     elif stac_type == 'Collection':
@@ -1840,6 +1848,15 @@ def _parse_stac_resource(context, repos, record):
         stype = 'collection'
         title = record.get('title')
         abstract = record.get('description')
+        _set(context, recobj, 'pycsw:CreationDate', record.get('created'))
+        _set(context, recobj, 'pycsw:Modified', record.get('updated'))
+        _set(context, recobj, 'pycsw:Platform', record.get('platform'))
+        instruments = record.get('instruments')
+        if instruments is not None:
+            _set(context, recobj, 'pycsw:Instrument', ','.join(instruments))
+        if record.get('gsd') is not None:
+            _set(context, recobj, 'pycsw:DistanceValue', record['gsd'])
+            _set(context, recobj, 'pycsw:DistanceUOM', 'm')
         if 'extent' in record and 'spatial' in record['extent']:
             bbox_csv = ','.join(str(t) for t in record['extent']['spatial']['bbox'][0])
             bbox_wkt = util.bbox2wktpolygon(bbox_csv)
@@ -1854,6 +1871,12 @@ def _parse_stac_resource(context, repos, record):
         stype = 'catalog'
         title = record.get('title')
         abstract = record.get('description')
+        _set(context, recobj, 'pycsw:CreationDate', record.get('created'))
+        _set(context, recobj, 'pycsw:Modified', record.get('updated'))
+        _set(context, recobj, 'pycsw:Platform', record.get('platform'))
+        instruments = record.get('instruments')
+        if instruments is not None:
+            _set(context, recobj, 'pycsw:Instrument', ','.join(instruments))
 
     _set(context, recobj, 'pycsw:Identifier', record['id'])
     _set(context, recobj, 'pycsw:Typename', typename)
@@ -1952,22 +1975,25 @@ def _parse_stac_resource(context, repos, record):
 
     return recobj
 
+
 def fgdccontact2iso(cnt, role='pointOfContact'):
     """Creates a iso format contact (owslib style) from fgdc format"""
 
-    return {'name': cnt.cntper, 
-            'organization': cnt.cntorg, 
-            'position': cnt.cntpos, 
-            'phone': cnt.voice, 
-            'address': cnt.address, 
-            'city': cnt.city, 
-            'region': cnt.state,
-            'postcode': cnt.postal, 
-            'country': cnt.country, 
-            'email': cnt.email, 
-            'role': role
+    return {
+        'name': cnt.cntper,
+        'organization': cnt.cntorg,
+        'position': cnt.cntpos,
+        'phone': cnt.voice,
+        'address': cnt.address,
+        'city': cnt.city,
+        'region': cnt.state,
+        'postcode': cnt.postal,
+        'country': cnt.country,
+        'email': cnt.email,
+        'role': role
     }
 
+
 def caps2iso(record, caps, context):
     """Creates ISO metadata from Capabilities XML"""
 

pycsw/core/pygeofilter_evaluate.py

@@ -84,6 +84,23 @@ def bbox(self, node, lhs):
         else:
             return text(f"query_spatial({geometry}, '{wkt}', 'bbox', 'false') = 'true'")  # noqa
 
+    @handle(ast.Equal)
+    def equal(self, node, lhs, rhs):
+        list_props = [
+            'dataset.keywords',
+            'dataset.instrument'
+        ]
+
+        if str(lhs.prop) in list_props:
+            LOGGER.debug(f'Overriding {lhs.prop} "=" with "ilike"')
+            node.pattern = f'%{rhs}%'
+            node.nocase = False
+            node.not_ = False
+
+            return self.ilike(node, lhs)
+
+        return filters.runop(lhs, rhs, '=')
+
     @handle(ast.Like)
     def ilike(self, node, lhs):
         LOGGER.debug('Overriding ILIKE filter handling')

pycsw/core/repository.py

@@ -159,6 +159,7 @@ def __init__(self, database, context, app_root=None, table='records', repo_filte
             'bbox': self.dataset.wkt_geometry,
             'date': self.dataset.date,
             'date_creation': self.dataset.date_creation,
+            'date_modified': self.dataset.date_modified,
             'datetime': self.dataset.date,
             'time_begin': self.dataset.time_begin,
             'time_end': self.dataset.time_end,

pycsw/stac/api.py

@@ -506,18 +506,45 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
 
         if not json_post_data and not cql_ops:
             LOGGER.debug('No JSON POST data or CQL ops')
+            args['type'] = 'item'
         elif not json_post_data and cql_ops:
             LOGGER.debug('No JSON POST data left')
+            cql_ops.append({
+                'op': '=',
+                'args': [
+                    {'property': 'type'},
+                    'item'
+                ]
+            })
             json_post_data = {
                 'op': 'and',
                 'args': cql_ops
             }
         else:
             LOGGER.debug('JSON POST data is CQL2 JSON')
-            if cql_ops:
-                print("JJ2", json_post_data)
-                LOGGER.debug('Adding STAC API query parameters to CQL2 JSON')
+            cql_ops.append({
+                'op': '=',
+                'args': [
+                    {'property': 'type'},
+                    'item'
+                ]
+            })
+            LOGGER.debug('Adding STAC API query parameters to CQL2 JSON')
+            if json_post_data.get('op') in ['and', 'or']:
                 json_post_data['args'].extend(cql_ops)
+            else:
+                op_, args_ = json_post_data.get('op'), json_post_data.get('args')
+                if None not in [op_, args_]:
+                    cql_ops.append({
+                        'op': op_,
+                        'args': args_,
+                    })
+                    json_post_data = {
+                        'op': 'and',
+                        'args': cql_ops
+                    }
+                else:
+                    json_post_data.update(*cql_ops)
 
         headers, status, response = super().items(headers_, json_post_data, args, collection)
 
@@ -526,7 +553,7 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
         response2['features'] = []
 
         LOGGER.debug('Filtering on STAC items')
-        for record in response['features']:
+        for record in response.get('features', []):
             if record.get('stac_version') is None:
                 record['links'].extend([{
                     'rel': 'self',
@@ -559,7 +586,7 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
 
         links2 = []
 
-        for link in response2['links']:
+        for link in response2.get('links', []):
             if link['rel'] in ['alternate', 'collection']:
                 continue
             link['href'] = link['href'].replace('collections/metadata:main/items', 'search')
@@ -574,6 +601,10 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'):
             }
         ])
 
+        if 'code' in response2:
+            response2.pop('features')
+            response2.pop('links')
+
         return self.get_response(status, headers, response2)
 
     def item(self, headers_, args, collection, item):

tests/functionaltests/suites/oarec/test_oarec_functional.py

@@ -108,7 +108,7 @@ def test_queryables(config):
     assert content['$id'] == 'http://localhost/pycsw/oarec/collections/metadata:main/queryables'  # noqa
     assert content['$schema'] == 'http://json-schema.org/draft/2019-09/schema'
 
-    assert len(content['properties']) == 16
+    assert len(content['properties']) == 17
 
     assert 'geometry' in content['properties']
     assert content['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Polygon.json'  # noqa

tests/functionaltests/suites/oarec/test_oarec_virtual_collections_functional.py

@@ -116,7 +116,7 @@ def test_queryables(config_virtual_collections):
     assert content['$id'] == 'http://localhost/pycsw/oarec/collections/metadata:main/queryables'  # noqa
     assert content['$schema'] == 'http://json-schema.org/draft/2019-09/schema'
 
-    assert len(content['properties']) == 16
+    assert len(content['properties']) == 17
 
     assert 'geometry' in content['properties']
     assert content['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Polygon.json'  # noqa