Skip to content

Commit 4b3e058

Browse files
tomkralidiskalxas
tomkralidis
and
kalxas
authored
set cloudcover and gsd as floats in repository (#1181)
* set cloudcover as float in repository * update distancevalue to float * Parse gsd as float * Parse md_identification.distance as float * add migrations and test DB updates --------- Co-authored-by: Angelos Tzotsos <gcpp.kalxas@gmail.com>
1 parent 31c0631 commit 4b3e058

File tree

10 files changed

+55
-34
lines changed

10 files changed

+55
-34
lines changed

‎etc/migrations/2.x-3.0/migrate.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,6 @@ alter table records add column edition TEXT;
44
alter table records add column contacts TEXT;
55
alter table records add column themes TEXT;
66
alter table records add column illuminationelevationangle TEXT;
7+
alter table records alter column cloudcover type real using cast(cloudcover as float);
8+
alter table records alter column distancevalue type real using cast(distancevalue as float);
79
vacuum;

‎pycsw/core/metadata.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1501,7 +1501,7 @@ def _parse_iso(context, repos, exml):
15011501
if len(md_identification.denominators) > 0:
15021502
_set(context, recobj, 'pycsw:Denominator', md_identification.denominators[0])
15031503
if len(md_identification.distance) > 0:
1504-
_set(context, recobj, 'pycsw:DistanceValue', md_identification.distance[0])
1504+
_set(context, recobj, 'pycsw:DistanceValue', float(md_identification.distance[0]))
15051505
if len(md_identification.uom) > 0:
15061506
_set(context, recobj, 'pycsw:DistanceUOM', md_identification.uom[0])
15071507
if len(md_identification.classification) > 0:
@@ -1536,7 +1536,7 @@ def _parse_iso(context, repos, exml):
15361536
if hasattr(md, 'contentinfo') and len(md.contentinfo) > 0:
15371537
for ci in md.contentinfo:
15381538
if isinstance(ci, MD_ImageDescription):
1539-
_set(context, recobj, 'pycsw:CloudCover', ci.cloud_cover)
1539+
_set(context, recobj, 'pycsw:CloudCover', float(ci.cloud_cover))
15401540
_set(context, recobj, 'pycsw:IlluminationElevationAngle', ci.illumination_elevation_angle)
15411541

15421542
keywords = _get(context, recobj, 'pycsw:Keywords')
@@ -1836,7 +1836,7 @@ def _parse_stac_resource(context, repos, record):
18361836
if instruments is not None:
18371837
_set(context, recobj, 'pycsw:Instrument', ','.join(instruments))
18381838
if record['properties'].get('gsd') is not None:
1839-
_set(context, recobj, 'pycsw:DistanceValue', record['properties']['gsd'])
1839+
_set(context, recobj, 'pycsw:DistanceValue', float(record['properties']['gsd']))
18401840
_set(context, recobj, 'pycsw:DistanceUOM', 'm')
18411841
if record.get('geometry') is not None:
18421842
bbox_wkt = util.bbox2wktpolygon(util.geojson_geometry2bbox(record['geometry']))
@@ -1855,7 +1855,7 @@ def _parse_stac_resource(context, repos, record):
18551855
if instruments is not None:
18561856
_set(context, recobj, 'pycsw:Instrument', ','.join(instruments))
18571857
if record.get('gsd') is not None:
1858-
_set(context, recobj, 'pycsw:DistanceValue', record['gsd'])
1858+
_set(context, recobj, 'pycsw:DistanceValue', float(record['gsd']))
18591859
_set(context, recobj, 'pycsw:DistanceUOM', 'm')
18601860
if 'extent' in record and 'spatial' in record['extent']:
18611861
bbox_csv = ','.join(str(t) for t in record['extent']['spatial']['bbox'][0])
@@ -1949,7 +1949,7 @@ def _parse_stac_resource(context, repos, record):
19491949
_set(context, recobj, 'pycsw:TempExtent_end', record['properties']['datetime'])
19501950

19511951
if 'eo:cloud_cover' in record['properties']:
1952-
_set(context, recobj, 'pycsw:CloudCover', record['properties']['eo:cloud_cover'])
1952+
_set(context, recobj, 'pycsw:CloudCover', float(record['properties']['eo:cloud_cover']))
19531953

19541954
if 'collection' in record:
19551955
_set(context, recobj, 'pycsw:ParentIdentifier', record['collection'])

‎pycsw/core/repository.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,8 @@ def __init__(self, database, context, app_root=None, table='records', repo_filte
167167
'cloudcover': self.dataset.cloudcover,
168168
'instrument': self.dataset.instrument,
169169
'sensortype': self.dataset.sensortype,
170-
'off_nadir': self.dataset.illuminationelevationangle
170+
'off_nadir': self.dataset.illuminationelevationangle,
171+
'gsd': self.dataset.distancevalue
171172
}
172173

173174
if self.dbtype == 'postgresql':
@@ -864,7 +865,7 @@ def setup(database, table, create_sfsql_tables=True, postgis_geometry_column='wk
864865
Column('crs', Text, index=True),
865866
Column('geodescode', Text, index=True),
866867
Column('denominator', Text, index=True),
867-
Column('distancevalue', Text, index=True),
868+
Column('distancevalue', Float, index=True),
868869
Column('distanceuom', Text, index=True),
869870
Column('wkt_geometry', Text),
870871
Column('vert_extent_min', Float, index=True),
@@ -893,7 +894,7 @@ def setup(database, table, create_sfsql_tables=True, postgis_geometry_column='wk
893894
Column('platform', Text, index=True),
894895
Column('instrument', Text, index=True),
895896
Column('sensortype', Text, index=True),
896-
Column('cloudcover', Text, index=True),
897+
Column('cloudcover', Float, index=True),
897898
# bands: JSON list of dicts with properties: name, units, min, max
898899
Column('bands', Text, index=True),
899900
# STAC: view:off_nadir

‎pycsw/ogc/fes/fes1.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -170,10 +170,6 @@ def _get_comparison_expression(elem):
170170
util.nspath_eval('ogc:UpperBoundary/ogc:Literal',
171171
nsmap)).text
172172

173-
if pname == queryables['pycsw:CloudCover']:
174-
LOGGER.debug("Casting queryables['pycsw:CloudCover'] as float")
175-
pname = 'cast(%s as float)' % pname
176-
177173
expression = "%s %s %s and %s" % \
178174
(pname, com_op, assign_param(), assign_param())
179175

@@ -209,10 +205,6 @@ def _get_comparison_expression(elem):
209205
else:
210206
LOGGER.debug('PostgreSQL non-FTS specific search')
211207

212-
if pname == queryables['pycsw:CloudCover']:
213-
LOGGER.debug("Casting queryables['pycsw:CloudCover'] as float")
214-
pname = 'cast(%s as float)' % pname
215-
216208
expression = "%s is null or not %s %s %s" % \
217209
(pname, pname, com_op, assign_param())
218210
else:
@@ -226,10 +218,6 @@ def _get_comparison_expression(elem):
226218
else:
227219
LOGGER.debug('PostgreSQL non-FTS specific search')
228220

229-
if pname == queryables['pycsw:CloudCover']:
230-
LOGGER.debug("Casting queryables['pycsw:CloudCover'] as float")
231-
pname = 'cast(%s as float)' % pname
232-
233221
expression = "%s %s %s" % (pname, com_op, assign_param())
234222

235223

‎pycsw/ogc/fes/fes2.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -189,10 +189,6 @@ def _get_comparison_expression(elem):
189189
util.nspath_eval('fes20:UpperBoundary/fes20:Literal',
190190
nsmap)).text
191191

192-
if pname == queryables['pycsw:CloudCover']:
193-
LOGGER.debug("Casting queryables['pycsw:CloudCover'] as float")
194-
pname = 'cast(%s as float)' % pname
195-
196192
expression = "%s %s %s and %s" % \
197193
(pname, com_op, assign_param(), assign_param())
198194

@@ -228,10 +224,6 @@ def _get_comparison_expression(elem):
228224
else:
229225
LOGGER.debug('PostgreSQL non-FTS specific search')
230226

231-
if pname == queryables['pycsw:CloudCover']:
232-
LOGGER.debug("Casting queryables['pycsw:CloudCover'] as float")
233-
pname = 'cast(%s as float)' % pname
234-
235227
expression = "%s is null or not %s %s %s" % \
236228
(pname, pname, com_op, assign_param())
237229
else:
@@ -245,10 +237,6 @@ def _get_comparison_expression(elem):
245237
else:
246238
LOGGER.debug('PostgreSQL non-FTS specific search')
247239

248-
if pname == queryables['pycsw:CloudCover']:
249-
LOGGER.debug("Casting queryables['pycsw:CloudCover'] as float")
250-
pname = 'cast(%s as float)' % pname
251-
252240
expression = "%s %s %s" % (pname, com_op, assign_param())
253241

254242
return expression

‎setup.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,6 @@ def get_package_version():
5858
DESCRIPTION = ('pycsw is an OGC API - Records and OGC CSW server '
5959
'implementation written in Python')
6060

61-
print("JJJ", DESCRIPTION)
62-
6361
# ensure a fresh MANIFEST file is generated
6462
if (os.path.exists('MANIFEST')):
6563
os.unlink('MANIFEST')

‎tests/functionaltests/suites/cite/data/cite-virtual-collections.db‎

20 KB
Binary file not shown.

‎tests/functionaltests/suites/cite/data/cite.db‎

20 KB
Binary file not shown.

‎tests/functionaltests/suites/stac_api/data/records.db‎

0 Bytes
Binary file not shown.

‎tests/functionaltests/suites/stac_api/test_stac_api_functional.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -734,14 +734,58 @@ def test_items(config):
734734
}
735735
}
736736
}
737+
737738
content = json.loads(api.items({}, cql_json, {})[2])
738739
assert content['numberMatched'] == 11
740+
739741
for link in content['links']:
740742
if link['rel'] == 'root':
741743
continue
742744
assert link['method'] == 'POST'
743745
assert link['body'] == cql_json
744746

747+
cql_json = {
748+
'filter-lang': 'cql2-json',
749+
'filter': {
750+
'op': 'and',
751+
'args': [
752+
{
753+
'op': '<=',
754+
'args': [
755+
{
756+
'property': 'cloudcover'
757+
},
758+
14
759+
]
760+
}
761+
]
762+
}
763+
}
764+
765+
content = json.loads(api.items({}, cql_json, {})[2])
766+
assert content['numberMatched'] == 2
767+
768+
cql_json = {
769+
'filter-lang': 'cql2-json',
770+
'filter': {
771+
'op': 'and',
772+
'args': [
773+
{
774+
'op': '<=',
775+
'args': [
776+
{
777+
'property': 'gsd'
778+
},
779+
0.66
780+
]
781+
}
782+
]
783+
}
784+
}
785+
786+
content = json.loads(api.items({}, cql_json, {})[2])
787+
assert content['numberMatched'] == 1
788+
745789

746790
def test_item(config):
747791
api = STACAPI(config)

0 commit comments

Comments
 (0)