Skip to content

Commit eacce01

Browse files
GeoSanderGeoSander
committed
Move key field computation to provider, add tests, add Joins (list/create) HTML template
1 parent c39071e commit eacce01

File tree

8 files changed

+341
-117
lines changed

8 files changed

+341
-117
lines changed

‎pygeoapi/api/joins.py‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
)
3737
from pygeoapi.openapi import get_visible_collections
3838
from pygeoapi.plugin import load_plugin
39-
from pygeoapi.provider.base import ProviderTypeError
39+
from pygeoapi.provider.base import ProviderTypeError, ProviderGenericError
4040
from pygeoapi.util import (
4141
get_provider_by_type, to_json, filter_providers_by_type,
4242
filter_dict_by_key_value, get_current_datetime
@@ -765,8 +765,8 @@ def create_join(api: API, request: APIRequest,
765765
LOGGER.error(f'Invalid parameter value: {e}', exc_info=True)
766766
return _not_found(api, request, headers, msg)
767767
except Exception as e:
768-
LOGGER.error(f'Failed to retrieve join: {e}', exc_info=True)
769-
msg = f'Failed to retrieve join: {str(e)}'
768+
LOGGER.error(f'Failed to create join: {e}', exc_info=True)
769+
msg = f'Failed to create join: {str(e)}'
770770
return _server_error(api, request, headers, msg)
771771

772772
# Set response language to requested provider locale
@@ -840,11 +840,16 @@ def key_fields(api: API, request: APIRequest,
840840
try:
841841
provider_def = get_provider_by_type(
842842
collections[dataset]['providers'], 'feature')
843+
provider = load_plugin('provider', provider_def)
843844
except ProviderTypeError:
844845
msg = f'Feature provider not found for collection: {dataset}'
845846
return _bad_request(api, request, headers, msg)
846847

847-
fields = join_util.collection_keys(provider_def, dataset)
848+
try:
849+
fields = provider.get_key_fields()
850+
except ProviderGenericError as e:
851+
LOGGER.error(f'Error retrieving key fields: {e}', exc_info=True)
852+
return _server_error(api, request, headers, str(e))
848853

849854
# Get provider locale (if any)
850855
prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale)
@@ -872,12 +877,12 @@ def key_fields(api: API, request: APIRequest,
872877
'keys': []
873878
}
874879

875-
for field in fields:
876-
field_id = field['id']
880+
for name, info in fields:
877881
content['keys'].append({
878-
'id': field_id,
879-
'isDefault': field.get('default', False),
880-
'language': prv_locale.language
882+
'id': name,
883+
'type': info.get('type'), # not always set (e.g. for ID)
884+
'isDefault': info['default'],
885+
'language': prv_locale.language # TODO: is this really useful?
881886
})
882887

883888
# Set response language to requested provider locale

‎pygeoapi/join_util.py‎

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import re
3333
import tempfile
3434
import uuid
35-
from copy import deepcopy
3635
from dataclasses import dataclass
3736
from datetime import datetime, timezone, timedelta
3837
from functools import lru_cache
@@ -241,39 +240,6 @@ def init(config: dict) -> bool:
241240
return True
242241

243242

244-
def collection_keys(provider: dict,
245-
collection_name: str) -> list[dict[str, str]]:
246-
"""
247-
Retrieves the key field configuration for the given feature provider.
248-
249-
:param provider: feature provider configuration
250-
:param collection_name: name of feature collection
251-
252-
:returns: list of key fields
253-
"""
254-
id_field = provider['id_field']
255-
key_fields = deepcopy(provider.get('key_fields', []))
256-
default_key = None
257-
id_field_found = False
258-
259-
for key in key_fields:
260-
if key.get('default', False):
261-
if default_key:
262-
raise ValueError(f'multiple default key fields configured for '
263-
f'feature collection \'{collection_name}\'')
264-
default_key = key['id']
265-
if key['id'] == id_field:
266-
id_field_found = True
267-
268-
if not id_field_found:
269-
key_fields.append({
270-
'id': id_field,
271-
'default': True if default_key in (None, id_field) else False,
272-
})
273-
274-
return key_fields
275-
276-
277243
def process_csv(collection_name: str, collection_provider: BaseProvider,
278244
form_data: dict) -> dict:
279245
"""
@@ -297,8 +263,7 @@ def process_csv(collection_name: str, collection_provider: BaseProvider,
297263
right_dataset_key = form_data['joinKey']
298264
csv_data = form_data['joinFile']
299265

300-
if not (left_dataset_key == collection_provider.id_field or
301-
left_dataset_key in (k['id'] for k in collection_provider.key_fields)): # noqa
266+
if left_dataset_key not in collection_provider.get_key_fields():
302267
raise ValueError(f'collectionKey \'{left_dataset_key}\' not found '
303268
f'in feature collection \'{collection_name}\'')
304269

‎pygeoapi/provider/base.py‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def __init__(self, provider_def):
7676
self.properties = provider_def.get('properties', [])
7777
self.file_types = provider_def.get('file_types', [])
7878
self.include_extra_query_parameters = provider_def.get('include_extra_query_parameters', False) # noqa
79-
self.key_fields = provider_def.get('key_fields', [])
79+
self._keys = provider_def.get('key_fields', [])
8080
self._fields = {}
8181
self.filename = None
8282

@@ -101,6 +101,67 @@ def get_fields(self):
101101

102102
raise NotImplementedError()
103103

104+
def get_key_fields(self) -> dict:
105+
"""
106+
Get provider key field information (names, types, default)
107+
108+
Example response:
109+
{
110+
'field1': {'type': 'string', 'default': True},
111+
'field2': {'type': 'integer', 'default': False}
112+
}
113+
114+
:returns: dict of key fields and their associated JSON Schema types
115+
"""
116+
default_key = None
117+
key_fields = {}
118+
if self.type != 'feature':
119+
# not a feature collection: there cannot be any key fields
120+
return {}
121+
if not self.fields:
122+
if not self.id_field:
123+
# not initialized with an ID field: return empty dict
124+
return {}
125+
else:
126+
# no fields: we can only return ID field as default key
127+
return {self.id_field: {'default': True}}
128+
129+
for key in self._keys:
130+
key_name = key['id']
131+
132+
is_default = key.get('default', False)
133+
if is_default:
134+
# check if admin configured multiple defaults
135+
if default_key:
136+
raise ProviderGenericError(
137+
f'multiple default key fields configured '
138+
f'for feature collection \'{self.name}\'')
139+
default_key = key_name
140+
141+
field_info = self.fields.get(key_name, {})
142+
if not field_info and key_name != self.id_field:
143+
# check if admin configured a key field that exists:
144+
# ID field is excluded from this check
145+
raise ProviderGenericError(
146+
f'key field \'{key_name}\' not found in '
147+
f'feature collection \'{self.name}\'')
148+
149+
# Add field to output
150+
field_info['default'] = is_default
151+
key_fields[key_name] = field_info
152+
153+
if self.id_field not in key_fields:
154+
# Always add ID field as key field, even if not configured
155+
field_info = self.fields.get(self.id_field, {})
156+
field_info['default'] = False
157+
key_fields[self.id_field] = field_info
158+
159+
if default_key is None:
160+
# If no default key was configured, use ID field as default
161+
key_fields[self.id_field]['default'] = True
162+
163+
return key_fields
164+
104165
@property
105166
def fields(self) -> dict:
106167
"""

‎pygeoapi/templates/collections/joins.html‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
{% extends "_base.html" %}
2+
{% block title %}{{ super() }} {{ data['title'] }} {% endblock %}
3+
{% block crumbs %}{{ super() }}
4+
/ <a href="{{ data['collections_path'] }}">{% trans %}Collections{% endtrans %}</a>
5+
/ <a href="{{ data['dataset_path'] }}">{{ data['title'] | truncate( 25 ) }}</a>
6+
/ <a href="{{ data['dataset_path'] }}/joins">{% trans %}Joins{% endtrans %}</a>
7+
{% endblock %}
8+
{% block body %}
9+
<section id="collection-joins">
10+
<h1>{{ data['title'] }}</h1>
11+
<p>{{ data['description'] }}</p>
12+
<p>
13+
{% for kw in data['keywords'] %}
14+
<span class="badge text-bg-primary bg-primary">{{ kw }}</span>
15+
{% endfor %}
16+
</p>
17+
<h3>{% trans %}Joins{% endtrans %}</h3>
18+
<p>
19+
{% trans %}
20+
Below is a list of all available join sources for this collection.<br/>
21+
If there aren't any joins, you can <a href="#create-join">create one</a>.
22+
{% endtrans %}
23+
</p>
24+
{% for join in data['joins'] %}
25+
<h4>{{ join['id'] }} - {{ join['timeStamp'] }}</h4>
26+
<ul>
27+
{% for link in join['links'] %}
28+
<li>
29+
<a title="{{ link['rel'] }}" href="{{ link['href'] }}">
30+
<span>{{ link['title'] }}</span> (<span>{{ link['type'] }}</span>)
31+
</a>
32+
</li>
33+
{% endfor %}
34+
</ul>
35+
{% endfor %}
36+
</section>
37+
<section id="create-join">
38+
<h4>{% trans %}Upload a join source (CSV){% endtrans %}</h4>
39+
<form method="POST" action="{{ data['dataset_path'] }}/joins" enctype="multipart/form-data" class="join-form">
40+
<!-- Required Fields -->
41+
<div class="form-group">
42+
<label for="collectionKey" class="required">
43+
{% trans %}Collection Key{% endtrans %} <span class="text-danger">*</span>
44+
</label>
45+
<select id="collectionKey"
46+
name="collectionKey"
47+
class="form-control"
48+
required
49+
aria-describedby="collectionKeyHelp">
50+
<option value="" disabled selected>{% trans %}Select a key field{% endtrans %}</option>
51+
{% for key_field in data['key_fields'] %}
52+
<option value="{{ key_field['id'] }}" {% if key_field.get('default', False) %}selected{% endif %}>
53+
{{ key_field['id'] }}{% if key_field.get('default', False) %} ({% trans %}default{% endtrans %}){% endif %}
54+
</option>
55+
{% endfor %}
56+
</select>
57+
<small id="collectionKeyHelp" class="form-text text-muted">
58+
{% trans %}
59+
Collection key field to join on.
60+
{% endtrans %}
61+
</small>
62+
</div>
63+
64+
<div class="form-group">
65+
<label for="joinFile" class="required">
66+
{% trans %}CSV File{% endtrans %} <span class="text-danger">*</span>
67+
</label>
68+
<input type="file"
69+
id="joinFile"
70+
name="joinFile"
71+
class="form-control-file"
72+
accept=".csv,text/csv,application/csv"
73+
required
74+
aria-describedby="joinFileHelp">
75+
<small id="joinFileHelp" class="form-text text-muted">
76+
{% trans %}CSV file to upload{% endtrans %}
77+
</small>
78+
</div>
79+
80+
<div class="form-group">
81+
<label for="joinKey" class="required">
82+
{% trans %}CSV Key{% endtrans %} <span class="text-danger">*</span>
83+
</label>
84+
<input type="text"
85+
id="joinKey"
86+
name="joinKey"
87+
class="form-control"
88+
required
89+
placeholder="{% trans %}e.g., city_id{% endtrans %}"
90+
aria-describedby="joinKeyHelp">
91+
<small id="joinKeyHelp" class="form-text text-muted">
92+
{% trans %}CSV key field to join on{% endtrans %}
93+
</small>
94+
</div>
95+
96+
<!-- Optional Fields -->
97+
<h5 class="mt-4">{% trans %}Optional Parameters{% endtrans %}</h5>
98+
99+
<div class="form-group">
100+
<label for="joinFields">
101+
{% trans %}CSV Fields{% endtrans %}
102+
</label>
103+
<input type="text"
104+
id="joinFields"
105+
name="joinFields"
106+
class="form-control"
107+
placeholder="{% trans %}e.g., name,population,area{% endtrans %}"
108+
aria-describedby="joinFieldsHelp">
109+
<small id="joinFieldsHelp" class="form-text text-muted">
110+
{% trans %}
111+
Comma-separated case-sensitive names of CSV fields to append when joining.
112+
Leave empty to include all (non-conflicting) fields.
113+
{% endtrans %}
114+
</small>
115+
</div>
116+
117+
<div class="form-group">
118+
<label for="csvDelimiter">
119+
{% trans %}CSV Delimiter{% endtrans %}
120+
</label>
121+
<input type="text"
122+
id="csvDelimiter"
123+
name="csvDelimiter"
124+
class="form-control"
125+
value=","
126+
maxlength="1"
127+
placeholder=","
128+
aria-describedby="csvDelimiterHelp">
129+
<small id="csvDelimiterHelp" class="form-text text-muted">
130+
{% trans %}CSV field delimiter (defaults to ','){% endtrans %}
131+
</small>
132+
</div>
133+
134+
<div class="form-group">
135+
<label for="csvHeaderRow">
136+
{% trans %}Header Row{% endtrans %}
137+
</label>
138+
<input type="number"
139+
id="csvHeaderRow"
140+
name="csvHeaderRow"
141+
class="form-control"
142+
value="1"
143+
min="1"
144+
placeholder="1"
145+
aria-describedby="csvHeaderRowHelp">
146+
<small id="csvHeaderRowHelp" class="form-text text-muted">
147+
{% trans %}Row number that contains the field names (defaults to 1){% endtrans %}
148+
</small>
149+
</div>
150+
151+
<div class="form-group">
152+
<label for="csvDataStartRow">
153+
{% trans %}Data Start Row{% endtrans %}
154+
</label>
155+
<input type="number"
156+
id="csvDataStartRow"
157+
name="csvDataStartRow"
158+
class="form-control"
159+
value="2"
160+
min="1"
161+
placeholder="2"
162+
aria-describedby="csvDataStartRowHelp">
163+
<small id="csvDataStartRowHelp" class="form-text text-muted">
164+
{% trans %}Row number from where to start reading the data (defaults to 2){% endtrans %}
165+
</small>
166+
</div>
167+
168+
<!-- Submit Button -->
169+
<div class="form-group mt-4">
170+
<button type="submit" class="btn btn-primary">
171+
{% trans %}Create Join{% endtrans %}
172+
</button>
173+
<button type="reset" class="btn btn-secondary ml-2">
174+
{% trans %}Reset{% endtrans %}
175+
</button>
176+
</div>
177+
</form>
178+
</section>
179+
180+
<style>
181+
.join-form {
182+
max-width: 800px;
183+
margin: 2rem 0;
184+
}
185+
.form-group {
186+
margin-bottom: 1.5rem;
187+
}
188+
.form-group label.required {
189+
font-weight: 600;
190+
}
191+
.form-control, .form-control-file {
192+
margin-top: 0.5rem;
193+
}
194+
.text-danger {
195+
color: #dc3545;
196+
}
197+
.ml-2 {
198+
margin-left: 0.5rem;
199+
}
200+
.mt-4 {
201+
margin-top: 1.5rem;
202+
}
203+
</style>
204+
{% endblock %}

0 commit comments

Comments
 (0)