diff --git a/ckanext/scheming/helpers.py b/ckanext/scheming/helpers.py index f118c747..3bd842c5 100644 --- a/ckanext/scheming/helpers.py +++ b/ckanext/scheming/helpers.py @@ -27,6 +27,9 @@ def lang(): from ckantoolkit import h return h.lang() +@helper +def scheming_composite_separator(): + return config.get('scheming.composite.separator','-') @helper def scheming_language_text(text, prefer_lang=None): @@ -420,16 +423,48 @@ def scheming_flatten_subfield(subfield, data): If data already contains flattened subfields (e.g. rendering values after a validation error) then they are returned as-is. """ + from ckantoolkit import h flat = dict(data) + sep = h.scheming_composite_separator() if subfield['field_name'] not in data: return flat for i, record in enumerate(data[subfield['field_name']]): - prefix = '{field_name}-{index}-'.format( + prefix = '{field_name}{sep}{index}{sep}'.format( field_name=subfield['field_name'], index=i, + sep=sep, ) for k in record: flat[prefix + k] = record[k] return flat + +@helper +def scheming_flatten_simple_subfield(subfield, data): + """ + Return flattened_data that converts all nested data for this subfield + into {field_name}-{subfield_name} values at the top level, + so that it matches the names of form fields submitted. + + If data already contains flattened subfields (e.g. rendering values + after a validation error) then they are returned as-is. + """ + from ckantoolkit import h + flat = dict(data) + sep = h.scheming_composite_separator() + + if subfield['field_name'] not in data: + return flat + + subdata = data[subfield['field_name']] + if(isinstance(subdata, list) and len(subdata) == 1): + subdata = subdata[0] + + for field, value in subdata.items(): + prefix = '{field_name}{sep}'.format( + field_name=subfield['field_name'], + sep=sep, + ) + flat[prefix + field] = value + return flat diff --git a/ckanext/scheming/plugins.py b/ckanext/scheming/plugins.py index 1484780d..1daaf055 100644 --- a/ckanext/scheming/plugins.py +++ b/ckanext/scheming/plugins.py @@ -244,7 +244,9 @@ def validate(self, context, data_dict, schema, action): if before: schema['__before'] = validation.validators_from_string( - before, None, scheming_schema) + before, None, scheming_schema) + validation.validators_from_string('scheming_simple_subfields', None, scheming_schema) + else: + schema['__before'] = validation.validators_from_string('scheming_simple_subfields', None, scheming_schema) if after: schema['__after'] = validation.validators_from_string( after, None, scheming_schema) @@ -262,7 +264,7 @@ def validate(self, context, data_dict, schema, action): scheming_schema, convert_this ) - if convert_this and 'repeating_subfields' in f: + if convert_this and ('repeating_subfields' in f or 'simple_subfields' in f): composite_convert_fields.append(f['field_name']) def composite_convert_to(key, data, errors, context): @@ -284,6 +286,14 @@ def composite_convert_to(key, data, errors, context): if ex['key'] not in composite_convert_fields ] else: + dataset_simple_composite = { + f['field_name'] + for f in scheming_schema['dataset_fields'] + if 'simple_subfields' in f + } + if dataset_simple_composite: + expand_form_simple_composite(data_dict, dataset_simple_composite) + dataset_composite = { f['field_name'] for f in scheming_schema['dataset_fields'] @@ -331,15 +341,16 @@ def expand_form_composite(data, fieldnames): when submitting dataset/resource form composite fields look like "field-0-subfield..." convert these to lists of dicts """ + sep = p.toolkit.h.scheming_composite_separator() # if "field" exists, don't look for "field-0-subfield" fieldnames -= set(data) if not fieldnames: return indexes = {} for key in sorted(data): - if '-' not in key: + if sep not in key: continue - parts = key.split('-') + parts = key.split(sep) if parts[0] not in fieldnames: continue if parts[1] not in indexes: @@ -348,16 +359,43 @@ def expand_form_composite(data, fieldnames): parts[1] = indexes[parts[1]] try: try: - comp[int(parts[1])]['-'.join(parts[2:])] = data[key] + comp[int(parts[1])][sep.join(parts[2:])] = data[key] del data[key] except IndexError: comp.append({}) - comp[int(parts[1])]['-'.join(parts[2:])] = data[key] + comp[int(parts[1])][sep.join(parts[2:])] = data[key] del data[key] except (IndexError, ValueError): pass # best-effort only +def expand_form_simple_composite(data, fieldnames): + """ + when submitting dataset/resource form composite fields look like + "field-subfield..." convert these to lists of dicts + """ + sep = p.toolkit.h.scheming_composite_separator() + # if "field" exists, don't look for "field-subfield" + fieldnames -= set(data) + if not fieldnames: + return + for key in sorted(data): + if sep not in key: + continue + parts = key.split(sep) + if parts[0] not in fieldnames: + continue + comp = data.setdefault(parts[0], []) + try: + try: + comp[0][sep.join(parts[1:])] = data[key] + del data[key] + except IndexError: + comp.append({}) + comp[0][sep.join(parts[1:])] = data[key] + del data[key] + except (IndexError, ValueError): + pass # best-effort only class SchemingGroupsPlugin(p.SingletonPlugin, _GroupOrganizationMixin, DefaultGroupForm, _SchemingMixin): @@ -445,12 +483,13 @@ def before_dataset_index(self, data_dict): for d in schemas[data_dict['type']]['dataset_fields']: if d['field_name'] not in data_dict: continue - if 'repeating_subfields' in d: + if 'simple_subfields' in d and isinstance(data_dict[d['field_name']], list): + data_dict[d['field_name']] = data_dict[d['field_name']][0] + if 'repeating_subfields' in d or 'simple_subfields' in d: data_dict[d['field_name']] = json.dumps(data_dict[d['field_name']]) return data_dict - def _load_schemas(schemas, type_field): out = {} for n in schemas: @@ -516,7 +555,12 @@ def _field_output_validators(f, schema, convert_extras, """ Return the output validators for a scheming field f """ - if 'repeating_subfields' in f: + if 'simple_subfields' in f: + validators = { + sf['field_name']: _field_output_validators(sf, schema, False) + for sf in f['simple_subfields'] + } + elif 'repeating_subfields' in f: validators = { sf['field_name']: _field_output_validators(sf, schema, False) for sf in f['repeating_subfields'] @@ -551,7 +595,12 @@ def _field_validators(f, schema, convert_extras): # If this field contains children, we need a special validator to handle # them. - if 'repeating_subfields' in f: + if 'simple_subfields' in f: + validators = { + sf['field_name']: _field_validators(sf, schema, False) + for sf in f['simple_subfields'] + } + elif 'repeating_subfields' in f: validators = { sf['field_name']: _field_validators(sf, schema, False) for sf in f['repeating_subfields'] @@ -579,7 +628,12 @@ def _field_create_validators(f, schema, convert_extras): # If this field contains children, we need a special validator to handle # them. - if 'repeating_subfields' in f: + if 'simple_subfields' in f: + validators = { + sf['field_name']: _field_create_validators(sf, schema, False) + for sf in f['simple_subfields'] + } + elif 'repeating_subfields' in f: validators = { sf['field_name']: _field_create_validators(sf, schema, False) for sf in f['repeating_subfields'] diff --git a/ckanext/scheming/templates/scheming/display_snippets/repeating_subfields.html b/ckanext/scheming/templates/scheming/display_snippets/repeating_subfields.html index f4948aa9..99756df9 100644 --- a/ckanext/scheming/templates/scheming/display_snippets/repeating_subfields.html +++ b/ckanext/scheming/templates/scheming/display_snippets/repeating_subfields.html @@ -9,6 +9,7 @@
{% for subfield in field.repeating_subfields %} + {% if subfield.field_name in field_data and field_data[subfield.field_name]|length and subfield.display_snippet is not none %}
{{ h.scheming_language_text(subfield.label) }}
@@ -20,6 +21,7 @@ object_type=object_type -%} + {% endif %} {% endfor %}
diff --git a/ckanext/scheming/templates/scheming/display_snippets/simple_subfields.html b/ckanext/scheming/templates/scheming/display_snippets/simple_subfields.html new file mode 100644 index 00000000..ff76f86b --- /dev/null +++ b/ckanext/scheming/templates/scheming/display_snippets/simple_subfields.html @@ -0,0 +1,26 @@ +{% set fields = data[field.field_name] %} +{% block subfield_display %} +
+
+ {% for field_data in fields %} +
+ {% for subfield in field.simple_subfields %} + {% if subfield.field_name in field_data and field_data[subfield.field_name]|length and subfield.display_snippet is not none %} +
+ {{ h.scheming_language_text(subfield.label) }} +
+
+ {%- snippet 'scheming/snippets/display_field.html', + field=subfield, + data=field_data, + entity_type=entity_type, + object_type=object_type + -%} +
+ {% endif %} + {% endfor %} +
+ {% endfor %} +
+
+{% endblock %} diff --git a/ckanext/scheming/templates/scheming/form_snippets/repeating_subfields.html b/ckanext/scheming/templates/scheming/form_snippets/repeating_subfields.html index f714a16e..ca7649b7 100644 --- a/ckanext/scheming/templates/scheming/form_snippets/repeating_subfields.html +++ b/ckanext/scheming/templates/scheming/form_snippets/repeating_subfields.html @@ -19,7 +19,7 @@ {% for subfield in field.repeating_subfields %} {% set sf = dict( subfield, - field_name=field.field_name ~ '-' ~ index ~ '-' ~ subfield.field_name) + field_name=field.field_name ~ h.scheming_composite_separator() ~ index ~ h.scheming_composite_separator() ~ subfield.field_name) %} {%- snippet 'scheming/snippets/form_field.html', field=sf, diff --git a/ckanext/scheming/templates/scheming/form_snippets/simple_subfields.html b/ckanext/scheming/templates/scheming/form_snippets/simple_subfields.html new file mode 100644 index 00000000..2d4f6c6e --- /dev/null +++ b/ckanext/scheming/templates/scheming/form_snippets/simple_subfields.html @@ -0,0 +1,69 @@ +{# A complex field with simple sub-fields #} + + +{% import 'macros/form.html' as form %} +{% include 'scheming/snippets/subfields_asset.html' %} +{% macro repeating_panel(index, index1) %} +
+
+
+ {% for subfield in field.simple_subfields %} + {% set sf = dict( + subfield, + field_name=field.field_name ~ h.scheming_composite_separator() ~ subfield.field_name) + %} + {%- snippet 'scheming/snippets/form_field.html', + field=sf, + data=flat, + errors=flaterr, + licenses=licenses, + entity_type=entity_type, + object_type=object_type + -%} + {% endfor %} +
+ +
+
+{% endmacro %} + +{% set flat = h.scheming_flatten_simple_subfield(field, data) %} +{% set flaterr = h.scheming_flatten_simple_subfield(field, errors) %} + +{% call form.input_block( + 'field-' + field.field_name, + h.scheming_language_text(field.label) or field.field_name, + [], + field.classes if 'classes' in field else ['control-medium'], + dict({"class": "form-control"}, **(field.get('form_attrs', {}))), + is_required=h.scheming_field_required(field)) %} +
+ {% set alert_warning = h.scheming_language_text(field.form_alert_warning) %} + {% if alert_warning %} +
+ {{ alert_warning|safe }} +
+ {% endif %} + +
+ {{ repeating_panel(0, 1) }} +
+
+ {% set help_text = h.scheming_language_text(field.help_text) %} + {% if help_text %} +
+ {{ help_text }} +
+ {% endif %} +
+ +
+{% endcall %} diff --git a/ckanext/scheming/templates/scheming/snippets/display_field.html b/ckanext/scheming/templates/scheming/snippets/display_field.html index 1ef4d0d9..189f99c7 100644 --- a/ckanext/scheming/templates/scheming/snippets/display_field.html +++ b/ckanext/scheming/templates/scheming/snippets/display_field.html @@ -5,6 +5,8 @@ {%- if not display_snippet -%} {%- if field.repeating_subfields -%} {%- set display_snippet = 'repeating_subfields.html' -%} + {%- elif field.simple_subfields -%} + {%- set display_snippet = 'simple_subfields.html' -%} {%- elif h.scheming_field_choices(field) -%} {%- set display_snippet = 'select.html' -%} {%- else -%} diff --git a/ckanext/scheming/templates/scheming/snippets/errors.html b/ckanext/scheming/templates/scheming/snippets/errors.html index a8298b27..23753bfe 100644 --- a/ckanext/scheming/templates/scheming/snippets/errors.html +++ b/ckanext/scheming/templates/scheming/snippets/errors.html @@ -54,6 +54,38 @@ {%- endif -%} {%- endfor -%} + {%- elif 'simple_subfields' in field %} + {%- for se in errors -%} + {%- if se -%} +
  • {{ + h.scheming_language_text(field.label) }}: + +
    • + {%- endif -%} + {%- endfor -%} {%- else -%}
  • {{ h.scheming_language_text(field.label) }}: diff --git a/ckanext/scheming/templates/scheming/snippets/form_field.html b/ckanext/scheming/templates/scheming/snippets/form_field.html index 8353f6e2..eaf986d8 100644 --- a/ckanext/scheming/templates/scheming/snippets/form_field.html +++ b/ckanext/scheming/templates/scheming/snippets/form_field.html @@ -1,7 +1,7 @@ {#- master snippet for all scheming form fields -#} {#- render the field the user requested, or use a default field -#} {%- set form_snippet = field.form_snippet|default( - 'repeating_subfields.html' if field.repeating_subfields else 'text.html') -%} + 'repeating_subfields.html' if field.repeating_subfields else 'simple_subfields.html' if field.simple_subfields else 'text.html') -%} {%- if '/' not in form_snippet -%} {%- set form_snippet = 'scheming/form_snippets/' + form_snippet -%} diff --git a/ckanext/scheming/tests/test_form.py b/ckanext/scheming/tests/test_form.py index e3f5b0ec..ab9af9b2 100644 --- a/ckanext/scheming/tests/test_form.py +++ b/ckanext/scheming/tests/test_form.py @@ -393,6 +393,56 @@ def test_dataset_form_update(self, app): assert dataset["contact_address"] == [{'address': 'home'}] +@pytest.mark.usefixtures("clean_db") +class TestSimpleSubfieldDatasetForm(object): + def test_dataset_form_includes_simple_subfields(self, app): + env, response = _get_package_new_page_as_sysadmin(app, 'test-subfields') + form = BeautifulSoup(response.body).select("form")[1] + assert form.select("fieldset[name=scheming-simple-subfields]") + + def test_dataset_form_create_simple_subfields(self, app, sysadmin_env): + data = {"save": "", "_ckan_phase": 1} + + data["name"] = "subfield_dataset_1" + data["temporal_extent-begin"] = '2000-01-23' + data["temporal_extent-end"] = '2021-12-30' + + url = '/test-subfields/new' + try: + app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) + except TypeError: + app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + + dataset = call_action("package_show", id="subfield_dataset_1") + assert dataset["temporal_extent"] == [{'begin': '2000-01-23', 'end': '2021-12-30'}] + + def test_dataset_form_update_simple_subfield(self, app): + dataset = Dataset( + type="test-subfields", + temporal_extent=[{'begin': '2000-01-23', 'end': '2021-12-30'}]) + + env, response = _get_package_update_page_as_sysadmin( + app, dataset["id"] + ) + form = BeautifulSoup(response.body).select_one("#dataset-edit") + assert form.select_one( + "input[name=temporal_extent-begin]" + ).attrs['value'] == '2000-01-23' + + data = {"save": ""} + data["temporal_extent-begin"] = '1989-04-13' + data["temporal_extent-end"] = '1995-05-15' + data["name"] = dataset["name"] + + url = '/test-subfields/edit/' + dataset["id"] + try: + app.post(url, environ_overrides=env, data=data, follow_redirects=False) + except TypeError: + app.post(url.encode('ascii'), params=data, extra_environ=env) + + dataset = call_action("package_show", id=dataset["id"]) + + assert dataset["temporal_extent"] == [{'begin': '1989-04-13', 'end': '1995-05-15'}] @pytest.mark.usefixtures("clean_db") class TestSubfieldResourceForm(object): diff --git a/ckanext/scheming/tests/test_subfields.yaml b/ckanext/scheming/tests/test_subfields.yaml index f4eaa78d..6490b985 100644 --- a/ckanext/scheming/tests/test_subfields.yaml +++ b/ckanext/scheming/tests/test_subfields.yaml @@ -44,6 +44,18 @@ dataset_fields: - field_name: country label: Country +- field_name: temporal_extent + label: Temporal Extent + simple_subfields: + - field_name: begin + label: Begin + required: true + preset: date + form_placeholder: yyyy-mm-dd + - field_name: end + label: End + preset: date + form_placeholder: yyyy-mm-dd resource_fields: diff --git a/ckanext/scheming/tests/test_validation.py b/ckanext/scheming/tests/test_validation.py index 675ec0ae..61ad0bcf 100644 --- a/ckanext/scheming/tests/test_validation.py +++ b/ckanext/scheming/tests/test_validation.py @@ -898,6 +898,73 @@ def test_invalid_bad_date_subfield(self): raise AssertionError("ValidationError not raised") + +@pytest.mark.usefixtures("clean_db") +class TestSimpleSubfieldDatasetValid(object): + def test_valid_simple_subfields(self): + lc = LocalCKAN() + dataset = lc.action.package_create( + type="test-subfields", + name="a_sf_1", + temporal_extent=[{'begin': '2000-01-23', 'end': ''}] + ) + + assert dataset["temporal_extent"] == [{'begin': '2000-01-23'}] + + def test_empty_simple_subfields(self): + lc = LocalCKAN() + dataset = lc.action.package_create( + type="test-subfields", + name="a_sf_1", + temporal_extent=[], + ) + + assert "temporal_extent" not in dataset + + def test_invalid_simple_subfields_length(self): + lc = LocalCKAN() + try: + dataset = lc.action.package_create( + type="test-subfields", + name="a_sf_1", + temporal_extent=[{'begin': '2000-01-23', 'end': '2021-09-13'}, {'begin': '2022-02-23', 'end': ''}] + ) + except ValidationError as e: + assert e.error_dict["temporal_extent_subfield_length"] == ["Too many items in simple subfield temporal_extent. Found 2 items, expected 1"] + else: + raise AssertionError("ValidationError not raised") + pass + +@pytest.mark.usefixtures("clean_db") +class TestSimpleSubfieldDatasetInvalid(object): + def test_invalid_missing_required_simple_subfield(self): + lc = LocalCKAN() + + try: + lc.action.package_create( + type="test-subfields", + name="b_sf_1", + temporal_extent=[{'begin': '', 'end': '2000-01-23'}] + ) + except ValidationError as e: + assert e.error_dict["temporal_extent"][0]["begin"] == ["Missing value"] + else: + raise AssertionError("ValidationError not raised") + + def test_invalid_bad_date_subfield(self): + lc = LocalCKAN() + + try: + lc.action.package_create( + type="test-subfields", + name="b_sf_1", + temporal_extent=[{'begin': '2000-01-23', 'end': 'THEN'}] + ) + except ValidationError as e: + assert e.error_dict["temporal_extent"][0]["end"] == ["Date format incorrect"] + else: + raise AssertionError("ValidationError not raised") + @pytest.mark.usefixtures("clean_db") class TestSubfieldResourceValid(object): def test_simple(self): diff --git a/ckanext/scheming/validation.py b/ckanext/scheming/validation.py index 6a6dcaf3..86164106 100644 --- a/ckanext/scheming/validation.py +++ b/ckanext/scheming/validation.py @@ -50,6 +50,27 @@ def scheming_validator(fn): register_validator(unicode_safe) +@scheming_validator +@register_validator +def scheming_simple_subfields(field, schema): + def validator(key, data, errors, context): + fields = schema.get('dataset_fields', []) + schema.get('resource_fields', []) + key_tuples = data.keys() + for f in fields: + if 'simple_subfields' in f: + error_fn = f['field_name'] + '_subfield_length' + iters = [tup[1] for tup in key_tuples if tup[0] == f['field_name']] + if iters and max(iters) > 0: + if (error_fn,) not in errors: + errors[(error_fn,)] = [] + errors[(error_fn,)].extend([ + _('Too many items in simple subfield %s. Found %s items, expected 1') + % (f['field_name'], (max(iters) + 1)) + ]) + raise StopOnError + return validator + + @scheming_validator @register_validator def scheming_choices(field, schema):