feat(logic): delete fields;

- Added delete_fields to recombinant update logic aciton method.

Author: JVickery-TBS

changes/140.changes

@@ -1 +1 @@
-`recombinant_update` with `force` will now DROP old columns from the DataStore for fields that do not exist in the Schema.
+`recombinant_update` with `force` and `delete_fields` will now DROP old columns from the DataStore for fields that do not exist in the Schema.

ckanext/recombinant/cli.py

@@ -116,19 +116,26 @@ def remove_empty(dataset_type: Optional[List[str]] = None,
     is_flag=True,
     help="Force update of tables (required for changes to only primary keys/indexes)",
 )
+@click.option(
+    "-d",
+    "--delete-fields",
+    is_flag=True,
+    help="Deletes fields that are no longer in the Schema (requires --force-update)",
+)
 @click.option('-v', '--verbose', is_flag=True,
               type=click.BOOL, help='Increase verbosity.')
 def update(dataset_type: Optional[List[str]] = None,
            all_types: bool = False,
            force_update: bool = False,
+           delete_fields: bool = False,
            verbose: bool = False):
     """
     Triggers recombinant update for recombinant resources
 
     Full Usage:\n
         recombinant update (-a | DATASET_TYPE ...) [-f]
     """
-    _update(dataset_type, all_types, force_update, verbose=verbose)
+    _update(dataset_type, all_types, force_update, delete_fields, verbose=verbose)
 
 
 @recombinant.command(short_help="Delete recombinant datasets and all their data.")
@@ -349,6 +356,7 @@ def _show(dataset_type: Optional[str],
 def _update(dataset_types: Optional[List[str]],
             all_types: bool = False,
             force_update: bool = False,
+            delete_fields: bool = False,
             verbose: bool = False):
     """
     Triggers recombinant update for recombinant resources
@@ -366,7 +374,8 @@ def _update(dataset_types: Optional[List[str]],
                 click.echo('%s %s updating' % (dtype, o))
                 lc.action.recombinant_update(
                     owner_org=o, dataset_type=dtype,
-                    force_update=force_update)
+                    force_update=force_update,
+                    delete_fields=delete_fields)
 
 
 def _expand_dataset_types(dataset_types: Optional[List[str]],

ckanext/recombinant/logic.py

@@ -62,6 +62,8 @@ def recombinant_update(context: Context, data_dict: DataDict):
     :param owner_org: organization name or id
     :param delete_resources: True to delete extra resources found
     :param force_update: True to force updating of datastore tables
+    :param delete_fields: True to delete old fields not in schema,
+                          requires force_update=True
     '''
     lc, geno, dataset = _action_get_dataset(context, data_dict)
 
@@ -70,7 +72,8 @@ def recombinant_update(context: Context, data_dict: DataDict):
         delete_resources=asbool(data_dict.get('delete_resources', False)))
     _update_datastore(
         lc, geno, dataset,
-        force_update=asbool(data_dict.get('force_update', False)))
+        force_update=asbool(data_dict.get('force_update', False)),
+        delete_fields=asbool(data_dict.get('delete_fields', False)))
 
 
 def recombinant_show(context: Context, data_dict: DataDict) -> Dict[str, Any]:
@@ -245,7 +248,8 @@ def _update_dataset(lc: LocalCKAN,
 def _update_datastore(lc: LocalCKAN,
                       geno: Dict[str, Any],
                       dataset: Dict[str, Any],
-                      force_update: bool = False):
+                      force_update: bool = False,
+                      delete_fields: bool = False):
     """
     call lc.action.datastore_create to create tables or add
     columns to existing datastore tables based on dataset definition
@@ -260,7 +264,7 @@ def _update_datastore(lc: LocalCKAN,
             chromo['resource_name'], dataset['id'])
         resource_id = resource_ids[chromo['resource_name']]
         fields = datastore_fields(chromo['fields'], datastore_text_types)
-        delete_fields = False
+        do_delete_fields = False
         try:
             ds = lc.action.datastore_search(resource_id=resource_id, limit=0)
         except NotFound:
@@ -275,17 +279,18 @@ def _update_datastore(lc: LocalCKAN,
             for f in datastore_fields(chromo['fields'], datastore_text_types):
                 if f['id'] not in seen:
                     fields.append(f)
-            # remove any fields from DS not in Schema
-            new_fields = []
-            schema_field_ids = set(
-                f['id'] for f in datastore_fields(chromo['fields'],
-                                                  datastore_text_types))
-            for f in fields:
-                if f['id'] not in schema_field_ids:
-                    delete_fields = True
-                    continue
-                new_fields.append(f)
-            fields = new_fields
+            if delete_fields:
+                # remove any fields from DS not in Schema
+                new_fields = []
+                schema_field_ids = set(
+                    f['id'] for f in datastore_fields(chromo['fields'],
+                                                    datastore_text_types))
+                for f in fields:
+                    if f['id'] not in schema_field_ids:
+                        do_delete_fields = True
+                        continue
+                    new_fields.append(f)
+                fields = new_fields
 
         trigger_names = _update_triggers(lc, chromo)
 
@@ -305,7 +310,7 @@ def _update_datastore(lc: LocalCKAN,
         lc.action.datastore_create(
             resource_id=resource_id,
             fields=fields,
-            delete_fields=delete_fields,
+            delete_fields=do_delete_fields,
             primary_key=chromo.get('datastore_primary_key', []),
             foreign_keys=foreign_keys,
             indexes=chromo.get('datastore_indexes', []),

ckanext/recombinant/templates/recombinant/resource_edit.html

@@ -45,13 +45,14 @@
   {% block action_panels %}
     {% if dataset %}
       {% if 'error' not in resource %}
-        {% if not resource.datastore_correct %}
+        {% if g.userobj.sysadmin and not resource.datastore_correct %}
+          {# only let sysadmins refresh the recombinant record via UI #}
           <div class="module-alert alert alert-warning">
-            <h3>{{ _("The Recombinant resource is out of date") }}</h3>
+            <h3>{{ _("The Recombinant resource is out of date") }}{% snippet 'snippets/sysadmin_only.html' %}</h3>
             <p style="margin-bottom: 10px;">{{ _('You can refresh your resource in the database to try to solve the problem.') }}</p>
             <form id="create-pd-resource" method="post">
               {{ h.csrf_input() }}
-              <button type="submit" class="btn btn-info mrgn-bttm-md m-b-3" name="refresh">{{_('Refresh…')}}</button>
+              <button type="submit" class="btn btn-danger mrgn-bttm-md m-b-3" name="refresh-hard">{{_('Refresh…')}}</button>
             </form>
           </div>
         {% endif %}

ckanext/recombinant/views.py

@@ -25,7 +25,7 @@
 
 from ckan.logic import ValidationError, NotAuthorized
 from ckan.model.group import Group
-from ckan.authz import has_user_permission_for_group_or_org
+from ckan.authz import has_user_permission_for_group_or_org, is_sysadmin
 
 from ckan.views.dataset import _get_package_type
 
@@ -533,10 +533,17 @@ def preview_table(resource_name: str,
                 if 'create' in request.form:
                     lc.action.recombinant_create(
                         dataset_type=chromo['dataset_type'], owner_org=owner_org)
-                else:
+                elif 'refresh-hard' in request.form or 'refresh' in request.form:
+                    if not is_sysadmin(g.user):
+                        # only sysadmins can refresh via UI
+                        return abort(404)
+                    delete_fields = False
+                    if 'refresh-hard' in request.form:
+                        delete_fields = True
                     lc.action.recombinant_update(
                         dataset_type=chromo['dataset_type'], owner_org=owner_org,
-                        force_update=True)
+                        force_update=True,
+                        delete_fields=delete_fields)
                     h.flash_success(_('Resources successfully refreshed.'))
             except NotAuthorized as e:
                 return abort(403, e.message or '')