Merge pull request #138 from open-data/fix/load-csv

Fix Load CSV Command

Author: JVickery-TBS

changes/138.bugfix

@@ -0,0 +1 @@
+Fixed an issue with the `load_csv` command in which it would raise exceptions when passing non-existing fields to the DataStore.

ckanext/recombinant/cli.py

@@ -519,11 +519,19 @@ def _load_one_csv_file(name: str) -> int:
 
         click.echo('- %s %s' % (org_name, len(records)))
 
+        # default org extras
+        org_extras = [
+            'owner_org',
+            'owner_org_title'
+        ]
         if 'csv_org_extras' in chromo:
-            # remove 'csv_org_extras' fields from records
-            for r in records:
-                for e in chromo['csv_org_extras']:
-                    del r[e]
+            org_extras += chromo['csv_org_extras']
+        # remove any org extras
+        for r in records:
+            for e in org_extras:
+                if e not in r:
+                    continue
+                del r[e]
 
         offset = 0
         while offset < len(records):
@@ -624,6 +632,7 @@ def _write_one_csv(lc: LocalCKAN,
                 chromo['resource_name'], pkg['owner_org']))
             continue
 
+        # default org extras
         org_extras = {
             'owner_org': pkg['owner_org'],
             'owner_org_title': pkg['org_title'],

ckanext/recombinant/read_csv.py

@@ -48,7 +48,7 @@ def csv_data_batch(csv_path: str,
                        if f['datastore_type'] != 'text']
 
         for row_dict in csv_in:
-            owner_org = row_dict.pop('owner_org')
+            owner_org = row_dict.pop('owner_org', None)
             if owner_org != current_owner_org:
                 if records:
                     yield (current_owner_org, records)