Merge branch 'master' into comment

Author: Bjwebb

flattentool/input.py

@@ -98,6 +98,19 @@ def convert_type(type_string, value, timezone = pytz.timezone('UTC')):
         raise ValueError('Unrecognised type: "{}"'.format(type_string))
 
 
+def warnings_for_ignored_columns(v, extra_message):
+    if isinstance(v, Cell):
+        warn('Column {} has been ignored, {}'.format(v.cell_location[3], extra_message))
+    elif isinstance(v, dict):
+        for x in v.values():
+            warnings_for_ignored_columns(x, extra_message)
+    elif isinstance(v, TemporaryDict):
+        for x in v.to_list():
+            warnings_for_ignored_columns(x, extra_message)
+    else:
+        raise ValueError()
+
+
 def merge(base, mergee, debug_info=None):
     if not debug_info:
         debug_info = {}
@@ -108,6 +121,9 @@ def merge(base, mergee, debug_info=None):
             value = v
         if key in base:
             if isinstance(value, TemporaryDict):
+                if not isinstance(base[key], TemporaryDict):
+                    warnings_for_ignored_columns(v, 'because it treats {} as an array, but another column does not'.format(key))
+                    continue
                 for temporarydict_key, temporarydict_value in value.items():
                     if temporarydict_key in base[key]:
                         merge(base[key][temporarydict_key], temporarydict_value, debug_info)
@@ -116,9 +132,18 @@ def merge(base, mergee, debug_info=None):
                         base[key][temporarydict_key] = temporarydict_value
                 for temporarydict_value in  value.items_no_keyfield:
                     base[key].items_no_keyfield.append(temporarydict_value)
-            elif isinstance(value, dict) and isinstance(base[key], dict):
-                merge(base[key], value, debug_info)
+            elif isinstance(value, dict):
+                if isinstance(base[key], dict):
+                    merge(base[key], value, debug_info)
+                else:
+                    warnings_for_ignored_columns(v, 'because it treats {} as an object, but another column does not'.format(key))
             else:
+                if not isinstance(base[key], Cell):
+                    id_info = '{} "{}"'.format(debug_info.get('id_name'), debug_info.get(debug_info.get('id_name')))
+                    if debug_info.get('root_id'):
+                        id_info = '{} "{}", '.format(debug_info.get('root_id'), debug_info.get('root_id_or_none'))+id_info
+                    warnings_for_ignored_columns(v, 'because another column treats it as an array or object'.format(key))
+                    continue
                 base_value = base[key].cell_value
                 if base_value != value:
                     id_info = '{} "{}"'.format(debug_info.get('id_name'), debug_info.get(debug_info.get('id_name')))
@@ -149,7 +174,7 @@ def convert_dict_titles(self, dicts, title_lookup=None):
 
         """
         if self.parser:
-            title_lookup = title_lookup or self.parser.title_lookup
+            title_lookup = self.parser.title_lookup
         for d in dicts:
             if title_lookup:
                 yield OrderedDict([(title_lookup.lookup_header(k), v) for k,v in d.items()])

flattentool/json_input.py

@@ -23,23 +23,21 @@ class BadlyFormedJSONError(ValueError):
     pass
 
 
-def sheet_key_field(sheet, key, id_key=None):
+def sheet_key_field(sheet, key):
     if key not in sheet:
         sheet.append(key)
     return key
 
-def sheet_key_title(sheet, key, id_key=None):
+def sheet_key_title(sheet, key):
     """
     If the key has a corresponding title, return that. If doesn't, create it in the sheet and return it.
 
     """
-    if id_key: # call sheet_key_field instead
-        return sheet_key_field(sheet, key, id_key)
-    title_lookup = {v: k for k, v in sheet.titles.items()}
-    if key in title_lookup:
-        return title_lookup[key]
+    if key in sheet.titles:
+        return sheet.titles[key]
     else:
-        sheet.append(key)
+        if key not in sheet:
+            sheet.append(key)
         return key
 
 
@@ -111,13 +109,13 @@ def parse_json_dict(self, json_dict, sheet, json_key=None, parent_name='', flatt
         if top_level_of_sub_sheet:
             # Only add the IDs for the top level of object in an array
             for k, v in parent_id_fields.items():
-                flattened_dict[sheet_key(sheet, k, id_key=json_key)] = v
+                flattened_dict[sheet_key(sheet, k)] = v
 
         if self.root_id and self.root_id in json_dict:
-            parent_id_fields[self.root_id] = json_dict[self.root_id]
+            parent_id_fields[sheet_key(sheet, self.root_id)] = json_dict[self.root_id]
 
         if 'id' in json_dict:
-            parent_id_fields[parent_name+'id'] = json_dict['id']
+            parent_id_fields[sheet_key(sheet, parent_name+'id')] = json_dict['id']
 
 
         for key, value in json_dict.items():
@@ -142,7 +140,12 @@ def parse_json_dict(self, json_dict, sheet, json_key=None, parent_name='', flatt
                     if self.rollup and parent_name == '': # Rollup only currently possible to main sheet
                         if len(value) == 1:
                             for k, v in value[0].items():
-                                if parent_name+key+'/0/'+k in self.schema_parser.main_sheet:
+                                if self.use_titles and parent_name+key+'/0/'+k in self.schema_parser.main_sheet.titles:
+                                    if type(v) in BASIC_TYPES:
+                                        flattened_dict[sheet_key_title(sheet, parent_name+key+'/0/'+k)] = v
+                                    else:
+                                        raise ValueError('Rolled up values must be basic types')
+                                elif not self.use_titles and parent_name+key+'/0/'+k in self.schema_parser.main_sheet:
                                     if type(v) in BASIC_TYPES:
                                         flattened_dict[sheet_key(sheet, parent_name+key+'/0/'+k)] = v
                                     else:

flattentool/schema.py

@@ -104,6 +104,7 @@ def parse(self):
                     warn('Field {} does not have a title, skipping.'.format(field))
                 else:
                     self.main_sheet.append(title)
+                    self.main_sheet.titles[field] = title
             else:
                 self.main_sheet.append(field)
 
@@ -174,26 +175,29 @@ def parse_schema_dict(self, parent_path, schema_dict, parent_id_fields=None, tit
 
                         for field in id_fields:
                             sub_sheet.add_field(field, id_field=True)
+                            sub_sheet.titles[title_lookup.lookup_header(field)] = field
                         fields = self.parse_schema_dict(
                                 parent_path+property_name+'/0',
                                 property_schema_dict['items'],
                                 parent_id_fields=id_fields,
                                 title_lookup=title_lookup.get(title),
                                 parent_title=parent_title+title+':' if parent_title is not None and title else None)
-
                         rolledUp = set()
 
                         for field, child_title in fields:
+                            full_path = parent_path+property_name+'/0/'+field
                             if self.use_titles:
                                 if not child_title or parent_title is None:
                                     warn('Field {}{}/0/{} is missing a title, skipping.'.format(parent_path, property_name, field))
                                 elif not title:
                                     warn('Field {}{} does not have a title, skipping it and all its children.'.format(parent_path, property_name))
                                 else:
                                     # This code only works for arrays that are at 0 or 1 layer of nesting
-                                    sub_sheet.add_field(parent_title+title+':'+child_title)
+                                    full_title = parent_title+title+':'+child_title
+                                    sub_sheet.add_field(full_title)
+                                    sub_sheet.titles[full_path] = full_title
                             else:
-                                sub_sheet.add_field(parent_path+property_name+'/0/'+field)
+                                sub_sheet.add_field(full_path)
                             if self.rollup and 'rollUp' in property_schema_dict and field in property_schema_dict['rollUp']:
                                 rolledUp.add(field)
                                 yield property_name+'/0/'+field, (title+':'+child_title if title and child_title else None)
@@ -204,7 +208,7 @@ def parse_schema_dict(self, parent_path, schema_dict, parent_id_fields=None, tit
                             if missedRollUp:
                                 warn('{} in rollUp but not in schema'.format(', '.join(missedRollUp)))
                     else:
-                        raise ValueError
+                        raise ValueError('Unknown type_set: {}, did you forget to explicity set the "type" key on "items"?'.format(type_set))
                 elif 'string' in property_type_set or not property_type_set:
                     self.flattened[parent_path.replace('/0/', '/')+property_name] = "string"
                     yield property_name, title