Merge pull request #86 from OpenDataServices/50-better-titles

Convert titles individually, and regardless of rollUp

Author: Bjwebb

flattentool/input.py

@@ -36,15 +36,19 @@ class SpreadsheetInput(object):
     or csv).
 
     """
-    def convert_dict_titles(self, dicts, titles):
+    def convert_dict_titles(self, dicts, title_lookup=None):
         """
-        Replace titles with field names in the given list of dictionaries (``dicts``) using the mapping in ``titles``.
+        Replace titles with field names in the given list of dictionaries
+        (``dicts``) using the titles lookup in the schema parser.
 
         """
-        titles = titles or {}
-        titles_map = {title.replace(' ', '').lower(): title for title in titles}
+        if self.parser:
+            title_lookup = title_lookup or self.parser.title_lookup
         for d in dicts:
-            yield { (titles[titles_map[k.replace(' ', '').lower()]] if k.replace(' ', '').lower() in titles_map else (k if '/' in k else k.replace(':','/'))):v for k,v in d.items() }
+            if title_lookup:
+                yield { title_lookup.lookup_header(k):v for k,v in d.items() }
+            else:
+                yield d
 
     def __init__(self, input_name='', main_sheet_name='', timezone_name='UTC', root_id='ocid', convert_titles=False):
         self.input_name = input_name
@@ -53,17 +57,19 @@ def __init__(self, input_name='', main_sheet_name='', timezone_name='UTC', root_
         self.timezone = pytz.timezone(timezone_name)
         self.root_id = root_id
         self.convert_titles = convert_titles
+        self.parser = None
 
     def get_main_sheet_lines(self):
         if self.convert_titles:
-            return self.convert_dict_titles(self.get_sheet_lines(self.main_sheet_name), self.parser.main_sheet.titles)
+            return self.convert_dict_titles(self.get_sheet_lines(self.main_sheet_name))
         else:
             return self.get_sheet_lines(self.main_sheet_name)
 
     def get_sub_sheets_lines(self):
         for sub_sheet_name in self.sub_sheet_names:
             if self.convert_titles:
-                yield sub_sheet_name, self.convert_dict_titles(self.get_sheet_lines(sub_sheet_name), self.parser.sub_sheets[sub_sheet_name].titles if sub_sheet_name in self.parser.sub_sheets else None)
+                yield sub_sheet_name, self.convert_dict_titles(self.get_sheet_lines(sub_sheet_name),
+                    self.parser.sub_sheets[sub_sheet_name].title_lookup if sub_sheet_name in self.parser.sub_sheets else None)
             else:
                 yield sub_sheet_name, self.get_sheet_lines(sub_sheet_name)
 

flattentool/schema.py

@@ -1,7 +1,9 @@
 """Classes for reading from a JSON schema"""
 
 from __future__ import print_function
+from __future__ import unicode_literals
 from collections import OrderedDict
+from six.moves import UserDict
 import jsonref
 from warnings import warn
 from flattentool.sheet import Sheet
@@ -15,6 +17,46 @@ def get_property_type_set(property_schema_dict):
         return set(property_type)
 
 
+class TitleLookup(UserDict):
+    property_name = None
+
+    def lookup_header(self, title_header):
+        # Ignore titles with a / in, as they may contain types
+        # https://github.com/OpenDataServices/flatten-tool/issues/56
+        if '/' in title_header:
+            return title_header
+        return self.lookup_header_list(title_header.split(':'))
+
+    def lookup_header_list(self, title_header_list):
+        first_title = title_header_list[0]
+        remaining_titles = title_header_list[1:]
+        if first_title in self:
+            if remaining_titles:
+                return self[first_title].property_name + '/' + self[first_title].lookup_header_list(remaining_titles)
+            else:
+                return self[first_title].property_name
+        else:
+            # If we can't look up the title, treat it and any children as
+            # field names directly.
+            # Strip spaces off these.
+            return '/'.join(x.strip(' ') for x in title_header_list)
+
+    def __setitem__(self, key, value):
+        self.data[key.replace(' ', '').lower()] = value
+
+    def __getitem__(self, key):
+        if key is None:
+            raise KeyError
+        else:
+            return self.data[key.replace(' ', '').lower()]
+    
+    def __contains__(self, key):
+        if key is None:
+            return False
+        else:
+            return key.replace(' ', '').lower() in self.data
+
+
 class SchemaParser(object):
     """Parse the fields of a JSON schema into a flattened structure."""
 
@@ -26,6 +68,7 @@ def __init__(self, schema_filename=None, root_schema_dict=None, main_sheet_name=
         self.rollup = rollup
         self.root_id = root_id
         self.use_titles = use_titles
+        self.title_lookup = TitleLookup()
 
         if root_schema_dict is None and schema_filename is  None:
             raise ValueError('One of schema_filename or root_schema_dict must be supplied')
@@ -52,11 +95,10 @@ def parse(self):
                     self.main_sheet.append(title)
             else:
                 self.main_sheet.append(field)
-            if title:
-                self.main_sheet.titles[title] = field
 
-    def parse_schema_dict(self, parent_name, schema_dict, parent_id_fields=None):
+    def parse_schema_dict(self, parent_name, schema_dict, parent_id_fields=None, title_lookup=None):
         parent_id_fields = parent_id_fields or []
+        title_lookup = self.title_lookup if title_lookup is None else title_lookup
         if 'properties' in schema_dict:
             if 'id' in schema_dict['properties']:
                 id_fields = parent_id_fields + [parent_name+'/id']
@@ -67,11 +109,21 @@ def parse_schema_dict(self, parent_name, schema_dict, parent_id_fields=None):
                 property_type_set = get_property_type_set(property_schema_dict)
 
                 title = property_schema_dict.get('title')
+                if title:
+                    title_lookup[title] = TitleLookup()
+                    title_lookup[title].property_name = property_name
 
                 if 'object' in property_type_set:
-                    for field, child_title in self.parse_schema_dict(parent_name+'/'+property_name, property_schema_dict,
-                                                        parent_id_fields=id_fields):
-                        yield property_name+'/'+field, (title+':'+child_title if title and child_title else None) # TODO ambiguous use of "title"
+                    for field, child_title in self.parse_schema_dict(
+                            parent_name+'/'+property_name,
+                            property_schema_dict,
+                            parent_id_fields=id_fields,
+                            title_lookup=title_lookup.get(title)):
+                        yield (
+                            property_name+'/'+field,
+                            # TODO ambiguous use of "title"
+                            (title+':'+child_title if title and child_title else None) 
+                        )
 
                 elif 'array' in property_type_set:
                     type_set = get_property_type_set(property_schema_dict['items'])
@@ -83,6 +135,8 @@ def parse_schema_dict(self, parent_name, schema_dict, parent_id_fields=None):
                         else:
                             raise ValueError
                     elif 'object' in type_set:
+                        if title:
+                            title_lookup[title].property_name = property_name+'[]'
                         if hasattr(property_schema_dict['items'], '__reference__'):
                             sub_sheet_name = property_schema_dict['items'].__reference__['$ref'].split('/')[-1]
                         else:
@@ -93,12 +147,14 @@ def parse_schema_dict(self, parent_name, schema_dict, parent_id_fields=None):
                         if sub_sheet_name not in self.sub_sheets:
                             self.sub_sheets[sub_sheet_name] = Sheet(root_id=self.root_id, name=sub_sheet_name)
                         sub_sheet = self.sub_sheets[sub_sheet_name]
+                        sub_sheet.title_lookup = title_lookup.get(title)
 
                         for field in id_fields:
                             sub_sheet.add_field(field+':'+property_name, id_field=True)
                         fields = self.parse_schema_dict(parent_name+'/'+property_name+'[]',
                                 property_schema_dict['items'],
-                                parent_id_fields=id_fields)
+                                parent_id_fields=id_fields,
+                                title_lookup=title_lookup.get(title))
 
                         rolledUp = set()
 
@@ -110,8 +166,6 @@ def parse_schema_dict(self, parent_name, schema_dict, parent_id_fields=None):
                                     sub_sheet.add_field(child_title)
                             else:
                                 sub_sheet.add_field(field)
-                            if child_title:
-                                self.sub_sheets[sub_sheet_name].titles[child_title] = field
                             if self.rollup and 'rollUp' in property_schema_dict and field in property_schema_dict['rollUp']:
                                 rolledUp.add(field)
                                 yield property_name+'[]/'+field, (title+':'+child_title if title and child_title else None)