Merge pull request #107 from OpenDataServices/96-deprecate-main-sheet-name

Changes to cli and Python function interface

Author: kindly

README.md

@@ -56,8 +56,8 @@ Can be converted to/from a spreadsheet like [examples/simple/main.csv](examples/
 Using the commands:
 
 ```
-flatten-tool unflatten -f csv examples/simple --main-sheet-name main --root-id='' -o examples/simple.json
-flatten-tool flatten -f csv examples/simple.json --main-sheet-name main --root-list-path main -o examples/simple
+flatten-tool unflatten -f csv examples/simple --root-id='' -o examples/simple.json
+flatten-tool flatten -f csv examples/simple.json -o examples/simple
 ```
 
 
@@ -123,8 +123,8 @@ These are also the spreadsheets that flatten-tool's `flatten` (JSON to Spreadshe
 Commands used to generate this:
 
 ```
-flatten-tool unflatten -f csv examples/array_multisheet --main-sheet-name main --root-id='' -o examples/array_multisheet.json
-flatten-tool flatten -f csv examples/array.json --main-sheet-name main --root-list-path main -o examples/array_multisheet
+flatten-tool unflatten -f csv examples/array_multisheet --root-id='' -o examples/array_multisheet.json
+flatten-tool flatten -f csv examples/array.json -o examples/array_multisheet
 ```
 
 However, there are other "shapes" of spreadsheet that can produce the same JSON.
@@ -137,7 +137,7 @@ New columns for each item of the array:
 |7|8|9|10|11|12|
 
 ```
-flatten-tool unflatten -f csv examples/array_pointer --main-sheet-name main --root-id='' -o examples/array.json
+flatten-tool unflatten -f csv examples/array_pointer --root-id='' -o examples/array.json
 ```
 
 Repeated rows:
@@ -151,7 +151,7 @@ Repeated rows:
 
 
 ```
-flatten-tool unflatten -f csv examples/array_repeat_rows --main-sheet-name main --root-id='' -o examples/array.json
+flatten-tool unflatten -f csv examples/array_repeat_rows --root-id='' -o examples/array.json
 ```
 
 
@@ -367,20 +367,20 @@ And populate this with the package information for your release.
 
 Then, for a populated xlsx template in (in release_populated.xlsx):
 
-    flatten-tool unflatten release_populated.xlsx --base-json base.json --input-format xlsx --output-name release.json
+    flatten-tool unflatten release_populated.xlsx --base-json base.json --input-format xlsx --output-name release.json --root-list-path='releases'
 
 Or for populated CSV files (in the release_populated directory):
 
-    flatten-tool unflatten release_populated --base-json base.json --input-format csv --output-name release.json
+    flatten-tool unflatten release_populated --base-json base.json --input-format csv --output-name release.json --root-list-path='releases'
 
 These produce a release.json file based on the data in the spreadsheets.
 
 
 ### Converting a JSON file to a spreadsheet
 
-    flatten-tool flatten input.json --main-sheet-name releases --output-name unflattened
+    flatten-tool flatten input.json --main-sheet-name releases --output-name flattened --root-list-path='releases'
 
-This will create `unflattened.xlsx` and a `unflattened/` directory of csv files.
+This will create `flattened.xlsx` and a `flattened/` directory of csv files.
 
 ## Usage for 360Giving
 
@@ -392,7 +392,7 @@ to the current directory.
 
     flatten-tool create-template --root-id='' --output-format all --output-name 360giving-template --schema 360-giving-schema.json --main-sheet-name grants --rollup --use-titles
 
-    flatten-tool unflatten --root-id='' -o out.json -f xlsx --main-sheet-name=grants input.xlsx --schema 360-giving-schema.json --convert-titles
+    flatten-tool unflatten --root-id='' -o out.json -f xlsx input.xlsx --schema 360-giving-schema.json --convert-titles --root-list-path='grants'
 
 
 Running the tests

flattentool/__init__.py

@@ -9,15 +9,15 @@
 from collections import OrderedDict
 
 
-def create_template(schema, output_name='releases', output_format='all', main_sheet_name='main', flatten=False, rollup=False, root_id='ocid', use_titles=False, **_):
+def create_template(schema, output_name='template', output_format='all', main_sheet_name='main', flatten=False, rollup=False, root_id='ocid', use_titles=False, **_):
     """
     Creates template file(s) from given inputs
     This function is built to deal with commandline input and arguments
     but to also be called from elswhere in future
 
     """
 
-    parser = SchemaParser(schema_filename=schema, main_sheet_name=main_sheet_name, rollup=rollup, root_id=root_id, use_titles=use_titles)
+    parser = SchemaParser(schema_filename=schema, rollup=rollup, root_id=root_id, use_titles=use_titles)
     parser.parse()
 
     def spreadsheet_output(spreadsheet_output_class, name):
@@ -38,7 +38,7 @@ def spreadsheet_output(spreadsheet_output_class, name):
         raise Exception('The requested format is not available')
 
 
-def flatten(input_name, schema=None, output_name='releases', output_format='all', main_sheet_name='main', root_list_path='releases', rollup=False, root_id='ocid', use_titles=False, **_):
+def flatten(input_name, schema=None, output_name='flattened', output_format='all', main_sheet_name='main', root_list_path='main', rollup=False, root_id='ocid', use_titles=False, **_):
     """
     Flatten a nested structure (JSON) to a flat structure (spreadsheet - csv or xlsx).
 
@@ -49,16 +49,14 @@ def flatten(input_name, schema=None, output_name='releases', output_format='all'
             schema_filename=schema,
             rollup=rollup,
             root_id=root_id,
-            use_titles=use_titles,
-            main_sheet_name=main_sheet_name)
+            use_titles=use_titles)
         schema_parser.parse()
     else:
         schema_parser = None
     parser = JSONParser(
         json_filename=input_name,
         root_list_path=root_list_path,
         schema_parser=schema_parser,
-        main_sheet_name=main_sheet_name,
         root_id=root_id,
         use_titles=use_titles)
     parser.parse()
@@ -102,8 +100,8 @@ def decimal_default(o):
     raise TypeError(repr(o) + " is not JSON serializable")
 
 
-def unflatten(input_name, base_json=None, input_format=None, output_name='releases.json',
-              main_sheet_name='releases', encoding='utf8', timezone_name='UTC',
+def unflatten(input_name, base_json=None, input_format=None, output_name='unflattened.json',
+              root_list_path='main', encoding='utf8', timezone_name='UTC',
               root_id='ocid', schema='', convert_titles=False, cell_source_map=None,
               heading_source_map=None, **_):
     """
@@ -119,11 +117,11 @@ def unflatten(input_name, base_json=None, input_format=None, output_name='releas
     spreadsheet_input = spreadsheet_input_class(
         input_name=input_name,
         timezone_name=timezone_name,
-        main_sheet_name=main_sheet_name,
+        root_list_path=root_list_path,
         root_id=root_id,
         convert_titles=convert_titles)
     if schema:
-        parser = SchemaParser(schema_filename=schema, main_sheet_name=main_sheet_name, rollup=True, root_id=root_id)
+        parser = SchemaParser(schema_filename=schema, rollup=True, root_id=root_id)
         parser.parse()
         spreadsheet_input.parser = parser
     spreadsheet_input.encoding = encoding
@@ -135,7 +133,7 @@ def unflatten(input_name, base_json=None, input_format=None, output_name='releas
         base = OrderedDict()
     if WITH_CELLS:
         result, cell_source_map_data, heading_source_map_data = spreadsheet_input.fancy_unflatten()
-        base[main_sheet_name] = list(result)
+        base[root_list_path] = list(result)
         with codecs.open(output_name, 'w', encoding='utf-8') as fp:
             json.dump(base, fp, indent=4, default=decimal_default, ensure_ascii=False)
         if cell_source_map:
@@ -146,7 +144,7 @@ def unflatten(input_name, base_json=None, input_format=None, output_name='releas
                 json.dump(heading_source_map_data, fp, indent=4, default=decimal_default, ensure_ascii=False)
     else:
         result = spreadsheet_input.unflatten()
-        base[main_sheet_name] = list(result)
+        base[root_list_path] = list(result)
         with codecs.open(output_name, 'w', encoding='utf-8') as fp:
             json.dump(base, fp, indent=4, default=decimal_default, ensure_ascii=False)
 

flattentool/cli.py

@@ -76,7 +76,7 @@ def create_parser():
         help="Name of the outputted file. Will have an extension appended if format is all.")
     parser_flatten.add_argument(
         "--root-list-path",
-        help="Path of the root list, defaults to releases")
+        help="Path of the root list, defaults to main")
     parser_flatten.add_argument(
         "--rollup",
         action='store_true',
@@ -101,16 +101,16 @@ def create_parser():
         required=True)
     parser_unflatten.add_argument(
         "-b", "--base-json",
-        help="A base json file to populate the releases key in.")
+        help="A base json file to populate with the unflattened data.")
     parser_unflatten.add_argument(
-        "-m", "--main-sheet-name",
-        help="The name of the main sheet. Defaults to releases")
+        "-m", "--root-list-path",
+        help="The path in the JSON that will contain the unflattened list. Defaults to main.")
     parser_unflatten.add_argument(
         "-e", "--encoding",
         help="Encoding of the input file(s) (only relevant for CSV). Defaults to utf8.")
     parser_unflatten.add_argument(
         "-o", "--output-name",
-        help="Name of the outputted file. Will have an extension appended as appropriate. Defaults to releases")
+        help="Name of the outputted file. Will have an extension appended as appropriate. Defaults to unflattened.json")
     parser_unflatten.add_argument(
         "-c", "--cell-source-map",
         help="Path to write a cell source map to. Will have an extension appended as appropriate.")

flattentool/input.py

@@ -143,21 +143,15 @@ def convert_dict_titles(self, dicts, title_lookup=None):
             else:
                 yield d
 
-    def __init__(self, input_name='', main_sheet_name='', timezone_name='UTC', root_id='ocid', convert_titles=False):
+    def __init__(self, input_name='', root_list_path='main', timezone_name='UTC', root_id='ocid', convert_titles=False):
         self.input_name = input_name
-        self.main_sheet_name = main_sheet_name
+        self.root_list_path = root_list_path
         self.sub_sheet_names = []
         self.timezone = pytz.timezone(timezone_name)
         self.root_id = root_id
         self.convert_titles = convert_titles
         self.parser = None
 
-    def get_main_sheet_lines(self):
-        if self.convert_titles:
-            return self.convert_dict_titles(self.get_sheet_lines(self.main_sheet_name))
-        else:
-            return self.get_sheet_lines(self.main_sheet_name)
-
     def get_sub_sheets_lines(self):
         for sub_sheet_name in self.sub_sheet_names:
             if self.convert_titles:
@@ -189,8 +183,7 @@ def convert_types(self, in_dict):
 
     def do_unflatten(self):
         main_sheet_by_ocid = OrderedDict()
-        # Eventually we should get rid of the concept of a "main sheet entirely"
-        sheets = [(self.main_sheet_name, self.get_main_sheet_lines())] + list(self.get_sub_sheets_lines())
+        sheets = list(self.get_sub_sheets_lines())
         for i, sheet in enumerate(sheets):
             sheet_name, lines = sheet
             try:
@@ -251,7 +244,7 @@ def fancy_unflatten(self):
             raise Exception('Can only do a fancy_unflatten() if WITH_CELLS=True')
         cell_tree = self.do_unflatten()
         result = extract_list_to_value(cell_tree)
-        cell_source_map = extract_list_to_error_path([self.main_sheet_name.lower()], cell_tree)
+        cell_source_map = extract_list_to_error_path([self.root_list_path], cell_tree)
         ordered_items = sorted(cell_source_map.items())
         ordered_cell_source_map = OrderedDict(( '/'.join(str(x) for x in path), location) for path, location in ordered_items)
         row_source_map = OrderedDict()
@@ -355,10 +348,6 @@ def get_sheet_headings(self, sheet_name):
 
     def read_sheets(self):
         sheet_file_names = os.listdir(self.input_name)
-        if self.main_sheet_name+'.csv' not in sheet_file_names:
-            raise ValueError('Main sheet "{}.csv" not found.'.format(self.main_sheet_name))
-        sheet_file_names.remove(self.main_sheet_name+'.csv')
-
         self.sub_sheet_names = sorted([fname[:-4] for fname in sheet_file_names if fname.endswith('.csv')])
 
     def get_sheet_lines(self, sheet_name):
@@ -380,17 +369,9 @@ class XLSXInput(SpreadsheetInput):
     def read_sheets(self):
         self.workbook = openpyxl.load_workbook(self.input_name, data_only=True)
 
-        self.sheet_names_map = {sheet_name: sheet_name for sheet_name in self.workbook.get_sheet_names()}
-        # allow main sheet to be any case
-        for sheet_name in list(self.sheet_names_map):
-            if sheet_name.lower() == self.main_sheet_name.lower():
-                self.sheet_names_map.pop(sheet_name)
-                self.sheet_names_map[self.main_sheet_name] = sheet_name
+        self.sheet_names_map = OrderedDict((sheet_name, sheet_name) for sheet_name in self.workbook.get_sheet_names())
 
         sheet_names = list(self.sheet_names_map.keys())
-        if self.main_sheet_name not in sheet_names:
-            raise ValueError('Main sheet "{}" not found in workbook.'.format(self.main_sheet_name))
-        sheet_names.remove(self.main_sheet_name)
         self.sub_sheet_names = sheet_names
 
     def get_sheet_headings(self, sheet_name):

flattentool/json_input.py

@@ -47,10 +47,9 @@ class JSONParser(object):
     # Named for consistency with schema.SchemaParser, but not sure it's the most appropriate name.
     # Similarily with methods like parse_json_dict
 
-    def __init__(self, json_filename=None, root_json_dict=None, main_sheet_name='main', schema_parser=None, root_list_path=None, root_id='ocid', use_titles=False):
+    def __init__(self, json_filename=None, root_json_dict=None, schema_parser=None, root_list_path=None, root_id='ocid', use_titles=False):
         self.sub_sheets = {}
         self.main_sheet = Sheet()
-        self.main_sheet_name = main_sheet_name
         self.root_list_path = root_list_path
         self.root_id = root_id
         self.use_titles = use_titles

flattentool/schema.py

@@ -62,11 +62,10 @@ def __contains__(self, key):
 class SchemaParser(object):
     """Parse the fields of a JSON schema into a flattened structure."""
 
-    def __init__(self, schema_filename=None, root_schema_dict=None, main_sheet_name='main', rollup=False, root_id='ocid', use_titles=False):
+    def __init__(self, schema_filename=None, root_schema_dict=None, rollup=False, root_id='ocid', use_titles=False):
         self.sub_sheets = {}
         self.main_sheet = Sheet()
         self.sub_sheet_mapping = {}
-        self.main_sheet_name = main_sheet_name
         self.rollup = rollup
         self.root_id = root_id
         self.use_titles = use_titles