Merge pull request #410 from OpenDataServices/csv-dialect

Line terminator option

Author: James (ODSC)

.gitignore

@@ -50,6 +50,8 @@ examples/flatten/rollup/direct/actual
 examples/flatten/rollup/direct/actual.*
 examples/flatten/rollup/file/actual
 examples/flatten/rollup/file/actual.*
+examples/flatten/line-terminator/actual
+examples/flatten/line-terminator/actual.*
 examples/receipt/source-map/actual
 examples/receipt/source-map/actual.*
 examples/bods/unflatten/actual

CHANGELOG.md

@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [0.20.0] - 2022-12-07
+
+### Changed
+
+- Add `--line-terminator` option to `flatten` and `create-template`
+
 ## [0.19.0] - 2022-11-16
 
 ### Fixed

examples/flatten/line-terminator/cmd.txt

@@ -0,0 +1 @@
+$ flatten-tool flatten --line-terminator=LF --root-list-path=cafe --main-sheet-name=cafe --schema=examples/receipt/cafe.schema examples/receipt/normalised/expected.json -o examples/flatten/line-terminator/actual

examples/flatten/line-terminator/expected/cafe.csv

@@ -0,0 +1,3 @@
+id,name,address
+CAFE-HEALTH,Healthy Cafe,
+CAFE-VEG,Vegetarian Cafe,

examples/flatten/line-terminator/expected/tab_dish.csv

@@ -0,0 +1,5 @@
+id,table/0/id,table/0/dish/0/id,table/0/dish/0/name,table/0/dish/0/cost
+CAFE-HEALTH,TABLE-1,,Fish and Chips,9.95
+CAFE-HEALTH,TABLE-1,,Pesto Pasta Salad,6.95
+CAFE-HEALTH,TABLE-3,,Fish and Chips,9.95
+CAFE-VEG,TABLE-16,,Large Glass Sauvignon,5.95

examples/flatten/line-terminator/expected/table.csv

@@ -0,0 +1,6 @@
+id,table/0/id,table/0/number
+CAFE-HEALTH,TABLE-1,1
+CAFE-HEALTH,TABLE-2,2
+CAFE-HEALTH,TABLE-3,3
+CAFE-VEG,TABLE-16,16
+CAFE-VEG,TABLE-17,17

examples/help/create-template/expected.txt

@@ -4,6 +4,7 @@ usage: flatten-tool create-template [-h] -s SCHEMA [-f {csv,ods,xlsx,all}]
                                     [--disable-local-refs]
                                     [--no-deprecated-fields]
                                     [--truncation-length TRUNCATION_LENGTH]
+                                    [--line-terminator LINE_TERMINATOR]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -32,3 +33,6 @@ optional arguments:
   --truncation-length TRUNCATION_LENGTH
                         The length of components of sub-sheet names (default
                         3).
+  --line-terminator LINE_TERMINATOR
+                        The line terminator to use when writing CSV files:
+                        CRLF or LF

examples/help/flatten/expected.txt

@@ -9,6 +9,7 @@ usage: flatten-tool flatten [-h] [-s SCHEMA] [-f {csv,ods,xlsx,all}] [--xml]
                             [--preserve-fields PRESERVE_FIELDS]
                             [--disable-local-refs]
                             [--remove-empty-schema-columns]
+                            [--line-terminator LINE_TERMINATOR]
                             input_name
 
 positional arguments:
@@ -61,3 +62,6 @@ optional arguments:
   --remove-empty-schema-columns
                         When using flatten with a schema, remove columns and
                         sheets from the output that contain no data.
+  --line-terminator LINE_TERMINATOR
+                        The line terminator to use when writing CSV files:
+                        CRLF or LF

flattentool/__init__.py

@@ -8,7 +8,7 @@
 from flattentool.json_input import JSONParser
 from flattentool.lib import parse_sheet_configuration
 from flattentool.output import FORMATS as OUTPUT_FORMATS
-from flattentool.output import FORMATS_SUFFIX
+from flattentool.output import FORMATS_SUFFIX, LINE_TERMINATORS
 from flattentool.schema import SchemaParser
 from flattentool.xml_output import toxml
 
@@ -24,7 +24,8 @@ def create_template(
     disable_local_refs=False,
     truncation_length=3,
     no_deprecated_fields=False,
-    **_
+    line_terminator="CRLF",
+    **_,
 ):
     """
     Creates template file(s) from given inputs
@@ -33,6 +34,9 @@ def create_template(
 
     """
 
+    if line_terminator not in LINE_TERMINATORS.keys():
+        raise Exception(f"{line_terminator} is not a valid line terminator")
+
     parser = SchemaParser(
         schema_filename=schema,
         rollup=rollup,
@@ -46,7 +50,10 @@ def create_template(
 
     def spreadsheet_output(spreadsheet_output_class, name):
         spreadsheet_output = spreadsheet_output_class(
-            parser=parser, main_sheet_name=main_sheet_name, output_name=name
+            parser=parser,
+            main_sheet_name=main_sheet_name,
+            output_name=name,
+            line_terminator=LINE_TERMINATORS[line_terminator],
         )
         spreadsheet_output.write_sheets()
 
@@ -87,7 +94,8 @@ def flatten(
     disable_local_refs=False,
     remove_empty_schema_columns=False,
     truncation_length=3,
-    **_
+    line_terminator="CRLF",
+    **_,
 ):
     """
     Flatten a nested structure (JSON) to a flat structure (spreadsheet - csv or xlsx).
@@ -99,6 +107,9 @@ def flatten(
     ):
         raise Exception("You must use filter_field and filter_value together")
 
+    if line_terminator not in LINE_TERMINATORS.keys():
+        raise Exception(f"{line_terminator} is not a valid line terminator")
+
     if schema:
         schema_parser = SchemaParser(
             schema_filename=schema,
@@ -136,6 +147,7 @@ def spreadsheet_output(spreadsheet_output_class, name):
                 main_sheet_name=main_sheet_name,
                 output_name=name,
                 sheet_prefix=sheet_prefix,
+                line_terminator=LINE_TERMINATORS[line_terminator],
             )
             spreadsheet_output.write_sheets()
 
@@ -206,7 +218,7 @@ def unflatten(
     disable_local_refs=False,
     xml_comment=None,
     truncation_length=3,
-    **_
+    **_,
 ):
     """
     Unflatten a flat structure (spreadsheet - csv or xlsx) into a nested structure (JSON).

flattentool/cli.py

@@ -96,6 +96,10 @@ def create_parser():
         default=3,
         help="The length of components of sub-sheet names (default 3).",
     )
+    parser_create_template.add_argument(
+        "--line-terminator",
+        help="The line terminator to use when writing CSV files: CRLF or LF",
+    )
 
     parser_flatten = subparsers.add_parser("flatten", help="Flatten a JSON file")
     parser_flatten.add_argument("input_name", help="Name of the input JSON file.")
@@ -177,7 +181,10 @@ def create_parser():
         action="store_true",
         help="When using flatten with a schema, remove columns and sheets from the output that contain no data.",
     )
-
+    parser_flatten.add_argument(
+        "--line-terminator",
+        help="The line terminator to use when writing CSV files: CRLF or LF",
+    )
     parser_unflatten = subparsers.add_parser(
         "unflatten", help="Unflatten a spreadsheet"
     )