ODSReader: Use datetime.fromisoformat to parse date values in .ods

(this method is new in Python 3.7, so use a backport for earlier
versions)

Author: Bjwebb

CHANGELOG.md

@@ -8,8 +8,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 
+- Fix parsing number formatting from .ods files https://github.com/OpenDataServices/flatten-tool/pull/373
+
 ## [0.15.3] - 2020-02-23
 
+### Fixed
+
 - use-titles: Use $ref'erring title if available https://github.com/OpenDataServices/flatten-tool/pull/368 
 - create-template --no-deprecated-fields: Did not work if deprecated element at same level as a $ref https://github.com/OpenDataServices/flatten-tool/issues/185#issuecomment-719587348
 

flattentool/ODSReader.py

@@ -15,14 +15,18 @@
 # Thanks to grt for the fixes
 # https://github.com/marcoconti83/read-ods-with-odfpy
 
-import re
 from collections import OrderedDict
 from datetime import datetime
 
+import backports.datetime_fromisoformat
 import odf.opendocument
 from odf.table import Table, TableCell, TableRow
 
 
+# Backport for datetime.fromisoformat, which is new in Python 3.7
+backports.datetime_fromisoformat.MonkeyPatch.patch_fromisoformat()
+
+
 # http://stackoverflow.com/a/4544699/1846474
 class GrowingList(list):
     def __setitem__(self, index, value):
@@ -86,14 +90,12 @@ def readSheet(self, sheet):
                                     "date-value",
                                 )
                             )
-                            # Add UTC timezone to naive datetime strings
-                            if re.match(
-                                r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d$", date_value
-                            ):
-                                date_value += "Z"
-                            elif re.match(r"^\d{4}-\d\d-\d\d$", date_value):
-                                datetime.strptime(date_value, "%Y-%m-%d").isoformat()
-                            arrCells[count] = date_value
+                            # fromisoformat assumes microseconds appear as 3 or
+                            # 6 digits, whereas ods drops trailing 0s, so can
+                            # have 1-6 digits, so pad some extra 0s
+                            if "." in date_value:
+                                date_value = date_value.ljust(26, "0")
+                            arrCells[count] = datetime.fromisoformat(date_value)
                         else:
                             arrCells[count] = str(cell)
                     count += 1

flattentool/tests/test_input_SpreadsheetInput.py

@@ -211,8 +211,8 @@ def test_ods_input_types(self):
             {
                 "colInt": 1,
                 "colFloat": 1.2,
-                "colDate": "2020-03-05",
-                "colDateTime": "2020-02-07T16:41:00Z",
+                "colDate": datetime.datetime(2020, 3, 5),
+                "colDateTime": datetime.datetime(2020, 2, 7, 16, 41),
             }
         ]
         assert type(list(odsinput.get_sheet_lines("main"))[0]["colInt"]) == int

setup.py

@@ -35,6 +35,7 @@ def run(self):
     "xmltodict",
     "lxml",
     "odfpy",
+    "backports-datetime-fromisoformat",
 ]
 
 setup(