Merge pull request #373 from OpenDataServices/342-last-modified-does-not-convert

ODSReader: Fix parsing date and number formatting from .ods files

Author: Bjwebb

CHANGELOG.md

@@ -6,9 +6,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [0.15.4] - 2021-03-08
+
 ### Fixed
 
-## [0.15.3] - 2020-02-23
+- Fix parsing date and number formatting from .ods files https://github.com/OpenDataServices/flatten-tool/pull/373
+
+## [0.15.3] - 2021-02-23
+
+### Fixed
 
 - use-titles: Use $ref'erring title if available https://github.com/OpenDataServices/flatten-tool/pull/368 
 - create-template --no-deprecated-fields: Did not work if deprecated element at same level as a $ref https://github.com/OpenDataServices/flatten-tool/issues/185#issuecomment-719587348

examples/iati.ods

@@ -1,7 +1,7 @@
 <?xml version='1.0' encoding='utf-8'?>
 <iati-activities>
   <!--XML generated by flatten-tool-->
-  <iati-activity>
+  <iati-activity last-updated-datetime="2011-10-01T00:00:00+00:00">
     <iati-identifier>AA-AAA-123456789-ABC123</iati-identifier>
     <reporting-org ref="AA-AAA-123456789" type="40">
       <narrative>Organisation name</narrative>
@@ -28,7 +28,7 @@
       <value value-date="2012-03-03">20</value>
     </transaction>
   </iati-activity>
-  <iati-activity>
+  <iati-activity last-updated-datetime="2016-01-01T00:00:00+00:00">
     <iati-identifier>AA-AAA-123456789-ABC124</iati-identifier>
     <reporting-org ref="AA-AAA-123456789" type="40">
       <narrative>Organisation name</narrative>

examples/iati.xlsx

@@ -1,3 +1,3 @@
-iati-identifier,reporting-org/@ref,reporting-org/@type,reporting-org/narrative,participating-org/@role,participating-org/@ref,activity-status/@code,activity-date/@type,activity-date/@iso-date,recipient-country/0/@code,recipient-country/0/@percentage,recipient-country/1/@code,recipient-country/1/@percentage,title/narrative,description/narrative
-AA-AAA-123456789-ABC123,AA-AAA-123456789,40,Organisation name,1,AA-AAA-123456789,2,1,2011-10-01,AF,40,XK,60,A title,A description
-AA-AAA-123456789-ABC124,AA-AAA-123456789,40,Organisation name,1,AA-AAA-123456789,3,2,2016-01-01,AG,30,XK,70,Another title,Another description
+iati-identifier,reporting-org/@ref,reporting-org/@type,reporting-org/narrative,participating-org/@role,participating-org/@ref,activity-status/@code,activity-date/@type,activity-date/@iso-date,recipient-country/0/@code,recipient-country/0/@percentage,recipient-country/1/@code,recipient-country/1/@percentage,title/narrative,description/narrative,@last-updated-datetime
+AA-AAA-123456789-ABC123,AA-AAA-123456789,40,Organisation name,1,AA-AAA-123456789,2,1,2011-10-01,AF,40,XK,60,A title,A description,2011-10-01T00:00:00+00:00
+AA-AAA-123456789-ABC124,AA-AAA-123456789,40,Organisation name,1,AA-AAA-123456789,3,2,2016-01-01,AG,30,XK,70,Another title,Another description,2016-01-01T00:00:00+00:00

examples/iati/expected.xml

@@ -15,12 +15,16 @@
 # Thanks to grt for the fixes
 # https://github.com/marcoconti83/read-ods-with-odfpy
 
-import re
 from collections import OrderedDict
+from datetime import datetime
 
+import backports.datetime_fromisoformat
 import odf.opendocument
 from odf.table import Table, TableCell, TableRow
 
+# Backport for datetime.fromisoformat, which is new in Python 3.7
+backports.datetime_fromisoformat.MonkeyPatch.patch_fromisoformat()
+
 
 # http://stackoverflow.com/a/4544699/1846474
 class GrowingList(list):
@@ -74,23 +78,29 @@ def readSheet(self, sheet):
                             )
                         )
                         if value_type == "float":
+                            value = cell.attributes.get(
+                                (
+                                    "urn:oasis:names:tc:opendocument:xmlns:office:1.0",
+                                    "value",
+                                )
+                            )
                             if "." in str(cell):
-                                arrCells[count] = float(str(cell))
+                                arrCells[count] = float(value)
                             else:
-                                arrCells[count] = int(str(cell))
+                                arrCells[count] = int(value)
                         elif value_type == "date":
                             date_value = cell.attributes.get(
                                 (
                                     "urn:oasis:names:tc:opendocument:xmlns:office:1.0",
                                     "date-value",
                                 )
                             )
-                            # Add UTC timezone to naive datetime strings
-                            if re.match(
-                                r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d$", date_value
-                            ):
-                                date_value += "Z"
-                            arrCells[count] = date_value
+                            # fromisoformat assumes microseconds appear as 3 or
+                            # 6 digits, whereas ods drops trailing 0s, so can
+                            # have 1-6 digits, so pad some extra 0s
+                            if "." in date_value:
+                                date_value = date_value.ljust(26, "0")
+                            arrCells[count] = datetime.fromisoformat(date_value)
                         else:
                             arrCells[count] = str(cell)
                     count += 1