Merge pull request #208 from OpenDataServices/cove-1030-xml-comma-array

Don't split text value of xml array

Author: Bjwebb

examples/iati_multilang/cmd.txt

@@ -0,0 +1 @@
+$ flatten-tool unflatten --xml --id-name iati-identifier --root-list-path iati-activity -f csv examples/iati_multilang

examples/iati_multilang/expected.xml

@@ -0,0 +1,59 @@
+<iati-activities>
+  <!--XML generated by flatten-tool-->
+  <iati-activity>
+    <iati-identifier>AA-AAA-123456789-ABC123</iati-identifier>
+    <reporting-org ref="AA-AAA-123456789" type="40">
+      <narrative>Organisation name</narrative>
+    </reporting-org>
+    <participating-org ref="AA-AAA-123456789" role="1"/>
+    <activity-status code="2"/>
+    <activity-date iso-date="2011-10-01" type="1"/>
+    <recipient-country code="AF" percentage="40"/>
+    <recipient-country code="XK" percentage="60"/>
+    <title>
+      <narrative xml:lang="en">A title, with comma</narrative>
+      <narrative xml:lang="fr">Un titre</narrative>
+    </title>
+    <description>
+      <narrative>A description</narrative>
+    </description>
+    <transaction>
+      <transaction-type code="2"/>
+      <transaction-date iso-date="2012-01-01"/>
+      <value value-date="2012-01-01">10</value>
+    </transaction>
+    <transaction>
+      <transaction-type code="3"/>
+      <transaction-date iso-date="2012-03-03"/>
+      <value value-date="2012-03-03">20</value>
+    </transaction>
+  </iati-activity>
+  <iati-activity>
+    <iati-identifier>AA-AAA-123456789-ABC124</iati-identifier>
+    <reporting-org ref="AA-AAA-123456789" type="40">
+      <narrative>Organisation name</narrative>
+    </reporting-org>
+    <participating-org ref="AA-AAA-123456789" role="1"/>
+    <activity-status code="3"/>
+    <activity-date iso-date="2016-01-01" type="2"/>
+    <recipient-country code="AG" percentage="30"/>
+    <recipient-country code="XK" percentage="70"/>
+    <title>
+      <narrative xml:lang="en">Another title; with semicolon</narrative>
+      <narrative xml:lang="fr">Un autre titre</narrative>
+    </title>
+    <description>
+      <narrative>Another description</narrative>
+    </description>
+    <transaction>
+      <transaction-type code="2"/>
+      <transaction-date iso-date="2013-04-04"/>
+      <value value-date="2013-04-04">30</value>
+    </transaction>
+    <transaction>
+      <transaction-type code="3"/>
+      <transaction-date iso-date="2013-05-05"/>
+      <value value-date="2013-05-05">40</value>
+    </transaction>
+  </iati-activity>
+</iati-activities>

examples/iati_multilang/main.csv

@@ -0,0 +1,3 @@
+iati-identifier,reporting-org/@ref,reporting-org/@type,reporting-org/narrative,participating-org/@role,participating-org/@ref,activity-status/@code,activity-date/@type,activity-date/@iso-date,recipient-country/0/@code,recipient-country/0/@percentage,recipient-country/1/@code,recipient-country/1/@percentage,title/narrative/0/@xml:lang,title/narrative/0,title/narrative/1/@xml:lang,title/narrative/1,description/narrative
+AA-AAA-123456789-ABC123,AA-AAA-123456789,40,Organisation name,1,AA-AAA-123456789,2,1,2011-10-01,AF,40,XK,60,en,"A title, with comma",fr,Un titre,A description
+AA-AAA-123456789-ABC124,AA-AAA-123456789,40,Organisation name,1,AA-AAA-123456789,3,2,2016-01-01,AG,30,XK,70,en,Another title; with semicolon,fr,Un autre titre,Another description

examples/iati_multilang/transactions.csv

@@ -0,0 +1,5 @@
+iati-identifier,transaction/0/transaction-type/@code,transaction/0/transaction-date/@iso-date,transaction/0/value/@value-date,transaction/0/value
+AA-AAA-123456789-ABC123,2,2012-01-01,2012-01-01,10
+AA-AAA-123456789-ABC123,3,2012-03-03,2012-03-03,20
+AA-AAA-123456789-ABC124,2,2013-04-04,2013-04-04,30
+AA-AAA-123456789-ABC124,3,2013-05-05,2013-05-05,40

flattentool/input.py

@@ -740,7 +740,14 @@ def unflatten_main_with_parser(parser, line, timezone, xml, id_name):
                 continue
 
             value = cell.cell_value
-            converted_value = convert_type(current_type or '', value, timezone)
+            if xml and current_type == 'array':
+                # In xml "arrays" can have text values, if they're the final element
+                # However the type of the text value itself should not be "array",
+                # as that would split the text on commas, which we don't want.
+                # https://github.com/OpenDataServices/cove/issues/1030
+                converted_value = convert_type('', value, timezone)
+            else:
+                converted_value = convert_type(current_type or '', value, timezone)
             cell.cell_value = converted_value
             if converted_value is not None and converted_value != '':
                 if xml:

flattentool/tests/test_docs.py

@@ -90,9 +90,9 @@ def test_examples_in_docs():
                 tests_passed += 1
     # Check that the number of tests were run that we expected
     if sys.version_info[:2] < (3,4):
-        assert tests_passed == 35
-    else:
         assert tests_passed == 36
+    else:
+        assert tests_passed == 37
 
 def _simplify_warnings(lines):
     return '\n'.join([_simplify_line(line) for line in lines.split('\n')])

flattentool/tests/test_unflatten.py

@@ -1,4 +1,6 @@
+import os
 import json
+import pytest
 
 from flattentool import unflatten
 
@@ -57,15 +59,16 @@ def test_360_fields_case_insensitive(tmpdir):
     assert output_json_grants == output_json_space_case
 
 
-def test_unflatten_xml(tmpdir):
+@pytest.mark.parametrize('dirname', ['examples/iati', 'examples/iati_multilang'])
+def test_unflatten_xml(tmpdir, dirname):
     unflatten(
-        input_name='examples/iati',
+        input_name=dirname,
         output_name=tmpdir.join('output.xml').strpath,
         input_format='csv',
         root_list_path='iati-activity',
         id_name='iati-identifier',
         xml=True)
-    assert open('examples/iati/expected.xml').read() == tmpdir.join('output.xml').read()
+    assert open(os.path.join(dirname, 'expected.xml')).read() == tmpdir.join('output.xml').read()
 
 
 def test_unflatten_org_xml(tmpdir):