[#151] Add XML roundtrip test

Author: Bjwebb

flattentool/json_input.py

@@ -54,6 +54,7 @@ def __init__(self, json_filename=None, root_json_dict=None, schema_parser=None,
         self.root_id = root_id
         self.use_titles = use_titles
         self.id_name = id_name
+        self.xml = xml
         if schema_parser:
             self.main_sheet = schema_parser.main_sheet
             self.sub_sheets = schema_parser.sub_sheets
@@ -63,9 +64,9 @@ def __init__(self, json_filename=None, root_json_dict=None, schema_parser=None,
         else:
             self.rollup = False
 
-        if xml:
+        if self.xml:
             with codecs.open(json_filename, 'rb') as xml_file:
-                root_json_dict = xmltodict.parse(xml_file, cdata_key='')['iati-activities']
+                root_json_dict = xmltodict.parse(xml_file)['iati-activities']
             json_filename = None
 
         if json_filename is None and root_json_dict is None:
@@ -128,6 +129,10 @@ def parse_json_dict(self, json_dict, sheet, json_key=None, parent_name='', flatt
 
         for key, value in json_dict.items():
             if type(value) in BASIC_TYPES:
+                if self.xml and key == '#text':
+                    # Handle the text output from xmltodict
+                    key = ''
+                    parent_name = parent_name.strip('/')
                 flattened_dict[sheet_key(sheet, parent_name+key)] = value
             elif hasattr(value, 'items'):
                 self.parse_json_dict(

flattentool/tests/test_roundtrip.py

@@ -1,8 +1,9 @@
 from flattentool import unflatten, flatten
 import json
-import pytest
 import sys
 import os
+import xmltodict
+import pytest
 
 
 @pytest.mark.parametrize('output_format', ['xlsx', 'csv'])
@@ -93,3 +94,37 @@ def test_roundtrip_360_rollup(tmpdir, use_titles):
     original_json = json.load(open(input_name))
     roundtripped_json = json.load(tmpdir.join('roundtrip.json'))
     assert original_json == roundtripped_json
+
+
+def to_dict(x):
+    ''' Converts a nested dictlike objects e.g. OrderedDict's, to a dicts. '''
+    if hasattr(x, 'items'):
+        return dict((k, to_dict(v)) for k,v in x.items())
+    elif isinstance(x, list):
+        return [to_dict(y) for y in x]
+    else:
+        return x
+
+
+@pytest.mark.parametrize('output_format', ['xlsx', 'csv'])
+def test_roundtrip_xml(tmpdir, output_format):
+    input_name = 'examples/iati/expected.xml'
+    flatten(
+        input_name=input_name,
+        output_name=tmpdir.join('flattened').strpath+'.'+output_format,
+        output_format=output_format,
+        root_list_path='iati-activity',
+        id_name='iati-identifier',
+        xml=True)
+    unflatten(
+        input_name=tmpdir.join('flattened').strpath+'.'+output_format,
+        output_name=tmpdir.join('roundtrip.xml').strpath,
+        input_format=output_format,
+        root_list_path='iati-activity',
+        id_name='iati-identifier',
+        xml=True)
+    original_xml = open(input_name, 'rb')
+    roundtripped_xml = tmpdir.join('roundtrip.xml').open('rb')
+
+    # Compare without ordering, by wrapping in to_dict
+    assert to_dict(xmltodict.parse(original_xml)) == to_dict(xmltodict.parse(roundtripped_xml))