Merge pull request #385 from OpenDataServices/ods-empty-column-header

input: Don't fail when an .ods column header is empty

Author: Bjwebb

.github/workflows/lint.yml

@@ -18,7 +18,6 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
     - run: pip install --upgrade -r requirements_dev.txt
-    - run: pip install black==19.10b0
     - run: black --check *.py */
     - run: isort --check-only *.py */
     - run: flake8

.github/workflows/test.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.5', '3.6', '3.7', '3.8']
+        python-version: [ '3.6', '3.7', '3.8']
     steps:
     - uses: actions/checkout@v2
     - name: Setup python

CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [0.17.0] - 2021-04-27
+
+### Removed
+
+- We no longer support Python 3.5
+
+### Fixed
+
+- Don't fail when an .ods column header is empty https://github.com/OpenDataServices/flatten-tool/issues/378
+
 ## [0.16.0] - 2021-03-24
 
 ### Fixed

docs/getting-started.rst

@@ -4,7 +4,7 @@ Getting Started
 Prerequisites
 -------------
 
-You will need Python 3.5 or later, including the venv module.
+You will need Python 3.6 or later, including the venv module.
 
 Generally the venv module should come with your default Python install, but not on Ubuntu.  On Ubuntu run:
 
@@ -41,9 +41,7 @@ will print help information specific to that sub-command.
 Python Version Support
 ----------------------
 
-This code supports Python 3.5 (and later).
+This code supports Python 3.6 (and later).
 
-Python 3.4 (and earlier Python 3 versions) is not supported, because one of the dependencies (openpyxl) does not
-support it.
-
-Python 2 is not supported because it is now end of life.
+Python 3.5 and earlier (including Python 2) are not supported, because they are
+end of life, and some of the dependencies do not support them.

flattentool/input.py

@@ -882,7 +882,10 @@ def get_sheet_lines(self, sheet_name):
             output_row = OrderedDict()
             for i, x in enumerate(row):
 
-                header = coli_to_header[i]
+                try:
+                    header = coli_to_header[i]
+                except KeyError:
+                    continue
                 value = x
                 if not header:
                     # None means that the cell will be ignored

flattentool/tests/fixtures/ods/empty_column_header.ods

@@ -312,6 +312,32 @@ def test_ods_input_formula(self):
             OrderedDict([("colC", 4), ("colD", 12)]),
         ]
 
+    def test_xlsx_empty_column_header(self):
+        xlsxinput = XLSXInput(
+            input_name="flattentool/tests/fixtures/xlsx/empty_column_header.xlsx"
+        )
+
+        xlsxinput.read_sheets()
+
+        assert list(xlsxinput.sub_sheet_names) == ["main"]
+        assert list(xlsxinput.get_sheet_lines("main")) == [
+            {"colA": "cell1", None: None},
+            {"colA": "cell3", None: None},
+        ]
+
+    def test_ods_empty_column_header(self):
+        odsinput = ODSInput(
+            input_name="flattentool/tests/fixtures/ods/empty_column_header.ods"
+        )
+
+        odsinput.read_sheets()
+
+        assert list(odsinput.sub_sheet_names) == ["main"]
+        assert list(odsinput.get_sheet_lines("main")) == [
+            {"colA": "cell1"},
+            {"colA": "cell3"},
+        ]
+
 
 class TestInputFailure(object):
     def test_csv_no_directory(self):

flattentool/tests/fixtures/xlsx/empty_column_header.xlsx

@@ -135,3 +135,27 @@ def test_unflatten_org_xml_xlsx(tmpdir, input_format):
         open("flattentool/tests/fixtures/iati-org.xml").read()
         == tmpdir.join("output.xml").read()
     )
+
+
+@pytest.mark.parametrize("input_format", ["xlsx", "ods"])
+def test_unflatten_empty_column_header(tmpdir, input_format):
+    unflatten(
+        input_name="flattentool/tests/fixtures/{}/empty_column_header.{}".format(
+            input_format, input_format
+        ),
+        output_name=tmpdir.join("output.json").strpath,
+        input_format=input_format,
+    )
+    assert (
+        tmpdir.join("output.json").read()
+        == """{
+    "main": [
+        {
+            "colA": "cell1"
+        },
+        {
+            "colA": "cell3"
+        }
+    ]
+}"""
+    )

flattentool/tests/test_input_SpreadsheetInput.py

@@ -8,4 +8,5 @@ sphinx
 sphinx_rtd_theme
 isort
 flake8
+black==19.10b0
 transifex-client