Merge pull request #34 from owid/fix-load-json

Fix load json

Author: pabloarosado

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.7
+current_version = 0.4.8
 commit = True
 tag = True
 

README.md

@@ -1,5 +1,5 @@
 # owid-datautils
-![version](https://img.shields.io/badge/version-0.4.7-blue)
+![version](https://img.shields.io/badge/version-0.4.8-blue)
 ![version](https://img.shields.io/badge/python-3.8|3.9|3.10-blue.svg?&logo=python&logoColor=yellow) [![codecov](https://codecov.io/gh/owid/owid-datautils-py/branch/main/graph/badge.svg?token=2emTQEJedw)](https://codecov.io/gh/owid/owid-datautils-py)
 [![Documentation Status](https://readthedocs.org/projects/owid-datautils/badge/?version=latest)](https://docs.owid.io/projects/owid-datautils/en/latest/?badge=latest)
 

docs/conf.py

@@ -22,7 +22,7 @@
 author = "Our World In Data"
 
 # The full version, including alpha/beta/rc tags
-release = "0.4.7"
+release = "0.4.8"
 
 
 # -- General configuration ---------------------------------------------------

owid/datautils/__init__.py

@@ -1,3 +1,3 @@
 """Library to support the work of the Data Team at Our World in Data."""
 
-__version__ = "0.4.7"
+__version__ = "0.4.8"

owid/datautils/io/local.py

@@ -2,30 +2,35 @@
 
 import json
 from pathlib import Path
-from typing import Any, cast, Dict, Hashable, List, Tuple, Union
+from typing import Any, Hashable, List, Tuple, Union
 
 from owid.datautils.common import warn_on_list_of_entities
 
 
 def _load_json_data_and_duplicated_keys(
     ordered_pairs: List[Tuple[Hashable, Any]]
-) -> Tuple[Dict[Any, Any], List[Any]]:
+) -> Any:
     clean_dict = {}
     duplicated_keys = []
     for key, value in ordered_pairs:
         if key in clean_dict:
             duplicated_keys.append(key)
         clean_dict[key] = value
+    if len(duplicated_keys) > 0:
+        warn_on_list_of_entities(
+            list_of_entities=duplicated_keys,
+            warning_message="Duplicated entities found.",
+            show_list=True,
+        )
 
-    return clean_dict, duplicated_keys
+    return clean_dict
 
 
-def load_json(
-    json_file: Union[str, Path], warn_on_duplicated_keys: bool = True
-) -> Dict[Any, Any]:
+def load_json(json_file: Union[str, Path], warn_on_duplicated_keys: bool = True) -> Any:
     """Load data from json file, and optionally warn if there are duplicated keys.
 
-    If json file contains duplicated keys, a warning is optionally raised, and only the latest value of the key is kept.
+    If json file contains duplicated keys, a warning is optionally raised, and only the value of the latest duplicated
+    key is kept.
 
     Parameters
     ----------
@@ -41,20 +46,15 @@ def load_json(
 
     """
     with open(json_file, "r") as _json_file:
+        json_content = _json_file.read()
         if warn_on_duplicated_keys:
-            data, duplicated_keys = json.loads(
-                _json_file.read(), object_pairs_hook=_load_json_data_and_duplicated_keys
+            data = json.loads(
+                json_content, object_pairs_hook=_load_json_data_and_duplicated_keys
             )
-            if len(duplicated_keys) > 0:
-                warn_on_list_of_entities(
-                    duplicated_keys,
-                    f"Duplicated entities found in {json_file}",
-                    show_list=True,
-                )
         else:
-            data = json.loads(_json_file.read())
+            data = json.loads(json_content)
 
-    return cast(Dict[Any, Any], data)
+    return data
 
 
 def save_json(data: Any, json_file: Union[str, Path], **kwargs: Any) -> None:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "owid-datautils"
-version = "0.4.7"
+version = "0.4.8"
 description = "Data utils library by the Data Team at Our World in Data"
 authors = ["Our World In Data <tech@ourworldindata.org>"]
 license = "MIT"

tests/io/test_local.py

@@ -13,22 +13,39 @@ class TestLoadJson:
     def test_load_json_without_duplicated_keys(self, _):
         assert load_json(_) == {"1": "10", "2": "20"}
 
+    @patch(
+        "builtins.open",
+        new_callable=mock_open,
+        read_data='{"1": {"1_1": "1", "1_2": "2"}, "2": "20"}',
+    )
+    def test_load_json_without_duplicated_keys_with_more_levels(self, _):
+        assert load_json(_) == {"1": {"1_1": "1", "1_2": "2"}, "2": "20"}
+
+    @patch(
+        "builtins.open",
+        new_callable=mock_open,
+        read_data='[{"1": {"1_1": "1", "1_2": "2"}, "2": "20"}, {"1": "10"}]',
+    )
+    def test_load_json_without_duplicated_keys_with_more_levels_in_a_list(self, _):
+        # Here the key "1" is repeated, however, it is in a different dictionary, so it is not a duplicated key.
+        assert load_json(_) == [{"1": {"1_1": "1", "1_2": "2"}, "2": "20"}, {"1": "10"}]
+
     @patch(
         "builtins.open",
         new_callable=mock_open,
         read_data='{"1": "10", "2": "20", "1": "100"}',
     )
-    def test_load_json_with_duplicated_keys(self, _):
+    def test_load_json_with_duplicated_keys_and_no_warning(self, _):
         assert load_json(_, warn_on_duplicated_keys=False) == {"1": "100", "2": "20"}
 
     @patch(
         "builtins.open",
         new_callable=mock_open,
         read_data='{"1": "10", "2": "20", "1": "100"}',
     )
-    def test_warn_on_load_json_with_duplicated_keys(self, _):
+    def test_warn_on_load_json_with_duplicated_keys_and_warning(self, _):
         with warns(UserWarning, match="Duplicated"):
-            load_json(_, warn_on_duplicated_keys=True)
+            assert load_json(_, warn_on_duplicated_keys=True) == {"1": "100", "2": "20"}
 
     @patch("builtins.open", new_callable=mock_open, read_data="{}")
     def test_load_empty_json(self, _):