Merge pull request #170 from jacebrowning/fix-notebooks

Restore custom list/dict types to fix notebooks

Author: jacebrowning

.travis.yml

@@ -21,6 +21,7 @@ install:
 script:
   - make test-repeat
   - make check
+  - make notebooks
   - make mkdocs
 
 after_success:

Makefile

@@ -89,7 +89,7 @@ test-profile: install
 MKDOCS_INDEX := site/index.html
 
 .PHONY: docs
-docs: mkdocs uml papermill  ## Generate documentation and UML
+docs: mkdocs uml notebooks  ## Generate documentation and UML
 
 .PHONY: mkdocs
 mkdocs: install $(MKDOCS_INDEX)
@@ -116,12 +116,12 @@ docs/*.png: $(MODULES)
 	- mv -f classes_$(PACKAGE).png docs/classes.png
 	- mv -f packages_$(PACKAGE).png docs/packages.png
 
-.PHONY: papermill
-papermill: install
+.PHONY: notebooks
+notebooks: install
 	@ cd notebooks; for filename in *.ipynb; do \
 	  poetry run papermill $$filename $$filename; \
 	done
-	git config filter.nbstripout.extrakeys 'metadata.papermill cell.metadata.papermill'
+	git config filter.nbstripout.extrakeys 'cell.metadata.execution cell.metadata.papermill metadata.papermill'
 	poetry run nbstripout --keep-output notebooks/*.ipynb
 
 # RELEASE #####################################################################

datafiles/formats.py

@@ -9,7 +9,7 @@
 
 import log
 
-from . import settings
+from . import settings, types
 
 
 _REGISTRY: Dict[str, type] = {}
@@ -96,6 +96,13 @@ def deserialize(cls, file_object):
     def serialize(cls, data):
         from ruamel import yaml
 
+        yaml.representer.RoundTripRepresenter.add_representer(
+            types.List, yaml.representer.RoundTripRepresenter.represent_list
+        )
+        yaml.representer.RoundTripRepresenter.add_representer(
+            types.Dict, yaml.representer.RoundTripRepresenter.represent_dict
+        )
+
         if settings.INDENT_YAML_BLOCKS:
             f = StringIO()
             y = yaml.YAML()

datafiles/hooks.py

@@ -3,9 +3,8 @@
 from functools import wraps
 
 import log
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
-from . import settings
+from . import settings, types
 from .mapper import create_mapper
 
 
@@ -49,10 +48,10 @@ def apply(instance, mapper):
         for attr_name in instance.datafile.attrs:
             attr = getattr(instance, attr_name)
             if isinstance(attr, list):
-                attr = CommentedSeq(attr)
+                attr = types.List(attr)
                 setattr(instance, attr_name, attr)
             elif isinstance(attr, dict):
-                attr = CommentedMap(attr)
+                attr = types.Dict(attr)
                 setattr(instance, attr_name, attr)
             elif not is_dataclass(attr):
                 continue

datafiles/mapper.py

@@ -13,14 +13,8 @@
 
 from . import config, formats, hooks
 from .converters import Converter, List, map_type
-from .utils import (
-    Missing,
-    Trilean,
-    display,
-    get_default_field_value,
-    recursive_update,
-    write,
-)
+from .types import Missing, Trilean
+from .utils import display, get_default_field_value, recursive_update, write
 
 
 class Mapper:

datafiles/tests/test_utils.py

@@ -70,15 +70,25 @@ def with_unchanged_immutables(expect):
             expect(old) == new
             expect(id(old['my_dict']['my_str'])) == previous_id
 
-        def with_updated_types(expect):
-            old = {'x': 1}
-            new = {'x': 1.0}
-            previous_id = id(old['x'])
+        def with_updated_numeric_types(expect):
+            old = {'value': 1}
+            new = {'value': 1.0}
+            previous_id = id(old['value'])
 
             recursive_update(old, new)
 
             expect(old) == new
-            expect(id(old['x'])) != previous_id
+            expect(id(old['value'])) != previous_id
+
+        def with_updated_boolean_types(expect):
+            old = {'value': 1}
+            new = {'value': True}
+            previous_id = id(old['value'])
+
+            recursive_update(old, new)
+
+            expect(old) == new
+            expect(id(old['value'])) != previous_id
 
     def describe_merge():
         def with_shoreter_list_into_longer(expect):

datafiles/types.py

@@ -0,0 +1,14 @@
+import dataclasses
+from typing import Optional
+
+
+Trilean = Optional[bool]
+Missing = dataclasses._MISSING_TYPE
+
+
+class List(list):
+    """Patchable `list` type."""
+
+
+class Dict(dict):
+    """Patchable `dict` type."""

datafiles/utils.py

@@ -6,13 +6,11 @@
 from pathlib import Path
 from pprint import pformat
 from shutil import get_terminal_size
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Union
 
 import log
 
-
-Trilean = Optional[bool]
-Missing = dataclasses._MISSING_TYPE
+from .types import Missing
 
 
 cached = lru_cache()
@@ -85,7 +83,7 @@ def _merge(old: Any, new: Any) -> Any:
 
         return old
 
-    if new == old and isinstance(new, type(old)):
+    if repr(new) == repr(old):
         return old
 
     return new

notebooks/file_inference.ipynb

@@ -157,7 +157,7 @@
       "script:\n",
       "  - make test-repeat\n",
       "  - make check\n",
-      "  - make mkdocs\n",
+      "  - make docs\n",
       "\n",
       "after_success:\n",
       "  - pip install coveralls\n",
@@ -237,7 +237,9 @@
       "  - 3.8\n",
       "cache:\n",
       "  pip: true\n",
-      "  directories: []\n",
+      "  directories:\n",
+      "    - ${VIRTUAL_ENV}\n",
+      "\n",
       "before_install:\n",
       "  - curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py\n",
       "    | python\n",
@@ -249,7 +251,7 @@
       "script:\n",
       "  - make test-repeat\n",
       "  - make check\n",
-      "  - make mkdocs\n",
+      "  - make docs\n",
       "after_success:\n",
       "  - pip install coveralls\n",
       "  - coveralls\n",
@@ -283,7 +285,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.2"
+   "version": "3.8.2"
   }
  },
  "nbformat": 4,

notebooks/format_options.ipynb

@@ -256,7 +256,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.2"
+   "version": "3.8.2"
   }
  },
  "nbformat": 4,