Merge branch 'dev' into antrg-fix-datatable

Author: antoinerg

.circleci/config.yml

@@ -4,6 +4,18 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 
 ## [Unreleased]
 
+- [#1745](https://github.com/plotly/dash/pull/1745):
+    Improve our `extras_require`: there are now five options here, each with a well-defined role:
+    - `dash[dev]`: for developing and building dash components.
+    - `dash[testing]`: for using the `pytest` plugins in the `dash.testing` module
+    - `dash[diskcache]`: required if you use `DiskcacheLongCallbackManager`
+    - `dash[celery]`: required if you use `CeleryLongCallbackManager`
+    - `dash[ci]`: mainly for internal use, these are additional requirements for the Dash CI tests, exposed for other component libraries to use a matching configuration.
+
+- [#1779](https://github.com/plotly/dash/pull/1779):
+    - Clean up our handling of serialization problems, including fixing `orjson` for Python 3.6
+    - Added the ability for `dash.testing` `percy_snapshot` methods to choose widths to generate.
+
 - [#1763](https://github.com/plotly/dash/pull/1763):
     ## Dash and Dash Renderer
 
@@ -29,7 +41,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
     @dash.callback(Output(my_output, 'children'), Input(my_input, 'value'))
     def update(value):
       return f'You have entered {value}'
-      
+
     ```
     ## Dash Core Components
 
@@ -138,7 +150,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
     ```python
     dcc.Checklist(inline=True)
     ```
-  
+
 ## [2.0.0] - 2021-08-03
 
 ## Dash and Dash Renderer

CHANGELOG.md

@@ -34,7 +34,7 @@
     "lint": "run-s private::lint.*",
     "test.server": "pytest --nopercyfinalize tests/selenium",
     "test.unit": "run-s private::test.python private::test.unit",
-    "test.visual": "build-storybook && percy-storybook",
+    "test.visual": "build-storybook && percy-storybook --widths=1280",
     "test.visual-local": "build-storybook"
   },
   "author": "Chris Parmer <[email protected]>",

components/dash-table/package.json

@@ -1,11 +1,11 @@
-import collections
+from collections.abc import MutableSequence
 import re
 from textwrap import dedent
 
 from ._grouping import grouping_len, map_grouping
 from .development.base_component import Component
 from . import exceptions
-from ._utils import patch_collections_abc, stringify_id
+from ._utils import patch_collections_abc, stringify_id, to_json
 
 
 def validate_callback(outputs, inputs, state, extra_args, types):
@@ -198,7 +198,8 @@ def validate_multi_return(outputs_list, output_value, callback_id):
 
 
 def fail_callback_output(output_value, output):
-    valid = (str, dict, int, float, type(None), Component)
+    valid_children = (str, int, float, type(None), Component)
+    valid_props = (str, int, float, type(None), tuple, MutableSequence)
 
     def _raise_invalid(bad_val, outer_val, path, index=None, toplevel=False):
         bad_type = type(bad_val).__name__
@@ -247,43 +248,74 @@ def _raise_invalid(bad_val, outer_val, path, index=None, toplevel=False):
             )
         )
 
-    def _value_is_valid(val):
-        return isinstance(val, valid)
+    def _valid_child(val):
+        return isinstance(val, valid_children)
+
+    def _valid_prop(val):
+        return isinstance(val, valid_props)
+
+    def _can_serialize(val):
+        if not (_valid_child(val) or _valid_prop(val)):
+            return False
+        try:
+            to_json(val)
+        except TypeError:
+            return False
+        return True
 
     def _validate_value(val, index=None):
         # val is a Component
         if isinstance(val, Component):
+            unserializable_items = []
             # pylint: disable=protected-access
             for p, j in val._traverse_with_paths():
                 # check each component value in the tree
-                if not _value_is_valid(j):
+                if not _valid_child(j):
                     _raise_invalid(bad_val=j, outer_val=val, path=p, index=index)
 
+                if not _can_serialize(j):
+                    # collect unserializable items separately, so we can report
+                    # only the deepest level, not all the parent components that
+                    # are just unserializable because of their children.
+                    unserializable_items = [
+                        i for i in unserializable_items if not p.startswith(i[0])
+                    ]
+                    if unserializable_items:
+                        # we already have something unserializable in a different
+                        # branch - time to stop and fail
+                        break
+                    if all(not i[0].startswith(p) for i in unserializable_items):
+                        unserializable_items.append((p, j))
+
                 # Children that are not of type Component or
                 # list/tuple not returned by traverse
                 child = getattr(j, "children", None)
-                if not isinstance(child, (tuple, collections.MutableSequence)):
-                    if child and not _value_is_valid(child):
+                if not isinstance(child, (tuple, MutableSequence)):
+                    if child and not _can_serialize(child):
                         _raise_invalid(
                             bad_val=child,
                             outer_val=val,
                             path=p + "\n" + "[*] " + type(child).__name__,
                             index=index,
                         )
+            if unserializable_items:
+                p, j = unserializable_items[0]
+                # just report the first one, even if there are multiple,
+                # as that's how all the other errors work
+                _raise_invalid(bad_val=j, outer_val=val, path=p, index=index)
 
             # Also check the child of val, as it will not be returned
             child = getattr(val, "children", None)
-            if not isinstance(child, (tuple, collections.MutableSequence)):
-                if child and not _value_is_valid(child):
+            if not isinstance(child, (tuple, MutableSequence)):
+                if child and not _can_serialize(val):
                     _raise_invalid(
                         bad_val=child,
                         outer_val=val,
                         path=type(child).__name__,
                         index=index,
                     )
 
-        # val is not a Component, but is at the top level of tree
-        elif not _value_is_valid(val):
+        if not _can_serialize(val):
             _raise_invalid(
                 bad_val=val,
                 outer_val=type(val).__name__,
@@ -301,13 +333,13 @@ def _validate_value(val, index=None):
     # if we got this far, raise a generic JSON error
     raise exceptions.InvalidCallbackReturnValue(
         """
-        The callback for property `{property:s}` of component `{id:s}`
+        The callback for output `{output}`
         returned a value which is not JSON serializable.
 
         In general, Dash properties can only be dash components, strings,
         dictionaries, numbers, None, or lists of those.
         """.format(
-            property=output.component_property, id=output.component_id
+            output=repr(output)
         )
     )
 

dash/_validate.py

@@ -25,10 +25,18 @@ def __init__(self, celery_app, cache_by=None, expire=None):
             for ``expire`` seconds.  If not provided, the lifetime of cache entries
             is determined by the default behavior of the celery result backend.
         """
-        import celery  # pylint: disable=import-outside-toplevel,import-error
-        from celery.backends.base import (  # pylint: disable=import-outside-toplevel,import-error
-            DisabledBackend,
-        )
+        try:
+            import celery  # pylint: disable=import-outside-toplevel,import-error
+            from celery.backends.base import (  # pylint: disable=import-outside-toplevel,import-error
+                DisabledBackend,
+            )
+        except ImportError as missing_imports:
+            raise ImportError(
+                """\
+CeleryLongCallbackManager requires extra dependencies which can be installed doing
+
+    $ pip install "dash[celery]"\n"""
+            ) from missing_imports
 
         if not isinstance(celery_app, celery.Celery):
             raise ValueError("First argument must be a celery.Celery object")

dash/long_callback/managers/celery_manager.py

@@ -4,14 +4,15 @@
 
 
 class DiskcacheLongCallbackManager(BaseLongCallbackManager):
-    def __init__(self, cache, cache_by=None, expire=None):
+    def __init__(self, cache=None, cache_by=None, expire=None):
         """
         Long callback manager that runs callback logic in a subprocess and stores
         results on disk using diskcache
 
         :param cache:
             A diskcache.Cache or diskcache.FanoutCache instance. See the diskcache
-            documentation for information on configuration options.
+            documentation for information on configuration options. If not provided,
+            a diskcache.Cache instance will be created with default values.
         :param cache_by:
             A list of zero-argument functions.  When provided, caching is enabled and
             the return values of these functions are combined with the callback
@@ -28,20 +29,22 @@ def __init__(self, cache, cache_by=None, expire=None):
         except ImportError as missing_imports:
             raise ImportError(
                 """\
-DiskcacheLongCallbackManager requires the multiprocess, diskcache, and psutil packages
-which can be installed using pip...
+DiskcacheLongCallbackManager requires extra dependencies which can be installed doing
 
-    $ pip install multiprocess diskcache psutil
-
-or conda.
-
-    $ conda install -c conda-forge multiprocess diskcache psutil\n"""
+    $ pip install "dash[diskcache]"\n"""
             ) from missing_imports
 
-        if not isinstance(cache, (diskcache.Cache, diskcache.FanoutCache)):
-            raise ValueError("First argument must be a diskcache.Cache object")
+        if cache is None:
+            self.handle = diskcache.Cache()
+        else:
+            if not isinstance(cache, (diskcache.Cache, diskcache.FanoutCache)):
+                raise ValueError(
+                    "First argument must be a diskcache.Cache "
+                    "or diskcache.FanoutCache object"
+                )
+            self.handle = cache
+
         super().__init__(cache_by)
-        self.handle = cache
         self.expire = expire
 
     def terminate_job(self, job):

dash/long_callback/managers/diskcache_manager.py

@@ -106,6 +106,7 @@ def visit_and_snapshot(
         convert_canvases=False,
         assert_check=True,
         stay_on_page=False,
+        widths=None,
     ):
         try:
             path = resource_path.lstrip("/")
@@ -119,6 +120,7 @@ def visit_and_snapshot(
                 path,
                 wait_for_callbacks=wait_for_callbacks,
                 convert_canvases=convert_canvases,
+                widths=widths,
             )
             if assert_check:
                 assert not self.driver.find_elements_by_css_selector(
@@ -130,10 +132,26 @@ def visit_and_snapshot(
             logger.exception("snapshot at resource %s error", path)
             raise e
 
-    def percy_snapshot(self, name="", wait_for_callbacks=False, convert_canvases=False):
+    def percy_snapshot(
+        self, name="", wait_for_callbacks=False, convert_canvases=False, widths=None
+    ):
         """percy_snapshot - visual test api shortcut to `percy_runner.snapshot`.
-        It also combines the snapshot `name` with the Python version.
+        It also combines the snapshot `name` with the Python version,
+        args:
+        - name: combined with the python version to give the final snapshot name
+        - wait_for_callbacks: default False, whether to wait for Dash callbacks,
+            after an extra second to ensure that any relevant callbacks have
+            been initiated
+        - convert_canvases: default False, whether to convert all canvas elements
+            in the DOM into static images for percy to see. They will be restored
+            after the snapshot is complete.
+        - widths: a list of pixel widths for percy to render the page with. Note
+            that this does not change the browser in which the DOM is constructed,
+            so the width will only affect CSS, not JS-driven layout.
+            Defaults to [375, 1280]
         """
+        if widths is None:
+            widths = [375, 1280]
         snapshot_name = "{} - py{}.{}".format(
             name, sys.version_info.major, sys.version_info.minor
         )
@@ -172,7 +190,7 @@ def percy_snapshot(self, name="", wait_for_callbacks=False, convert_canvases=Fal
             """
             )
 
-            self.percy_runner.snapshot(name=snapshot_name)
+            self.percy_runner.snapshot(name=snapshot_name, widths=widths)
 
             self.driver.execute_script(
                 """
@@ -189,7 +207,7 @@ def percy_snapshot(self, name="", wait_for_callbacks=False, convert_canvases=Fal
             )
 
         else:
-            self.percy_runner.snapshot(name=snapshot_name)
+            self.percy_runner.snapshot(name=snapshot_name, widths=widths)
 
     def take_snapshot(self, name):
         """Hook method to take snapshot when a selenium test fails. The

dash/testing/browser.py

@@ -192,7 +192,5 @@ def diskcache_manager():
     from dash.long_callback import (  # pylint: disable=import-outside-toplevel
         DiskcacheLongCallbackManager,
     )
-    import diskcache  # pylint: disable=import-outside-toplevel
 
-    cache = diskcache.Cache()
-    return DiskcacheLongCallbackManager(cache)
+    return DiskcacheLongCallbackManager()

dash/testing/plugin.py

@@ -0,0 +1,3 @@
+# Dependencies used by the CeleryLongCallbackManager
+redis>=3.5.3
+celery[redis]>=5.1.2

requires-celery.txt

@@ -0,0 +1,14 @@
+# Dependencies used by CI on github.com/plotly/dash
+black==21.6b0
+dash-flow-example==0.0.5
+dash-dangerously-set-inner-html
+flake8==3.9.2
+flaky==3.7.0
+flask-talisman==0.8.1
+isort==4.3.21;python_version<"3.7"
+mock==4.0.3
+orjson==3.5.4;python_version<"3.7"
+orjson==3.6.3;python_version>="3.7"
+pylint==2.10.2
+pytest-mock==3.2.0
+pytest-sugar==0.9.4