Merge pull request #87 from predict-idlab/os_matrix

📦 serialization support +  🎚️ update OS & python version in test-matrix

Author: jonasvdd

.github/workflows/test.yml

@@ -12,11 +12,12 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        os: ['windows-latest', 'macOS-latest', 'ubuntu-latest']
+        python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:
     - uses: actions/checkout@v2

plotly_resampler/figure_resampler/figure_resampler.py

@@ -11,7 +11,7 @@
 __author__ = "Jonas Van Der Donckt, Jeroen Van Der Donckt, Emiel Deprost"
 
 import warnings
-from typing import Tuple
+from typing import Tuple, List
 
 import dash
 import plotly.graph_objects as go
@@ -41,26 +41,98 @@ def __init__(
         show_mean_aggregation_size: bool = True,
         convert_traces_kwargs: dict | None = None,
         verbose: bool = False,
+        show_dash_kwargs: dict | None = None,
     ):
+        """Initialize a dynamic aggregation data mirror using a dash web app.
+
+        Parameters
+        ----------
+        figure: BaseFigure
+            The figure that will be decorated. Can be either an empty figure
+            (e.g., ``go.Figure()``, ``make_subplots()``, ``go.FigureWidget``) or an
+            existing figure.
+        convert_existing_traces: bool
+            A bool indicating whether the high-frequency traces of the passed ``figure``
+            should be resampled, by default True. Hence, when set to False, the
+            high-frequency traces of the passed ``figure`` will not be resampled.
+        default_n_shown_samples: int, optional
+            The default number of samples that will be shown for each trace,
+            by default 1000.\n
+            .. note::
+                * This can be overridden within the :func:`add_trace` method.
+                * If a trace withholds fewer datapoints than this parameter,
+                  the data will *not* be aggregated.
+        default_downsampler: AbstractSeriesDownsampler
+            An instance which implements the AbstractSeriesDownsampler interface and
+            will be used as default downsampler, by default ``EfficientLTTB`` with
+            _interleave_gaps_ set to True. \n
+            .. note:: This can be overridden within the :func:`add_trace` method.
+        resampled_trace_prefix_suffix: str, optional
+            A tuple which contains the ``prefix`` and ``suffix``, respectively, which
+            will be added to the trace its legend-name when a resampled version of the
+            trace is shown. By default a bold, orange ``[R]`` is shown as prefix
+            (no suffix is shown).
+        show_mean_aggregation_size: bool, optional
+            Whether the mean aggregation bin size will be added as a suffix to the trace
+            its legend-name, by default True.
+        convert_traces_kwargs: dict, optional
+            A dict of kwargs that will be passed to the :func:`add_traces` method and
+            will be used to convert the existing traces. \n
+            .. note::
+                This argument is only used when the passed ``figure`` contains data and
+                ``convert_existing_traces`` is set to True.
+        verbose: bool, optional
+            Whether some verbose messages will be printed or not, by default False.
+        show_dash_kwargs: dict, optional
+            A dict that will be used as default kwargs for the :func:`show_dash` method. 
+            Note that the passed kwargs will be take precedence over these defaults.
+
+        """
         # Parse the figure input before calling `super`
-        if is_figure(figure) and not is_fr(figure):  # go.Figure
-            # Base case, the figure does not need to be adjusted
+        if is_figure(figure) and not is_fr(figure):
+            # A go.Figure
+            # => base case: the figure does not need to be adjusted
             f = figure
         else:
             # Create a new figure object and make sure that the trace uid will not get
             # adjusted when they are added.
             f = self._get_figure_class(go.Figure)()
             f._data_validator.set_uid = False
 
-            if isinstance(figure, BaseFigure):  # go.FigureWidget or AbstractFigureAggregator
-                # A base figure object, we first copy the layout and grid ref
+            if isinstance(figure, BaseFigure):
+                # A base figure object, can be;
+                # - a go.FigureWidget
+                # - a plotly-resampler figure: subclass of AbstractFigureAggregator
+                # => we first copy the layout, grid_str and grid ref
                 f.layout = figure.layout
+                f._grid_str = figure._grid_str
                 f._grid_ref = figure._grid_ref
                 f.add_traces(figure.data)
+            elif isinstance(figure, dict) and (
+                "data" in figure or "layout" in figure # or "frames" in figure  # TODO
+            ):
+                # A figure as a dict, can be;
+                # - a plotly figure as a dict (after calling `fig.to_dict()`)
+                # - a pickled (plotly-resampler) figure (after loading a pickled figure)
+                # => we first copy the layout, grid_str and grid ref
+                f.layout = figure.get("layout")
+                f._grid_str = figure.get("_grid_str")
+                f._grid_ref = figure.get("_grid_ref")
+                f.add_traces(figure.get("data"))
+                # `pr_props` is not None when loading a pickled plotly-resampler figure
+                f._pr_props = figure.get("pr_props")
+                # `f._pr_props`` is an attribute to store properties of a
+                # plotly-resampler figure. This attribute is only used to pass
+                # information to the super() constructor. Once the super constructor is
+                # called, the attribute is removed.
+
+                # f.add_frames(figure.get("frames")) TODO
             elif isinstance(figure, (dict, list)):
                 # A single trace dict or a list of traces
                 f.add_traces(figure)
 
+        self._show_dash_kwargs = show_dash_kwargs if show_dash_kwargs is not None else {}
+
         super().__init__(
             f,
             convert_existing_traces,
@@ -129,7 +201,9 @@ def show_dash(
             ``config`` parameter for this property in this method.
             See more https://dash.plotly.com/dash-core-components/graph
         **kwargs: dict
-            Additional app.run_server() kwargs. e.g.: port
+            Additional app.run_server() kwargs. e.g.: port, ...
+            Also note that these kwargs take precedence over the ones passed to the
+            constructor via the ``show_dash_kwargs`` argument.
 
         """
         graph_properties = {} if graph_properties is None else graph_properties
@@ -150,14 +224,19 @@ def show_dash(
 
         # 2. Run the app
         if (
-            self.layout.height is not None
-            and mode == "inline"
+            mode == "inline"
             and "height" not in kwargs
         ):
-            # If figure height is specified -> re-use is for inline dash app height
-            kwargs["height"] = self.layout.height + 18
+            # If app height is not specified -> re-use figure height for inline dash app
+            #  Note: default layout height is 450 (whereas default app height is 650)
+            #  See: https://plotly.com/python/reference/layout/#layout-height
+            fig_height = self.layout.height if self.layout.height is not None else 450
+            kwargs["height"] = fig_height + 18
+
+        # kwargs take precedence over the show_dash_kwargs
+        kwargs = {**self._show_dash_kwargs, **kwargs}
 
-        # store the app information, so it can be killed
+        # Store the app information, so it can be killed
         self._app = app
         self._host = kwargs.get("host", "127.0.0.1")
         self._port = kwargs.get("port", "8050")
@@ -213,3 +292,11 @@ def register_update_graph_callback(
             dash.dependencies.Input(graph_id, "relayoutData"),
             prevent_initial_call=True,
         )(self.construct_update_data)
+
+    def _get_pr_props_keys(self) -> List[str]:
+        # Add the additional plotly-resampler properties of this class
+        return super()._get_pr_props_keys() + ["_show_dash_kwargs"]
+
+    def _ipython_display_(self):
+        # To display the figure inline as a dash app
+        self.show_dash(mode="inline")

plotly_resampler/figure_resampler/figure_resampler_interface.py

@@ -109,10 +109,21 @@ def __init__(
         assert not issubclass(type(figure), AbstractFigureAggregator)
         self._figure_class = figure.__class__
 
+        # Overwrite the passed arguments with the property dict values
+        # (this is the case when the PR figure is created from a pickled object)
+        if hasattr(figure, "_pr_props"):
+            pr_props = figure._pr_props  # a dict of PR properties
+            if pr_props is not None:
+                # Overwrite the default arguments with the serialized properties
+                for k, v in pr_props.items():
+                    setattr(self, k, v)
+            delattr(figure, "_pr_props")  # should not be stored anymore
+
         if convert_existing_traces:
             # call __init__ with the correct layout and set the `_grid_ref` of the
             # to-be-converted figure
             f_ = self._figure_class(layout=figure.layout)
+            f_._grid_str = figure._grid_str
             f_._grid_ref = figure._grid_ref
             super().__init__(f_)
 
@@ -682,7 +693,7 @@ def _parse_get_trace_props(
             if hf_y.dtype == "object":
                 # But first, we try to parse to a numeric dtype (as this is the
                 # behavior that plotly supports)
-                # Note that a bool array of type object will remain a bool array (and 
+                # Note that a bool array of type object will remain a bool array (and
                 # not will be transformed to an array of ints (0, 1))
                 try:
                     hf_y = pd.to_numeric(hf_y, errors="raise")
@@ -1256,3 +1267,42 @@ def _re_matches(regex: re.Pattern, strings: Iterable[str]) -> List[str]:
             if m is not None:
                 matches.append(m.string)
         return sorted(matches)
+
+    ## Magic methods (to use plotly.py words :grin:)
+
+    def _get_pr_props_keys(self) -> List[str]:
+        """Returns the keys (i.e., the names) of the plotly-resampler properties.
+
+        Note
+        ----
+        This method is used to serialize the object in the `__reduce__` method.
+
+        """
+        return [
+            "_hf_data",
+            "_global_n_shown_samples",
+            "_print_verbose",
+            "_show_mean_aggregation_size",
+            "_prefix",
+            "_suffix",
+            "_global_downsampler",
+        ]
+
+    def __reduce__(self):
+        """Overwrite the reduce method (which is used to support deep copying and
+        pickling).
+
+        Note
+        ----
+        We do not overwrite the `to_dict` method, as this is used to send the figure
+        to the frontend (and thus should not capture the plotly-resampler properties).
+        """
+        _, props = super().__reduce__()
+        assert len(props) == 1  # I don't know why this would be > 1
+        props = props[0]
+
+        # Add the plotly-resampler properties
+        props["pr_props"] = {}
+        for k in self._get_pr_props_keys():
+            props["pr_props"][k] = getattr(self, k)
+        return (self.__class__, (props,))  # (props,) to comply with plotly magic

plotly_resampler/figure_resampler/figurewidget_resampler.py

@@ -56,11 +56,33 @@ def __init__(
         f = self._get_figure_class(go.FigureWidget)()
         f._data_validator.set_uid = False
 
-        if isinstance(figure, BaseFigure):  # go.Figure or go.FigureWidget or AbstractFigureAggregator
-            # A base figure object, we first copy the layout and grid ref
+        if isinstance(figure, BaseFigure):
+            # A base figure object, can be;
+            # - a base plotly figure: go.Figure or go.FigureWidget
+            # - a plotly-resampler figure: subclass of AbstractFigureAggregator
+            # => we first copy the layout, grid_str and grid ref
             f.layout = figure.layout
+            f._grid_str = figure._grid_str
             f._grid_ref = figure._grid_ref
             f.add_traces(figure.data)
+        elif isinstance(figure, dict) and (
+            "data" in figure or "layout" in figure # or "frames" in figure  # TODO
+        ):
+            # A figure as a dict, can be;
+            # - a plotly figure as a dict (after calling `fig.to_dict()`)
+            # - a pickled (plotly-resampler) figure (after loading a pickled figure)
+            f.layout = figure.get("layout")
+            f._grid_str = figure.get("_grid_str")
+            f._grid_ref = figure.get("_grid_ref")
+            f.add_traces(figure.get("data"))
+            # `pr_props` is not None when loading a pickled plotly-resampler figure
+            f._pr_props = figure.get("pr_props")
+            # `f._pr_props`` is an attribute to store properties of a plotly-resampler
+            # figure. This attribute is only used to pass information to the super()
+            # constructor. Once the super constructor is called, the attribute is
+            # removed.
+
+            # f.add_frames(figure.get("frames")) TODO
         elif isinstance(figure, (dict, list)):
             # A single trace dict or a list of traces
             f.add_traces(figure)

tests/conftest.py

@@ -3,13 +3,14 @@
 
 from typing import Union
 
+import os
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 import pytest
 from plotly.subplots import make_subplots
 
-from plotly_resampler import FigureResampler, LTTB, EveryNthPoint, register_plotly_resampler, unregister_plotly_resampler
+from plotly_resampler import FigureResampler, LTTB, EveryNthPoint, unregister_plotly_resampler
 
 # hyperparameters
 _nb_samples = 10_000
@@ -26,12 +27,27 @@ def registering_cleanup():
     unregister_plotly_resampler()
 
 
+def _remove_file(file_path):
+    if os.path.exists(file_path):
+        os.remove(file_path)
+
+@pytest.fixture
+def pickle_figure():
+    FIG_PATH = "fig.pkl"
+    _remove_file(FIG_PATH)
+    yield FIG_PATH
+    _remove_file(FIG_PATH)
+
+
 @pytest.fixture
 def driver():
     from seleniumwire import webdriver
     from webdriver_manager.chrome import ChromeDriverManager, ChromeType
     from selenium.webdriver.chrome.options import Options
 
+    import time
+    time.sleep(1)
+    
     options = Options()
     if not TESTING_LOCAL:
         if headless:
@@ -144,7 +160,7 @@ def example_figure() -> FigureResampler:
                 name=f"room {i+1}",
             ),
             hf_x=df_data_pc.index,
-            hf_y=df_data_pc[c],
+            hf_y=df_data_pc[c].astype(np.float32),
             row=2,
             col=1,
             downsampler=LTTB(interleave_gaps=True),
@@ -216,7 +232,7 @@ def example_figure_fig() -> go.Figure:
             go.Scattergl(
                 name=f"room {i+1}",
                 x=df_data_pc.index,
-                y=df_data_pc[c],
+                y=df_data_pc[c].astype(np.float32),
             ),
             row=2,
             col=1,

tests/fr_selenium.py

@@ -9,8 +9,9 @@
 
 from __future__ import annotations
 
-__author__ = "Jonas Van Der Donckt"
+__author__ = "Jonas Van Der Donckt, Jeroen Van Der Donckt"
 
+import sys
 import json
 import time
 from datetime import datetime, timedelta
@@ -25,8 +26,20 @@
 from selenium.webdriver.support.ui import WebDriverWait
 
 
+def not_on_linux():
+    """Return True if the current platform is not Linux.
+    
+    Note: this will be used to add more waiting time to windows & mac os tests as 
+    - on these OS's serialization of the figure is necessary (to start the dash app in a 
+      multiprocessing.Process)
+      https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
+    - on linux, the browser (i.e., sending & getting requests) goes a lot faster
+    """
+    return not sys.platform.startswith("linux")
+
+
 # https://www.blazemeter.com/blog/improve-your-selenium-webdriver-tests-with-pytest
-# and credate a parameterized driver.get method
+# and create a parameterized driver.get method
 
 
 class RequestParser:
@@ -173,12 +186,14 @@ def go_to_page(self):
         time.sleep(1)
         self.driver.get("http://localhost:{}".format(self.port))
         self.on_page = True
+        if not_on_linux(): time.sleep(7)  # bcs serialization of multiprocessing
 
     def clear_requests(self, sleep_time_s=1):
         time.sleep(1)
         del self.driver.requests
 
     def get_requests(self, delete: bool = True):
+        if not_on_linux(): time.sleep(2)  # bcs slower browser 
         requests = self.driver.requests
         if delete:
             self.clear_requests()