Generic hf properties (#355)

* 💪 Somewhat generic fix for #354

- Added support for new downsamplable properties: `marker_symbol` and `hf_marker_symbol`.
- Refactored the handling of high-frequency properties to be more dynamic and extensible.
- Updated the `_hf_data_container` to include new properties and adjusted parsing logic accordingly.
- Introduced tests to validate the functionality of dynamic properties and their priority over trace attributes.

* 🔍 Add tests similar to #354

* 💨  updating ruff and fixing linting

* 🔍 fix linting in properties test

* 💪 support marker angle and opacity

* 🖍️ upating basic example with other hf marker properties

* 🕵️ review code

* 💨 removing todo & fixing small bug

* 🙈 fixing formatting (command) and updating poetry lock hash

* 🙈 fixing undefined property

* 🙏 increase webdriver wait time for failing MacOS tests

* ✨ Enhance Chrome WebDriver options for improved stability in testing

* 🙏 try to fix failing selenium test on macos

* 🙏

* 🙏

* 🙏 :prayl

* 🙏

* 💨 skipping macos tests

---------

Co-authored-by: jeroen <[email protected]>

Author: jonasvdd

Makefile

@@ -2,12 +2,12 @@ black = black plotly_resampler examples tests
 
 .PHONY: format
 format:
-	ruff --fix plotly_resampler tests
-	$(black)
+	poetry run $(black)
+	poetry run ruff check plotly_resampler tests
 
 .PHONY: lint
 lint:
-	poetry run ruff plotly_resampler tests
+	poetry run ruff check plotly_resampler tests
 	poetry run $(black) --check --diff
 
 .PHONY: test

examples/basic_example.ipynb

@@ -29,14 +29,31 @@
 from ..aggregation.plotly_aggregator_parser import PlotlyAggregatorParser
 from .utils import round_number_str, round_td_str
 
+# Configuration for properties that can be downsampled
+# Each entry is a tuple of (property_name, trace_path, hf_param_name)
+# - property_name: the name in the _hf_data_container
+# - trace_path: the path to access the property in a trace (e.g., "marker.color" -> ["marker", "color"])
+# - hf_param_name: the parameter name for high-frequency data (e.g., "hf_marker_color")
+DOWNSAMPLABLE_PROPERTIES = [
+    ("text", ["text"], "hf_text"),
+    ("hovertext", ["hovertext"], "hf_hovertext"),
+    ("marker_angle", ["marker", "angle"], "hf_marker_angle"),
+    ("marker_opacity", ["marker", "opacity"], "hf_marker_opacity"),
+    ("marker_size", ["marker", "size"], "hf_marker_size"),
+    ("marker_color", ["marker", "color"], "hf_marker_color"),
+    ("marker_symbol", ["marker", "symbol"], "hf_marker_symbol"),
+    ("customdata", ["customdata"], "hf_customdata"),
+]
+
 # A high-frequency data container
 # NOTE: the attributes must all be valid trace attributes, with attribute levels
 # separated by an '_' (e.g., 'marker_color' is valid) as the
 # `_hf_data_container._asdict()` function is used in
 #  `AbstractFigureAggregator._construct_hf_data_dict`.
+# Create the _hf_data_container dynamically from the configuration
 _hf_data_container = namedtuple(
     "DataContainer",
-    ["x", "y", "text", "hovertext", "marker_size", "marker_color", "customdata"],
+    ["x", "y"] + [prop[0] for prop in DOWNSAMPLABLE_PROPERTIES],
 )
 
 
@@ -219,8 +236,7 @@ def _get_current_graph(self) -> dict:
             "data": [
                 {
                     k: copy(trace[k])
-                    # TODO: why not "text" as well? -> we can use _hf_data_container.fields then
-                    for k in set(trace.keys()).difference({"x", "y", "hovertext"})
+                    for k in set(trace.keys()).difference({_hf_data_container._fields})
                 }
                 for trace in self._data
             ],
@@ -385,16 +401,18 @@ def _nest_dict_rec(k: str, v: any, out: dict) -> None:
             else:
                 out[k] = v
 
-        # Check if (hover)text also needs to be downsampled
-        for k in ["text", "hovertext", "marker_size", "marker_color", "customdata"]:
-            k_val = hf_trace_data.get(k)
+        # Check if downsamplable properties also need to be downsampled
+        for prop_name, _, _ in DOWNSAMPLABLE_PROPERTIES:
+            k_val = hf_trace_data.get(prop_name)
             if isinstance(k_val, (np.ndarray, pd.Series)):
                 assert isinstance(
                     hf_trace_data["downsampler"], DataPointSelector
                 ), "Only DataPointSelector can downsample non-data trace array props."
-                _nest_dict_rec(k, k_val[start_idx + indices], trace)
+                # Use the same indices that were used for x and y aggregation
+                # indices are relative to the slice, so we need to add start_idx
+                _nest_dict_rec(prop_name, k_val[start_idx + indices], trace)
             elif k_val is not None:
-                trace[k] = k_val
+                trace[prop_name] = k_val
 
         return trace
 
@@ -549,10 +567,7 @@ def _parse_get_trace_props(
         trace: BaseTraceType,
         hf_x: Iterable = None,
         hf_y: Iterable = None,
-        hf_text: Iterable = None,
-        hf_hovertext: Iterable = None,
-        hf_marker_size: Iterable = None,
-        hf_marker_color: Iterable = None,
+        **hf_properties,
     ) -> _hf_data_container:
         """Parse and capture the possibly high-frequency trace-props in a datacontainer.
 
@@ -603,47 +618,19 @@ def _parse_get_trace_props(
         if not hasattr(hf_y, "dtype"):
             hf_y: np.ndarray = np.asarray(hf_y)
 
-        hf_text = (
-            hf_text
-            if hf_text is not None
-            else (
-                trace["text"]
-                if hasattr(trace, "text") and trace["text"] is not None
-                else None
-            )
-        )
-
-        hf_hovertext = (
-            hf_hovertext
-            if hf_hovertext is not None
-            else (
-                trace["hovertext"]
-                if hasattr(trace, "hovertext") and trace["hovertext"] is not None
-                else None
-            )
-        )
+        # Parse downsamplable properties dynamically
+        parsed_properties = {}
+        for prop_name, trace_path, hf_param_name in DOWNSAMPLABLE_PROPERTIES:
+            # Get the high-frequency value from parameters if provided
+            hf_value = hf_properties.get(hf_param_name)
 
-        hf_marker_size = (
-            trace["marker"]["size"]
-            if (
-                hf_marker_size is None
-                and hasattr(trace, "marker")
-                and "size" in trace["marker"]
-            )
-            else hf_marker_size
-        )
-
-        hf_marker_color = (
-            trace["marker"]["color"]
-            if (
-                hf_marker_color is None
-                and hasattr(trace, "marker")
-                and "color" in trace["marker"]
-            )
-            else hf_marker_color
-        )
-
-        hf_customdata = trace["customdata"] if hasattr(trace, "customdata") else None
+            if hf_value is not None:
+                # Use the provided high-frequency value
+                parsed_properties[prop_name] = hf_value
+            else:
+                # Try to get the value from the trace
+                trace_value = self._get_trace_property(trace, trace_path)
+                parsed_properties[prop_name] = trace_value
 
         if trace["type"].lower() in self._high_frequency_traces:
             if hf_x is None:  # if no data as x or hf_x is passed
@@ -664,15 +651,11 @@ def _parse_get_trace_props(
                 "(i.e., x and y, or hf_x and hf_y) to be <= 1 dimensional!"
             )
 
-            # Note: this converts the hf property to a np.ndarray
-            if isinstance(hf_text, (tuple, list, np.ndarray, pd.Series)):
-                hf_text = np.asarray(hf_text)
-            if isinstance(hf_hovertext, (tuple, list, np.ndarray, pd.Series)):
-                hf_hovertext = np.asarray(hf_hovertext)
-            if isinstance(hf_marker_size, (tuple, list, np.ndarray, pd.Series)):
-                hf_marker_size = np.asarray(hf_marker_size)
-            if isinstance(hf_marker_color, (tuple, list, np.ndarray, pd.Series)):
-                hf_marker_color = np.asarray(hf_marker_color)
+            # Note: this converts the hf properties to np.ndarray
+            for prop_name, _, _ in DOWNSAMPLABLE_PROPERTIES:
+                prop_value = parsed_properties.get(prop_name)
+                if isinstance(prop_value, (tuple, list, np.ndarray, pd.Series)):
+                    parsed_properties[prop_name] = np.asarray(prop_value)
 
             # Try to parse the hf_x data if it is of object type or
             if len(hf_x) and (hf_x.dtype.type is np.str_ or hf_x.dtype == "object"):
@@ -719,26 +702,77 @@ def _parse_get_trace_props(
             if hasattr(trace, "y"):
                 trace["y"] = hf_y
 
-            if hasattr(trace, "text"):
-                trace["text"] = hf_text
-
-            if hasattr(trace, "hovertext"):
-                trace["hovertext"] = hf_hovertext
-            if hasattr(trace, "marker"):
-                if hasattr(trace.marker, "size"):
-                    trace.marker.size = hf_marker_size
-                if hasattr(trace.marker, "color"):
-                    trace.marker.color = hf_marker_color
-
-        return _hf_data_container(
-            hf_x,
-            hf_y,
-            hf_text,
-            hf_hovertext,
-            hf_marker_size,
-            hf_marker_color,
-            hf_customdata,
-        )
+            # Set downsamplable properties if they exist
+            for prop_name, trace_path, _ in DOWNSAMPLABLE_PROPERTIES:
+                prop_value = parsed_properties.get(prop_name)
+                if prop_value is not None:
+                    self._set_trace_property(trace, trace_path, prop_value)
+
+        # Build the container with all properties
+        container_args = [hf_x, hf_y]
+        for prop_name, _, _ in DOWNSAMPLABLE_PROPERTIES:
+            container_args.append(parsed_properties.get(prop_name))
+
+        return _hf_data_container(*container_args)
+
+    def _get_trace_property(self, trace: BaseTraceType, trace_path: List[str]) -> any:
+        """Get a property from a trace using a path.
+
+        Parameters
+        ----------
+        trace : BaseTraceType
+            The trace to get the property from.
+        trace_path : List[str]
+            The path to the property (e.g., ["marker", "color"]).
+
+        Returns
+        -------
+        any
+            The property value or None if not found.
+        """
+        current = trace
+        for path_component in trace_path:
+            if hasattr(current, path_component):
+                current = getattr(current, path_component)
+            elif isinstance(current, dict) and path_component in current:
+                current = current[path_component]
+            else:
+                return None
+        return current
+
+    def _set_trace_property(
+        self, trace: BaseTraceType, trace_path: List[str], value: any
+    ) -> None:
+        """Set a property on a trace using a path.
+
+        Parameters
+        ----------
+        trace : BaseTraceType
+            The trace to set the property on.
+        trace_path : List[str]
+            The path to the property (e.g., ["marker", "color"]).
+        value : any
+            The value to set.
+        """
+        current = trace
+        for path_component in trace_path[:-1]:
+            if hasattr(current, path_component):
+                current = getattr(current, path_component)
+            elif isinstance(current, dict):
+                if path_component not in current:
+                    current[path_component] = {}
+                current = current[path_component]
+            else:
+                # Create the path if it doesn't exist
+                setattr(current, path_component, {})
+                current = getattr(current, path_component)
+
+        # Set the final property
+        final_component = trace_path[-1]
+        if hasattr(current, final_component):
+            setattr(current, final_component, value)
+        elif isinstance(current, dict):
+            current[final_component] = value
 
     def _construct_hf_data_dict(
         self,
@@ -853,10 +887,6 @@ def add_trace(
         # Use these if you want some speedups (and are working with really large data)
         hf_x: Iterable = None,
         hf_y: Iterable = None,
-        hf_text: Union[str, Iterable] = None,
-        hf_hovertext: Union[str, Iterable] = None,
-        hf_marker_size: Union[str, Iterable] = None,
-        hf_marker_color: Union[str, Iterable] = None,
         **trace_kwargs,
     ):
         """Add a trace to the figure.
@@ -900,22 +930,21 @@ def add_trace(
         hf_y: Iterable, optional
             The original high frequency values. If set, this has priority over the
             trace its data.
-        hf_text: Iterable, optional
-            The original high frequency text. If set, this has priority over the trace
-            its ``text`` argument.
-        hf_hovertext: Iterable, optional
-            The original high frequency hovertext. If set, this has priority over the
-            trace its ```hovertext`` argument.
-        hf_marker_size: Iterable, optional
-            The original high frequency marker size. If set, this has priority over the
-            trace its ``marker.size`` argument.
-        hf_marker_color: Iterable, optional
-            The original high frequency marker color. If set, this has priority over the
-            trace its ``marker.color`` argument.
         **trace_kwargs: dict
             Additional trace related keyword arguments.
             e.g.: row=.., col=..., secondary_y=...
 
+            High-frequency property parameters can also be passed:
+            - hf_text: High-frequency text data
+            - hf_hovertext: High-frequency hovertext data
+            - hf_marker_size: High-frequency marker size data
+            - hf_marker_color: High-frequency marker color data
+            - hf_marker_symbol: High-frequency marker symbol data
+            - hf_marker_angle: High-frequency marker angle data
+            - hf_customdata: High-frequency customdata
+
+            These have priority over the corresponding trace properties.
+
             !!! info "See Also"
                 [`Figure.add_trace`](https://plotly.com/python-api-reference/generated/plotly.graph_objects.Figure.html#plotly.graph_objects.Figure.add_trace>) docs.
 
@@ -985,17 +1014,14 @@ def add_trace(
         # These traces will determine the autoscale its RANGE!
         #   -> so also store when `limit_to_view` is set.
         if trace["type"].lower() in self._high_frequency_traces:
+            # Extract hf_* parameters from trace_kwargs
+            hf_properties = {}
+            for _, _, hf_param_name in DOWNSAMPLABLE_PROPERTIES:
+                if hf_param_name in trace_kwargs:
+                    hf_properties[hf_param_name] = trace_kwargs.pop(hf_param_name)
+
             # construct the hf_data_container
-            # TODO in future version -> maybe regex on kwargs which start with `hf_`
-            dc = self._parse_get_trace_props(
-                trace,
-                hf_x,
-                hf_y,
-                hf_text,
-                hf_hovertext,
-                hf_marker_size,
-                hf_marker_color,
-            )
+            dc = self._parse_get_trace_props(trace, hf_x, hf_y, **hf_properties)
 
             n_samples = len(dc.x)
             if n_samples > max_out_s or limit_to_view:
@@ -1368,10 +1394,14 @@ def _construct_update_data(
         layout_traces_list: List[dict] = [relayout_data]
 
         # 2. Create the additional trace data for the frond-end
-        relevant_keys = list(_hf_data_container._fields) + ["name", "marker"]
+        relevant_keys = ["name", "marker", "x", "y"] + [
+            prop_name for prop_name, _, _ in DOWNSAMPLABLE_PROPERTIES
+        ]
+        # self._print("relevant keys", relevant_keys)
         # Note that only updated trace-data will be sent to the client
         for idx in updated_trace_indices:
             trace = current_graph["data"][idx]
+            # self._print("trace keys", dict(trace).keys())
             # TODO: check if we can reduce even more
             trace_reduced = {k: trace[k] for k in relevant_keys if k in trace}