Skip to content

Commit 418ffd1

Browse files
authored
Merge pull request #53 from predict-idlab/figurewidget
🔍 investigating gap-detection methodology
2 parents 5135b60 + b2e1599 commit 418ffd1

File tree

12 files changed

+1136
-120
lines changed

12 files changed

+1136
-120
lines changed

docs/sphinx/getting_started.rst

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,12 @@ Working example ⬇️:
111111
112112
.. Note::
113113

114-
`hf_data` only withholds high-frequency traces (i.e., traces that are aggregated)
114+
`hf_data` only withholds high-frequency traces (i.e., traces that are aggregated).
115+
To add non high-frequency traces (i.e., traces with fewer data points than
116+
*max_n_samples*), you need to set the ``limit_to_view`` argument to *True* when adding
117+
the corresponding trace with the :func:`add_trace <plotly_resampler.figure_resampler.figure_resampler_interface.AbstractFigureAggregator.add_trace>` function.
118+
119+
115120

116121
.. tip::
117122

examples/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ which `plotly-resampler` is integrated
2424
| --- | --- |
2525
| [file visualization](dash_apps/dash_app.py) | load and visualize multiple `.parquet` files with plotly-resampler |
2626
| [dynamic sine generator](dash_apps/construct_dynamic_figures.py) | expeonential sine generator which uses [pattern matching callbacks](https://dash.plotly.com/pattern-matching-callbacks) to remove and construct plotly-resampler graphs dynamically |
27+
| [dynamic static graph](dash_apps/dash_app_coarse_fine.py) | Visualization dashboard in which a dynamic (i.e., plotly-resampler graph) and static graph (i.e., go.Figure) are shown (made for [this issue](https://github.com/predict-idlab/plotly-resampler/issues/56)). Relayout events on the coarse graph update the dynamic graph.

examples/dash_apps/callback_helpers.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ def _register_selection_callbacks(app, ids=None):
4141

4242

4343
def multiple_folder_file_selector(
44-
app, name_folders_list: List[Dict[str, dict]]
44+
app, name_folders_list: List[Dict[str, dict]], multi=True
4545
) -> dbc.Card:
4646
"""Constructs a folder user date selector
4747
@@ -78,7 +78,7 @@ def multiple_folder_file_selector(
7878
id=f"file-selector{i}",
7979
options=[],
8080
clearable=True,
81-
multi=True,
81+
multi=multi,
8282
),
8383
html.Br(),
8484
]
Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
"""Minimal dash app example.
2+
3+
In this usecase, we have dropdowns which allows the end-user to select files, which are
4+
visualized using FigureResampler after clicking on a button.
5+
6+
There a two graphs displayed a coarse and a dynamic graph.
7+
Interactions with the coarse will affect the dynamic graph range.
8+
Note that the autosize of the coarse graph is not linked.
9+
10+
TODO: add an rectangle on the coarse graph
11+
12+
"""
13+
14+
__author__ = "Jonas Van Der Donckt"
15+
16+
import re
17+
import dash
18+
import dash_bootstrap_components as dbc
19+
import pandas as pd
20+
import plotly.graph_objects as go
21+
import trace_updater
22+
from pathlib import Path
23+
from typing import List, Union
24+
from dash import Input, Output, State, dcc, html
25+
26+
from plotly_resampler import FigureResampler
27+
28+
from callback_helpers import multiple_folder_file_selector
29+
30+
31+
# Globals
32+
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.LUX])
33+
fr_fig: FigureResampler = FigureResampler()
34+
# NOTE, this reference to a FigureResampler is essential to preserve throughout the
35+
# whole dash app! If your dash apps want to create a new go.Figure(), you should not
36+
# construct a new FigureResampler object, but replace the figure of this FigureResampler
37+
# object by using the FigureResampler.replace() method.
38+
# Example: see the plot_multiple_files function in this file.
39+
40+
41+
# ------------------------------- File selection logic -------------------------------
42+
name_folder_list = [
43+
{
44+
# the key-string below is the title which will be shown in the dash app
45+
"dash example data": {"folder": Path("../data")},
46+
"other name same folder": {"folder": Path("../data")},
47+
},
48+
# NOTE: A new item om this level creates a new file-selector card.
49+
# { "PC data": { "folder": Path("/home/jonas/data/wesad/empatica/") } }
50+
# TODO: change the folder path above to a location where you have some
51+
# `.parquet` files stored on your machine.
52+
]
53+
54+
55+
# ------------------------------------ DASH logic -------------------------------------
56+
# First we construct the app layout
57+
def serve_layout() -> dbc.Container:
58+
"""Constructs the app's layout.
59+
60+
Returns
61+
-------
62+
dbc.Container
63+
A Container withholding the layout.
64+
65+
"""
66+
return dbc.Container(
67+
[
68+
dbc.Container(
69+
html.H1("Data visualization - coarse & dynamic graph"),
70+
style={"textAlign": "center"},
71+
),
72+
html.Hr(),
73+
dbc.Row(
74+
[
75+
# Add file selection layout (+ assign callbacks)
76+
dbc.Col(
77+
multiple_folder_file_selector(
78+
app, name_folder_list, multi=False
79+
),
80+
md=2,
81+
),
82+
# Add the graphs and the trace updater component
83+
dbc.Col(
84+
[
85+
# The coarse graph whose updates will fetch data for the
86+
# broad graph
87+
dcc.Graph(
88+
id="coarse-graph",
89+
figure=go.Figure(),
90+
config={"modeBarButtonsToAdd": ["drawrect"]},
91+
),
92+
dcc.Graph(id="plotly-resampler-graph", figure=go.Figure()),
93+
# The broad graph
94+
trace_updater.TraceUpdater(
95+
id="trace-updater", gdID="plotly-resampler-graph"
96+
),
97+
],
98+
md=10,
99+
),
100+
],
101+
align="center",
102+
),
103+
],
104+
fluid=True,
105+
)
106+
107+
108+
app.layout = serve_layout()
109+
110+
111+
# Register the graph update callbacks to the layout
112+
@app.callback(
113+
Output("trace-updater", "updateData"),
114+
Input("coarse-graph", "relayoutData"),
115+
Input("plotly-resampler-graph", "relayoutData"),
116+
prevent_initial_call=True,
117+
)
118+
def update_dynamic_fig(coarse_grained_relayout, fine_grained_relayout):
119+
global fr_fig
120+
121+
ctx = dash.callback_context
122+
trigger_id = ctx.triggered[0].get("prop_id", "").split(".")[0]
123+
124+
if trigger_id == "plotly-resampler-graph":
125+
return fr_fig.construct_update_data(fine_grained_relayout)
126+
elif trigger_id == "coarse-graph":
127+
if "shapes" in coarse_grained_relayout:
128+
print(coarse_grained_relayout)
129+
cl_k = coarse_grained_relayout.keys()
130+
# We do not resample when and autorange / autosize event takes place
131+
matches = fr_fig._re_matches(re.compile(r"xaxis\d*.range\[0]"), cl_k)
132+
if len(matches):
133+
return fr_fig.construct_update_data(coarse_grained_relayout)
134+
135+
return dash.no_update
136+
137+
138+
# ------------------------------ Visualization logic ---------------------------------
139+
def plot_multiple_files(file_list: List[Union[str, Path]]) -> FigureResampler:
140+
"""Code to create the visualizations.
141+
142+
Parameters
143+
----------
144+
file_list: List[Union[str, Path]]
145+
146+
Returns
147+
-------
148+
FigureResampler
149+
Returns a view of the existing, global FigureResampler object.
150+
151+
"""
152+
global fr_fig
153+
154+
# NOTE, we do not construct a new FigureResampler object, but replace the figure of
155+
# the figureResampler object. Otherwise the coupled callbacks would be lost and it
156+
# is not (straightforward) to construct dynamic callbacks in dash.
157+
fr_fig._global_n_shown_samples = 3000
158+
fr_fig.replace(go.Figure())
159+
fr_fig.update_layout(height=min(900, 350 * len(file_list)))
160+
161+
for f in file_list:
162+
df = pd.read_parquet(f) # should be replaced by more generic data loading code
163+
if "timestamp" in df.columns:
164+
df = df.set_index("timestamp")
165+
166+
for c in df.columns:
167+
fr_fig.add_trace(go.Scatter(name=c), hf_x=df.index, hf_y=df[c])
168+
return fr_fig
169+
170+
171+
# Note: the list sum-operations flattens the list
172+
selector_states = list(
173+
sum(
174+
[
175+
(
176+
State(f"folder-selector{i}", "value"),
177+
State(f"file-selector{i}", "value"),
178+
)
179+
for i in range(1, len(name_folder_list) + 1)
180+
],
181+
(),
182+
)
183+
)
184+
185+
186+
@app.callback(
187+
Output("coarse-graph", "figure"),
188+
Output("plotly-resampler-graph", "figure"),
189+
[Input("plot-button", "n_clicks"), *selector_states],
190+
prevent_initial_call=True,
191+
)
192+
def plot_graph(
193+
n_clicks,
194+
*folder_list,
195+
):
196+
it = iter(folder_list)
197+
file_list: List[Path] = []
198+
for folder, files in zip(it, it):
199+
if not all((folder, files)):
200+
continue
201+
else:
202+
files = [files] if not isinstance(files, list) else file_list
203+
for file in files:
204+
file_list.append((Path(folder).joinpath(file)))
205+
206+
ctx = dash.callback_context
207+
if len(ctx.triggered) and "plot-button" in ctx.triggered[0]["prop_id"]:
208+
if len(file_list):
209+
dynamic_fig = plot_multiple_files(file_list)
210+
coarse_fig = go.Figure(dynamic_fig)
211+
coarse_fig.update_layout(title="<b>coarse view</b>", height=250)
212+
coarse_fig.update_layout(margin=dict(l=0, r=0, b=0, t=40, pad=10))
213+
coarse_fig.update_layout(showlegend=False)
214+
coarse_fig._config = coarse_fig._config.update(
215+
{"modeBarButtonsToAdd": ["drawrect"]}
216+
)
217+
218+
dynamic_fig._global_n_shown_samples = 1000
219+
dynamic_fig.update_layout(title="<b>dynamic view<b>", height=450)
220+
dynamic_fig.update_layout(margin=dict(l=0, r=0, b=40, t=40, pad=10))
221+
dynamic_fig.update_layout(
222+
legend=dict(
223+
orientation="h", y=-0.11, xanchor="right", x=1, font_size=18
224+
)
225+
)
226+
227+
# coarse_fig['layout'].update(dict(title='coarse view', title_x=0.5, height=250))
228+
return coarse_fig, dynamic_fig
229+
else:
230+
raise dash.exceptions.PreventUpdate()
231+
232+
233+
# --------------------------------- Running the app ---------------------------------
234+
if __name__ == "__main__":
235+
app.run_server(debug=True, port=9023)

plotly_resampler/aggregation/aggregation_interface.py

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ class AbstractSeriesAggregator(ABC):
1616
def __init__(
1717
self,
1818
interleave_gaps: bool = True,
19+
nan_position: str = "end",
1920
dtype_regex_list: List[str] = None,
2021
):
2122
"""Constructor of AbstractSeriesAggregator.
@@ -26,12 +27,24 @@ def __init__(
2627
Whether None values should be added when there are gaps / irregularly
2728
sampled data. A quantile-based approach is used to determine the gaps /
2829
irregularly sampled data. By default, True.
30+
nan_position: str, optional
31+
Indicates where nans must be placed when gaps are detected. \n
32+
If ``'end'``, the first point after a gap will be replaced with a
33+
nan-value \n
34+
If ``'begin'``, the last point before a gap will be replaced with a
35+
nan-value \n
36+
If ``'both'``, both the encompassing gap datapoints are replaced with
37+
nan-values \n
38+
.. note::
39+
This parameter only has an effect when ``interleave_gaps`` is set
40+
to *True*.
2941
dtype_regex_list: List[str], optional
3042
List containing the regex matching the supported datatypes, by default None.
3143
3244
"""
3345
self.interleave_gaps = interleave_gaps
3446
self.dtype_regex_list = dtype_regex_list
47+
self.nan_position = nan_position.lower()
3548
super().__init__()
3649

3750
@abstractmethod
@@ -61,18 +74,16 @@ def _calc_med_diff(s: pd.Series) -> Tuple[float, np.ndarray]:
6174
# To do so - use a quantile-based (median) approach where we reshape the data
6275
# into `n_blocks` blocks and calculate the min
6376
n_blcks = 128
64-
if s.shape[0] > n_blcks:
77+
if s.shape[0] > 5 * n_blcks:
6578
blck_size = s_idx_diff.shape[0] // n_blcks
6679

6780
# convert the index series index diff into a reshaped view (i.e., sid_v)
6881
sid_v: np.ndarray = s_idx_diff[: blck_size * n_blcks].reshape(n_blcks, -1)
6982

7083
# calculate the min and max and calculate the median on that
71-
med_diff = np.quantile(
72-
np.concatenate((sid_v.min(axis=0), sid_v.max(axis=0))), q=0.55
73-
)
84+
med_diff = np.median(np.mean(sid_v, axis=1))
7485
else:
75-
med_diff = np.quantile(s_idx_diff, q=0.55)
86+
med_diff = np.median(s_idx_diff)
7687

7788
return med_diff, s_idx_diff
7889

@@ -108,7 +119,15 @@ def _replace_gap_end_none(self, s: pd.Series) -> pd.Series:
108119
med_diff, s_idx_diff = self._calc_med_diff(s)
109120
if med_diff is not None:
110121
# Replace data-points with None where the gaps occur
111-
s.loc[s_idx_diff > 3 * med_diff] = None
122+
# The default is the end of a gap
123+
nan_mask = s_idx_diff > 4 * med_diff
124+
if self.nan_position == "begin":
125+
# Replace the last non-gap datapoint (begin of gap) with Nan
126+
nan_mask = np.roll(nan_mask, -1)
127+
elif self.nan_position == "both":
128+
# Replace the encompassing gap datapoints with Nan
129+
nan_mask |= np.roll(nan_mask, -1)
130+
s.loc[nan_mask] = None
112131
return s
113132

114133
def aggregate(self, s: pd.Series, n_out: int) -> pd.Series:

0 commit comments

Comments
 (0)