Skip to content

Commit 31e03b9

Browse files
ianthomas23martinRenou
ianthomas23
authored and
martinRenou
committed
Sorting and filtering on backend when streaming
Signed-off-by: Ian Thomas <[email protected]>
1 parent 33798af commit 31e03b9

File tree

7 files changed

+272
-60
lines changed

7 files changed

+272
-60
lines changed

‎ipydatagrid/datagrid.py‎

Lines changed: 98 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1001,6 +1001,7 @@ def __init__(self, *args, debounce_delay=160, **kwargs):
10011001
super().__init__(*args, **kwargs)
10021002

10031003
self._debounce_delay = debounce_delay
1004+
self._transformed_data = None
10041005

10051006
self.on_msg(self._handle_comm_msg)
10061007

@@ -1041,6 +1042,66 @@ def tick(self):
10411042
"""Notify that the underlying dataframe has changed."""
10421043
self.send({"event_type": "tick"})
10431044

1045+
def _apply_frontend_transforms(self, frontend_transforms, dataframe):
1046+
for transform in frontend_transforms:
1047+
if transform["type"] == "sort":
1048+
column = transform["column"]
1049+
desc = transform.get("desc", False)
1050+
dataframe = dataframe.sort_values(
1051+
by=[column], ascending=not desc
1052+
)
1053+
elif transform["type"] == "filter":
1054+
column = transform["column"]
1055+
operator = transform["operator"]
1056+
value = transform["value"]
1057+
if operator == "<":
1058+
dataframe = dataframe[dataframe[column] < value]
1059+
elif operator == ">":
1060+
dataframe = dataframe[dataframe[column] > value]
1061+
elif operator == "=":
1062+
dataframe = dataframe[dataframe[column] == value]
1063+
elif operator == "<=":
1064+
dataframe = dataframe[dataframe[column] <= value]
1065+
elif operator == ">=":
1066+
dataframe = dataframe[dataframe[column] >= value]
1067+
elif operator == "!=":
1068+
dataframe = dataframe[dataframe[column] != value]
1069+
elif operator == "empty":
1070+
dataframe = dataframe[dataframe[column].isna()]
1071+
elif operator == "notempty":
1072+
dataframe = dataframe[dataframe[column].notna()]
1073+
elif operator == "in":
1074+
dataframe = dataframe[dataframe[column].isin(value)]
1075+
elif operator == "between":
1076+
dataframe = dataframe[
1077+
dataframe[column].between(value[0], value[1])
1078+
]
1079+
elif operator == "startswith":
1080+
dataframe = dataframe[
1081+
dataframe[column].str.startswith(value)
1082+
]
1083+
elif operator == "endswith":
1084+
dataframe = dataframe[dataframe[column].str.endswith(value)]
1085+
elif operator == "stringContains":
1086+
dataframe = dataframe[dataframe[column].str.contains(value)]
1087+
elif operator == "contains":
1088+
dataframe = dataframe[dataframe[column].str.contains(value)]
1089+
elif operator == "!contains":
1090+
dataframe = dataframe[
1091+
not dataframe[column].str.contains(value)
1092+
]
1093+
elif operator == "isOnSameDay":
1094+
value = pd.to_datetime(value).date()
1095+
dataframe = dataframe[
1096+
pd.to_datetime(dataframe[column]).dt.date == value
1097+
]
1098+
else:
1099+
raise RuntimeError(
1100+
f"Unrecognised filter operator '{operator}'"
1101+
)
1102+
1103+
return dataframe
1104+
10441105
def _handle_comm_msg(self, _, content, _buffs):
10451106
event_type = content.get("type", "")
10461107

@@ -1050,7 +1111,14 @@ def _handle_comm_msg(self, _, content, _buffs):
10501111
c1 = content.get("c1")
10511112
c2 = content.get("c2")
10521113

1053-
value = self.__dataframe_reference.iloc[r1 : r2 + 1, c1 : c2 + 1]
1114+
# Filter/sort whole dataset before selecting rows/cols of interest
1115+
if self._transformed_data is not None:
1116+
# Use existing transformed data.
1117+
value = self._transformed_data
1118+
else:
1119+
value = self.__dataframe_reference
1120+
1121+
value = value.iloc[r1 : r2 + 1, c1 : c2 + 1]
10541122

10551123
# Primary key used
10561124
index_key = self.get_dataframe_index(value)
@@ -1079,3 +1147,32 @@ def _handle_comm_msg(self, _, content, _buffs):
10791147
}
10801148

10811149
self.send(answer, buffers)
1150+
1151+
elif event_type == "frontend-transforms":
1152+
# Transforms is an array of dicts.
1153+
frontend_transforms = content.get("transforms")
1154+
1155+
self._transformed_data = None
1156+
data = self.__dataframe_reference
1157+
1158+
if frontend_transforms:
1159+
self._transformed_data = self._apply_frontend_transforms(
1160+
frontend_transforms, data
1161+
)
1162+
data = self._transformed_data
1163+
1164+
self._row_count = len(data) # Sync to frontend.
1165+
1166+
# Should only request a tick if the transforms have changed.
1167+
self.tick()
1168+
1169+
elif event_type == "unique-values-request":
1170+
column = content.get("column")
1171+
unique = (
1172+
self.__dataframe_reference[column].drop_duplicates().to_numpy()
1173+
)
1174+
answer = {
1175+
"event_type": "unique-values-reply",
1176+
"values": unique,
1177+
}
1178+
self.send(answer)

‎js/core/filterMenu.ts‎

Lines changed: 56 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -251,32 +251,33 @@ export class InteractiveFilterDialog extends BoxPanel {
251251
* Displays the unique values of a column.
252252
*/
253253
_renderUniqueVals() {
254-
const uniqueVals = this._model.uniqueValues(this._region, this._column);
255-
const data = new DataSource(
256-
{ index: [...uniqueVals.keys()], uniqueVals },
257-
[{ index: null }, { uniqueVals: null }],
258-
{
259-
fields: [
260-
{ name: 'index', type: 'integer', rows: [] },
261-
{ name: 'uniqueVals', type: 'number', rows: [] },
262-
],
263-
primaryKey: ['index'],
264-
primaryKeyUuid: 'index',
265-
},
266-
);
267-
this._uniqueValueGrid.dataModel = new ViewBasedJSONModel({
268-
datasource: data,
254+
this._model.uniqueValues(this._region, this._column).then((uniqueVals) => {
255+
const data = new DataSource(
256+
{ index: [...uniqueVals.keys()], uniqueVals },
257+
[{ index: null }, { uniqueVals: null }],
258+
{
259+
fields: [
260+
{ name: 'index', type: 'integer', rows: [] },
261+
{ name: 'uniqueVals', type: 'number', rows: [] },
262+
],
263+
primaryKey: ['index'],
264+
primaryKeyUuid: 'index',
265+
},
266+
);
267+
this._uniqueValueGrid.dataModel = new ViewBasedJSONModel({
268+
datasource: data,
269+
});
270+
const sortTransform: Transform.Sort = {
271+
type: 'sort',
272+
column: 'uniqueVals',
273+
columnIndex: this.model.getSchemaIndex(this._region, 0),
274+
desc: false,
275+
};
276+
// Sort items in filter-by-value menu in ascending order
277+
(<ViewBasedJSONModel>this._uniqueValueGrid.dataModel).addTransform(
278+
sortTransform,
279+
);
269280
});
270-
const sortTransform: Transform.Sort = {
271-
type: 'sort',
272-
column: 'uniqueVals',
273-
columnIndex: this.model.getSchemaIndex(this._region, 0),
274-
desc: false,
275-
};
276-
// Sort items in filter-by-value menu in ascending order
277-
(<ViewBasedJSONModel>this._uniqueValueGrid.dataModel).addTransform(
278-
sortTransform,
279-
);
280281
}
281282

282283
/**
@@ -292,20 +293,20 @@ export class InteractiveFilterDialog extends BoxPanel {
292293
return;
293294
}
294295

295-
const uniqueVals = this._model.uniqueValues(this._region, this._column);
296-
297-
let showAsChecked = true;
298-
for (const value of uniqueVals) {
299-
// If there is a unique value which is not present in the state then it is
300-
// not ticked, and therefore we should not tick the "Select all" checkbox.
301-
if (
302-
!this._uniqueValueStateManager.has(this._region, this._column, value)
303-
) {
304-
showAsChecked = false;
305-
break;
296+
this._model.uniqueValues(this._region, this._column).then((uniqueVals) => {
297+
let showAsChecked = true;
298+
for (const value of uniqueVals) {
299+
// If there is a unique value which is not present in the state then it is
300+
// not ticked, and therefore we should not tick the "Select all" checkbox.
301+
if (
302+
!this._uniqueValueStateManager.has(this._region, this._column, value)
303+
) {
304+
showAsChecked = false;
305+
break;
306+
}
306307
}
307-
}
308-
this._selectAllCheckbox.checked = showAsChecked;
308+
this._selectAllCheckbox.checked = showAsChecked;
309+
});
309310
}
310311

311312
/**
@@ -714,7 +715,11 @@ export class InteractiveFilterDialog extends BoxPanel {
714715
* values of a column.
715716
*/
716717
protected async createUniqueValueNodes(): Promise<VirtualElement> {
717-
const uniqueVals = this._model.uniqueValues(this._region, this._column);
718+
const uniqueVals = await this._model.uniqueValues(
719+
this._region,
720+
this._column,
721+
);
722+
718723
const optionElems = uniqueVals.map((val) => {
719724
return h.option({ value: val }, String(val));
720725
});
@@ -1124,15 +1129,19 @@ export class InteractiveFilterDialog extends BoxPanel {
11241129
}
11251130

11261131
addRemoveAllUniqueValuesToState(add: boolean) {
1127-
const uniqueVals = this.model.uniqueValues(this._region, this._column);
1128-
1129-
for (const value of uniqueVals) {
1130-
if (add) {
1131-
this._uniqueValueStateManager.add(this._region, this._column, value);
1132-
} else {
1133-
this._uniqueValueStateManager.remove(this._region, this._column, value);
1132+
this.model.uniqueValues(this._region, this._column).then((uniqueVals) => {
1133+
for (const value of uniqueVals) {
1134+
if (add) {
1135+
this._uniqueValueStateManager.add(this._region, this._column, value);
1136+
} else {
1137+
this._uniqueValueStateManager.remove(
1138+
this._region,
1139+
this._column,
1140+
value,
1141+
);
1142+
}
11341143
}
1135-
}
1144+
});
11361145
}
11371146

11381147
/**

‎js/core/streamingview.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ export class StreamingView extends View {
9191
c2: number,
9292
value: DataSource,
9393
) {
94+
// Columns need to be set some time, not sure if this is the best place.
95+
this._data.setColumns(value.columns);
96+
9497
let field: DataSource.IField;
9598

9699
// Update body
@@ -135,8 +138,12 @@ export class StreamingView extends View {
135138
this._streamed_data[field.name][row] = value;
136139
}
137140

141+
setRowCount(rowCount: number) {
142+
this._rowCount = rowCount;
143+
}
144+
138145
private _streamed_data: Dict<any[]> = {};
139-
private readonly _rowCount: number;
146+
private _rowCount: number;
140147
}
141148

142149
export namespace StreamingView {

‎js/core/streamingviewbasedjsonmodel.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,17 @@
1+
import { PromiseDelegate } from '@lumino/coreutils';
2+
import { DataModel } from '@lumino/datagrid';
13
import { StreamingView } from './streamingview';
4+
import { TransformStateManager } from './transformStateManager';
25
import { ViewBasedJSONModel } from './viewbasedjsonmodel';
6+
import { DataGridModel } from '../datagrid';
37
import { DataSource } from '../datasource';
48

9+
interface IUnique {
10+
region: DataModel.CellRegion;
11+
column: string;
12+
values: any[];
13+
}
14+
515
/**
616
* A view based data model implementation for in-memory JSON data.
717
*/
@@ -38,7 +48,42 @@ export class StreamingViewBasedJSONModel extends ViewBasedJSONModel {
3848
});
3949
}
4050

51+
/**
52+
* Returns a Promise that resolves to an array of unique values contained in
53+
* the provided column index.
54+
*
55+
* @param region - The CellRegion to retrieve unique values for.
56+
* @param column - The column to retrieve unique values for.
57+
*/
58+
uniqueValues(region: DataModel.CellRegion, column: string): Promise<any[]> {
59+
if (
60+
this._unique &&
61+
region == this._unique.region &&
62+
column == this._unique.column
63+
) {
64+
return Promise.resolve(this._unique.values);
65+
}
66+
67+
const promiseDelegate = new PromiseDelegate<any[]>();
68+
this._dataModel.on('msg:custom', (content) => {
69+
// when message received, want to drop this handler...
70+
// Or keep it going but need a way of identifying where to put the received data??????
71+
if (content.event_type === 'unique-values-reply') {
72+
this._unique = { region, column, values: content.values };
73+
promiseDelegate.resolve(this._unique.values);
74+
}
75+
76+
// Do I need to cancel this callback?????????
77+
});
78+
79+
const msg = { type: 'unique-values-request', column: column };
80+
this._dataModel.send(msg);
81+
82+
return promiseDelegate.promise;
83+
}
84+
4185
updateDataset(options: StreamingViewBasedJSONModel.IOptions): void {
86+
this._dataModel = options.dataModel;
4287
this._dataset = options.datasource;
4388
this._updatePrimaryKeyMap();
4489
const view = new StreamingView({
@@ -63,7 +108,23 @@ export class StreamingViewBasedJSONModel extends ViewBasedJSONModel {
63108
super.currentView = view;
64109
}
65110

111+
/**
112+
* Handler for transformState.changed events.
113+
*
114+
* @param sender - TransformStateManager
115+
*
116+
* @param value - Event.
117+
*/
118+
protected _transformStateChangedHandler(
119+
sender: TransformStateManager,
120+
value: TransformStateManager.IEvent,
121+
) {
122+
this._transformSignal.emit(value);
123+
}
124+
66125
protected _currentView: StreamingView;
126+
protected _dataModel: DataGridModel;
127+
protected _unique?: IUnique;
67128
}
68129

69130
export namespace StreamingViewBasedJSONModel {
@@ -75,5 +136,7 @@ export namespace StreamingViewBasedJSONModel {
75136
* The row number of the grid.
76137
*/
77138
rowCount: number;
139+
140+
dataModel: DataGridModel;
78141
}
79142
}

0 commit comments

Comments
 (0)