perf(filtering): Optimize filtering performance

projects/igniteui-angular/src/lib/data-operations/data-util.ts

@@ -5,7 +5,7 @@ import { IPagingState, PagingError } from './paging-state.interface';
 import { IGroupByKey } from './groupby-expand-state.interface';
 import { IGroupByRecord } from './groupby-record.interface';
 import { IGroupingState } from './groupby-state.interface';
-import { mergeObjects } from '../core/utils';
+import { cloneArray, mergeObjects } from '../core/utils';
 import { Transaction, TransactionType, HierarchicalTransaction } from '../services/transaction/transaction';
 import { getHierarchy, isHierarchyMatch } from './operations';
 import { ColumnType, GridType } from '../grids/common/grid.interface';
@@ -21,6 +21,8 @@ import {
 import { DefaultDataCloneStrategy, IDataCloneStrategy } from '../data-operations/data-clone-strategy';
 import { IGroupingExpression } from './grouping-expression.interface';
 import { DefaultMergeStrategy, IGridMergeStrategy } from './merge-strategy';
+import { IFilteringExpressionsTree } from './filtering-expressions-tree';
+import { FilteringStrategy, FilterUtil } from './filtering-strategy';
 
 /**
  * @hidden
@@ -278,6 +280,15 @@ export class DataUtil {
         return value;
     }
 
+    public static filterDataByExpressions(data: any[], expressionsTree: IFilteringExpressionsTree, grid: GridType): any {
+        if (expressionsTree.filteringOperands.length) {
+            const state = { expressionsTree, strategy: FilteringStrategy.instance() };
+            data = FilterUtil.filter(cloneArray(data), state, grid);
+        }
+
+        return data;
+    }
+
     private static findParentFromPath(data: any[], primaryKey: any, childDataKey: any, path: any[]): any {
         let collection: any[] = data;
         let result: any;

projects/igniteui-angular/src/lib/data-operations/filtering-strategy.ts

@@ -2,12 +2,13 @@ import { FilteringLogic, type IFilteringExpression } from './filtering-expressio
 import { FilteringExpressionsTree, type IFilteringExpressionsTree } from './filtering-expressions-tree';
 import { resolveNestedPath, parseDate, formatDate, formatCurrency, columnFieldPath } from '../core/utils';
 import type { ColumnType, EntityType, GridType } from '../grids/common/grid.interface';
-import { GridColumnDataType } from './data-util';
+import { DataUtil, GridColumnDataType } from './data-util';
 import { SortingDirection } from './sorting-strategy';
 import { formatNumber, formatPercent, getLocaleCurrencyCode } from '@angular/common';
 import type { IFilteringState } from './filtering-state.interface';
 import { isTree } from './expressions-tree-util';
 import type { IgxHierarchicalGridComponent } from '../grids/hierarchical-grid/hierarchical-grid.component';
+import { IgxSorting } from '../grids/common/strategy';
 
 const DateType = 'date';
 const DateTimeType = 'dateTime';
@@ -132,27 +133,35 @@ export abstract class BaseFilteringStrategy implements IFilteringStrategy {
     public getFilterItems(column: ColumnType, tree: IFilteringExpressionsTree): Promise<IgxFilterItem[]> {
         const applyFormatter = column.formatter && this.shouldFormatFilterValues(column);
 
-        let data = column.grid.gridAPI.filterDataByExpressions(tree);
-        data = column.grid.gridAPI.sortDataByExpressions(data,
-            [{ fieldName: column.field, dir: SortingDirection.Asc, ignoreCase: column.sortingIgnoreCase }]);
-
+        const data = this.getFilteredData(column, tree);
 
         const pathParts = columnFieldPath(column.field)
-        let filterItems: IgxFilterItem[] = data.map(record => {
-            const value = applyFormatter ?
-                column.formatter(resolveNestedPath(record, pathParts), record) :
-                resolveNestedPath(record, pathParts);
-
-            return {
-                value,
-                label: this.getFilterItemLabel(column, value, !applyFormatter, record)
-            };
-        });
-        filterItems = this.getUniqueFilterItems(column, filterItems);
+        const seenFormattedFilterItems = new Map<any, IgxFilterItem>()
+
+        for (let i = 0; i < data.length; ++i) {
+            const record = data[i]
+            const rawValue = resolveNestedPath(record, pathParts);
+            const formattedValue = applyFormatter ? column.formatter(rawValue, record) : rawValue;
+            const { key, finalValue } = this.getFilterItemKeyValue(formattedValue, column);
+            // Deduplicate by normalized key
+            if (!seenFormattedFilterItems.has(key)) {
+                const label = this.getFilterItemLabel(column, finalValue, !applyFormatter, record);
+                seenFormattedFilterItems.set(key, { value: finalValue, label });
+            }
+        }
+
+        let filterItems: IgxFilterItem[] = Array.from(seenFormattedFilterItems.values());
+
+        filterItems = DataUtil.sort(filterItems,
+            [{ fieldName: 'value', dir: SortingDirection.Asc, ignoreCase: column.sortingIgnoreCase }], new IgxSorting())
 
         return Promise.resolve(filterItems);
     }
 
+    protected getFilteredData(column: ColumnType, tree: IFilteringExpressionsTree) {
+        return column.grid.gridAPI.filterDataByExpressions(tree);
+    }
+
     protected getFilterItemLabel(column: ColumnType, value: any, applyFormatter: boolean, data: any) {
         if (column.formatter) {
             if (applyFormatter) {
@@ -180,30 +189,33 @@ export abstract class BaseFilteringStrategy implements IFilteringStrategy {
         }
     }
 
-    protected getUniqueFilterItems(column: ColumnType, filterItems: IgxFilterItem[]) {
-        const filteredUniqueValues = filterItems.reduce((map, item) => {
-            let key = item.value;
-
-            if (column.dataType === GridColumnDataType.String && column.filteringIgnoreCase) {
-                key = key?.toString().toLowerCase();
-            } else if (column.dataType === GridColumnDataType.DateTime) {
-                key = item.value?.toString();
-                item.value = key ? new Date(key) : key;
-            } else if (column.dataType === GridColumnDataType.Time) {
-                const date = key ? new Date(key) : key;
-                key = date ? new Date().setHours(date.getHours(), date.getMinutes(), date.getSeconds(), date.getMilliseconds()) : key;
-                item.value = key ? new Date(key) : key;
-            } else if (column.dataType === GridColumnDataType.Date) {
-                const date = key ? new Date(key) : key;
-                key = date ? new Date(date.getFullYear(), date.getMonth(), date.getDate()).toISOString() : key;
-                item.value = date;
-            }
-
-            return map.has(key) ? map : map.set(key, item)
-        }, new Map());
-        const uniqueValues = Array.from(filteredUniqueValues.values());
-
-        return uniqueValues;
+    protected getFilterItemKeyValue(value: any, column: ColumnType) {
+        let key: any = value;
+        let finalValue = value;
+        if (column.dataType === GridColumnDataType.String && column.filteringIgnoreCase) {
+            key = key?.toString().toLowerCase();
+        } else if (column.dataType === GridColumnDataType.DateTime) {
+            key = value?.toString();
+            finalValue = key ? new Date(key) : key;
+        } else if (column.dataType === GridColumnDataType.Time) {
+            const date = key ? new Date(key) : key;
+            key = date
+                ? new Date().setHours(
+                    date.getHours(),
+                    date.getMinutes(),
+                    date.getSeconds(),
+                    date.getMilliseconds()
+                )
+                : date;
+            finalValue = key ? new Date(key) : key;
+        } else if (column.dataType === GridColumnDataType.Date) {
+            const date = key ? new Date(key) : key;
+            key = date
+                ? new Date(date.getFullYear(), date.getMonth(), date.getDate()).toISOString()
+                : date;
+            finalValue = date;
+        }
+        return { key, finalValue };
     }
 
     protected shouldFormatFilterValues(_column: ColumnType): boolean {

projects/igniteui-angular/src/lib/grids/filtering/excel-style/base-filtering.component.ts

@@ -23,6 +23,7 @@ export abstract class BaseFilteringComponent {
     public abstract columnChange: EventEmitter<any>;
     public abstract sortingChanged: EventEmitter<undefined>;
     public abstract listDataLoaded: EventEmitter<undefined>;
+    public abstract filterCleared: EventEmitter<undefined>;
 
     constructor(
         protected cdr: ChangeDetectorRef,

projects/igniteui-angular/src/lib/grids/filtering/excel-style/excel-style-clear-filters.component.ts

@@ -34,6 +34,7 @@ export class IgxExcelStyleClearFiltersComponent {
      */
     public clearFilter() {
         this.esf.grid.filteringService.clearFilter(this.esf.column.field);
+        this.esf.filterCleared.emit();
         this.selectAllFilterItems();
     }
 

projects/igniteui-angular/src/lib/grids/filtering/excel-style/excel-style-filtering.component.ts

@@ -128,6 +128,12 @@ export class IgxGridExcelStyleFilteringComponent extends BaseFilteringComponent
     @Output()
     public listDataLoaded = new EventEmitter();
 
+    /**
+     * @hidden @internal
+     */
+    @Output()
+    public filterCleared = new EventEmitter();
+
     @ViewChild('mainDropdown', { read: ElementRef })
     public mainDropdown: ElementRef<HTMLElement>;
 
@@ -160,29 +166,13 @@ export class IgxGridExcelStyleFilteringComponent extends BaseFilteringComponent
      */
     @Input()
     public set column(value: ColumnType) {
-        this._column = value;
-        this.listData = new Array<FilterListItem>();
-        this.columnChange.emit(this._column);
-
-        this.subscriptions?.unsubscribe();
-
-        if (this._column) {
-            this.grid.filteringService.registerSVGIcons();
-            this.init();
-            this.sortingChanged.emit();
-
-            this.subscriptions = this.grid.columnPin.subscribe(() => {
-                requestAnimationFrame(() => {
-                    if (!(this.cdr as ViewRef).destroyed) {
-                        this.cdr.detectChanges();
-                    }
-                });
-            });
-
-            this.subscriptions.add(this.grid.columnVisibilityChanged.subscribe(() => this.detectChanges()));
-            this.subscriptions.add(this.grid.sortingExpressionsChange.subscribe(() => this.sortingChanged.emit()));
-            this.subscriptions.add(this.grid.filteringExpressionsTreeChange.subscribe(() => this.init()));
-            this.subscriptions.add(this.grid.columnMovingEnd.subscribe(() => this.cdr.markForCheck()));
+        if (value) {
+            this._column = value;
+            this.columnChange.emit(this._column);
+            if (this.inline) {
+                // In case external filtering
+                this.populateData();
+            }
         }
     }
 
@@ -326,6 +316,16 @@ export class IgxGridExcelStyleFilteringComponent extends BaseFilteringComponent
         this.inline = false;
         this.column = column;
         this.overlayService = overlayService;
+
+    }
+
+    /**
+     * @hidden @internal
+     */
+    public populateData() {
+        if (this.column) {
+            this.afterColumnChange();
+        }
         if (this._originalDisplay) {
             this.element.nativeElement.style.display = this._originalDisplay;
         }
@@ -421,6 +421,34 @@ export class IgxGridExcelStyleFilteringComponent extends BaseFilteringComponent
         return this.computedStyles?.getPropertyValue('--component-size');
     }
 
+    protected afterColumnChange() {
+        this.listData = new Array<FilterListItem>();
+        this.subscriptions?.unsubscribe();
+
+        if (this._column) {
+            this.grid.filteringService.registerSVGIcons();
+            this.init();
+            this.sortingChanged.emit();
+
+            this.subscriptions = this.grid.columnPin.subscribe(() => {
+                requestAnimationFrame(() => {
+                    if (!(this.cdr as ViewRef).destroyed) {
+                        this.cdr.detectChanges();
+                    }
+                });
+            });
+
+            this.subscriptions.add(this.grid.columnVisibilityChanged.subscribe(() => this.detectChanges()));
+            this.subscriptions.add(this.grid.sortingExpressionsChange.subscribe(() => this.sortingChanged.emit()));
+            this.subscriptions.add(this.grid.filteringExpressionsTreeChange.subscribe(() => {
+                this.expressionsList = new Array<ExpressionUI>();
+                generateExpressionsList(this.column.filteringExpressionsTree, this.grid.filteringLogic, this.expressionsList);
+                this.cdr.detectChanges();
+            }));
+            this.subscriptions.add(this.grid.columnMovingEnd.subscribe(() => this.cdr.markForCheck()));
+        }
+    }
+
     private init() {
         this.expressionsList = new Array<ExpressionUI>();
         generateExpressionsList(this.column.filteringExpressionsTree, this.grid.filteringLogic, this.expressionsList);
@@ -510,12 +538,6 @@ export class IgxGridExcelStyleFilteringComponent extends BaseFilteringComponent
     private renderValues() {
         this.filterValues = this.generateFilterValues();
         this.generateListData();
-        this.expressionsList.forEach(expr => {
-            if (this.column.dataType === GridColumnDataType.String && this.column.filteringIgnoreCase
-                && expr.expression.searchVal && expr.expression.searchVal instanceof Set) {
-                this.modifyExpression(expr);
-            }
-        });
     }
 
     private generateFilterValues() {
@@ -546,16 +568,6 @@ export class IgxGridExcelStyleFilteringComponent extends BaseFilteringComponent
         return filterValues;
     }
 
-    private modifyExpression(expr: ExpressionUI) {
-        const lowerCaseFilterValues = new Set(Array.from(expr.expression.searchVal).map((value: string) => value.toLowerCase()));
-
-        this.grid.data.forEach(item => {
-            if (typeof item[this.column.field] === "string" && lowerCaseFilterValues.has(item[this.column.field]?.toLowerCase())) {
-                expr.expression.searchVal.add(item[this.column.field]);
-            }
-        });
-    }
-
     private generateListData() {
         this.listData = new Array<FilterListItem>();
         const shouldUpdateSelection = this.areExpressionsSelectable();