Skip to content

fix: DH-20680: Reindexing filters to support barriers #7547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lbooker42 merged 3 commits into from
Jan 14, 2026
Merged

fix: DH-20680: Reindexing filters to support barriers #7547

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fbebcd1
Initial commit of Reindexing filter barrier support.
lbooker42 Jan 6, 2026
1cb319e
Added test and correction for Copilot PR comment.
lbooker42 Jan 6, 2026
052b956
Restrict ReindexingFilter inversion and create tests asserting RI tre…
lbooker42 Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 73 additions & 29 deletions engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,7 @@
import io.deephaven.engine.table.hierarchical.TreeTable;
import io.deephaven.engine.table.impl.MemoizedOperationKey.SelectUpdateViewOrUpdateView.Flavor;
import io.deephaven.engine.table.impl.by.*;
import io.deephaven.engine.table.impl.filter.ExtractBarriers;
import io.deephaven.engine.table.impl.filter.ExtractInnerConjunctiveFilters;
import io.deephaven.engine.table.impl.filter.ExtractShiftedColumnDefinitions;
import io.deephaven.engine.table.impl.filter.ExtractRespectedBarriers;
import io.deephaven.engine.table.impl.filter.*;
import io.deephaven.engine.table.impl.hierarchical.RollupTableImpl;
import io.deephaven.engine.table.impl.hierarchical.TreeTableImpl;
import io.deephaven.engine.table.impl.indexer.DataIndexer;
Expand Down Expand Up @@ -1332,6 +1329,21 @@ private void initializeFilters(@NotNull final WhereFilter[] filters) {
compilationProcessor.compile();
}

/**
* Create a copy of the array of filters, introducing the declared barriers into the first filter if any are
* provided.
*/
private WhereFilter[] cloneFiltersForReindexing(final WhereFilter[] filters,
final Collection<Object> declaredBarriers) {
if (filters.length == 0 || declaredBarriers.isEmpty()) {
return filters;
}
final WhereFilter[] toUse = filters.clone();
// Force the first filter to declare all the barriers we have already processed.
toUse[0] = toUse[0].withDeclaredBarriers(declaredBarriers.toArray());
return toUse;
}

private QueryTable whereInternal(final WhereFilter... filters) {
if (filters.length == 0) {
return (QueryTable) prepareReturnThis();
Expand All @@ -1346,32 +1358,64 @@ private QueryTable whereInternal(final WhereFilter... filters) {
() -> {
initializeFilters(extractedFilters);

for (int fi = 0; fi < extractedFilters.length; ++fi) {
if (!(extractedFilters[fi] instanceof ReindexingFilter)) {
continue;
}
final ReindexingFilter reindexingFilter = (ReindexingFilter) extractedFilters[fi];
final boolean first = fi == 0;
final boolean last = fi == extractedFilters.length - 1;
if (last && !reindexingFilter.requiresSorting()) {
// If this is the last (or only) filter, we can just run it as normal unless it requires
// sorting.
break;
}
QueryTable result = this;
if (!first) {
result = result.whereInternal(Arrays.copyOf(extractedFilters, fi));
}
if (reindexingFilter.requiresSorting()) {
result = (QueryTable) result.sort(reindexingFilter.getSortColumns());
reindexingFilter.sortingDone();
}
result = result.whereInternal(reindexingFilter);
if (!last) {
result = result.whereInternal(
Arrays.copyOfRange(extractedFilters, fi + 1, extractedFilters.length));
final boolean hasReindexingFilter = Arrays.stream(extractedFilters)
.anyMatch(filter -> !ExtractReindexingFilters.of(filter).isEmpty());
if (hasReindexingFilter) {
// Filters will be processed in subsets of the provided list. Need to track all the
// declared barriers that we have encountered.
final Set<Object> declaredBarriers = new HashSet<>();

for (int fi = 0; fi < extractedFilters.length; ++fi) {
final WhereFilter filter = extractedFilters[fi];
final Collection<ReindexingFilter> reindexingFilters = ExtractReindexingFilters.of(filter);
if (reindexingFilters.isEmpty()) {
continue;
}
if (reindexingFilters.size() > 1) {
throw new IllegalStateException(
"Expected at most one reindexing filter per component filter: " + filter);
}
final ReindexingFilter reindexingFilter = reindexingFilters.iterator().next();
final boolean first = fi == 0;
final boolean last = fi == extractedFilters.length - 1;
if (last && !reindexingFilter.requiresSorting()) {
// If this is the last (or only) filter, we can just run it as normal unless it requires
// sorting.
break;
}
QueryTable result = this;
if (!first) {
final WhereFilter[] subset = Arrays.copyOf(extractedFilters, fi);
final WhereFilter[] toUse = cloneFiltersForReindexing(subset, declaredBarriers);
result = result.whereInternal(toUse);
// collect all the newly declared barriers
for (final WhereFilter priorFilter : subset) {
declaredBarriers.addAll(ExtractBarriers.of(priorFilter));
}
}
if (reindexingFilter.requiresSorting()) {
result = (QueryTable) result.sort(reindexingFilter.getSortColumns());
reindexingFilter.sortingDone();
}
// Execute the current filter (which is or wraps a reindexing filter)
// adding all previous encountered barriers as declared barriers.
if (!declaredBarriers.isEmpty()) {
result = result.whereInternal(filter.withDeclaredBarriers(declaredBarriers.toArray()));
} else {
result = result.whereInternal(filter);
}
if (!last) {
final WhereFilter[] subset =
Arrays.copyOfRange(extractedFilters, fi + 1, extractedFilters.length);
final WhereFilter[] toUse = cloneFiltersForReindexing(subset, declaredBarriers);
result = result.whereInternal(toUse);
// collect all the new barriers that may have been processed
for (final WhereFilter priorFilter : subset) {
declaredBarriers.addAll(ExtractBarriers.of(priorFilter));
}
}
return result;
}
return result;
}

boolean hasConstArrayOffsetFilter = false;
Expand Down
68 changes: 68 additions & 0 deletions ...e/table/src/main/java/io/deephaven/engine/table/impl/filter/ExtractReindexingFilters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
//
// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending
//
package io.deephaven.engine.table.impl.filter;

import io.deephaven.engine.table.impl.select.*;

import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;

/**
* This will return the ReindexingFilter instances contained within a WhereFilter, if any exist. Note that this removes
* all inversion, barrier and serial wrappers and the returned filters should not be used directly for filtering.
*/
public enum ExtractReindexingFilters implements WhereFilter.Visitor<Collection<ReindexingFilter>> {
EXTRACT_REINDEXING_FILTERS;

public static Collection<ReindexingFilter> of(final WhereFilter filter) {
return filter.walk(EXTRACT_REINDEXING_FILTERS);
}

@Override
public Collection<ReindexingFilter> visitOther(final WhereFilter filter) {
if (filter instanceof ReindexingFilter) {
return List.of((ReindexingFilter) filter);
}
return List.of();
}

@Override
public Collection<ReindexingFilter> visit(final WhereFilterInvertedImpl filter) {
return of(filter.getWrappedFilter());
}

@Override
public Collection<ReindexingFilter> visit(final WhereFilterSerialImpl filter) {
return of(filter.getWrappedFilter());
}

@Override
public Collection<ReindexingFilter> visit(final WhereFilterWithDeclaredBarriersImpl filter) {
return of(filter.getWrappedFilter());
}

@Override
public Collection<ReindexingFilter> visit(final WhereFilterWithRespectedBarriersImpl filter) {
return of(filter.getWrappedFilter());
}

@Override
public Collection<ReindexingFilter> visit(final DisjunctiveFilter filter) {
// DisjunctiveFilter should disallow reindexing filter components, but visit the children anyway because
// this could change.
return filter.getFilters().stream()
.flatMap(whereFilter -> of(whereFilter).stream())
.collect(Collectors.toList());
}

@Override
public Collection<ReindexingFilter> visit(final ConjunctiveFilter filter) {
// ConjunctiveFilter should disallow reindexing filter components, but visit the children anyway because
// this could change.
return filter.getFilters().stream()
.flatMap(whereFilter -> of(whereFilter).stream())
.collect(Collectors.toList());
}
}
Loading
Loading