Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
482aad8
Added test and instrumentation
Mikep86 Jul 18, 2025
43174f1
Store original query in InterceptedQueryBuilderWrapper
Mikep86 Jul 18, 2025
a922ed5
Spotless
Mikep86 Jul 18, 2025
d9f37b7
Added SemanticCrossClusterSearchIT
Mikep86 Jul 18, 2025
b94b41c
test development
Mikep86 Jul 18, 2025
ed58e78
Allow cross-cluster search
Mikep86 Jul 18, 2025
bc1ef89
Fix test
Mikep86 Jul 18, 2025
9a7d6f4
Send pre-intercepted request to remote cluster
Mikep86 Jul 18, 2025
4740f0c
Added match query test
Mikep86 Jul 18, 2025
83923c4
Added stub classes for match query builder wrapper
Mikep86 Jul 18, 2025
a132f5d
Merge branch 'main' into semantic-text_ccs-discovery
Mikep86 Jul 31, 2025
f00a520
Fix build error
Mikep86 Jul 31, 2025
87a1eaf
Fixed entitlement policy
Mikep86 Jul 31, 2025
38af314
Add PIT integration test
Mikep86 Aug 1, 2025
4935515
Added CCS minimize round-trips to query rewrite context
Mikep86 Aug 1, 2025
9c251dc
Code cleanup
Mikep86 Aug 1, 2025
1e5cb6f
Spotless
Mikep86 Aug 1, 2025
8db87ab
Add model registry to semantic query builder
Mikep86 Aug 1, 2025
0b8a1a9
Remove unused code
Mikep86 Aug 1, 2025
c55628c
Added the map embeddings provider
Mikep86 Aug 1, 2025
65b20bd
Added the single embeddings provider
Mikep86 Aug 1, 2025
a335f34
Update semantic query builder to use embeddings providers
Mikep86 Aug 1, 2025
071f6c5
Added TODOs
Mikep86 Aug 1, 2025
0b945d4
Make ccsMinimizeRoundtrips nullable
Mikep86 Aug 1, 2025
33d1be1
Fix build errors
Mikep86 Aug 1, 2025
42d5eab
Fix test errors
Mikep86 Aug 4, 2025
43dda15
Check that model registry is set
Mikep86 Aug 4, 2025
cd878bd
Allow CCS only when ccs_minimize_roundtrips=true
Mikep86 Aug 4, 2025
d3937c1
Perform inference on remote cluster when necessary
Mikep86 Aug 4, 2025
659a79f
Update integration test to use different inference IDs across clusters
Mikep86 Aug 4, 2025
b407a4f
Set model registry for each semantic query builder instance
Mikep86 Aug 4, 2025
10ec96d
Revert CrossClusterSearchIT changes
Mikep86 Aug 4, 2025
7e2d973
Revert InterceptedQueryBuilderWrapper changes
Mikep86 Aug 4, 2025
c39adbe
Revert MatchQueryBuilder changes
Mikep86 Aug 4, 2025
9c58c11
Adjust PIT test
Mikep86 Aug 4, 2025
4fb6f42
Remove match query test
Mikep86 Aug 4, 2025
39b56cf
Merge branch 'main' into semantic-text_ccs-discovery
Mikep86 Aug 19, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,7 @@ static TransportVersion def(int id) {
public static final TransportVersion ESQL_LOOKUP_JOIN_ON_MANY_FIELDS = def(9_139_0_00);
public static final TransportVersion SIMULATE_INGEST_EFFECTIVE_MAPPING = def(9_140_0_00);
public static final TransportVersion RESOLVE_INDEX_MODE_ADDED = def(9_141_0_00);
public static final TransportVersion SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS = def(9_142_0_00);

/*
* STOP! READ THIS FIRST! No, really,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ protected void doExecute(Task task, ValidateQueryRequest request, ActionListener
} else {
Rewriteable.rewriteAndFetch(
request.query(),
searchService.getRewriteContext(timeProvider, resolvedIndices, null),
searchService.getRewriteContext(timeProvider, resolvedIndices, null, null),
rewriteListener
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,11 @@ protected void doExecute(Task task, ExplainRequest request, ActionListener<Expla

assert request.query() != null;
LongSupplier timeProvider = () -> request.nowInMillis;
Rewriteable.rewriteAndFetch(request.query(), searchService.getRewriteContext(timeProvider, resolvedIndices, null), rewriteListener);
Rewriteable.rewriteAndFetch(
request.query(),
searchService.getRewriteContext(timeProvider, resolvedIndices, null, null),
rewriteListener
);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,7 @@ public void onFailure(Exception e) {
timeProvider::absoluteStartMillis,
resolvedIndices,
original.pointInTimeBuilder(),
shouldMinimizeRoundtrips(original),
isExplain
),
rewriteListener
Expand Down Expand Up @@ -787,6 +788,7 @@ public void onFailure(Exception e) {
String clusterAlias = entry.getKey();
boolean skipUnavailable = remoteClusterService.isSkipUnavailable(clusterAlias);
OriginalIndices indices = entry.getValue();

SearchRequest ccsSearchRequest = SearchRequest.subSearchRequest(
parentTaskId,
searchRequest,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ public void searchShards(Task task, SearchShardsRequest searchShardsRequest, Act

Rewriteable.rewriteAndFetch(
original,
searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices, null),
searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices, null, original.isCcsMinimizeRoundtrips()),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there precedent for having CCS-specific knobs like this in generic search code?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure, but there's a good case to be made why this is necessary. The CCS mode affects the query rewrite cycle, thus we need a way to know about it within that context. Info is passed to query rewrite via QueryRewriteContext, thus this implementation.

listener.delegateFailureAndWrap((delegate, searchRequest) -> {
Index[] concreteIndices = resolvedIndices.getConcreteLocalIndices();
final Set<ResolvedExpression> indicesAndAliases = indexNameExpressionResolver.resolveExpressions(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -836,6 +836,7 @@ public QueryRewriteContext newQueryRewriteContext(
null,
null,
null,
null,
false
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ public CoordinatorRewriteContext(
null,
null,
null,
null,
false
);
this.dateFieldRangeInfo = dateFieldRangeInfo;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o instanceof InterceptedQueryBuilderWrapper == false) return false;
return Objects.equals(queryBuilder, ((InterceptedQueryBuilderWrapper) o).queryBuilder);
if (o == null || getClass() != o.getClass()) return false;
InterceptedQueryBuilderWrapper that = (InterceptedQueryBuilderWrapper) o;
return Objects.equals(queryBuilder, that.queryBuilder);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ public class QueryRewriteContext {
private final ResolvedIndices resolvedIndices;
private final PointInTimeBuilder pit;
private QueryRewriteInterceptor queryRewriteInterceptor;
private final Boolean ccsMinimizeRoundtrips;
private final boolean isExplain;

public QueryRewriteContext(
Expand All @@ -91,6 +92,7 @@ public QueryRewriteContext(
final ResolvedIndices resolvedIndices,
final PointInTimeBuilder pit,
final QueryRewriteInterceptor queryRewriteInterceptor,
final Boolean ccsMinimizeRoundtrips,
final boolean isExplain
) {

Expand All @@ -111,6 +113,7 @@ public QueryRewriteContext(
this.resolvedIndices = resolvedIndices;
this.pit = pit;
this.queryRewriteInterceptor = queryRewriteInterceptor;
this.ccsMinimizeRoundtrips = ccsMinimizeRoundtrips;
this.isExplain = isExplain;
}

Expand All @@ -132,6 +135,7 @@ public QueryRewriteContext(final XContentParserConfiguration parserConfiguration
null,
null,
null,
null,
false
);
}
Expand All @@ -142,9 +146,10 @@ public QueryRewriteContext(
final LongSupplier nowInMillis,
final ResolvedIndices resolvedIndices,
final PointInTimeBuilder pit,
final QueryRewriteInterceptor queryRewriteInterceptor
final QueryRewriteInterceptor queryRewriteInterceptor,
final Boolean ccsMinimizeRoundtrips
) {
this(parserConfiguration, client, nowInMillis, resolvedIndices, pit, queryRewriteInterceptor, false);
this(parserConfiguration, client, nowInMillis, resolvedIndices, pit, queryRewriteInterceptor, ccsMinimizeRoundtrips, false);
}

public QueryRewriteContext(
Expand All @@ -154,6 +159,7 @@ public QueryRewriteContext(
final ResolvedIndices resolvedIndices,
final PointInTimeBuilder pit,
final QueryRewriteInterceptor queryRewriteInterceptor,
final Boolean ccsMinimizeRoundtrips,
final boolean isExplain
) {
this(
Expand All @@ -173,6 +179,7 @@ public QueryRewriteContext(
resolvedIndices,
pit,
queryRewriteInterceptor,
ccsMinimizeRoundtrips,
isExplain
);
}
Expand Down Expand Up @@ -279,6 +286,10 @@ public void setMapUnmappedFieldAsString(boolean mapUnmappedFieldAsString) {
this.mapUnmappedFieldAsString = mapUnmappedFieldAsString;
}

public Boolean isCcsMinimizeRoundtrips() {
return ccsMinimizeRoundtrips;
}

public boolean isExplain() {
return this.isExplain;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,7 @@ private SearchExecutionContext(
null,
null,
null,
null,
false
);
this.shardId = shardId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1849,9 +1849,19 @@ public QueryRewriteContext getRewriteContext(
LongSupplier nowInMillis,
ResolvedIndices resolvedIndices,
PointInTimeBuilder pit,
final Boolean ccsMinimizeRoundtrips,
final boolean isExplain
) {
return new QueryRewriteContext(parserConfig, client, nowInMillis, resolvedIndices, pit, queryRewriteInterceptor, isExplain);
return new QueryRewriteContext(
parserConfig,
client,
nowInMillis,
resolvedIndices,
pit,
queryRewriteInterceptor,
ccsMinimizeRoundtrips,
isExplain
);
}

public DataRewriteContext getDataRewriteContext(LongSupplier nowInMillis) {
Expand Down
12 changes: 9 additions & 3 deletions server/src/main/java/org/elasticsearch/search/SearchService.java
Original file line number Diff line number Diff line change
Expand Up @@ -2127,8 +2127,13 @@ private void rewriteAndFetchShardRequest(IndexShard shard, ShardSearchRequest re
/**
* Returns a new {@link QueryRewriteContext} with the given {@code now} provider
*/
public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, ResolvedIndices resolvedIndices, PointInTimeBuilder pit) {
return getRewriteContext(nowInMillis, resolvedIndices, pit, false);
public QueryRewriteContext getRewriteContext(
LongSupplier nowInMillis,
ResolvedIndices resolvedIndices,
PointInTimeBuilder pit,
final Boolean ccsMinimizeRoundtrips
) {
return getRewriteContext(nowInMillis, resolvedIndices, pit, ccsMinimizeRoundtrips, false);
}

/**
Expand All @@ -2138,9 +2143,10 @@ public QueryRewriteContext getRewriteContext(
LongSupplier nowInMillis,
ResolvedIndices resolvedIndices,
PointInTimeBuilder pit,
final Boolean ccsMinimizeRoundtrips,
final boolean isExplain
) {
return indicesService.getRewriteContext(nowInMillis, resolvedIndices, pit, isExplain);
return indicesService.getRewriteContext(nowInMillis, resolvedIndices, pit, ccsMinimizeRoundtrips, isExplain);
}

public CoordinatorRewriteContextProvider getCoordinatorRewriteContextProvider(LongSupplier nowInMillis) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1784,8 +1784,8 @@ protected void doWriteTo(StreamOutput out) throws IOException {
NodeClient client = new NodeClient(settings, threadPool, TestProjectResolvers.alwaysThrow());

SearchService searchService = mock(SearchService.class);
when(searchService.getRewriteContext(any(), any(), any(), anyBoolean())).thenReturn(
new QueryRewriteContext(null, null, null, null, null, null)
when(searchService.getRewriteContext(any(), any(), any(), anyBoolean(), anyBoolean())).thenReturn(
new QueryRewriteContext(null, null, null, null, null, null, null)
);
ClusterService clusterService = new ClusterService(
settings,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ public void testGetTierPreference() {
null,
null,
null,
null,
false
);

Expand Down Expand Up @@ -83,6 +84,7 @@ public void testGetTierPreference() {
null,
null,
null,
null,
false
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,7 @@ QueryRewriteContext createQueryRewriteContext() {
createMockResolvedIndices(),
null,
createMockQueryRewriteInterceptor(),
null,
false
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ private static QueryRewriteContext queryRewriteContext(TransportActionServices s
System.currentTimeMillis()
);

return services.searchService().getRewriteContext(System::currentTimeMillis, resolvedIndices, null);
return services.searchService().getRewriteContext(System::currentTimeMillis, resolvedIndices, null, null);
}

private static Set<String> indexNames(LogicalPlan plan) {
Expand Down
Loading