Skip to content

Commit 9903035

Browse files
mark-vieiramark-vieira
committed
Add percolator field fallback compatibility (#137466)
1 parent c1a46d5 commit 9903035

File tree

3 files changed

+92
-84
lines changed

3 files changed

+92
-84
lines changed

‎modules/percolator/src/main/java/module-info.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@
1616
requires org.apache.lucene.memory;
1717
requires org.apache.lucene.queries;
1818
requires org.apache.lucene.sandbox;
19+
requires org.elasticsearch.logging;
1920
}

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java‎

Lines changed: 75 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.elasticsearch.TransportVersion;
3838
import org.elasticsearch.TransportVersions;
3939
import org.elasticsearch.action.get.GetRequest;
40+
import org.elasticsearch.common.CheckedSupplier;
4041
import org.elasticsearch.common.bytes.BytesReference;
4142
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
4243
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@@ -63,6 +64,11 @@
6364
import org.elasticsearch.index.query.SearchExecutionContext;
6465
import org.elasticsearch.indices.breaker.CircuitBreakerService;
6566
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
67+
import org.elasticsearch.logging.LogManager;
68+
import org.elasticsearch.logging.Logger;
69+
import org.elasticsearch.search.lookup.Source;
70+
import org.elasticsearch.search.lookup.SourceFilter;
71+
import org.elasticsearch.search.lookup.SourceProvider;
6672
import org.elasticsearch.xcontent.ConstructingObjectParser;
6773
import org.elasticsearch.xcontent.NamedXContentRegistry;
6874
import org.elasticsearch.xcontent.ParseField;
@@ -86,6 +92,8 @@
8692
import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
8793

8894
public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBuilder> {
95+
private static final Logger LOGGER = LogManager.getLogger(PercolateQueryBuilder.class);
96+
8997
public static final String NAME = "percolate";
9098

9199
static final ParseField DOCUMENT_FIELD = new ParseField("document");
@@ -557,41 +565,81 @@ static PercolateQuery.QueryStore createStore(MappedFieldType queryBuilderFieldTy
557565
return docId -> {
558566
if (binaryDocValues.advanceExact(docId)) {
559567
BytesRef qbSource = binaryDocValues.binaryValue();
560-
try (
561-
InputStream in = new ByteArrayInputStream(qbSource.bytes, qbSource.offset, qbSource.length);
562-
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in, qbSource.length), registry)
563-
) {
564-
// Query builder's content is stored via BinaryFieldMapper, which has a custom encoding
565-
// to encode multiple binary values into a single binary doc values field.
566-
// This is the reason we need to first read the number of values and
567-
// then the length of the field value in bytes.
568-
int numValues = input.readVInt();
569-
assert numValues == 1;
570-
int valueLength = input.readVInt();
571-
assert valueLength > 0;
572-
573-
TransportVersion transportVersion;
574-
if (indexVersion.before(IndexVersions.V_8_8_0)) {
575-
transportVersion = TransportVersion.fromId(indexVersion.id());
576-
} else {
577-
transportVersion = TransportVersion.readVersion(input);
568+
QueryBuilder queryBuilder = readQueryBuilder(qbSource, registry, indexVersion, () -> {
569+
// query builder is written in an incompatible format, fall-back to reading it from source
570+
if (context.isSourceEnabled() == false) {
571+
throw new ElasticsearchException(
572+
"Unable to read percolator query. Original transport version is incompatible and source is "
573+
+ "unavailable on index [{}].",
574+
context.index().getName()
575+
);
578576
}
579-
// set the transportversion here - only read vints so far, so can change the version freely at this point
580-
input.setTransportVersion(transportVersion);
581-
582-
QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
583-
assert in.read() == -1;
584-
queryBuilder = Rewriteable.rewrite(queryBuilder, context);
585-
return queryBuilder.toQuery(context);
586-
}
587-
577+
LOGGER.warn(
578+
"Reading percolator query from source. For best performance, reindexing of index [{}] is required.",
579+
context.index().getName()
580+
);
581+
SourceProvider sourceProvider = context.createSourceProvider(new SourceFilter(null, null));
582+
Source source = sourceProvider.getSource(ctx, docId);
583+
SourceToParse sourceToParse = new SourceToParse(
584+
String.valueOf(docId),
585+
source.internalSourceRef(),
586+
source.sourceContentType()
587+
);
588+
589+
return context.parseDocument(sourceToParse).rootDoc().getBinaryValue(queryBuilderFieldType.name());
590+
});
591+
592+
queryBuilder = Rewriteable.rewrite(queryBuilder, context);
593+
return queryBuilder.toQuery(context);
588594
} else {
589595
return null;
590596
}
591597
};
592598
};
593599
}
594600

601+
private static QueryBuilder readQueryBuilder(
602+
BytesRef bytesRef,
603+
NamedWriteableRegistry registry,
604+
IndexVersion indexVersion,
605+
CheckedSupplier<BytesRef, IOException> fallbackSource
606+
) throws IOException {
607+
try (
608+
InputStream in = new ByteArrayInputStream(bytesRef.bytes, bytesRef.offset, bytesRef.length);
609+
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in, bytesRef.length), registry)
610+
) {
611+
// Query builder's content is stored via BinaryFieldMapper, which has a custom encoding
612+
// to encode multiple binary values into a single binary doc values field.
613+
// This is the reason we need to first read the number of values and
614+
// then the length of the field value in bytes.
615+
int numValues = input.readVInt();
616+
assert numValues == 1;
617+
int valueLength = input.readVInt();
618+
assert valueLength > 0;
619+
620+
TransportVersion transportVersion;
621+
if (indexVersion.before(IndexVersions.V_8_8_0)) {
622+
transportVersion = TransportVersion.fromId(indexVersion.id());
623+
} else {
624+
transportVersion = TransportVersion.readVersion(input);
625+
}
626+
627+
QueryBuilder queryBuilder;
628+
629+
if (TransportVersion.isCompatible(transportVersion) || fallbackSource == null) {
630+
// set the transportversion here - only read vints so far, so can change the version freely at this point
631+
input.setTransportVersion(transportVersion);
632+
queryBuilder = input.readNamedWriteable(QueryBuilder.class);
633+
assert in.read() == -1;
634+
} else {
635+
// incompatible transport version, try the fallback
636+
queryBuilder = readQueryBuilder(fallbackSource.get(), registry, indexVersion, null);
637+
}
638+
639+
return queryBuilder;
640+
}
641+
}
642+
595643
static SearchExecutionContext wrap(SearchExecutionContext delegate) {
596644
return new SearchExecutionContext(delegate) {
597645

‎qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java‎

Lines changed: 16 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,10 @@
1111

1212
import com.carrotsearch.randomizedtesting.annotations.Name;
1313

14-
import org.elasticsearch.TransportVersion;
15-
import org.elasticsearch.Version;
1614
import org.elasticsearch.client.Request;
1715
import org.elasticsearch.client.Response;
1816
import org.elasticsearch.common.Strings;
19-
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
20-
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
21-
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
22-
import org.elasticsearch.common.io.stream.StreamInput;
23-
import org.elasticsearch.common.settings.Settings;
2417
import org.elasticsearch.common.unit.Fuzziness;
25-
import org.elasticsearch.core.UpdateForV10;
2618
import org.elasticsearch.index.query.BoolQueryBuilder;
2719
import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
2820
import org.elasticsearch.index.query.DisMaxQueryBuilder;
@@ -37,33 +29,23 @@
3729
import org.elasticsearch.index.query.SpanTermQueryBuilder;
3830
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
3931
import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
40-
import org.elasticsearch.search.SearchModule;
4132
import org.elasticsearch.test.cluster.ElasticsearchCluster;
4233
import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
4334
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
4435
import org.elasticsearch.xcontent.XContentBuilder;
4536
import org.junit.ClassRule;
4637

47-
import java.io.ByteArrayInputStream;
48-
import java.io.InputStream;
4938
import java.util.ArrayList;
50-
import java.util.Base64;
51-
import java.util.Collections;
5239
import java.util.List;
53-
import java.util.Map;
5440

55-
import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS;
5641
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
5742

5843
/**
5944
* An integration test that tests whether percolator queries stored in older supported ES version can still be read by the
6045
* current ES version. Percolator queries are stored in the binary format in a dedicated doc values field (see
61-
* PercolatorFieldMapper#createQueryBuilderField(...) method). Using the query builders writable contract. This test
62-
* does best effort verifying that we don't break bwc for query builders between the first previous major version and
63-
* the latest current major release.
64-
*
65-
* The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the
66-
* json format of a query being tested here then feel free to change this.
46+
* PercolatorFieldMapper#createQueryBuilderField(...) method). We don't attempt to assert anything on results here, simply executing
47+
* a percolator query will force deserialization of the old query builder. This also verifies that our fallback compatibility
48+
* functionality is working correctly, otherwise the search request will throw an exception.
6749
*/
6850
public class QueryBuilderBWCIT extends ParameterizedFullClusterRestartTestCase {
6951
private static final List<Object[]> CANDIDATES = new ArrayList<>();
@@ -227,43 +209,20 @@ public void testQueryBuilderBWC() throws Exception {
227209
assertEquals(201, rsp.getStatusLine().getStatusCode());
228210
}
229211
} else {
230-
NamedWriteableRegistry registry = new NamedWriteableRegistry(
231-
new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables()
232-
);
233-
234-
for (int i = 0; i < CANDIDATES.size(); i++) {
235-
QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1];
236-
Request request = new Request("GET", "/" + index + "/_search");
237-
request.setJsonEntity(Strings.format("""
238-
{"query": {"ids": {"values": ["%s"]}}, "docvalue_fields": [{"field":"query.query_builder_field"}]}
239-
""", i));
240-
Response rsp = client().performRequest(request);
241-
assertEquals(200, rsp.getStatusLine().getStatusCode());
242-
var hitRsp = (Map<?, ?>) ((List<?>) ((Map<?, ?>) responseAsMap(rsp).get("hits")).get("hits")).get(0);
243-
String queryBuilderStr = (String) ((List<?>) ((Map<?, ?>) hitRsp.get("fields")).get("query.query_builder_field")).get(0);
244-
byte[] qbSource = Base64.getDecoder().decode(queryBuilderStr);
245-
try (
246-
InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length);
247-
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)
248-
) {
249-
@UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) // won't need to read <8.8 data anymore
250-
boolean originalClusterHasTransportVersion = parseLegacyVersion(getOldClusterVersion()).map(
251-
v -> v.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS)
252-
).orElse(true);
253-
TransportVersion transportVersion;
254-
if (originalClusterHasTransportVersion == false) {
255-
transportVersion = TransportVersion.fromId(
256-
parseLegacyVersion(getOldClusterVersion()).map(Version::id).orElse(TransportVersion.minimumCompatible().id())
257-
);
258-
} else {
259-
transportVersion = TransportVersion.readVersion(input);
212+
Request request = new Request("GET", "/" + index + "/_search");
213+
request.setJsonEntity("""
214+
{
215+
"query": {
216+
"percolate": {
217+
"field": "query",
218+
"document": {
219+
"foo": "bar"
220+
}
260221
}
261-
input.setTransportVersion(transportVersion);
262-
QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
263-
assert in.read() == -1;
264-
assertEquals(expectedQueryBuilder, queryBuilder);
265-
}
266-
}
222+
}
223+
}""");
224+
Response rsp = client().performRequest(request);
225+
assertEquals(200, rsp.getStatusLine().getStatusCode());
267226
}
268227
}
269228
}

0 commit comments

Comments
 (0)