Skip to content

Commit b4086c9

Browse files
mark-vieiramark-vieira
authored
Add percolator field fallback compatibility (elastic#137466) (elastic#138314)
1 parent 1f21eb7 commit b4086c9

File tree

3 files changed

+92
-84
lines changed

3 files changed

+92
-84
lines changed

‎modules/percolator/src/main/java/module-info.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@
1616
requires org.apache.lucene.memory;
1717
requires org.apache.lucene.queries;
1818
requires org.apache.lucene.sandbox;
19+
requires org.elasticsearch.logging;
1920
}

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java‎

Lines changed: 75 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.elasticsearch.ResourceNotFoundException;
3737
import org.elasticsearch.TransportVersion;
3838
import org.elasticsearch.action.get.GetRequest;
39+
import org.elasticsearch.common.CheckedSupplier;
3940
import org.elasticsearch.common.bytes.BytesReference;
4041
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
4142
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@@ -62,6 +63,11 @@
6263
import org.elasticsearch.index.query.SearchExecutionContext;
6364
import org.elasticsearch.indices.breaker.CircuitBreakerService;
6465
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
66+
import org.elasticsearch.logging.LogManager;
67+
import org.elasticsearch.logging.Logger;
68+
import org.elasticsearch.search.lookup.Source;
69+
import org.elasticsearch.search.lookup.SourceFilter;
70+
import org.elasticsearch.search.lookup.SourceProvider;
6571
import org.elasticsearch.xcontent.ConstructingObjectParser;
6672
import org.elasticsearch.xcontent.NamedXContentRegistry;
6773
import org.elasticsearch.xcontent.ParseField;
@@ -85,6 +91,8 @@
8591
import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
8692

8793
public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBuilder> {
94+
private static final Logger LOGGER = LogManager.getLogger(PercolateQueryBuilder.class);
95+
8896
public static final String NAME = "percolate";
8997

9098
static final ParseField DOCUMENT_FIELD = new ParseField("document");
@@ -540,41 +548,81 @@ static PercolateQuery.QueryStore createStore(MappedFieldType queryBuilderFieldTy
540548
return docId -> {
541549
if (binaryDocValues.advanceExact(docId)) {
542550
BytesRef qbSource = binaryDocValues.binaryValue();
543-
try (
544-
InputStream in = new ByteArrayInputStream(qbSource.bytes, qbSource.offset, qbSource.length);
545-
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in, qbSource.length), registry)
546-
) {
547-
// Query builder's content is stored via BinaryFieldMapper, which has a custom encoding
548-
// to encode multiple binary values into a single binary doc values field.
549-
// This is the reason we need to first read the number of values and
550-
// then the length of the field value in bytes.
551-
int numValues = input.readVInt();
552-
assert numValues == 1;
553-
int valueLength = input.readVInt();
554-
assert valueLength > 0;
555-
556-
TransportVersion transportVersion;
557-
if (indexVersion.before(IndexVersions.V_8_8_0)) {
558-
transportVersion = TransportVersion.fromId(indexVersion.id());
559-
} else {
560-
transportVersion = TransportVersion.readVersion(input);
551+
QueryBuilder queryBuilder = readQueryBuilder(qbSource, registry, indexVersion, () -> {
552+
// query builder is written in an incompatible format, fall-back to reading it from source
553+
if (context.isSourceEnabled() == false) {
554+
throw new ElasticsearchException(
555+
"Unable to read percolator query. Original transport version is incompatible and source is "
556+
+ "unavailable on index [{}].",
557+
context.index().getName()
558+
);
561559
}
562-
// set the transportversion here - only read vints so far, so can change the version freely at this point
563-
input.setTransportVersion(transportVersion);
564-
565-
QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
566-
assert in.read() == -1;
567-
queryBuilder = Rewriteable.rewrite(queryBuilder, context);
568-
return queryBuilder.toQuery(context);
569-
}
570-
560+
LOGGER.warn(
561+
"Reading percolator query from source. For best performance, reindexing of index [{}] is required.",
562+
context.index().getName()
563+
);
564+
SourceProvider sourceProvider = context.createSourceProvider(new SourceFilter(null, null));
565+
Source source = sourceProvider.getSource(ctx, docId);
566+
SourceToParse sourceToParse = new SourceToParse(
567+
String.valueOf(docId),
568+
source.internalSourceRef(),
569+
source.sourceContentType()
570+
);
571+
572+
return context.parseDocument(sourceToParse).rootDoc().getBinaryValue(queryBuilderFieldType.name());
573+
});
574+
575+
queryBuilder = Rewriteable.rewrite(queryBuilder, context);
576+
return queryBuilder.toQuery(context);
571577
} else {
572578
return null;
573579
}
574580
};
575581
};
576582
}
577583

584+
private static QueryBuilder readQueryBuilder(
585+
BytesRef bytesRef,
586+
NamedWriteableRegistry registry,
587+
IndexVersion indexVersion,
588+
CheckedSupplier<BytesRef, IOException> fallbackSource
589+
) throws IOException {
590+
try (
591+
InputStream in = new ByteArrayInputStream(bytesRef.bytes, bytesRef.offset, bytesRef.length);
592+
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in, bytesRef.length), registry)
593+
) {
594+
// Query builder's content is stored via BinaryFieldMapper, which has a custom encoding
595+
// to encode multiple binary values into a single binary doc values field.
596+
// This is the reason we need to first read the number of values and
597+
// then the length of the field value in bytes.
598+
int numValues = input.readVInt();
599+
assert numValues == 1;
600+
int valueLength = input.readVInt();
601+
assert valueLength > 0;
602+
603+
TransportVersion transportVersion;
604+
if (indexVersion.before(IndexVersions.V_8_8_0)) {
605+
transportVersion = TransportVersion.fromId(indexVersion.id());
606+
} else {
607+
transportVersion = TransportVersion.readVersion(input);
608+
}
609+
610+
QueryBuilder queryBuilder;
611+
612+
if (TransportVersion.isCompatible(transportVersion) || fallbackSource == null) {
613+
// set the transportversion here - only read vints so far, so can change the version freely at this point
614+
input.setTransportVersion(transportVersion);
615+
queryBuilder = input.readNamedWriteable(QueryBuilder.class);
616+
assert in.read() == -1;
617+
} else {
618+
// incompatible transport version, try the fallback
619+
queryBuilder = readQueryBuilder(fallbackSource.get(), registry, indexVersion, null);
620+
}
621+
622+
return queryBuilder;
623+
}
624+
}
625+
578626
static SearchExecutionContext wrap(SearchExecutionContext delegate) {
579627
return new SearchExecutionContext(delegate) {
580628

‎qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java‎

Lines changed: 16 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,10 @@
1111

1212
import com.carrotsearch.randomizedtesting.annotations.Name;
1313

14-
import org.elasticsearch.TransportVersion;
15-
import org.elasticsearch.Version;
1614
import org.elasticsearch.client.Request;
1715
import org.elasticsearch.client.Response;
1816
import org.elasticsearch.common.Strings;
19-
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
20-
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
21-
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
22-
import org.elasticsearch.common.io.stream.StreamInput;
23-
import org.elasticsearch.common.settings.Settings;
2417
import org.elasticsearch.common.unit.Fuzziness;
25-
import org.elasticsearch.core.UpdateForV10;
2618
import org.elasticsearch.index.query.BoolQueryBuilder;
2719
import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
2820
import org.elasticsearch.index.query.DisMaxQueryBuilder;
@@ -37,33 +29,23 @@
3729
import org.elasticsearch.index.query.SpanTermQueryBuilder;
3830
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
3931
import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
40-
import org.elasticsearch.search.SearchModule;
4132
import org.elasticsearch.test.cluster.ElasticsearchCluster;
4233
import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
4334
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
4435
import org.elasticsearch.xcontent.XContentBuilder;
4536
import org.junit.ClassRule;
4637

47-
import java.io.ByteArrayInputStream;
48-
import java.io.InputStream;
4938
import java.util.ArrayList;
50-
import java.util.Base64;
51-
import java.util.Collections;
5239
import java.util.List;
53-
import java.util.Map;
5440

55-
import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS;
5641
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
5742

5843
/**
5944
* An integration test that tests whether percolator queries stored in older supported ES version can still be read by the
6045
* current ES version. Percolator queries are stored in the binary format in a dedicated doc values field (see
61-
* PercolatorFieldMapper#createQueryBuilderField(...) method). Using the query builders writable contract. This test
62-
* does best effort verifying that we don't break bwc for query builders between the first previous major version and
63-
* the latest current major release.
64-
*
65-
* The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the
66-
* json format of a query being tested here then feel free to change this.
46+
* PercolatorFieldMapper#createQueryBuilderField(...) method). We don't attempt to assert anything on results here, simply executing
47+
* a percolator query will force deserialization of the old query builder. This also verifies that our fallback compatibility
48+
* functionality is working correctly, otherwise the search request will throw an exception.
6749
*/
6850
public class QueryBuilderBWCIT extends ParameterizedFullClusterRestartTestCase {
6951
private static final List<Object[]> CANDIDATES = new ArrayList<>();
@@ -227,43 +209,20 @@ public void testQueryBuilderBWC() throws Exception {
227209
assertEquals(201, rsp.getStatusLine().getStatusCode());
228210
}
229211
} else {
230-
NamedWriteableRegistry registry = new NamedWriteableRegistry(
231-
new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables()
232-
);
233-
234-
for (int i = 0; i < CANDIDATES.size(); i++) {
235-
QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1];
236-
Request request = new Request("GET", "/" + index + "/_search");
237-
request.setJsonEntity(Strings.format("""
238-
{"query": {"ids": {"values": ["%s"]}}, "docvalue_fields": [{"field":"query.query_builder_field"}]}
239-
""", i));
240-
Response rsp = client().performRequest(request);
241-
assertEquals(200, rsp.getStatusLine().getStatusCode());
242-
var hitRsp = (Map<?, ?>) ((List<?>) ((Map<?, ?>) responseAsMap(rsp).get("hits")).get("hits")).get(0);
243-
String queryBuilderStr = (String) ((List<?>) ((Map<?, ?>) hitRsp.get("fields")).get("query.query_builder_field")).get(0);
244-
byte[] qbSource = Base64.getDecoder().decode(queryBuilderStr);
245-
try (
246-
InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length);
247-
StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)
248-
) {
249-
@UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) // won't need to read <8.8 data anymore
250-
boolean originalClusterHasTransportVersion = parseLegacyVersion(getOldClusterVersion()).map(
251-
v -> v.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS)
252-
).orElse(true);
253-
TransportVersion transportVersion;
254-
if (originalClusterHasTransportVersion == false) {
255-
transportVersion = TransportVersion.fromId(
256-
parseLegacyVersion(getOldClusterVersion()).map(Version::id).orElse(TransportVersion.minimumCompatible().id())
257-
);
258-
} else {
259-
transportVersion = TransportVersion.readVersion(input);
212+
Request request = new Request("GET", "/" + index + "/_search");
213+
request.setJsonEntity("""
214+
{
215+
"query": {
216+
"percolate": {
217+
"field": "query",
218+
"document": {
219+
"foo": "bar"
220+
}
260221
}
261-
input.setTransportVersion(transportVersion);
262-
QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
263-
assert in.read() == -1;
264-
assertEquals(expectedQueryBuilder, queryBuilder);
265-
}
266-
}
222+
}
223+
}""");
224+
Response rsp = client().performRequest(request);
225+
assertEquals(200, rsp.getStatusLine().getStatusCode());
267226
}
268227
}
269228
}

0 commit comments

Comments
 (0)