Merge branch 'main' into fix/data-stream-get-effective-settings

Author: masseyke

BUILDING.md

@@ -144,7 +144,7 @@ To wire this registered cluster into a `TestClusterAware` task (e.g. `RestIntegT
 Additional integration tests for a certain Elasticsearch modules that are specific to certain cluster configuration can be declared in a separate so called `qa` subproject of your module.
 
 The benefit of a dedicated project for these tests are:
-- `qa` projects are dedicated two specific use-cases and easier to maintain
+- `qa` projects are dedicated to specific use-cases and easier to maintain
 - It keeps the specific test logic separated from the common test logic.
 - You can run those tests in parallel to other projects of the build.
 

docs/changelog/127229.yaml

@@ -0,0 +1,6 @@
+pr: 127229
+summary: Return BAD_REQUEST when a field scorer references a missing field
+area: Ranking
+type: bug
+issues:
+ - 127162

docs/changelog/127414.yaml

@@ -0,0 +1,5 @@
+pr: 127414
+summary: Fix npe when using source confirmed text query against missing field
+area: Search
+type: bug
+issues: []

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java

@@ -267,7 +267,11 @@ public boolean isCacheable(LeafReaderContext ctx) {
             @Override
             public Explanation explain(LeafReaderContext context, int doc) throws IOException {
                 NumericDocValues norms = context.reader().getNormValues(field);
-                RuntimePhraseScorer scorer = (RuntimePhraseScorer) scorerSupplier(context).get(0);
+                ScorerSupplier scorerSupplier = scorerSupplier(context);
+                if (scorerSupplier == null) {
+                    return Explanation.noMatch("No matching phrase");
+                }
+                RuntimePhraseScorer scorer = (RuntimePhraseScorer) scorerSupplier.get(0);
                 if (scorer == null) {
                     return Explanation.noMatch("No matching phrase");
                 }
@@ -277,6 +281,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                 }
                 float phraseFreq = scorer.freq();
                 Explanation freqExplanation = Explanation.match(phraseFreq, "phraseFreq=" + phraseFreq);
+                assert simScorer != null;
                 Explanation scoreExplanation = simScorer.explain(freqExplanation, getNormValue(norms, doc));
                 return Explanation.match(
                     scoreExplanation.getValue(),
@@ -321,7 +326,11 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException {
                     Weight innerWeight = in.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, 1);
                     return innerWeight.matches(context, doc);
                 }
-                RuntimePhraseScorer scorer = (RuntimePhraseScorer) scorerSupplier(context).get(0L);
+                ScorerSupplier scorerSupplier = scorerSupplier(context);
+                if (scorerSupplier == null) {
+                    return null;
+                }
+                RuntimePhraseScorer scorer = (RuntimePhraseScorer) scorerSupplier.get(0L);
                 if (scorer == null) {
                     return null;
                 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java

@@ -24,6 +24,7 @@
 import org.apache.lucene.queries.spans.SpanTermQuery;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Matches;
@@ -101,6 +102,26 @@ public void testTerm() throws Exception {
         }
     }
 
+    public void testMissingPhrase() throws Exception {
+        try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) {
+
+            Document doc = new Document();
+            doc.add(new TextField("body", "a b c b a b c", Store.YES));
+            w.addDocument(doc);
+
+            try (IndexReader reader = DirectoryReader.open(w)) {
+                IndexSearcher searcher = newSearcher(reader);
+                PhraseQuery query = new PhraseQuery("missing_field", "b", "c");
+                Query sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER);
+                Explanation explanation = searcher.explain(sourceConfirmedPhraseQuery, 0);
+                assertFalse(explanation.isMatch());
+
+                Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE, 1);
+                assertNull(weight.matches(getOnlyLeafReader(reader).getContext(), 0));
+            }
+        }
+    }
+
     public void testPhrase() throws Exception {
         try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) {
 

plugins/examples/rescore/src/yamlRestTest/resources/rest-api-spec/test/example-rescore/30_factor_field.yml

@@ -48,6 +48,24 @@ setup:
   - match: { hits.hits.1._score: 20 }
   - match: { hits.hits.2._score: 10 }
 
+---
+"referencing a missing field returns bad request":
+  - requires:
+      cluster_features: [ "search.rescorer.missing.field.bad.request" ]
+      reason: "Testing the behaviour change with this feature"
+  - do:
+      catch: bad_request
+      search:
+        index: test
+        body:
+          rescore:
+            example:
+              factor: 1
+              factor_field: missing
+  - match: { status: 400 }
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "Missing value for field [missing]" }
+
 ---
 "sorted based on a numeric field and rescored based on a factor field using a window size":
   - do:

server/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java

@@ -11,7 +11,6 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Explanation;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -73,7 +72,7 @@ public double score(int docId, float subQueryScore) throws IOException {
                     if (missing != null) {
                         value = missing;
                     } else {
-                        throw new ElasticsearchException("Missing value for field [" + field + "]");
+                        throw new IllegalArgumentException("Missing value for field [" + field + "]");
                     }
                 }
                 double val = value * boostFactor;

server/src/main/java/org/elasticsearch/search/SearchFeatures.java

@@ -28,9 +28,10 @@ public Set<NodeFeature> getFeatures() {
     public static final NodeFeature COMPLETION_FIELD_SUPPORTS_DUPLICATE_SUGGESTIONS = new NodeFeature(
         "search.completion_field.duplicate.support"
     );
+    public static final NodeFeature RESCORER_MISSING_FIELD_BAD_REQUEST = new NodeFeature("search.rescorer.missing.field.bad.request");
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
-        return Set.of(RETRIEVER_RESCORER_ENABLED, COMPLETION_FIELD_SUPPORTS_DUPLICATE_SUGGESTIONS);
+        return Set.of(RETRIEVER_RESCORER_ENABLED, COMPLETION_FIELD_SUPPORTS_DUPLICATE_SUGGESTIONS, RESCORER_MISSING_FIELD_BAD_REQUEST);
     }
 }

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/multiproject/action/DeleteProjectActionIT.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.multiproject.action;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.not;
+
+public class ProjectCrudActionIT extends ESRestTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster CLUSTER = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("test.multi_project.enabled", "true")
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return CLUSTER.getHttpAddresses();
+    }
+
+    public void testCreateAndDeleteProject() throws IOException {
+        final var projectId = randomUniqueProjectId();
+        var request = new Request("PUT", "/_project/" + projectId);
+
+        final int numberOfRequests = between(1, 8);
+        final var successCount = new AtomicInteger();
+        final var errorCount = new AtomicInteger();
+
+        runInParallel(numberOfRequests, ignore -> {
+            try {
+                var response = client().performRequest(request);
+                assertAcknowledged(response);
+                successCount.incrementAndGet();
+            } catch (IOException e) {
+                if (e instanceof ResponseException responseException) {
+                    assertThat(responseException.getMessage(), containsString("project [" + projectId + "] already exists"));
+                    errorCount.incrementAndGet();
+                    return;
+                }
+                fail(e, "unexpected exception");
+            }
+        });
+
+        assertThat(successCount.get(), equalTo(1));
+        assertThat(errorCount.get(), equalTo(numberOfRequests - 1));
+        assertThat(getProjectIdsFromClusterState(), hasItem(projectId.id()));
+
+        final Response response = client().performRequest(new Request("DELETE", "/_project/" + projectId));
+        assertAcknowledged(response);
+        assertThat(getProjectIdsFromClusterState(), not(hasItem(projectId.id())));
+    }
+
+    private Set<String> getProjectIdsFromClusterState() throws IOException {
+        final Response response = client().performRequest(new Request("GET", "/_cluster/state?multi_project=true"));
+        final ObjectPath clusterState = assertOKAndCreateObjectPath(response);
+
+        final Set<String> projectIdsFromMetadata = extractProjectIds(clusterState, "metadata.projects");
+        final Set<String> projectIdsFromRoutingTable = extractProjectIds(clusterState, "routing_table.projects");
+
+        assertThat(projectIdsFromMetadata, equalTo(projectIdsFromRoutingTable));
+        return projectIdsFromMetadata;
+    }
+
+    @SuppressWarnings("unchecked")
+    private Set<String> extractProjectIds(ObjectPath clusterState, String path) throws IOException {
+        final int numberProjects = ((List<Object>) clusterState.evaluate(path)).size();
+        return IntStream.range(0, numberProjects).mapToObj(i -> {
+            try {
+                return (String) clusterState.evaluate(path + "." + i + ".id");
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }).collect(Collectors.toUnmodifiableSet());
+    }
+}