Merge branch 'main' into esql-dls-opt-in-ccs-test

Author: n1v0lg

benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java

@@ -63,7 +63,7 @@ public void setup() throws IOException {
         );
         includesSet = Set.of(fetchContext.includes());
         excludesSet = Set.of(fetchContext.excludes());
-        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet, false);
+        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, includesSet, excludesSet, false);
     }
 
     private BytesReference read300BytesExample() throws IOException {

benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java

@@ -170,7 +170,7 @@ private XContentParserConfiguration buildParseConfig(boolean matchDotsInFieldNam
             includes = null;
             excludes = filters;
         }
-        return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchDotsInFieldNames);
+        return XContentParserConfiguration.EMPTY.withFiltering(null, includes, excludes, matchDotsInFieldNames);
     }
 
     private BytesReference filter(XContentParserConfiguration contentParserConfiguration) throws IOException {

docs/changelog/113827.yaml

@@ -0,0 +1,5 @@
+pr: 113827
+summary: Add Optional Source Filtering to Source Loaders
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/117939.yaml

@@ -0,0 +1,5 @@
+pr: 117939
+summary: Adding default endpoint for Elastic Rerank
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/118370.yaml

@@ -0,0 +1,6 @@
+pr: 118370
+summary: Fix concurrency issue with `ReinitializingSourceProvider`
+area: Mapping
+type: bug
+issues:
+ - 118238

docs/changelog/118380.yaml

@@ -0,0 +1,5 @@
+pr: 118380
+summary: Restore original "is within leaf" value in `SparseVectorFieldMapper`
+area: Mapping
+type: bug
+issues: []

docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc

@@ -36,7 +36,7 @@ the `cosine` measures are equivalent.
 ------------------------------------------------------------
 PUT _inference/sparse_embedding/elser_embeddings <1>
 {
-  "service": "elser",
+  "service": "elasticsearch",
   "service_settings": {
     "num_allocations": 1,
     "num_threads": 1
@@ -206,7 +206,7 @@ PUT _inference/text_embedding/google_vertex_ai_embeddings <1>
 <2> A valid service account in JSON format for the Google Vertex AI API.
 <3> For the list of the available models, refer to the https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api[Text embeddings API] page.
 <4> The name of the location to use for the {infer} task. Refer to https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations[Generative AI on Vertex AI locations] for available locations.
-<5> The name of the project to use for the {infer} task. 
+<5> The name of the project to use for the {infer} task.
 
 // end::google-vertex-ai[]
 

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java

@@ -19,6 +19,8 @@
 import org.elasticsearch.xcontent.provider.filtering.FilterPathBasedFilter;
 import org.elasticsearch.xcontent.support.filtering.FilterPath;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Set;
 
 public class XContentParserConfigurationImpl implements XContentParserConfiguration {
@@ -106,12 +108,41 @@ public XContentParserConfiguration withFiltering(
         Set<String> excludeStrings,
         boolean filtersMatchFieldNamesWithDots
     ) {
+        return withFiltering(null, includeStrings, excludeStrings, filtersMatchFieldNamesWithDots);
+    }
+
+    public XContentParserConfiguration withFiltering(
+        String prefixPath,
+        Set<String> includeStrings,
+        Set<String> excludeStrings,
+        boolean filtersMatchFieldNamesWithDots
+    ) {
+        FilterPath[] includePaths = FilterPath.compile(includeStrings);
+        FilterPath[] excludePaths = FilterPath.compile(excludeStrings);
+
+        if (prefixPath != null) {
+            if (includePaths != null) {
+                List<FilterPath> includeFilters = new ArrayList<>();
+                for (var incl : includePaths) {
+                    incl.matches(prefixPath, includeFilters, true);
+                }
+                includePaths = includeFilters.isEmpty() ? null : includeFilters.toArray(FilterPath[]::new);
+            }
+
+            if (excludePaths != null) {
+                List<FilterPath> excludeFilters = new ArrayList<>();
+                for (var excl : excludePaths) {
+                    excl.matches(prefixPath, excludeFilters, true);
+                }
+                excludePaths = excludeFilters.isEmpty() ? null : excludeFilters.toArray(FilterPath[]::new);
+            }
+        }
         return new XContentParserConfigurationImpl(
             registry,
             deprecationHandler,
             restApiVersion,
-            FilterPath.compile(includeStrings),
-            FilterPath.compile(excludeStrings),
+            includePaths,
+            excludePaths,
             filtersMatchFieldNamesWithDots
         );
     }

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java

@@ -49,10 +49,27 @@ public interface XContentParserConfiguration {
 
     RestApiVersion restApiVersion();
 
+    // TODO: Remove when serverless uses the new API
+    XContentParserConfiguration withFiltering(
+        Set<String> includeStrings,
+        Set<String> excludeStrings,
+        boolean filtersMatchFieldNamesWithDots
+    );
+
     /**
      * Replace the configured filtering.
+     *
+     * @param prefixPath                    The path to be prepended to each sub-path before applying the include/exclude rules.
+     *                                      Specify {@code null} if parsing starts from the root.
+     * @param includeStrings                A set of strings representing paths to include during filtering.
+     *                                      If specified, only these paths will be included in parsing.
+     * @param excludeStrings                A set of strings representing paths to exclude during filtering.
+     *                                      If specified, these paths will be excluded from parsing.
+     * @param filtersMatchFieldNamesWithDots Indicates whether filters should match field names containing dots ('.')
+     *                                      as part of the field name.
      */
     XContentParserConfiguration withFiltering(
+        String prefixPath,
         Set<String> includeStrings,
         Set<String> excludeStrings,
         boolean filtersMatchFieldNamesWithDots

libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java

@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.IntStream;
 
@@ -332,6 +333,24 @@ protected final void testFilter(Builder expected, Builder sample, Collection<Str
     private void testFilter(Builder expected, Builder sample, Set<String> includes, Set<String> excludes, boolean matchFieldNamesWithDots)
         throws IOException {
         assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes, matchFieldNamesWithDots));
+
+        String rootPrefix = "root.path.random";
+        if (includes != null) {
+            Set<String> rootIncludes = new HashSet<>();
+            for (var incl : includes) {
+                rootIncludes.add(rootPrefix + (randomBoolean() ? "." : "*.") + incl);
+            }
+            includes = rootIncludes;
+        }
+
+        if (excludes != null) {
+            Set<String> rootExcludes = new HashSet<>();
+            for (var excl : excludes) {
+                rootExcludes.add(rootPrefix + (randomBoolean() ? "." : "*.") + excl);
+            }
+            excludes = rootExcludes;
+        }
+        assertFilterResult(expected.apply(createBuilder()), filterSub(sample, rootPrefix, includes, excludes, matchFieldNamesWithDots));
     }
 
     public void testArrayWithEmptyObjectInInclude() throws IOException {
@@ -413,21 +432,36 @@ private XContentBuilder filter(Builder sample, Set<String> includes, Set<String>
             && matchFieldNamesWithDots == false) {
             return filterOnBuilder(sample, includes, excludes);
         }
-        return filterOnParser(sample, includes, excludes, matchFieldNamesWithDots);
+        return filterOnParser(sample, null, includes, excludes, matchFieldNamesWithDots);
+    }
+
+    private XContentBuilder filterSub(
+        Builder sample,
+        String root,
+        Set<String> includes,
+        Set<String> excludes,
+        boolean matchFieldNamesWithDots
+    ) throws IOException {
+        return filterOnParser(sample, root, includes, excludes, matchFieldNamesWithDots);
     }
 
     private XContentBuilder filterOnBuilder(Builder sample, Set<String> includes, Set<String> excludes) throws IOException {
         return sample.apply(XContentBuilder.builder(getXContentType(), includes, excludes));
     }
 
-    private XContentBuilder filterOnParser(Builder sample, Set<String> includes, Set<String> excludes, boolean matchFieldNamesWithDots)
-        throws IOException {
+    private XContentBuilder filterOnParser(
+        Builder sample,
+        String rootPath,
+        Set<String> includes,
+        Set<String> excludes,
+        boolean matchFieldNamesWithDots
+    ) throws IOException {
         try (XContentBuilder builtSample = sample.apply(createBuilder())) {
             BytesReference sampleBytes = BytesReference.bytes(builtSample);
             try (
                 XContentParser parser = getXContentType().xContent()
                     .createParser(
-                        XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchFieldNamesWithDots),
+                        XContentParserConfiguration.EMPTY.withFiltering(rootPath, includes, excludes, matchFieldNamesWithDots),
                         sampleBytes.streamInput()
                     )
             ) {