Merge remote-tracking branch 'upstream/main' into security-stats-with-dls

* upstream/main: (188 commits)
  XContent parsing for DocStats and ShardFieldStats (elastic#135016)
  ESQL: Fix multi cluster tests not working with @repeat (elastic#135208)
  ESQL: Enable Lookup Join on Expression Tech Preview (elastic#134952)
  Use gradleProperty instead of findProperty (elastic#135203)
  Update 133980.yaml
  Small fixes
  Adjust commit sizes to reflect change in TieredMergePolicy
  Update 133980.yaml
  Update docs/changelog/133980.yaml
  Use released lucene 10.3.0
  Update to Lucene 10.3.0 RC2
  Aggs: Fix test (elastic#134308)
  correct docs version
  Use last lucene 10.3 snapshot
  [Automated] Update Lucene snapshot to 10.4.0-snapshot-28bd476b1a1
  Update ValuesSourceReaderOperatorTests test condition for changes in lucene (elastic#133807)
  [Automated] Update Lucene snapshot to 10.3.0-snapshot-878a3db9c2d
  [Automated] Update Lucene snapshot to 10.3.0-snapshot-878a3db9c2d
  [Automated] Update Lucene snapshot to 10.3.0-snapshot-d664abb181a
  [Automated] Update Lucene snapshot to 10.3.0-snapshot-de669618c3c
  ...

Author: szybia

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -26,7 +26,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
+import org.elasticsearch.index.codec.Elasticsearch92Lucene103Codec;
 import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -63,7 +63,6 @@
 public class TSDBDocValuesMergeBenchmark {
 
     static {
-        // For Elasticsearch900Lucene101Codec:
         LogConfigurator.loadLog4jPlugins();
         LogConfigurator.configureESLogging();
         LogConfigurator.setNodeName("test");
@@ -259,8 +258,7 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
         var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
-        config.setCodec(new Elasticsearch900Lucene101Codec() {
-
+        config.setCodec(new Elasticsearch92Lucene103Codec() {
             @Override
             public DocValuesFormat getDocValuesFormatForField(String field) {
                 return docValuesFormat;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java

@@ -19,6 +19,7 @@
 import org.gradle.api.Project;
 import org.gradle.api.file.Directory;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.Copy;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
@@ -49,9 +50,9 @@ public void apply(Project project) {
                 Directory transportResources = project.getLayout().getProjectDirectory().dir("src/main/resources/" + resourceRoot);
                 spec.getParameters().getTransportResourcesDirectory().set(transportResources);
                 spec.getParameters().getRootDirectory().set(project.getLayout().getSettingsDirectory().getAsFile());
-                Object upstreamRef = project.findProperty("org.elasticsearch.transport.upstreamRef");
-                if (upstreamRef != null) {
-                    spec.getParameters().getUpstreamRefOverride().set(upstreamRef.toString());
+                Provider<String> upstreamRef = project.getProviders().gradleProperty("org.elasticsearch.transport.upstreamRef");
+                if (upstreamRef.isPresent()) {
+                    spec.getParameters().getUpstreamRefOverride().set(upstreamRef.get());
                 }
             });
 
@@ -120,10 +121,7 @@ public void apply(Project project) {
     }
 
     private static String getResourceRoot(Project project) {
-        var resourceRoot = project.findProperty("org.elasticsearch.transport.resourceRoot");
-        if (resourceRoot == null) {
-            resourceRoot = "transport";
-        }
-        return resourceRoot.toString();
+        Provider<String> resourceRootProperty = project.getProviders().gradleProperty("org.elasticsearch.transport.resourceRoot");
+        return resourceRootProperty.isPresent() ? resourceRootProperty.get() : "transport";
     }
 }

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.2.0
-lucene            = 10.2.2
+lucene            = 10.3.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 25+36@bd75d5f9689641da8e1daabeccb5528b

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -60,13 +60,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dio.netty.noUnsafe=true",
                 "-Dio.netty.noKeySetOptimization=true",
                 "-Dio.netty.recycler.maxCapacityPerThread=0",
-                // temporary until we get off-heap vector stats in Lucene 10.3
-                "--add-opens=org.apache.lucene.core/org.apache.lucene.codecs.lucene99=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene90=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene91=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene92=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene94=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene95=org.elasticsearch.server",
                 // log4j 2
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.2.2
-:lucene_version_path:   10_2_2
+:lucene_version:        10.3.0
+:lucene_version_path:   10_3_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/133980.yaml

@@ -0,0 +1,12 @@
+pr: 133980
+summary: Upgrade elasticsearch to lucene 10.3.0
+area: Search
+type: upgrade
+issues: []
+highlight:
+  title: "New lucene 10.3.0 release"
+  body: |-
+    - Improved performance for lexical, vector and primary-key searches
+    - Use optimistic-with-checking KNN Query execution strategy in place of cross-thread global queue min-score checking. Improves performance and consistency.
+    - Bulk scoring added for floating point vectors in HNSW. Improving query latency and indexing throughput
+    - Multiple improvements to HNSW graph traversal and storage

docs/changelog/134952.yaml

@@ -0,0 +1,11 @@
+pr: 134952
+summary: Add support for expressions with LOOKUP JOIN in tech preview
+area: ES|QL
+type: enhancement
+issues: [ ]
+highlight:
+  title: Add support for expressions with LOOKUP JOIN in tech preview
+  body: |-
+    Enable Lookup Join on Expression Tech Preview
+    FROM index1 | LOOKUP JOIN lookup_index on left_field1 > right_field1 AND left_field2 <= right_field2
+  notable: true

docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md

@@ -19,17 +19,22 @@ FROM <source_index>
 FROM <source_index>
 | LOOKUP JOIN <lookup_index> ON <field_name1>, <field_name2>, <field_name3>
 ```
+```esql
+FROM <source_index>
+| LOOKUP JOIN <lookup_index> ON <left_field1> >= <lookup_field1> AND <left_field2> == <lookup_field2>
+```
 
 **Parameters**
 
 `<lookup_index>`
 :   The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported. Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting).
 
-`<field_name>` or `<field_name1>, <field_name2>, <field_name3>`
-:   The field(s) to join on. Can be either:
-  * A single field name
-  * A comma-separated list of field names {applies_to}`stack: ga 9.2`
-:   These fields must exist in both your current query results and in the lookup index. If the fields contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows).
+`<field_name>` or `<field_name1>, <field_name2>, <field_name3>` or `<left_field1> >= <lookup_field1> AND <left_field2> == <lookup_field2>`
+:   The join condition. Can be one of the following:
+   * A single field name
+   * A comma-separated list of field names {applies_to}`stack: ga 9.2`
+   * An expression with one or more join conditions linked by `AND`. Each condition compares a field from the left index with a field from the lookup index using [binary operators](/reference/query-languages/esql/functions-operators/operators.md#esql-binary-operators) (`==`, `>=`, `<=`, `>`, `<`, `!=`). Each field name in the join condition must exist in only one of the indexes. Use RENAME to resolve naming conflicts. {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
+:   If using join on a single field or a field list, the fields used must exist in both your current query results and in the lookup index. If the fields contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows).
 
 
 **Description**

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -34,13 +34,15 @@ The `LOOKUP JOIN` command adds fields from the lookup index as new columns to yo
 
 The command requires two parameters:
 * The name of the lookup index (which must have the `lookup` [`index.mode setting`](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting))
-* The field(s) to join on. Can be either:
-  * A single field name
-  * A comma-separated list of field names {applies_to}`stack: ga 9.2`
+* The join condition. Can be one of the following:
+   * A single field name
+   * A comma-separated list of field names {applies_to}`stack: ga 9.2`
+   * An expression with one or more join conditions linked by `AND`. Each condition compares a field from the left index with a field from the lookup index using [binary operators](/reference/query-languages/esql/functions-operators/operators.md#esql-binary-operators) (`==`, `>=`, `<=`, `>`, `<`, `!=`). Each field name in the join condition must exist in only one of the indexes. Use RENAME to resolve naming conflicts. {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
 
 ```esql
 LOOKUP JOIN <lookup_index> ON <field_name>  # Join on a single field
 LOOKUP JOIN <lookup_index> ON <field_name1>, <field_name2>, <field_name3>  # Join on multiple fields
+LOOKUP JOIN <lookup_index> ON <left_field1> >= <lookup_field1> AND <left_field2> == <lookup_field2>  # Join on expression
 ```
 
 :::{image} ../images/esql-lookup-join.png