Merge remote-tracking branch 'origin/main' into mapper/analyzer-params

Author: romseygeek

.ci/jobs.t/elastic+elasticsearch+dra-staging-update.yml

@@ -0,0 +1,51 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+dra-staging-update
+    display-name: "elastic / elasticsearch # %BRANCH% - DRA staging update"
+    description: "Scheduled job to check for staging dependency updates"
+    node: master
+    scm: []
+    properties: []
+    parameters: []
+    publishers: []
+    triggers:
+      - timed: "*/30 * * * *"
+    builders:
+      - conditional-step:
+          condition-kind: shell
+          condition-command: |
+            #!/usr/bin/env bash
+            set -e
+            # Don't publish main branch to staging
+            if [ "%BRANCH%" == "main" ]; then
+              exit 1
+            fi
+
+            RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
+            BEATS_MANIFEST=$(curl -sS https://artifacts-staging.elastic.co/beats/latest/${RM_BRANCH}.json | jq -r '.manifest_url')
+            ML_MANIFEST=$(curl -sS https://artifacts-staging.elastic.co/ml-cpp/latest/${RM_BRANCH}.json | jq -r '.manifest_url')
+            ES_MANIFEST=$(curl -sS https://artifacts-staging.elastic.co/elasticsearch/latest/${RM_BRANCH}.json | jq -r '.manifest_url')
+            ES_BEATS_DEPENDENCY=$(curl -sS $ES_MANIFEST | jq -r '.projects.elasticsearch.dependencies[] | select(.prefix == "beats") | .build_uri')
+            ES_ML_DEPENDENCY=$(curl -sS $ES_MANIFEST | jq -r '.projects.elasticsearch.dependencies[] | select(.prefix == "ml-cpp") | .build_uri')
+
+            if [ "$BEATS_MANIFEST" = "$ES_BEATS_DEPENDENCY" ]; then
+              echo "ES has the latest beats"
+            else
+              echo "Need to trigger a build, $BEATS_MANIFEST available but ES has $ES_BEATS_DEPENDENCY"
+              exit 0
+            fi
+
+            if [ "$ML_MANIFEST" = "$ES_ML_DEPENDENCY" ]; then
+              echo "ES has the latest ml-cpp"
+            else
+              echo "Need to trigger a build, $ML_MANIFEST available but ES has $ES_ML_DEPENDENCY"
+              exit 0
+            fi
+
+            # A non-zero exit value means the trigger step will be skipped
+            exit 1
+          steps:
+            - trigger-builds:
+                - project: "elastic+elasticsearch+%BRANCH%+triggers/elastic+elasticsearch+%BRANCH%+dra-staging-trigger"
+                  current-parameters: false
+                  git-revision: false

.ci/packer_cache.sh

@@ -36,7 +36,7 @@ else
   rm -rf checkout/6.8
   git clone --reference $(dirname "${SCRIPT}")/../.git https://github.com/elastic/elasticsearch.git --branch 6.8 --single-branch checkout/6.8
   export JAVA_HOME="${JAVA11_HOME}"
-  ./checkout/6.8/gradlew --project-dir ./checkout/6.8 --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true --stacktrace resolveAllDependencies
+  ./checkout/6.8/gradlew --project-dir ./checkout/6.8 --parallel clean --stacktrace resolveAllDependencies
   rm -rf ./checkout/6.8
 fi
 

CONTRIBUTING.md

@@ -69,9 +69,8 @@ cycle.
 * Lines that are not part of your change should not be edited (e.g. don't format
   unchanged lines, don't reorder existing imports)
 * Add the appropriate [license headers](#license-headers) to any new files
-* For contributions involving the elasticsearch build you can find (details about the build setup in the
-  [BUILDING](BUILDING.md) file)
-
+* For contributions involving the elasticsearch build you can find details about the build setup in the
+  [BUILDING](BUILDING.md) file
 
 ### Submitting your changes
 

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -52,10 +52,9 @@ public static MapperService create(String mappings) {
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
         MapperService mapperService = new MapperService(
             indexSettings,
-            new IndexAnalyzers(
+            IndexAnalyzers.of(
                 Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())),
-                Map.of("lowercase", new NamedAnalyzer("lowercase", AnalyzerScope.INDEX, new LowercaseNormalizer())),
-                Map.of()
+                Map.of("lowercase", new NamedAnalyzer("lowercase", AnalyzerScope.INDEX, new LowercaseNormalizer()))
             ),
             XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(ClusterModule.getNamedXWriteables()))
                 .withDeprecationHandler(LoggingDeprecationHandler.INSTANCE),

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -58,7 +58,6 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -85,7 +84,6 @@ public class ScriptScoreBenchmark {
         Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null))
     );
     private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None();
-    private final Map<String, Set<String>> sourcePaths = Map.of("n", Set.of("n"));
     private final CircuitBreakerService breakerService = new NoneCircuitBreakerService();
     private final SearchLookup lookup = new SearchLookup(
         fieldTypes::get,

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -9,7 +9,6 @@
 package org.elasticsearch.benchmark.search;
 
 import org.apache.logging.log4j.util.Strings;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
@@ -23,13 +22,11 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalyzerScope;
-import org.elasticsearch.index.analysis.IndexAnalyzers;
-import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
@@ -173,11 +170,7 @@ protected final MapperService createMapperService(String mappings) {
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
         MapperService mapperService = new MapperService(
             indexSettings,
-            new IndexAnalyzers(
-                Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())),
-                Map.of(),
-                Map.of()
-            ),
+            (type, name) -> Lucene.STANDARD_ANALYZER,
             XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(ClusterModule.getNamedXWriteables()))
                 .withDeprecationHandler(LoggingDeprecationHandler.INSTANCE),
             similarityService,

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -58,10 +58,8 @@ static List<String> systemJvmOptions() {
             /*
              * Temporarily suppress illegal reflective access in searchable snapshots shared cache preallocation; this is temporary while we
              * explore alternatives. See org.elasticsearch.xpack.searchablesnapshots.preallocate.Preallocate.
-             *
-             * TODO: either modularlize Elasticsearch so that we can limit the opening of this module, or find an alternative
              */
-            "--add-opens=java.base/java.io=ALL-UNNAMED",
+            "--add-opens=java.base/java.io=org.elasticsearch.blobcache.preallocate",
             maybeOverrideDockerCgroup()
         ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }

docs/changelog/93524.yaml

@@ -0,0 +1,6 @@
+pr: 93524
+summary: Reduce the likelihood of writing small segments due to an oversize translog
+area: Engine
+type: enhancement
+issues:
+ - 75611

docs/changelog/94543.yaml

@@ -0,0 +1,6 @@
+pr: 94543
+summary: Report transport message size per action
+area: Network
+type: enhancement
+issues:
+ - 88151

docs/changelog/94702.yaml

@@ -0,0 +1,6 @@
+pr: 94702
+summary: Fix async enrich execution prematurely releases enrich policy lock
+area: Ingest Node
+type: bug
+issues:
+ - 94690