Merge remote-tracking branch 'upstream/main' into bwc-elastic#125599

Author: benwtrent

.buildkite/scripts/dra-workflow.trigger.sh

@@ -2,10 +2,34 @@
 
 set -euo pipefail
 
-echo "steps:"
-
 source .buildkite/scripts/branches.sh
 
+# We use that filtering to keep different schedule for different branches
+if [ -n "${INCLUDED_BRANCHES:-}" ]; then
+  # If set, only trigger the pipeline for the specified branches
+  IFS=',' read -r -a BRANCHES <<< "${INCLUDED_BRANCHES}"
+elif [ -n "${EXCLUDED_BRANCHES:-}" ]; then
+  # If set, listed branches will be excluded from the list of branches in branches.json
+  IFS=',' read -r -a EXCLUDED_BRANCHES <<< "${EXCLUDED_BRANCHES}"
+  FILTERED_BRANCHES=()
+    for BRANCH in "${BRANCHES[@]}"; do
+      EXCLUDE=false
+      for EXCLUDED_BRANCH in "${EXCLUDED_BRANCHES[@]}"; do
+        if [ "$BRANCH" == "$EXCLUDED_BRANCH" ]; then
+          EXCLUDE=true
+          break
+        fi
+      done
+      if [ "$EXCLUDE" = false ]; then
+        FILTERED_BRANCHES+=("$BRANCH")
+      fi
+    done
+    BRANCHES=("${FILTERED_BRANCHES[@]}")
+fi
+
+
+echo "steps:"
+
 for BRANCH in "${BRANCHES[@]}"; do
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
   BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')

plugins/examples/build.gradle

@@ -22,22 +22,16 @@ subprojects {
 
   repositories {
     // Only necessary when building plugins against SNAPSHOT versions of Elasticsearch
-    maven {
-      url = 'https://snapshots.elastic.co/maven/'
-      mavenContent {
-        if (gradle.includedBuilds) {
-          // When building in a composite, restrict this to only resolve the Java REST client
-          includeModule 'co.elastic.clients', 'elasticsearch-java'
-        }
-      }
-    }
     if (gradle.includedBuilds.isEmpty()) {
       maven {
         url = "https://artifacts-snapshot.elastic.co/elasticsearch/${elasticsearchVersion}/maven"
         mavenContent {
           includeModule 'org.elasticsearch', 'elasticsearch'
         }
       }
+      maven {
+        url = 'https://snapshots.elastic.co/maven/'
+      }
     }
 
     // Same for Lucene, add the snapshot repo based on the currently used Lucene version

plugins/examples/security-authorization-engine/build.gradle

@@ -16,7 +16,7 @@ dependencies {
   testImplementation "org.elasticsearch.plugin:x-pack-core:${elasticsearchVersion}"
   javaRestTestImplementation "org.elasticsearch.plugin:x-pack-core:${elasticsearchVersion}"
   javaRestTestImplementation "org.apache.logging.log4j:log4j-core:${log4jVersion}"
-  javaRestTestImplementation("co.elastic.clients:elasticsearch-java:${elasticsearchVersion}")
+  javaRestTestImplementation "co.elastic.clients:elasticsearch-java:[9.0,10.0)"
   javaRestTestImplementation "org.elasticsearch.client:elasticsearch-rest-client:${elasticsearchVersion}"
   javaRestTestImplementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3'
   javaRestTestImplementation "org.elasticsearch.test:framework:${elasticsearchVersion}"

server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java

@@ -29,8 +29,6 @@
 import java.util.Set;
 import java.util.function.Function;
 
-import static java.util.Collections.emptyMap;
-
 /**
  * Base {@link StoredFieldVisitor} that retrieves all non-redundant metadata.
  */
@@ -40,9 +38,9 @@ public class FieldsVisitor extends FieldNamesProvidingStoredFieldsVisitor {
     private final boolean loadSource;
     final String sourceFieldName;
     private final Set<String> requiredFields;
-    protected BytesReference source;
-    protected String id;
-    protected Map<String, List<Object>> fieldsValues;
+    private BytesReference source;
+    private String id;
+    private final Map<String, List<Object>> fieldsValues = new HashMap<>();
 
     public FieldsVisitor(boolean loadSource) {
         this(loadSource, SourceFieldMapper.NAME);
@@ -58,24 +56,29 @@ public FieldsVisitor(boolean loadSource, String sourceFieldName) {
 
     @Override
     public Status needsField(FieldInfo fieldInfo) {
-        if (requiredFields.remove(fieldInfo.name)) {
+        final String name = fieldInfo.name;
+        var requiredFields = this.requiredFields;
+        if (requiredFields.remove(name)) {
             return Status.YES;
         }
         // Always load _ignored to be explicit about ignored fields
         // This works because _ignored is added as the first metadata mapper,
         // so its stored fields always appear first in the list.
         // Note that _ignored is also multi-valued, which is why it can't be removed from the set like other fields
-        if (IgnoredFieldMapper.NAME.equals(fieldInfo.name)) {
+        if (IgnoredFieldMapper.NAME.equals(name)) {
             return Status.YES;
         }
+        // All these fields are single-valued so we can stop when the set is empty
+        if (requiredFields.isEmpty()) {
+            return Status.STOP;
+        }
         // support _uid for loading older indices
-        if ("_uid".equals(fieldInfo.name)) {
+        if ("_uid".equals(name)) {
             if (requiredFields.remove(IdFieldMapper.NAME) || requiredFields.remove(LegacyTypeFieldMapper.NAME)) {
                 return Status.YES;
             }
         }
-        // All these fields are single-valued so we can stop when the set is empty
-        return requiredFields.isEmpty() ? Status.STOP : Status.NO;
+        return Status.NO;
     }
 
     @Override
@@ -98,33 +101,31 @@ public final void postProcess(Function<String, MappedFieldType> fieldTypeLookup)
 
     @Override
     public void binaryField(FieldInfo fieldInfo, byte[] value) {
-        binaryField(fieldInfo, new BytesRef(value));
-    }
-
-    private void binaryField(FieldInfo fieldInfo, BytesRef value) {
-        if (sourceFieldName.equals(fieldInfo.name)) {
+        final String name = fieldInfo.name;
+        if (sourceFieldName.equals(name)) {
             source = new BytesArray(value);
-        } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
-            id = Uid.decodeId(value.bytes, value.offset, value.length);
+        } else if (IdFieldMapper.NAME.equals(name)) {
+            id = Uid.decodeId(value, 0, value.length);
         } else {
-            addValue(fieldInfo.name, value);
+            addValue(name, new BytesRef(value));
         }
     }
 
     @Override
     public void stringField(FieldInfo fieldInfo, String value) {
-        assert sourceFieldName.equals(fieldInfo.name) == false : "source field must go through binaryField";
-        if ("_uid".equals(fieldInfo.name)) {
+        final String name = fieldInfo.name;
+        assert sourceFieldName.equals(name) == false : "source field must go through binaryField";
+        if ("_uid".equals(name)) {
             // 5.x-only
             int delimiterIndex = value.indexOf('#'); // type is not allowed to have # in it..., ids can
             String type = value.substring(0, delimiterIndex);
             id = value.substring(delimiterIndex + 1);
             addValue(LegacyTypeFieldMapper.NAME, type);
-        } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
+        } else if (IdFieldMapper.NAME.equals(name)) {
             // only applies to 5.x indices that have single_type = true
             id = value;
         } else {
-            addValue(fieldInfo.name, value);
+            addValue(name, value);
         }
     }
 
@@ -157,25 +158,20 @@ public String id() {
     }
 
     public String routing() {
-        if (fieldsValues == null) {
-            return null;
-        }
         List<Object> values = fieldsValues.get(RoutingFieldMapper.NAME);
         if (values == null || values.isEmpty()) {
             return null;
         }
         assert values.size() == 1;
-        return values.get(0).toString();
+        return values.getFirst().toString();
     }
 
     public Map<String, List<Object>> fields() {
-        return fieldsValues != null ? fieldsValues : emptyMap();
+        return fieldsValues;
     }
 
     public void reset() {
-        if (fieldsValues != null) {
-            fieldsValues.clear();
-        }
+        fieldsValues.clear();
         source = null;
         id = null;
 
@@ -186,11 +182,6 @@ public void reset() {
     }
 
     void addValue(String name, Object value) {
-        if (fieldsValues == null) {
-            fieldsValues = new HashMap<>();
-        }
-
-        List<Object> values = fieldsValues.computeIfAbsent(name, k -> new ArrayList<>(2));
-        values.add(value);
+        fieldsValues.computeIfAbsent(name, k -> new ArrayList<>(2)).add(value);
     }
 }

test/external-modules/multi-project/build.gradle

@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
-
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.test-with-dependencies'
 apply plugin: 'elasticsearch.internal-java-rest-test'
@@ -12,12 +10,10 @@ esplugin {
 dependencies {
   testImplementation project(path: ':test:test-clusters')
   clusterModules project(':test:external-modules:test-multi-project')
-}
-
-tasks.withType(StandaloneRestIntegTestTask).configureEach {
-  usesDefaultDistribution("to be triaged")
+  clusterModules project(':modules:analysis-common')
 }
 
 tasks.named("javaRestTest").configure {
   enabled = buildParams.snapshotBuild
+  systemProperty "tests.multi_project.enabled", true
 }

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/admin/indices/IndexMultiProjectCRUDIT.java

@@ -46,7 +46,7 @@ public class IndexMultiProjectCRUDIT extends MultiProjectRestTestCase {
     private static ElasticsearchCluster createCluster() {
         LocalClusterSpecBuilder<ElasticsearchCluster> clusterBuilder = ElasticsearchCluster.local()
             .nodes(NODE_NUM)
-            .distribution(DistributionType.INTEG_TEST) // TODO multi-project: make this test suite work under the default distrib
+            .distribution(DistributionType.INTEG_TEST)
             .module("test-multi-project")
             .setting("test.multi_project.enabled", "true")
             .setting("xpack.security.enabled", "false") // TODO multi-project: make this test suite work with Security enabled

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/action/index/IndexDocumentMultiProjectIT.java

@@ -42,7 +42,7 @@ public class IndexDocumentMultiProjectIT extends MultiProjectRestTestCase {
     private static ElasticsearchCluster createCluster() {
         LocalClusterSpecBuilder<ElasticsearchCluster> clusterBuilder = ElasticsearchCluster.local()
             .nodes(NODE_NUM)
-            .distribution(DistributionType.INTEG_TEST) // TODO multi-project: make this test suite work under the default distrib
+            .distribution(DistributionType.INTEG_TEST)
             .module("test-multi-project")
             .setting("test.multi_project.enabled", "true")
             .setting("xpack.security.enabled", "false") // TODO multi-project: make this test suite work with Security enabled

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/multiproject/MultiProjectClusterStateActionIT.java

@@ -25,7 +25,7 @@ public class MultiProjectClusterStateActionIT extends MultiProjectRestTestCase {
 
     @ClassRule
     public static ElasticsearchCluster CLUSTER = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
+        .distribution(DistributionType.INTEG_TEST)
         .setting("test.multi_project.enabled", "true")
         .setting("xpack.security.http.ssl.enabled", "false")
         .setting("xpack.security.enabled", "false")

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/multiproject/MultiProjectRestTestCase.java

@@ -11,77 +11,32 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.xcontent.ObjectPath;
 import org.junit.After;
 
 import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 public abstract class MultiProjectRestTestCase extends ESRestTestCase {
-    protected static Request setRequestProjectId(Request request, String projectId) {
-        RequestOptions.Builder options = request.getOptions().toBuilder();
-        options.removeHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER);
-        options.addHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, projectId);
-        request.setOptions(options);
-        return request;
-    }
-
-    protected static void clearRequestProjectId(Request request) {
-        RequestOptions options = request.getOptions();
-        if (options.containsHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER)) {
-            request.setOptions(options.toBuilder().removeHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER).build());
-        }
-    }
-
-    protected void createProject(String projectId) throws IOException {
-        Request request = new Request("PUT", "/_project/" + projectId);
-        try {
-            Response response = adminClient().performRequest(request);
-            assertOK(response);
-            logger.info("Created project {} : {}", projectId, response.getStatusLine());
-        } catch (ResponseException e) {
-            logger.error("Failed to create project: {}", projectId);
-            throw e;
-        }
-    }
-
-    protected void deleteProject(String projectId) throws IOException {
-        final Request request = new Request("DELETE", "/_project/" + projectId);
-        try {
-            final Response response = adminClient().performRequest(request);
-            logger.info("Deleted project {} : {}", projectId, response.getStatusLine());
-        } catch (ResponseException e) {
-            logger.error("Failed to delete project: {}", projectId);
-            throw e;
-        }
-    }
 
-    protected Set<String> listProjects() throws IOException {
-        final Request request = new Request("GET", "/_cluster/state/metadata?multi_project");
-        final Response response = adminClient().performRequest(request);
-        final List<Map<String, ?>> projects = ObjectPath.eval("metadata.projects", entityAsMap(response));
-        return projects.stream().map(m -> String.valueOf(m.get("id"))).collect(Collectors.toSet());
+    @Override
+    protected boolean shouldConfigureProjects() {
+        return false;
     }
 
     @After
     public void removeNonDefaultProjects() throws IOException {
         if (preserveClusterUponCompletion() == false) {
-            final Set<String> projects = listProjects();
-            logger.info("Removing non-default projects from {}", projects);
-            for (String projectId : projects) {
-                if (projectId.equals(Metadata.DEFAULT_PROJECT_ID.id()) == false) {
-                    deleteProject(projectId);
-                }
-            }
+            cleanUpProjects();
         }
     }
 
+    protected static Request setRequestProjectId(Request request, String projectId) {
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.removeHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER);
+        options.addHeader(Task.X_ELASTIC_PROJECT_ID_HTTP_HEADER, projectId);
+        request.setOptions(options);
+        return request;
+    }
+
 }

test/external-modules/multi-project/src/javaRestTest/java/org/elasticsearch/multiproject/MultiProjectRestartIT.java

@@ -23,7 +23,7 @@ public class MultiProjectRestartIT extends MultiProjectRestTestCase {
 
     @ClassRule
     public static ElasticsearchCluster CLUSTER = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
+        .distribution(DistributionType.INTEG_TEST)
         .setting("test.multi_project.enabled", "true")
         .setting("xpack.security.http.ssl.enabled", "false")
         .setting("xpack.security.enabled", "false")