Merge branch 'main' into migrate-docs

Author: Mpdreamz

.buildkite/pipelines/periodic-fwc.template.yml

@@ -0,0 +1,15 @@
+steps:
+  - label: $FWC_VERSION / fwc
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
+      preemptible: true
+    matrix:
+      setup:
+        FWC_VERSION: $FWC_LIST
+    env:
+      FWC_VERSION: $FWC_VERSION

.buildkite/pipelines/periodic-fwc.yml

@@ -0,0 +1,16 @@
+# This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml
+steps:
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
+      preemptible: true
+    matrix:
+      setup:
+        FWC_VERSION: []
+    env:
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/scripts/branches.sh

@@ -2,3 +2,7 @@
 
 # This determines which branches will have pipelines triggered periodically, for dra workflows.
 BRANCHES=( $(cat branches.json | jq -r '.branches[].branch') )
+
+# Sort them to make ordering predictable
+IFS=$'\n' BRANCHES=($(sort <<<"${BRANCHES[*]}"))
+unset IFS

.buildkite/scripts/periodic.trigger.sh

@@ -46,4 +46,15 @@ EOF
       branch: "$BRANCH"
       commit: "$LAST_GOOD_COMMIT"
 EOF
+# Include forward compatibility tests only for the bugfix branch
+if [[ "${BRANCH}" == "${BRANCHES[2]}" ]]; then
+  cat <<EOF
+  - trigger: elasticsearch-periodic-fwc
+    label: Trigger periodic-fwc pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
+EOF
+fi
 done

benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
@@ -84,7 +85,11 @@ public void setUp() {
         }
         int mid = indices.length / 2;
         clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build();
-        resolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(List.of()));
+        resolver = new IndexNameExpressionResolver(
+            new ThreadContext(Settings.EMPTY),
+            new SystemIndices(List.of()),
+            DefaultProjectResolver.INSTANCE
+        );
         indexListRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), indices);
         starRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), "*");
         String[] mixed = indices.clone();

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -138,7 +138,7 @@ public void setUp() throws Exception {
         Metadata metadata = mb.build();
         RoutingTable.Builder rb = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 1; i <= numIndices; i++) {
-            rb.addAsNew(metadata.index("test_" + i));
+            rb.addAsNew(metadata.getProject().index("test_" + i));
         }
         RoutingTable routingTable = rb.build();
         DiscoveryNodes.Builder nb = DiscoveryNodes.builder();

build-tools-internal/src/main/groovy/elasticsearch.fwc-test.gradle

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+
+def fwcVersions = buildParams.bwcVersions.released.findAll { it.major == VersionProperties.elasticsearchVersion.major &&  it.minor == VersionProperties.elasticsearchVersion.minor }
+def previousMinorSnapshot = buildParams.bwcVersions.unreleased.find { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor - 1 }
+
+fwcVersions.each { fwcVersion ->
+  tasks.register("v${fwcVersion}#fwcTest", StandaloneRestIntegTestTask) {
+    usesBwcDistribution(previousMinorSnapshot)
+    usesBwcDistribution(fwcVersion)
+    systemProperty("tests.old_cluster_version", previousMinorSnapshot)
+    systemProperty("tests.new_cluster_version", fwcVersion)
+    nonInputProperties.systemProperty 'tests.fwc', 'true'
+  }
+}
+
+gradle.taskGraph.whenReady { graph ->
+  if (graph.allTasks.any { it.name.endsWith('#fwcTest') } && Boolean.parseBoolean(System.getProperty("tests.bwc.snapshot", "true"))) {
+    throw new GradleException("Running forward compatibility tests requires passing `-Dtests.bwc.snapshot=false`.")
+  }
+
+  if (graph.allTasks.any { it.name.endsWith('#fwcTest') } && graph.allTasks.any { it.name.endsWith('#bwcTest') }) {
+    throw new GradleException("Backward compatibility and forward compatibility tests cannot be executed in the same build.")
+  }
+}

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -145,18 +145,23 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     }
   }
 
-  // modifies the idea module config to enable preview features on ':libs:native' module
+  // modifies the idea module config to enable preview features on module that need them
   tasks.register("enablePreviewFeatures", EnablePreviewFeaturesTask) {
     group = 'ide'
-    description = 'Enables preview features on native library module'
+    description = 'Enables preview features on modules that need them'
     dependsOn tasks.named("enableExternalConfiguration")
     doLast {
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.test.iml', 'JDK_21_PREVIEW')
+      // due to org.elasticsearch.plugins.PluginsLoader
+      enablePreview('.idea/modules/server/elasticsearch.server.main.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/server/elasticsearch.server.test.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/entitlement/elasticsearch.libs.entitlement.main.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/entitlement/elasticsearch.libs.entitlement.test.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/entitlement/bridge/elasticsearch.libs.entitlement.bridge.main.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/entitlement/bridge/elasticsearch.libs.entitlement.bridge.test.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/libs/entitlement/qa/entitlement-test-plugin/elasticsearch.libs.entitlement.qa.entitlement-test-plugin.main.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/libs/entitlement/qa/entitlement-test-plugin/elasticsearch.libs.entitlement.qa.entitlement-test-plugin.test.iml', 'JDK_21_PREVIEW')
     }
   }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -145,7 +145,14 @@ public void execute(Task task) {
                 loggedExec.args("-DisCI");
             }
 
-            loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED");
+            loggedExec.args("-Dscan.tag.NESTED");
+
+            if (System.getProperty("tests.bwc.snapshot", "true").equals("false")) {
+                loggedExec.args("-Dbuild.snapshot=false", "-Dlicense.key=x-pack/plugin/core/snapshot.key");
+            } else {
+                loggedExec.args("-Dbuild.snapshot=true");
+            }
+
             final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel();
             List<LogLevel> nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG);
             if (nonDefaultLogLevels.contains(logLevel)) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -355,8 +355,9 @@ static void createBuildBwcTask(
         String bwcTaskName = buildBwcTaskName(projectName);
         bwcSetupExtension.bwcTask(bwcTaskName, c -> {
             boolean useNativeExpanded = projectArtifact.expandedDistDir != null;
+            boolean isReleaseBuild = System.getProperty("tests.bwc.snapshot", "true").equals("false");
             File expectedOutputFile = useNativeExpanded
-                ? new File(projectArtifact.expandedDistDir, "elasticsearch-" + bwcVersion.get() + "-SNAPSHOT")
+                ? new File(projectArtifact.expandedDistDir, "elasticsearch-" + bwcVersion.get() + (isReleaseBuild ? "" : "-SNAPSHOT"))
                 : projectArtifact.distFile;
             c.getInputs().file(new File(project.getBuildDir(), "refspec")).withPathSensitivity(PathSensitivity.RELATIVE);
             if (useNativeExpanded) {