Merge branch 'main' into esql-command-tech-preview

Author: jan-elastic

.buildkite/pipelines/intake.template.yml

@@ -72,6 +72,8 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
+  - label: bc-upgrade
+    command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
   - group: lucene-compat
     steps:
       - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

.buildkite/pipelines/intake.yml

@@ -72,6 +72,11 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
+  - label: bc-upgrade
+    command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      memory: "4G"
   - group: lucene-compat
     steps:
       - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+set -euo pipefail
+
+echo "Selecting the most recent build from branch [$BUILDKITE_BRANCH]."
+
+# Select the most recent build from the current branch.
+# We collect snapshots, order by date, then collect BCs, order by date, and concat them; then we select the last.
+# So if we have one (or more) BC, we will always prefer to use that. Otherwise we will use the latest snapshot.
+MANIFEST_URL="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json |
+jq ".releases[] |
+select(.branch == \"$BUILDKITE_BRANCH\") |
+select(.active_release == true) |
+((.snapshots | to_entries | sort_by(.value.completed_at)) +
+(.build_candidates | to_entries | sort_by(.value.completed_at))) |
+last | .value.manifest_url")"
+
+if [[ -z "$MANIFEST_URL" ]]; then
+   echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]."
+   echo "Skipping BC upgrade tests."
+   exit 0
+fi
+
+echo "Getting build manifest from [$MANIFEST_URL]"
+
+# Note: we use eval to perform variable substitution for the curl arguments, and command substitution to
+# set the output variable. Double quotes are not enough in this case.
+MANIFEST="$(eval "curl -s $MANIFEST_URL")"
+if [[ -z "$MANIFEST" ]]; then
+   echo "Cannot get the build manifest from [$MANIFEST_URL]"
+   exit 1
+fi
+
+CURRENT_VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
+
+BC_VERSION=$(echo "$MANIFEST" | jq -r .version)
+BC_BUILD_ID=$(echo "$MANIFEST" | jq -r .build_id)
+BC_COMMIT_HASH=$(echo "$MANIFEST" | jq -r .projects.elasticsearch.commit_hash)
+
+if [ "$CURRENT_VERSION-SNAPSHOT" != "$BC_VERSION" ]; then
+    echo "Version [$BC_VERSION] of BC (or snapshot) does not match current version [$CURRENT_VERSION] of branch [$BUILDKITE_BRANCH]."
+    echo "Skipping BC upgrade tests."
+    exit 0
+fi
+
+echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or snapshot) build of commit [$BC_COMMIT_HASH] with build id [$BC_BUILD_ID]."
+
+cat <<EOF | buildkite-agent pipeline upload
+steps:
+    - label: bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH
+      command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
+      timeout_in_minutes: 300
+      agents:
+        provider: gcp
+        image: family/elasticsearch-ubuntu-2004
+        machineType: n1-standard-32
+        buildDirectory: /dev/shm/bk
+        preemptible: true
+      retry:
+        automatic:
+          - exit_status: "-1"
+            limit: 3
+            signal_reason: none
+          - signal_reason: agent_stop
+            limit: 3
+EOF

.github/workflows/docs-preview-comment.yml

@@ -35,8 +35,8 @@ jobs:
                 !/(^|\/)_snippets\//i.test(f.filename)
               )
               .map(f => {
-                let p = f.filename.replace(/\/index\.md$/i, '/');
-                if (p === f.filename) p = p.replace(/\.md$/i, '');
+                let p = f.filename.replace(/^docs\//, '').replace(/\/index\.md$/i, '/');
+                if (p === f.filename.replace(/^docs\//, '')) p = p.replace(/\.md$/i, '');
                 return `- [\`${f.filename}\`](${base}/${p})`;
               });
             if (!links.length) return;  // nothing to do
@@ -68,4 +68,4 @@ jobs:
                 issue_number: prNum,
                 body
               });
-            }
+            }

CONTRIBUTING.md

@@ -203,6 +203,18 @@ Alternative manual steps for IntelliJ.
    3. Navigate to the file `build-conventions/formatterConfig.xml`
    4. Click "OK"
 
+#### Options
+
+When importing to IntelliJ, we offer a few options that can be used to
+configure the behaviour of the import:
+
+| Property                                   | Description                                                                                          | Values (* = default) |
+|--------------------------------------------|------------------------------------------------------------------------------------------------------|----------------------|
+| `org.elasticsearch.idea-configuration-cache` | Should IntelliJ enable the Gradle Configuration cache to speed up builds when generating run configs | *`true`, `false`         |
+| `org.elasticsearch.idea-delegate-to-gradle`  | Should IntelliJ use Gradle for all generated run / test configs or prompt each time                  | `true`, *`false`         |
+
+These options can be set anywhere on the Gradle config path including in `~/.gradle/gradle.properties`
+
 ### REST endpoint conventions
 
 Elasticsearch typically uses singular nouns rather than plurals in URLs.

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -180,6 +180,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   // this path is produced by the extractLibs task above
   String testLibraryPath = TestUtil.getTestLibraryPath("${elasticsearchProject.left()}/libs/native/libraries/build/platform")
   def enableIdeaCC = providers.gradleProperty("org.elasticsearch.idea-configuration-cache").getOrElse("true").toBoolean()
+  def delegateToGradle = providers.gradleProperty("org.elasticsearch.idea-delegate-to-gradle").getOrElse("false").toBoolean()
   idea {
     project {
       vcs = 'Git'
@@ -188,7 +189,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       settings {
         delegateActions {
           delegateBuildRunToGradle = false
-          testRunner = 'choose_per_test'
+          testRunner = delegateToGradle ? 'gradle' : 'choose_per_test'
         }
         taskTriggers {
           afterSync tasks.named('configureIdeCheckstyle'),

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java

@@ -63,7 +63,7 @@ public TransportGetDataStreamSettingsAction(
 
     @Override
     protected ClusterBlockException checkBlock(GetDataStreamSettingsAction.Request request, ProjectState state) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        return state.blocks().globalBlockedException(state.projectId(), ClusterBlockLevel.METADATA_WRITE);
     }
 
     @Override

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -410,6 +410,6 @@ static List<DataStream> getDataStreams(ProjectMetadata project, IndexNameExpress
 
     @Override
     protected ClusterBlockException checkBlock(GetDataStreamAction.Request request, ProjectState state) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        return state.blocks().globalBlockedException(state.projectId(), ClusterBlockLevel.METADATA_WRITE);
     }
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportMigrateToDataStreamAction.java

@@ -77,6 +77,6 @@ protected void masterOperation(
 
     @Override
     protected ClusterBlockException checkBlock(MigrateToDataStreamAction.Request request, ProjectState state) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        return state.blocks().globalBlockedException(state.projectId(), ClusterBlockLevel.METADATA_WRITE);
     }
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportModifyDataStreamsAction.java

@@ -67,7 +67,7 @@ protected void masterOperation(
 
     @Override
     protected ClusterBlockException checkBlock(ModifyDataStreamsAction.Request request, ProjectState state) {
-        ClusterBlockException globalBlock = state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        ClusterBlockException globalBlock = state.blocks().globalBlockedException(state.projectId(), ClusterBlockLevel.METADATA_WRITE);
         if (globalBlock != null) {
             return globalBlock;
         }