Merge branch 'main' of github.com:elastic/elasticsearch into inference-eis-acl

Author: jonathan-buttner

.buildkite/scripts/dra-workflow.sh

@@ -6,7 +6,7 @@ WORKFLOW="${DRA_WORKFLOW:-snapshot}"
 BRANCH="${BUILDKITE_BRANCH:-}"
 
 # Don't publish main branch to staging
-if [[ ("$BRANCH" == "main" || "$BRANCH" == *.x) && "$WORKFLOW" == "staging" ]]; then
+if [[ "$BRANCH" == *.x && "$WORKFLOW" == "staging" ]]; then
   exit 0
 fi
 
@@ -22,13 +22,19 @@ if [[ "$BRANCH" == "main" ]]; then
 fi
 
 ES_VERSION=$(grep elasticsearch build-tools-internal/version.properties | sed "s/elasticsearch *= *//g")
+BASE_VERSION="$ES_VERSION"
 echo "ES_VERSION=$ES_VERSION"
 
 VERSION_SUFFIX=""
 if [[ "$WORKFLOW" == "snapshot" ]]; then
   VERSION_SUFFIX="-SNAPSHOT"
 fi
 
+if [[ -n "${VERSION_QUALIFIER:-}" ]]; then
+  ES_VERSION="${ES_VERSION}-${VERSION_QUALIFIER}"
+  echo "Version qualifier specified. ES_VERSION=${ES_VERSION}."
+fi
+
 BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
 echo "BEATS_BUILD_ID=$BEATS_BUILD_ID"
 
@@ -37,6 +43,7 @@ echo "ML_CPP_BUILD_ID=$ML_CPP_BUILD_ID"
 
 LICENSE_KEY_ARG=""
 BUILD_SNAPSHOT_ARG=""
+VERSION_QUALIFIER_ARG=""
 
 if [[ "$WORKFLOW" == "staging" ]]; then
   LICENSE_KEY=$(mktemp -d)/license.key
@@ -47,6 +54,10 @@ if [[ "$WORKFLOW" == "staging" ]]; then
   BUILD_SNAPSHOT_ARG="-Dbuild.snapshot=false"
 fi
 
+if [[ -n "${VERSION_QUALIFIER:-}" ]]; then
+  VERSION_QUALIFIER_ARG="-Dbuild.version_qualifier=$VERSION_QUALIFIER"
+fi
+
 echo --- Building release artifacts
 
 .ci/scripts/run-gradle.sh -Ddra.artifacts=true \
@@ -56,12 +67,17 @@ echo --- Building release artifacts
   -Dcsv="$WORKSPACE/build/distributions/dependencies-${ES_VERSION}${VERSION_SUFFIX}.csv" \
   $LICENSE_KEY_ARG \
   $BUILD_SNAPSHOT_ARG \
+  $VERSION_QUALIFIER_ARG \
   buildReleaseArtifacts \
   exportCompressedDockerImages \
   :distribution:generateDependenciesReport
 
 PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script
+if [[ -z "${VERSION_QUALIFIER:-}" ]]; then
 x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="$VERSION_SUFFIX"
+else
+x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="-$VERSION_QUALIFIER"
+fi
 
 # we regenerate this file as part of the release manager invocation
 rm "build/distributions/elasticsearch-jdbc-${ES_VERSION}${VERSION_SUFFIX}.taco.sha512"
@@ -88,7 +104,8 @@ docker run --rm \
   --branch "$RM_BRANCH" \
   --commit "$BUILDKITE_COMMIT" \
   --workflow "$WORKFLOW" \
-  --version "$ES_VERSION" \
+  --qualifier "${VERSION_QUALIFIER:-}" \
+  --version "$BASE_VERSION" \
   --artifact-set main \
   --dependency "beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
   --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json"

.buildkite/scripts/dra-workflow.trigger.sh

@@ -8,7 +8,7 @@ source .buildkite/scripts/branches.sh
 
 for BRANCH in "${BRANCHES[@]}"; do
   if [[ "$BRANCH" == "main" ]]; then
-    continue
+    export VERSION_QUALIFIER="alpha1"
   fi
 
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
@@ -24,5 +24,6 @@ for BRANCH in "${BRANCHES[@]}"; do
       commit: "$LAST_GOOD_COMMIT"
       env:
         DRA_WORKFLOW: staging
+        VERSION_QUALIFIER: ${VERSION_QUALIFIER:-}
 EOF
 done

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadLongBenchmark.java

@@ -47,7 +47,7 @@ public class BytesArrayReadLongBenchmark {
     @Setup
     public void initResults() throws IOException {
         final BytesStreamOutput tmp = new BytesStreamOutput();
-        final long bytes = new ByteSizeValue(dataMb, ByteSizeUnit.MB).getBytes();
+        final long bytes = ByteSizeValue.of(dataMb, ByteSizeUnit.MB).getBytes();
         for (int i = 0; i < bytes / 8; i++) {
             tmp.writeLong(i);
         }

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadLongBenchmark.java

@@ -47,7 +47,7 @@ public class PagedBytesReferenceReadLongBenchmark {
     @Setup
     public void initResults() throws IOException {
         final BytesStreamOutput tmp = new BytesStreamOutput();
-        final long bytes = new ByteSizeValue(dataMb, ByteSizeUnit.MB).getBytes();
+        final long bytes = ByteSizeValue.of(dataMb, ByteSizeUnit.MB).getBytes();
         for (int i = 0; i < bytes / 8; i++) {
             tmp.writeLong(i);
         }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java

@@ -53,7 +53,7 @@ public void apply(Project project) {
     }
 
     // List of root tasks, by name, whose compileJava task should not use the module path. These are test related sources.
-    static final Set<String> EXCLUDES = Set.of(":test:framework", ":x-pack:plugin:eql:qa:common");
+    static final Set<String> EXCLUDES = Set.of(":test:framework", ":x-pack:plugin:eql:qa:common", ":x-pack:plugin:esql:compute:test");
 
     void configureCompileModulePath(Project project) {
         // first disable Gradle's builtin module path inference

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -26,7 +26,7 @@ public interface TestClustersAware extends Task {
     Collection<ElasticsearchCluster> getClusters();
 
     @ServiceReference(REGISTRY_SERVICE_NAME)
-    Property<TestClustersRegistry> getRegistery();
+    Property<TestClustersRegistry> getRegistry();
 
     @ServiceReference(TEST_CLUSTER_TASKS_SERVICE)
     Property<TestClustersPlugin.TaskEventsService> getTasksService();
@@ -47,6 +47,14 @@ default void useCluster(ElasticsearchCluster cluster) {
         getClusters().add(cluster);
     }
 
+    default Provider<TestClusterInfo> getClusterInfo(String clusterName) {
+        return getProject().getProviders().of(TestClusterValueSource.class, source -> {
+            source.getParameters().getService().set(getRegistry());
+            source.getParameters().getClusterName().set(clusterName);
+            source.getParameters().getPath().set(getProject().getIsolated().getPath());
+        });
+    }
+
     default void useCluster(Provider<ElasticsearchCluster> cluster) {
         useCluster(cluster.get());
     }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -249,7 +249,7 @@ private void configureStartClustersHook(Gradle gradle) {
                     .forEach(awareTask -> {
                         awareTask.doFirst(task -> {
                             awareTask.beforeStart();
-                            awareTask.getClusters().forEach(awareTask.getRegistery().get()::maybeStartCluster);
+                            awareTask.getClusters().forEach(awareTask.getRegistry().get()::maybeStartCluster);
                         });
                     });
             });

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java

@@ -109,6 +109,23 @@ public void restart(String path, String clusterName) {
         cluster.restart();
     }
 
+    public void nextNodeToNextVersion(Provider<ElasticsearchCluster> cluster) {
+        nextNodeToNextVersion(cluster.get());
+    }
+
+    public void nextNodeToNextVersion(ElasticsearchCluster cluster) {
+        nextNodeToNextVersion(cluster.getPath(), cluster.getName());
+    }
+
+    public void nextNodeToNextVersion(String path, String clusterName) {
+        ElasticsearchCluster cluster = runningClusters.stream()
+            .filter(c -> c.getPath().equals(path))
+            .filter(c -> c.getName().equals(clusterName))
+            .findFirst()
+            .orElseThrow();
+        cluster.nextNodeToNextVersion();
+    }
+
     public void storeProcess(String id, Process esProcess) {
         nodeProcesses.put(id, esProcess);
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -187,12 +187,20 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
     static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
         final Set<String> settingNames = secrets.getSettingNames();
         for (String key : List.of("api_key", "secret_token")) {
-            String prefix = "telemetry.";
-            if (settingNames.contains(prefix + key)) {
-                try (SecureString token = secrets.getString(prefix + key)) {
-                    propertiesMap.put(key, token.toString());
+            for (String prefix : List.of("telemetry.", "tracing.apm.")) {
+                if (settingNames.contains(prefix + key)) {
+                    if (propertiesMap.containsKey(key)) {
+                        throw new IllegalStateException(
+                            Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
+                        );
+                    }
+
+                    try (SecureString token = secrets.getString(prefix + key)) {
+                        propertiesMap.put(key, token.toString());
+                    }
                 }
             }
+
         }
     }
 
@@ -219,12 +227,44 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
     static Map<String, String> extractApmSettings(Settings settings) throws UserException {
         final Map<String, String> propertiesMap = new HashMap<>();
 
+        // tracing.apm.agent. is deprecated by telemetry.agent.
         final String telemetryAgentPrefix = "telemetry.agent.";
+        final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";
 
         final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
         telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));
 
+        final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
+        for (String key : apmAgentSettings.keySet()) {
+            if (propertiesMap.containsKey(key)) {
+                throw new IllegalStateException(
+                    Strings.format(
+                        "Duplicate telemetry setting: [%s%s] and [%s%s]",
+                        telemetryAgentPrefix,
+                        key,
+                        deprecatedTelemetryAgentPrefix,
+                        key
+                    )
+                );
+            }
+            propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
+        }
+
         StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
+        if (globalLabels.length() == 0) {
+            globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
+        } else {
+            StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
+            if (tracingGlobalLabels.length() != 0) {
+                throw new IllegalArgumentException(
+                    "Cannot have global labels with tracing.agent prefix ["
+                        + globalLabels
+                        + "] and telemetry.apm.agent prefix ["
+                        + tracingGlobalLabels
+                        + "]"
+                );
+            }
+        }
         if (globalLabels.length() > 0) {
             propertiesMap.put("global_labels", globalLabels.toString());
         }
@@ -234,7 +274,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
             if (propertiesMap.containsKey(key)) {
                 throw new UserException(
                     ExitCodes.CONFIG,
-                    "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
+                    "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
                 );
             }
         }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -171,8 +171,8 @@ private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlement
             throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e);
         }
         // We instrument classes in these modules to call the bridge. Because the bridge gets patched
-        // into java.base, we must export the bridge from java.base to these modules.
-        String modulesContainingEntitlementInstrumentation = "java.logging";
+        // into java.base, we must export the bridge from java.base to these modules, as a comma-separated list
+        String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming";
         return Stream.of(
             "-Des.entitlements.enabled=true",
             "-XX:+EnableDynamicAgentLoading",