Merge branch 'main' into saml-issuer-in-warn-logs

Author: richard-dennehy

benchmarks/README.md

@@ -82,19 +82,21 @@ To get realistic results, you should exercise care when running benchmarks. Here
 NOTE: Linux only. Sorry Mac and Windows.
 
 Disassembling is fun! Maybe not always useful, but always fun! Generally, you'll want to install `perf` and the JDK's `hsdis`.
-`perf` is generally available via `apg-get install perf` or `pacman -S perf`. `hsdis` you'll want to compile from source. is a little more involved. This worked
+`perf` is generally available via `apg-get install perf` or `pacman -S perf linux-tools`. `hsdis` you'll want to compile from source. is a little more involved. This worked
 on 2020-08-01:
 
 ```
 git clone [email protected]:openjdk/jdk.git
 cd jdk
-git checkout jdk-17-ga
-cd src/utils/hsdis
+git checkout jdk-24-ga
 # Get a known good binutils
 wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.gz
 tar xf binutils-2.35.tar.gz
-make BINUTILS=binutils-2.35 ARCH=amd64
-sudo cp build/linux-amd64/hsdis-amd64.so /usr/lib/jvm/java-17-openjdk/lib/server/
+bash configure --with-hsdis=binutils --with-binutils-src=binutils-2.35 \
+    --with-boot-jdk=~/.gradle/jdks/oracle_corporation-24-amd64-linux.2
+make build-hsdis
+cp ./build/linux-x86_64-server-release/jdk/lib/hsdis-amd64.so \
+    ~/.gradle/jdks/oracle_corporation-24-amd64-linux.2/lib/hsdis.so
 ```
 
 If you want to disassemble a single method do something like this:
@@ -105,6 +107,30 @@ gradlew -p benchmarks run --args ' MemoryStatsBenchmark -jvmArgs "-XX:+UnlockDia
 
 If you want `perf` to find the hot methods for you, then do add `-prof perfasm`.
 
+NOTE: `perfasm` will need more access:
+```
+sudo bash
+echo -1 > /proc/sys/kernel/perf_event_paranoid
+exit
+```
+
+If you get warnings like:
+```
+The perf event count is suspiciously low (0).
+```
+then check if you are bumping into [this](https://man.archlinux.org/man/perf-stat.1.en#INTEL_HYBRID_SUPPORT)
+by running:
+```
+perf stat -B dd if=/dev/zero of=/dev/null count=1000000
+```
+
+If you see lines like:
+```
+         765019980      cpu_atom/cycles/                 #    1.728 GHz                         (0.60%)
+        2258845959      cpu_core/cycles/                 #    5.103 GHz                         (99.18%)
+```
+then `perf` is just not going to work for you.
+
 ## Async Profiler
 
 Note: Linux and Mac only. Sorry Windows.

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ParseIpBenchmark.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.compute.operator;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ParseIp;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.net.InetAddress;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 5)
+@Measurement(iterations = 7)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+@Fork(1)
+public class ParseIpBenchmark {
+    private final BytesRef ip = new BytesRef("192.168.0.1");
+    private final BreakingBytesRefBuilder scratch = ParseIp.buildScratch(new NoopCircuitBreaker("request"));
+
+    @Benchmark
+    public BytesRef leadingZerosRejected() {
+        return ParseIp.leadingZerosRejected(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef leadingZerosAreDecimal() {
+        return ParseIp.leadingZerosAreDecimal(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef leadingZerosAreOctal() {
+        return ParseIp.leadingZerosAreOctal(ip, scratch);
+    }
+
+    @Benchmark
+    public BytesRef original() {
+        InetAddress inetAddress = InetAddresses.forString(ip.utf8ToString());
+        return new BytesRef(InetAddressPoint.encode(inetAddress));
+    }
+}

docs/changelog/125562.yaml

@@ -0,0 +1,6 @@
+pr: 125562
+summary: Improve handling of empty response
+area: Infra/REST API
+type: bug
+issues:
+ - 57639

docs/changelog/126338.yaml

@@ -0,0 +1,5 @@
+pr: 126338
+summary: Speed up TO_IP
+area: ES|QL
+type: enhancement
+issues: []

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml

@@ -180,6 +180,11 @@ setup:
   - requires:
       cluster_features: ["gte_v8.8.0"]
       reason: "reset API added in in 8.8.0"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: DELETE
+          path: /_internal/desired_balance
+          capabilities: [ plain_text_empty_response ]
 
   - do:
       _internal.delete_desired_balance: { }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/10_basic.yml

@@ -3,6 +3,11 @@ setup:
   - requires:
       cluster_features: ["gte_v8.13.0"]
       reason: "API added in in 8.1.0 but modified in 8.13 (node_version field removed)"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: DELETE
+          path: /_internal/desired_nodes
+          capabilities: [ plain_text_empty_response ]
 ---
 teardown:
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/11_old_format.yml

@@ -6,6 +6,11 @@ setup:
   - requires:
       cluster_features: ["gte_v8.3.0"]
       reason: "API added in in 8.1.0 but modified in 8.3"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: DELETE
+          path: /_internal/desired_nodes
+          capabilities: [ plain_text_empty_response ]
 ---
 teardown:
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/20_dry_run.yml

@@ -3,6 +3,11 @@ setup:
   - requires:
       cluster_features: ["gte_v8.4.0"]
       reason: "Support for the dry run option was added in in 8.4.0"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: DELETE
+          path: /_internal/desired_nodes
+          capabilities: [ plain_text_empty_response ]
 ---
 teardown:
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.voting_config_exclusions/10_basic.yml

@@ -1,3 +1,16 @@
+setup:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_cluster/voting_config_exclusions
+          capabilities: [ plain_text_empty_response ]
+        - method: DELETE
+          path: /_cluster/voting_config_exclusions
+          capabilities: [ plain_text_empty_response ]
+      reason: needs these capabilities
+
+---
 teardown:
   - do:
       cluster.delete_voting_config_exclusions: {}

server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java

@@ -9,22 +9,24 @@
 package org.elasticsearch.cluster.coordination;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
-import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
 
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
@@ -38,18 +40,39 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singletonList(MockTransportService.TestPlugin.class);
     }
 
-    public void testAbdicateAfterVotingConfigExclusionAdded() throws ExecutionException, InterruptedException {
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable HTTP
+    }
+
+    public void testAbdicateAfterVotingConfigExclusionAdded() throws IOException {
         internalCluster().setBootstrapMasterNodeIndex(0);
         internalCluster().startNodes(2);
         final String originalMaster = internalCluster().getMasterName();
+        final var restClient = getRestClient();
 
         logger.info("--> excluding master node {}", originalMaster);
-        client().execute(
-            TransportAddVotingConfigExclusionsAction.TYPE,
-            new AddVotingConfigExclusionsRequest(TEST_REQUEST_TIMEOUT, originalMaster)
-        ).get();
+        final var excludeRequest = new Request("POST", "/_cluster/voting_config_exclusions");
+        excludeRequest.addParameter("node_names", originalMaster);
+        assertEmptyResponse(restClient.performRequest(excludeRequest));
+
         clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get();
         assertNotEquals(originalMaster, internalCluster().getMasterName());
+
+        final var clearRequest = new Request("DELETE", "/_cluster/voting_config_exclusions");
+        clearRequest.addParameter("wait_for_removal", "false");
+        assertEmptyResponse(restClient.performRequest(clearRequest));
+
+        assertThat(
+            internalCluster().getInstance(ClusterService.class).state().metadata().coordinationMetadata().getVotingConfigExclusions(),
+            empty()
+        );
+    }
+
+    private void assertEmptyResponse(Response response) throws IOException {
+        assertEquals("text/plain; charset=UTF-8", response.getHeader("content-type"));
+        assertEquals(0, response.getEntity().getContentLength());
+        assertEquals(0, response.getEntity().getContent().readAllBytes().length);
     }
 
     public void testElectsNodeNotInVotingConfiguration() throws Exception {