Merge branch 'main' into eut2

Author: jdconrad

server/src/main/java/org/elasticsearch/common/text/Text.java renamed to libs/x-content/src/main/java/org/elasticsearch/xcontent/Text.java

@@ -6,22 +6,17 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-package org.elasticsearch.common.text;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.xcontent.ToXContentFragment;
-import org.elasticsearch.xcontent.XContentBuilder;
+package org.elasticsearch.xcontent;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 
 /**
- * Both {@link String} and {@link BytesReference} representation of the text. Starts with one of those, and if
+ * Both {@link String} and {@link ByteBuffer} representation of the text. Starts with one of those, and if
  * the other is requests, caches the other one in a local reference so no additional conversion will be needed.
  */
-public final class Text implements Comparable<Text>, ToXContentFragment {
+public final class Text implements XContentString, Comparable<Text>, ToXContentFragment {
 
     public static final Text[] EMPTY_ARRAY = new Text[0];
 
@@ -36,31 +31,43 @@ public static Text[] convertFromStringArray(String[] strings) {
         return texts;
     }
 
-    private BytesReference bytes;
+    private ByteBuffer bytes;
     private String text;
     private int hash;
+    private int stringLength = -1;
+
+    /**
+     * Construct a Text from a UTF-8 encoded ByteBuffer. Since no string length is specified, {@link #stringLength()}
+     * will perform a string conversion to measure the string length.
+     */
+    public Text(ByteBuffer bytes) {
+        this.bytes = bytes;
+    }
 
-    public Text(BytesReference bytes) {
+    /**
+     * Construct a Text from a UTF-8 encoded ByteBuffer and an explicit string length. Used to avoid string conversion
+     * in {@link #stringLength()}.
+     */
+    public Text(ByteBuffer bytes, int stringLength) {
         this.bytes = bytes;
+        this.stringLength = stringLength;
     }
 
     public Text(String text) {
         this.text = text;
     }
 
     /**
-     * Whether a {@link BytesReference} view of the data is already materialized.
+     * Whether a {@link ByteBuffer} view of the data is already materialized.
      */
     public boolean hasBytes() {
         return bytes != null;
     }
 
-    /**
-     * Returns a {@link BytesReference} view of the data.
-     */
-    public BytesReference bytes() {
+    @Override
+    public ByteBuffer bytes() {
         if (bytes == null) {
-            bytes = new BytesArray(text.getBytes(StandardCharsets.UTF_8));
+            bytes = StandardCharsets.UTF_8.encode(text);
         }
         return bytes;
     }
@@ -72,11 +79,20 @@ public boolean hasString() {
         return text != null;
     }
 
-    /**
-     * Returns a {@link String} view of the data.
-     */
+    @Override
     public String string() {
-        return text == null ? bytes.utf8ToString() : text;
+        if (text == null) {
+            text = StandardCharsets.UTF_8.decode(bytes).toString();
+        }
+        return text;
+    }
+
+    @Override
+    public int stringLength() {
+        if (stringLength < 0) {
+            stringLength = string().length();
+        }
+        return stringLength;
     }
 
     @Override
@@ -115,8 +131,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         } else {
             // TODO: TextBytesOptimization we can use a buffer here to convert it? maybe add a
             // request to jackson to support InputStream as well?
-            BytesRef br = this.bytes().toBytesRef();
-            return builder.utf8Value(br.bytes, br.offset, br.length);
+            assert bytes.hasArray();
+            return builder.utf8Value(bytes.array(), bytes.arrayOffset() + bytes.position(), bytes.remaining());
         }
     }
 }

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentString.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.xcontent;
+
+import java.nio.ByteBuffer;
+
+public interface XContentString {
+    /**
+     * Returns a {@link String} view of the data.
+     */
+    String string();
+
+    /**
+     * Returns a UTF8-encoded {@link ByteBuffer} view of the data.
+     */
+    ByteBuffer bytes();
+
+    /**
+     * Returns the number of characters in the represented string.
+     */
+    int stringLength();
+}

muted-tests.yml

@@ -432,9 +432,6 @@ tests:
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=search/350_point_in_time/point-in-time with index filter}
   issue: https://github.com/elastic/elasticsearch/issues/127741
-- class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityFcActionAuthorizationIT
-  method: testIndicesPrivilegesAreEnforcedForCcrRestoreSessionActions
-  issue: https://github.com/elastic/elasticsearch/issues/127782
 - class: org.elasticsearch.compute.aggregation.FilteredGroupingAggregatorFunctionTests
   method: testSimpleCircuitBreaking
   issue: https://github.com/elastic/elasticsearch/issues/127833

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -76,8 +76,8 @@ static TransportVersion def(int id) {
      * READ THE COMMENT BELOW THIS BLOCK OF DECLARATIONS BEFORE ADDING NEW TRANSPORT VERSIONS
      * Detached transport versions added below here.
      */
-    public static final TransportVersion V_8_9_X = def(8_500_020);
-    public static final TransportVersion V_8_10_X = def(8_500_061);
+    public static final TransportVersion V_8_9_X = def(8_500_0_20);
+    public static final TransportVersion V_8_10_X = def(8_500_0_61);
     public static final TransportVersion V_8_11_X = def(8_512_0_01);
     public static final TransportVersion V_8_12_0 = def(8_560_0_00);
     public static final TransportVersion V_8_12_1 = def(8_560_0_01);
@@ -242,16 +242,16 @@ static TransportVersion def(int id) {
     public static final TransportVersion RANDOM_SAMPLER_QUERY_BUILDER = def(9_063_0_00);
     public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_064_0_00);
     public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = def(9_065_0_00);
-    public static final TransportVersion PROJECT_METADATA_SETTINGS = def(9_066_00_0);
-    public static final TransportVersion AGGREGATE_METRIC_DOUBLE_BLOCK = def(9_067_00_0);
+    public static final TransportVersion PROJECT_METADATA_SETTINGS = def(9_066_0_00);
+    public static final TransportVersion AGGREGATE_METRIC_DOUBLE_BLOCK = def(9_067_0_00);
     public static final TransportVersion PINNED_RETRIEVER = def(9_068_0_00);
     public static final TransportVersion ML_INFERENCE_SAGEMAKER = def(9_069_0_00);
-    public static final TransportVersion WRITE_LOAD_INCLUDES_BUFFER_WRITES = def(9_070_00_0);
+    public static final TransportVersion WRITE_LOAD_INCLUDES_BUFFER_WRITES = def(9_070_0_00);
     public static final TransportVersion INTRODUCE_FAILURES_DEFAULT_RETENTION = def(9_071_0_00);
     public static final TransportVersion FILE_SETTINGS_HEALTH_INFO = def(9_072_0_00);
     public static final TransportVersion FIELD_CAPS_ADD_CLUSTER_ALIAS = def(9_073_0_00);
-    public static final TransportVersion INFERENCE_ADD_TIMEOUT_PUT_ENDPOINT = def(9_074_00_0);
-    public static final TransportVersion ESQL_FIELD_ATTRIBUTE_DROP_TYPE = def(9_075_00_0);
+    public static final TransportVersion INFERENCE_ADD_TIMEOUT_PUT_ENDPOINT = def(9_074_0_00);
+    public static final TransportVersion ESQL_FIELD_ATTRIBUTE_DROP_TYPE = def(9_075_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/service/MasterService.java

@@ -36,7 +36,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.CountDown;
@@ -56,6 +55,7 @@
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.Text;
 
 import java.util.ArrayList;
 import java.util.Collections;

server/src/main/java/org/elasticsearch/cluster/service/PendingClusterTask.java

@@ -13,8 +13,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xcontent.Text;
 
 import java.io.IOException;
 

server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java

@@ -23,12 +23,12 @@
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.CharArrays;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xcontent.Text;
 
 import java.io.EOFException;
 import java.io.FilterInputStream;
@@ -391,13 +391,23 @@ public Text readOptionalText() throws IOException {
         if (length == -1) {
             return null;
         }
-        return new Text(readBytesReference(length));
+        byte[] bytes = new byte[length];
+        if (length > 0) {
+            readBytes(bytes, 0, length);
+        }
+        var byteBuff = ByteBuffer.wrap(bytes);
+        return new Text(byteBuff);
     }
 
     public Text readText() throws IOException {
-        // use StringAndBytes so we can cache the string if it's ever converted to it
+        // use Text so we can cache the string if it's ever converted to it
         int length = readInt();
-        return new Text(readBytesReference(length));
+        byte[] bytes = new byte[length];
+        if (length > 0) {
+            readBytes(bytes, 0, length);
+        }
+        var byteBuff = ByteBuffer.wrap(bytes);
+        return new Text(byteBuff);
     }
 
     @Nullable

server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

@@ -22,11 +22,11 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.Writeable.Writer;
 import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.core.CharArrays;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xcontent.Text;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
@@ -419,7 +419,7 @@ public void writeText(Text text) throws IOException {
             writeInt(spare.length());
             write(spare.bytes(), 0, spare.length());
         } else {
-            BytesReference bytes = text.bytes();
+            BytesReference bytes = BytesReference.fromByteBuffer(text.bytes());
             writeInt(bytes.length());
             bytes.writeTo(this);
         }

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -526,6 +526,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         IndexingMemoryController.MAX_INDEX_BUFFER_SIZE_SETTING,
         IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING,
         IndexingMemoryController.SHARD_MEMORY_INTERVAL_TIME_SETTING,
+        IndexingMemoryController.PAUSE_INDEXING_ON_THROTTLE,
         ResourceWatcherService.ENABLED,
         ResourceWatcherService.RELOAD_INTERVAL_HIGH,
         ResourceWatcherService.RELOAD_INTERVAL_MEDIUM,

server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsFormat.java

@@ -46,7 +46,7 @@
  */
 public class IVFVectorsFormat extends KnnVectorsFormat {
 
-    static final FeatureFlag IVF_FORMAT_FEATURE_FLAG = new FeatureFlag("ivf_format");
+    public static final FeatureFlag IVF_FORMAT_FEATURE_FLAG = new FeatureFlag("ivf_format");
     public static final String IVF_VECTOR_COMPONENT = "IVF";
     public static final String NAME = "IVFVectorsFormat";
     // centroid ordinals -> centroid values, offsets