Merge remote-tracking branch 'upstream/main' into counted_keyword_multi_subobject_fix

Author: jordan-powers

.buildkite/pipelines/periodic.template.yml

@@ -86,6 +86,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
+              - openjdk24
             GRADLE_TASK:
               - checkPart1
               - checkPart2

.buildkite/pipelines/periodic.yml

@@ -505,6 +505,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
+              - openjdk24
             GRADLE_TASK:
               - checkPart1
               - checkPart2

docs/changelog/120751.yaml

@@ -0,0 +1,5 @@
+pr: 120751
+summary: Adding support for binary embedding type to Cohere service embedding type
+area: Machine Learning
+type: enhancement
+issues: []

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java

@@ -52,9 +52,10 @@
  * <p>
  * A bit like Mockito but way more painful.
  */
-public class DummyImplementations {
+class DummyImplementations {
+
+    static class DummyLocaleServiceProvider extends LocaleServiceProvider {
 
-    public static class DummyLocaleServiceProvider extends LocaleServiceProvider {
         @Override
         public Locale[] getAvailableLocales() {
             throw unexpected();

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java

@@ -96,9 +96,6 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
 
     private static final Map<String, CheckAction> checkActions = Stream.concat(
         Stream.<Entry<String, CheckAction>>of(
-            entry("static_reflection", deniedToPlugins(RestEntitlementsCheckAction::staticMethodNeverEntitledViaReflection)),
-            entry("nonstatic_reflection", deniedToPlugins(RestEntitlementsCheckAction::nonstaticMethodNeverEntitledViaReflection)),
-            entry("constructor_reflection", deniedToPlugins(RestEntitlementsCheckAction::constructorNeverEntitledViaReflection)),
             entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
             entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
             entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
@@ -341,11 +338,6 @@ private static void systemExit() {
         System.exit(123);
     }
 
-    private static void staticMethodNeverEntitledViaReflection() throws Exception {
-        Method systemExit = System.class.getMethod("exit", int.class);
-        systemExit.invoke(null, 123);
-    }
-
     private static void createClassLoader() throws IOException {
         try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
             logger.info("Created URLClassLoader [{}]", classLoader.getName());
@@ -356,11 +348,6 @@ private static void processBuilder_start() throws IOException {
         new ProcessBuilder("").start();
     }
 
-    private static void nonstaticMethodNeverEntitledViaReflection() throws Exception {
-        Method processBuilderStart = ProcessBuilder.class.getMethod("start");
-        processBuilderStart.invoke(new ProcessBuilder(""));
-    }
-
     private static void processBuilder_startPipeline() throws IOException {
         ProcessBuilder.startPipeline(List.of());
     }
@@ -399,10 +386,6 @@ private static void setHttpsConnectionProperties() {
         new DummyLocaleServiceProvider();
     }
 
-    private static void constructorNeverEntitledViaReflection() throws Exception {
-        DummyLocaleServiceProvider.class.getConstructor().newInstance();
-    }
-
     private static void breakIteratorProvider$() {
         new DummyBreakIteratorProvider();
     }

muted-tests.yml

@@ -46,9 +46,6 @@ tests:
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test81JavaOptsInJvmOptions
   issue: https://github.com/elastic/elasticsearch/issues/113313
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=mtermvectors/10_basic/Tests catching other exceptions per item}
-  issue: https://github.com/elastic/elasticsearch/issues/113325
 - class: org.elasticsearch.xpack.transform.integration.TransformIT
   method: testStopWaitForCheckpoint
   issue: https://github.com/elastic/elasticsearch/issues/106113

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mtermvectors/10_basic.yml

@@ -3,6 +3,7 @@ setup:
   - do:
       indices.create:
           index: testidx
+          wait_for_active_shards: all
           body:
             mappings:
               properties:
@@ -80,6 +81,7 @@ setup:
   - do:
       indices.create:
           index: testidx2
+          wait_for_active_shards: all
 
   - do:
       indices.put_alias:

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -173,7 +173,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_0_00);
     public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
     public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_0_00);
-
+    public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceByteEmbedding.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ *
+ * this file was contributed to by a generative AI
+ */
+
+package org.elasticsearch.xpack.core.inference.results;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+public record InferenceByteEmbedding(byte[] values) implements Writeable, ToXContentObject, EmbeddingInt {
+    public static final String EMBEDDING = "embedding";
+
+    public InferenceByteEmbedding(StreamInput in) throws IOException {
+        this(in.readByteArray());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeByteArray(values);
+    }
+
+    public static InferenceByteEmbedding of(List<Byte> embeddingValuesList) {
+        byte[] embeddingValues = new byte[embeddingValuesList.size()];
+        for (int i = 0; i < embeddingValuesList.size(); i++) {
+            embeddingValues[i] = embeddingValuesList.get(i);
+        }
+        return new InferenceByteEmbedding(embeddingValues);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.startArray(EMBEDDING);
+        for (byte value : values) {
+            builder.value(value);
+        }
+        builder.endArray();
+
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    float[] toFloatArray() {
+        float[] floatArray = new float[values.length];
+        for (int i = 0; i < values.length; i++) {
+            floatArray[i] = ((Byte) values[i]).floatValue();
+        }
+        return floatArray;
+    }
+
+    double[] toDoubleArray() {
+        double[] doubleArray = new double[values.length];
+        for (int i = 0; i < values.length; i++) {
+            doubleArray[i] = ((Byte) values[i]).doubleValue();
+        }
+        return doubleArray;
+    }
+
+    @Override
+    public int getSize() {
+        return values().length;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        InferenceByteEmbedding embedding = (InferenceByteEmbedding) o;
+        return Arrays.equals(values, embedding.values);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(values);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingBitResults.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ *
+ * this file was contributed to by a generative AI
+ */
+
+package org.elasticsearch.xpack.core.inference.results;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.inference.InferenceResults;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Writes a text embedding result in the follow json format
+ * {
+ *     "text_embedding_bytes": [
+ *         {
+ *             "embedding": [
+ *                 23
+ *             ]
+ *         },
+ *         {
+ *             "embedding": [
+ *                 -23
+ *             ]
+ *         }
+ *     ]
+ * }
+ */
+public record InferenceTextEmbeddingBitResults(List<InferenceByteEmbedding> embeddings) implements InferenceServiceResults, TextEmbedding {
+    public static final String NAME = "text_embedding_service_bit_results";
+    public static final String TEXT_EMBEDDING_BITS = "text_embedding_bits";
+
+    public InferenceTextEmbeddingBitResults(StreamInput in) throws IOException {
+        this(in.readCollectionAsList(InferenceByteEmbedding::new));
+    }
+
+    @Override
+    public int getFirstEmbeddingSize() {
+        return TextEmbeddingUtils.getFirstEmbeddingSize(new ArrayList<>(embeddings));
+    }
+
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.array(TEXT_EMBEDDING_BITS, embeddings.iterator());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeCollection(embeddings);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public List<? extends InferenceResults> transformToCoordinationFormat() {
+        return embeddings.stream()
+            .map(embedding -> new MlTextEmbeddingResults(TEXT_EMBEDDING_BITS, embedding.toDoubleArray(), false))
+            .toList();
+    }
+
+    @Override
+    @SuppressWarnings("deprecation")
+    public List<? extends InferenceResults> transformToLegacyFormat() {
+        var legacyEmbedding = new LegacyTextEmbeddingResults(
+            embeddings.stream().map(embedding -> new LegacyTextEmbeddingResults.Embedding(embedding.toFloatArray())).toList()
+        );
+
+        return List.of(legacyEmbedding);
+    }
+
+    public Map<String, Object> asMap() {
+        Map<String, Object> map = new LinkedHashMap<>();
+        map.put(TEXT_EMBEDDING_BITS, embeddings);
+
+        return map;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        InferenceTextEmbeddingBitResults that = (InferenceTextEmbeddingBitResults) o;
+        return Objects.equals(embeddings, that.embeddings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(embeddings);
+    }
+}