Merge branch 'main' into multi-project/search-it

Author: tvernum

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java renamed to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/MethodReplacement.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
+package org.elasticsearch.gradle.internal.dependencies.patches;
 
 import org.objectweb.asm.MethodVisitor;
 import org.objectweb.asm.Opcodes;
@@ -16,7 +16,7 @@ public class MethodReplacement extends MethodVisitor {
     private final MethodVisitor delegate;
     private final Runnable bodyWriter;
 
-    MethodReplacement(MethodVisitor delegate, Runnable bodyWriter) {
+    public MethodReplacement(MethodVisitor delegate, Runnable bodyWriter) {
         super(Opcodes.ASM9);
         this.delegate = delegate;
         this.bodyWriter = bodyWriter;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/azurecore/AzureCoreClassPatcher.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.dependencies.patches.azurecore;
+
+import org.elasticsearch.gradle.internal.dependencies.patches.PatcherInfo;
+import org.elasticsearch.gradle.internal.dependencies.patches.Utils;
+import org.gradle.api.artifacts.transform.CacheableTransform;
+import org.gradle.api.artifacts.transform.InputArtifact;
+import org.gradle.api.artifacts.transform.TransformAction;
+import org.gradle.api.artifacts.transform.TransformOutputs;
+import org.gradle.api.artifacts.transform.TransformParameters;
+import org.gradle.api.file.FileSystemLocation;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.Classpath;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.gradle.internal.dependencies.patches.PatcherInfo.classPatcher;
+
+@CacheableTransform
+public abstract class AzureCoreClassPatcher implements TransformAction<TransformParameters.None> {
+
+    private static final String JAR_FILE_TO_PATCH = "azure-core-[\\d.]*\\.jar";
+
+    private static final List<PatcherInfo> CLASS_PATCHERS = List.of(
+        classPatcher(
+            "com/azure/core/implementation/ImplUtils.class",
+            "7beda5bdff5ea460cfc08721a188cf07d16e0c987dae45401fca7abf4e6e6c0e",
+            ImplUtilsPatcher::new
+        )
+    );
+
+    @Classpath
+    @InputArtifact
+    public abstract Provider<FileSystemLocation> getInputArtifact();
+
+    @Override
+    public void transform(@NotNull TransformOutputs outputs) {
+        File inputFile = getInputArtifact().get().getAsFile();
+
+        if (Pattern.matches(JAR_FILE_TO_PATCH, inputFile.getName())) {
+            System.out.println("Patching " + inputFile.getName());
+            File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar"));
+            Utils.patchJar(inputFile, outputFile, CLASS_PATCHERS, true);
+        } else {
+            System.out.println("Skipping " + inputFile.getName());
+            outputs.file(getInputArtifact());
+        }
+    }
+
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/azurecore/ImplUtilsPatcher.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.dependencies.patches.azurecore;
+
+import org.elasticsearch.gradle.internal.dependencies.patches.MethodReplacement;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+
+class ImplUtilsPatcher extends ClassVisitor {
+    ImplUtilsPatcher(ClassVisitor classVisitor) {
+        super(Opcodes.ASM9, classVisitor);
+    }
+
+    public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+        MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
+        // `addShutdownHook` invokes `java.lang.Runtime.addShutdownHook`, which is forbidden (i.e. it will throw an Entitlements error).
+        // We replace the method body here with `return null`.
+        if (name.equals("addShutdownHookSafely")) {
+            return new MethodReplacement(mv, () -> {
+                mv.visitInsn(Opcodes.ACONST_NULL);
+                mv.visitInsn(Opcodes.ARETURN);
+            });
+        }
+        return mv;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
 
+import org.elasticsearch.gradle.internal.dependencies.patches.MethodReplacement;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.MethodVisitor;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.gradle.internal.dependencies.patches.hdfs;
 
+import org.elasticsearch.gradle.internal.dependencies.patches.MethodReplacement;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.MethodVisitor;

distribution/docker/build.gradle

@@ -117,6 +117,7 @@ dependencies {
   log4jConfig project(path: ":distribution", configuration: 'log4jConfig')
   tini "krallin:tini:0.19.0:${tiniArch}"
   allPlugins project(path: ':plugins', configuration: 'allPlugins')
+  allPlugins project(path: ':x-pack:extras:plugins', configuration: 'allPlugins')
   filebeat_aarch64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
   filebeat_x86_64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
   filebeat_fips_aarch64 "beats:filebeat-fips:${VersionProperties.elasticsearch}:[email protected]"

docs/changelog/128293.yaml

@@ -0,0 +1,5 @@
+pr: 128396
+summary: Delegated authorization using Microsoft Graph (SDK)
+area: Authorization
+type: feature
+issues: []

docs/changelog/128396.yaml

@@ -0,0 +1,6 @@
+pr: 129150
+summary: Add `none` chunking strategy to disable automatic chunking for inference
+  endpoints
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/129150.yaml

@@ -117,15 +117,16 @@ If specified, these will override the chunking settings set in the {{infer-cap}}
 endpoint associated with `inference_id`.
 If chunking settings are updated, they will not be applied to existing documents
 until they are reindexed.
+To completely disable chunking, use the `none` chunking strategy.
 
     **Valid values for `chunking_settings`**:
 
     `type`
-    :   Indicates the type of chunking strategy to use. Valid values are `word` or
+    :   Indicates the type of chunking strategy to use. Valid values are `none`, `word` or
     `sentence`. Required.
 
     `max_chunk_size`
-    :   The maximum number of works in a chunk. Required.
+    :   The maximum number of words in a chunk. Required for `word` and `sentence` strategies.
 
     `overlap`
     :   The number of overlapping words allowed in chunks. This cannot be defined as
@@ -136,6 +137,12 @@ until they are reindexed.
     :   The number of overlapping sentences allowed in chunks. Valid values are `0`
     or `1`. Required for `sentence` type chunking settings
 
+::::{warning}
+If the input exceeds the maximum token limit of the underlying model,  some services (such as OpenAI) may return an 
+error. In contrast, the `elastic` and `elasticsearch` services  will automatically truncate the input to fit within the 
+model's limit.
+::::
+
 ## {{infer-cap}} endpoint validation [infer-endpoint-validation]
 
 The `inference_id` will not be validated when the mapping is created, but when
@@ -166,10 +173,49 @@ For more details on chunking and how to configure chunking settings,
 see [Configuring chunking](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-inference)
 in the Inference API documentation.
 
+You can pre-chunk the input by sending it to Elasticsearch as an array of strings.
+Example:
+
+```console
+PUT test-index
+{
+  "mappings": {
+    "properties": {
+      "my_semantic_field": {
+        "type": "semantic_text",
+        "chunking_settings": {
+          "strategy": "none"    <1>
+        }
+      }
+    }
+  }
+}
+```
+
+1. Disable chunking on `my_semantic_field`.
+
+```console
+PUT test-index/_doc/1
+{
+    "my_semantic_field": ["my first chunk", "my second chunk", ...]    <1>
+    ...
+}
+```
+
+1. The text is pre-chunked and provided as an array of strings.
+   Each element in the array represents a single chunk that will be sent directly to the inference service without further chunking.
+
+**Important considerations**:
+
+* When providing pre-chunked input, ensure that you set the chunking strategy to `none` to avoid additional processing.
+* Each chunk should be sized carefully, staying within the token limit of the inference service and the underlying model.
+* If a chunk exceeds the model's token limit, the behavior depends on the service:
+  * Some services (such as OpenAI) will return an error.
+  * Others (such as `elastic` and `elasticsearch`) will automatically truncate the input.
+
 Refer
 to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md)
-to learn more about semantic search using `semantic_text` and the `semantic`
-query.
+to learn more about semantic search using `semantic_text`.
 
 ## Extracting Relevant Fragments from Semantic Text [semantic-text-highlighting]