Merge branch 'main' into fix/134809-1

Author: prwhelan

docs/changelog/134952.yaml

@@ -0,0 +1,11 @@
+pr: 134952
+summary: Add support for expressions with LOOKUP JOIN in tech preview
+area: ES|QL
+type: enhancement
+issues: [ ]
+highlight:
+  title: Add support for expressions with LOOKUP JOIN in tech preview
+  body: |-
+    Enable Lookup Join on Expression Tech Preview
+    FROM index1 | LOOKUP JOIN lookup_index on left_field1 > right_field1 AND left_field2 <= right_field2
+  notable: true

docs/changelog/134960.yaml

@@ -0,0 +1,5 @@
+pr: 134960
+summary: Adding custom headers support openai text embeddings
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/135235.yaml

@@ -0,0 +1,5 @@
+pr: 135235
+summary: Fix systemd notify to use a shared arena
+area: Infra/Node Lifecycle
+type: bug
+issues: []

docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md

@@ -19,17 +19,22 @@ FROM <source_index>
 FROM <source_index>
 | LOOKUP JOIN <lookup_index> ON <field_name1>, <field_name2>, <field_name3>
 ```
+```esql
+FROM <source_index>
+| LOOKUP JOIN <lookup_index> ON <left_field1> >= <lookup_field1> AND <left_field2> == <lookup_field2>
+```
 
 **Parameters**
 
 `<lookup_index>`
 :   The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported. Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting).
 
-`<field_name>` or `<field_name1>, <field_name2>, <field_name3>`
-:   The field(s) to join on. Can be either:
-  * A single field name
-  * A comma-separated list of field names {applies_to}`stack: ga 9.2`
-:   These fields must exist in both your current query results and in the lookup index. If the fields contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows).
+`<field_name>` or `<field_name1>, <field_name2>, <field_name3>` or `<left_field1> >= <lookup_field1> AND <left_field2> == <lookup_field2>`
+:   The join condition. Can be one of the following:
+   * A single field name
+   * A comma-separated list of field names {applies_to}`stack: ga 9.2`
+   * An expression with one or more join conditions linked by `AND`. Each condition compares a field from the left index with a field from the lookup index using [binary operators](/reference/query-languages/esql/functions-operators/operators.md#esql-binary-operators) (`==`, `>=`, `<=`, `>`, `<`, `!=`). Each field name in the join condition must exist in only one of the indexes. Use RENAME to resolve naming conflicts. {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
+:   If using join on a single field or a field list, the fields used must exist in both your current query results and in the lookup index. If the fields contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows).
 
 
 **Description**

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -34,13 +34,15 @@ The `LOOKUP JOIN` command adds fields from the lookup index as new columns to yo
 
 The command requires two parameters:
 * The name of the lookup index (which must have the `lookup` [`index.mode setting`](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting))
-* The field(s) to join on. Can be either:
-  * A single field name
-  * A comma-separated list of field names {applies_to}`stack: ga 9.2`
+* The join condition. Can be one of the following:
+   * A single field name
+   * A comma-separated list of field names {applies_to}`stack: ga 9.2`
+   * An expression with one or more join conditions linked by `AND`. Each condition compares a field from the left index with a field from the lookup index using [binary operators](/reference/query-languages/esql/functions-operators/operators.md#esql-binary-operators) (`==`, `>=`, `<=`, `>`, `<`, `!=`). Each field name in the join condition must exist in only one of the indexes. Use RENAME to resolve naming conflicts. {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
 
 ```esql
 LOOKUP JOIN <lookup_index> ON <field_name>  # Join on a single field
 LOOKUP JOIN <lookup_index> ON <field_name1>, <field_name2>, <field_name3>  # Join on multiple fields
+LOOKUP JOIN <lookup_index> ON <left_field1> >= <lookup_field1> AND <left_field2> == <lookup_field2>  # Join on expression
 ```
 
 :::{image} ../images/esql-lookup-join.png

libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java

@@ -46,9 +46,15 @@ public Zstd getZstd() {
     }
 
     @Override
-    public CloseableByteBuffer newBuffer(int len) {
+    public CloseableByteBuffer newSharedBuffer(int len) {
         assert len > 0;
-        return javaLib.newBuffer(len);
+        return javaLib.newSharedBuffer(len);
+    }
+
+    @Override
+    public CloseableByteBuffer newConfinedBuffer(int len) {
+        assert len > 0;
+        return javaLib.newConfinedBuffer(len);
     }
 
     @Override

libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java

@@ -98,7 +98,7 @@ record Arch(
             this.systemd = null; // not running under systemd
         } else {
             logger.debug("Systemd socket path: {}", socketPath);
-            var buffer = newBuffer(64);
+            var buffer = newSharedBuffer(64);
             this.systemd = new Systemd(libraryProvider.getLibrary(PosixCLibrary.class), socketPath, buffer);
         }
     }

libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java

@@ -88,12 +88,20 @@ default WindowsFunctions getWindowsFunctions() {
     Optional<VectorSimilarityFunctions> getVectorSimilarityFunctions();
 
     /**
-     * Creates a new {@link CloseableByteBuffer}. The buffer must be used within the same thread
-     * that it is created.
+     * Creates a new {@link CloseableByteBuffer} using a shared arena. The buffer can be used
+     * across multiple threads.
      * @param len the number of bytes the buffer should allocate
      * @return the buffer
      */
-    CloseableByteBuffer newBuffer(int len);
+    CloseableByteBuffer newSharedBuffer(int len);
+
+    /**
+     * Creates a new {@link CloseableByteBuffer} using a confined arena. The buffer must be
+     * used within the same thread that it is created.
+     * @param len the number of bytes the buffer should allocate
+     * @return the buffer
+     */
+    CloseableByteBuffer newConfinedBuffer(int len);
 
     /**
      * Possible stats for execution filtering.

libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java

@@ -78,7 +78,13 @@ public Zstd getZstd() {
     }
 
     @Override
-    public CloseableByteBuffer newBuffer(int len) {
+    public CloseableByteBuffer newSharedBuffer(int len) {
+        logger.warn("cannot allocate buffer because native access is not available");
+        return null;
+    }
+
+    @Override
+    public CloseableByteBuffer newConfinedBuffer(int len) {
         logger.warn("cannot allocate buffer because native access is not available");
         return null;
     }

libs/native/src/main/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java

@@ -20,8 +20,16 @@ class JdkCloseableByteBuffer implements CloseableByteBuffer {
     final MemorySegment segment;
     private final ByteBuffer bufferView;
 
-    JdkCloseableByteBuffer(int len) {
-        this.arena = Arena.ofConfined();
+    static JdkCloseableByteBuffer ofShared(int len) {
+        return new JdkCloseableByteBuffer(len, true);
+    }
+
+    static JdkCloseableByteBuffer ofConfined(int len) {
+        return new JdkCloseableByteBuffer(len, false);
+    }
+
+    private JdkCloseableByteBuffer(int len, boolean shared) {
+        this.arena = shared ? Arena.ofShared() : Arena.ofConfined();
         this.segment = arena.allocate(len);
         this.bufferView = segment.asByteBuffer();
     }