Merge branch 'main' into lucene_snapshot

Author: javanna

docs/changelog/112933.yaml

@@ -0,0 +1,5 @@
+pr: 112933
+summary: "Allow incubating Panama Vector in simdvec, and add vectorized `ipByteBin`"
+area: Search
+type: enhancement
+issues: []

docs/changelog/113251.yaml

@@ -0,0 +1,5 @@
+pr: 113251
+summary: Span term query to convert to match no docs when unmapped field is targeted
+area: Search
+type: bug
+issues: []

docs/changelog/114177.yaml

@@ -0,0 +1,5 @@
+pr: 114177
+summary: "Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]`"
+area: Infra/Metrics
+type: bug
+issues: []

docs/reference/connector/apis/create-connector-api.asciidoc

@@ -116,7 +116,7 @@ PUT _connector/my-connector
   "name": "My Connector",
   "description": "My Connector to sync data to Elastic index from Google Drive",
   "service_type": "google_drive",
-  "language": "english"
+  "language": "en"
 }
 ----
 

docs/reference/connector/docs/connectors-zoom.asciidoc

@@ -63,18 +63,22 @@ To connect to Zoom you need to https://developers.zoom.us/docs/internal-apps/s2s
 6. Click on the "Create" button to create the app registration.
 7. After the registration is complete, you will be redirected to the app's overview page. Take note of the "App Credentials" value, as you'll need it later.
 8. Navigate to the "Scopes" section and click on the "Add Scopes" button.
-9. The following scopes need to be added to the app.
+9. The following granular scopes need to be added to the app.
 +
 [source,bash]
 ----
-user:read:admin
-meeting:read:admin
-chat_channel:read:admin
-recording:read:admin
-chat_message:read:admin
-report:read:admin
+user:read:list_users:admin
+meeting:read:list_meetings:admin
+meeting:read:list_past_participants:admin
+cloud_recording:read:list_user_recordings:admin
+team_chat:read:list_user_channels:admin
+team_chat:read:list_user_messages:admin
 ----
-
+[NOTE]
+====
+The connector requires a minimum scope of `user:read:list_users:admin` to ingest data into Elasticsearch.
+====
++
 10. Click on the "Done" button to add the selected scopes to your app.
 11. Navigate to the "Activation" section and input the necessary information to activate the app.
 
@@ -220,18 +224,22 @@ To connect to Zoom you need to https://developers.zoom.us/docs/internal-apps/s2s
 6. Click on the "Create" button to create the app registration.
 7. After the registration is complete, you will be redirected to the app's overview page. Take note of the "App Credentials" value, as you'll need it later.
 8. Navigate to the "Scopes" section and click on the "Add Scopes" button.
-9. The following scopes need to be added to the app.
+9. The following granular scopes need to be added to the app.
 +
 [source,bash]
 ----
-user:read:admin
-meeting:read:admin
-chat_channel:read:admin
-recording:read:admin
-chat_message:read:admin
-report:read:admin
+user:read:list_users:admin
+meeting:read:list_meetings:admin
+meeting:read:list_past_participants:admin
+cloud_recording:read:list_user_recordings:admin
+team_chat:read:list_user_channels:admin
+team_chat:read:list_user_messages:admin
 ----
-
+[NOTE]
+====
+The connector requires a minimum scope of `user:read:list_users:admin` to ingest data into Elasticsearch.
+====
++
 10. Click on the "Done" button to add the selected scopes to your app.
 11. Navigate to the "Activation" section and input the necessary information to activate the app.
 

docs/reference/mapping/runtime.asciidoc

@@ -821,8 +821,6 @@ address.
 [[lookup-runtime-fields]]
 ==== Retrieve fields from related indices
 
-experimental[]
-
 The <<search-fields,`fields`>> parameter on the `_search` API can also be used to retrieve fields from
 the related indices via runtime fields with a type of `lookup`.
 

libs/simdvec/build.gradle

@@ -23,6 +23,20 @@ dependencies {
   }
 }
 
+// compileMain21Java does not exist within idea (see MrJarPlugin) so we cannot reference directly by name
+tasks.matching { it.name == "compileMain21Java" }.configureEach {
+  options.compilerArgs << '--add-modules=jdk.incubator.vector'
+  // we remove Werror, since incubating suppression (-Xlint:-incubating)
+  // is only support since JDK 22
+  options.compilerArgs -= '-Werror'
+}
+
+tasks.named('test').configure {
+  if (JavaVersion.current().majorVersion.toInteger() >= 21) {
+    jvmArgs '--add-modules=jdk.incubator.vector'
+  }
+}
+
 tasks.withType(CheckForbiddenApisTask).configureEach {
   replaceSignatureFiles 'jdk-signatures'
 }

libs/simdvec/src/main/java/module-info.java

@@ -10,6 +10,7 @@
 module org.elasticsearch.simdvec {
     requires org.elasticsearch.nativeaccess;
     requires org.apache.lucene.core;
+    requires org.elasticsearch.logging;
 
     exports org.elasticsearch.simdvec to org.elasticsearch.server;
 }

libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.simdvec;
+
+import org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport;
+import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
+
+import static org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport.B_QUERY;
+
+public class ESVectorUtil {
+
+    private static final ESVectorUtilSupport IMPL = ESVectorizationProvider.getInstance().getVectorUtilSupport();
+
+    public static long ipByteBinByte(byte[] q, byte[] d) {
+        if (q.length != d.length * B_QUERY) {
+            throw new IllegalArgumentException("vector dimensions incompatible: " + q.length + "!= " + B_QUERY + " x " + d.length);
+        }
+        return IMPL.ipByteBinByte(q, d);
+    }
+}

libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.simdvec.internal.vectorization;
+
+import org.apache.lucene.util.BitUtil;
+
+final class DefaultESVectorUtilSupport implements ESVectorUtilSupport {
+
+    DefaultESVectorUtilSupport() {}
+
+    @Override
+    public long ipByteBinByte(byte[] q, byte[] d) {
+        return ipByteBinByteImpl(q, d);
+    }
+
+    public static long ipByteBinByteImpl(byte[] q, byte[] d) {
+        long ret = 0;
+        int size = d.length;
+        for (int i = 0; i < B_QUERY; i++) {
+            int r = 0;
+            long subRet = 0;
+            for (final int upperBound = d.length & -Integer.BYTES; r < upperBound; r += Integer.BYTES) {
+                subRet += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(q, i * size + r) & (int) BitUtil.VH_NATIVE_INT.get(d, r));
+            }
+            for (; r < d.length; r++) {
+                subRet += Integer.bitCount((q[i * size + r] & d[r]) & 0xFF);
+            }
+            ret += subRet << i;
+        }
+        return ret;
+    }
+}