Merge branch 'main' into unknown-query-parameter-error-message

Author: fang-xing-esql

branches.json

@@ -7,6 +7,12 @@
     {
       "branch": "8.16"
     },
+    {
+      "branch": "9.0"
+    },
+    {
+      "branch": "8.18"
+    },
     {
       "branch": "8.17"
     },

docs/changelog/121193.yaml

@@ -0,0 +1,18 @@
+pr: 121193
+summary: Enable LOOKUP JOIN in non-snapshot builds
+area: ES|QL
+type: enhancement
+issues:
+ - 121185
+highlight:
+  title: Enable LOOKUP JOIN in non-snapshot builds
+  body: |-
+    This effectively releases LOOKUP JOIN into tech preview. Docs will
+    follow in a separate PR.
+
+    - Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds.
+    - Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages.
+    - Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries.
+
+    Also fix https://github.com/elastic/elasticsearch/issues/121185
+  notable: true

docs/reference/search/search-your-data/semantic-text-hybrid-search

@@ -113,6 +113,7 @@ POST _tasks/<task_id>/_cancel
 ==== Perform hybrid search
 
 After reindexing the data into the `semantic-embeddings` index, you can perform hybrid search by using <<rrf,reciprocal rank fusion (RRF)>>. RRF is a technique that merges the rankings from both semantic and lexical queries, giving more weight to results that rank high in either search. This ensures that the final results are balanced and relevant.
+To extract the most relevant fragments from the original text and query, you can use the <<highlighting,highlight parameter>>:
 
 [source,console]
 ------------------------------------------------------------
@@ -142,6 +143,13 @@ GET semantic-embeddings/_search
         }
       ]
     }
+  },
+  "highlight": {
+    "fields": {
+        "semantic_text": {
+            "number_of_fragments": 2  <5>
+        }
+    }
   }
 }
 ------------------------------------------------------------
@@ -150,7 +158,7 @@ GET semantic-embeddings/_search
 <2> Lexical search is performed on the `content` field using the specified phrase.
 <3> The second `standard` retriever refers to the semantic search.
 <4> The `semantic_text` field is used to perform the semantic search.
-
+<5> Specifies the maximum number of fragments to return. See <<semantic-text-highlighting, semantic text highlighting>> for a more complete example.
 
 After performing the hybrid search, the query will return the top 10 documents that match both semantic and lexical search criteria. The results include detailed information about each document:
 
@@ -178,36 +186,14 @@ After performing the hybrid search, the query will return the top 10 documents t
         "_score": 0.032786883,
         "_rank": 1,
         "_source": {
-          "semantic_text": {
-            "inference": {
-              "inference_id": "my-elser-endpoint",
-              "model_settings": {
-                "task_type": "sparse_embedding"
-              },
-              "chunks": [
-                {
-                  "text": "What so many out there do not realize is the importance of what you do after you work out. You may have done the majority of the work, but how you treat your body in the minutes and hours after you exercise has a direct effect on muscle soreness, muscle strength and growth, and staying hydrated. Cool Down. After your last exercise, your workout is not over. The first thing you need to do is cool down. Even if running was all that you did, you still should do light cardio for a few minutes. This brings your heart rate down at a slow and steady pace, which helps you avoid feeling sick after a workout.",
-                  "embeddings": {
-                    "exercise": 1.571044,
-                    "after": 1.3603843,
-                    "sick": 1.3281639,
-                    "cool": 1.3227621,
-                    "muscle": 1.2645415,
-                    "sore": 1.2561599,
-                    "cooling": 1.2335974,
-                    "running": 1.1750668,
-                    "hours": 1.1104802,
-                    "out": 1.0991782,
-                    "##io": 1.0794281,
-                    "last": 1.0474665,
-                   (...) 
-                  }
-                }
-              ]
-            }
-          },
           "id": 8408852,
           "content": "What so many out there do not realize is the importance of (...)"
+        },
+        "highlight" : {
+            "semantic_text" : [
+                "... fragment_1 ...",
+                "... fragment_2 ..."
+            ]
         }
       }
     ]

libs/entitlement/src/main/java/module-info.java

@@ -19,6 +19,7 @@
 
     exports org.elasticsearch.entitlement.runtime.api;
     exports org.elasticsearch.entitlement.runtime.policy;
+    exports org.elasticsearch.entitlement.runtime.policy.entitlements to org.elasticsearch.server;
     exports org.elasticsearch.entitlement.instrumentation;
     exports org.elasticsearch.entitlement.bootstrap to org.elasticsearch.server;
     exports org.elasticsearch.entitlement.initialization to java.base;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -18,15 +18,15 @@
 import org.elasticsearch.entitlement.instrumentation.MethodKey;
 import org.elasticsearch.entitlement.instrumentation.Transformer;
 import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker;
-import org.elasticsearch.entitlement.runtime.policy.CreateClassLoaderEntitlement;
-import org.elasticsearch.entitlement.runtime.policy.Entitlement;
-import org.elasticsearch.entitlement.runtime.policy.ExitVMEntitlement;
-import org.elasticsearch.entitlement.runtime.policy.InboundNetworkEntitlement;
-import org.elasticsearch.entitlement.runtime.policy.LoadNativeLibrariesEntitlement;
-import org.elasticsearch.entitlement.runtime.policy.OutboundNetworkEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
 import org.elasticsearch.entitlement.runtime.policy.Scope;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement;
 
 import java.lang.instrument.Instrumentation;
 import java.lang.reflect.Constructor;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
+import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement;
+
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.entitlement.runtime.policy;
 
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.FileEntitlement;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -18,13 +19,13 @@
 import java.util.List;
 import java.util.Objects;
 
-final class FileAccessTree {
-    static final FileAccessTree EMPTY = new FileAccessTree(List.of());
+public final class FileAccessTree {
+    public static final FileAccessTree EMPTY = new FileAccessTree(List.of());
 
     private final String[] readPaths;
     private final String[] writePaths;
 
-    FileAccessTree(List<FileEntitlement> fileEntitlements) {
+    private FileAccessTree(List<FileEntitlement> fileEntitlements) {
         List<String> readPaths = new ArrayList<>();
         List<String> writePaths = new ArrayList<>();
         for (FileEntitlement fileEntitlement : fileEntitlements) {
@@ -42,6 +43,10 @@ final class FileAccessTree {
         this.writePaths = writePaths.toArray(new String[0]);
     }
 
+    public static FileAccessTree of(List<FileEntitlement> fileEntitlements) {
+        return new FileAccessTree(fileEntitlements);
+    }
+
     boolean canRead(Path path) {
         return checkPath(normalize(path), readPaths);
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -13,6 +13,15 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.instrumentation.InstrumentationService;
 import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.FileEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 
@@ -55,7 +64,7 @@ public static ModuleEntitlements from(List<Entitlement> entitlements) {
                 .toList();
             return new ModuleEntitlements(
                 entitlements.stream().collect(groupingBy(Entitlement::getClass)),
-                new FileAccessTree(fileEntitlements)
+                FileAccessTree.of(fileEntitlements)
             );
         }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java

@@ -9,6 +9,14 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
+import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.FileEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement;
 import org.elasticsearch.xcontent.XContentLocation;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyValidationException.java

@@ -15,13 +15,13 @@
  * parser is able to wrap this exception with a line/character number for
  * additional useful error information.
  */
-class PolicyValidationException extends RuntimeException {
+public class PolicyValidationException extends RuntimeException {
 
-    PolicyValidationException(String message) {
+    public PolicyValidationException(String message) {
         super(message);
     }
 
-    PolicyValidationException(String message, Throwable cause) {
+    public PolicyValidationException(String message, Throwable cause) {
         super(message, cause);
     }
 }