Merge remote-tracking branch 'upstream/main' into enable-entitlements

Author: prdoyle

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -11,11 +11,14 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.DoubleVector;
 import org.elasticsearch.compute.data.LongBlock;
@@ -40,9 +43,13 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.planner.Layout;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -56,8 +63,10 @@
 import org.openjdk.jmh.annotations.Warmup;
 
 import java.time.Duration;
+import java.time.ZoneOffset;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -106,7 +115,9 @@ public class EvalBenchmark {
             "long_equal_to_int",
             "mv_min",
             "mv_min_ascending",
-            "rlike" }
+            "rlike",
+            "to_lower",
+            "to_upper" }
     )
     public String operation;
 
@@ -169,7 +180,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     new Coalesce(Source.EMPTY, lhs, List.of(f2)),
                     layout(f1, f2)
                 ).get(driverContext);
-                String desc = operation.endsWith("lazy") ? "CoalesceLazyEvaluator" : "CoalesceEagerEvaluator";
+                String desc = operation.endsWith("lazy") ? "CoalesceLongLazyEvaluator" : "CoalesceLongEagerEvaluator";
                 if (evaluator.toString().contains(desc) == false) {
                     throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
                 }
@@ -214,6 +225,16 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
+            case "to_lower" -> {
+                FieldAttribute keywordField = keywordField();
+                ToLower toLower = new ToLower(Source.EMPTY, keywordField, configuration());
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, toLower, layout(keywordField)).get(driverContext);
+            }
+            case "to_upper" -> {
+                FieldAttribute keywordField = keywordField();
+                ToUpper toUpper = new ToUpper(Source.EMPTY, keywordField, configuration());
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, toUpper, layout(keywordField)).get(driverContext);
+            }
             default -> throw new UnsupportedOperationException();
         };
     }
@@ -234,6 +255,23 @@ private static FieldAttribute keywordField() {
         return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataType.KEYWORD, Map.of(), true));
     }
 
+    private static Configuration configuration() {
+        return new Configuration(
+            ZoneOffset.UTC,
+            Locale.ROOT,
+            null,
+            null,
+            null,
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.get(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY),
+            null,
+            false,
+            Map.of(),
+            0,
+            false
+        );
+    }
+
     private static Layout layout(FieldAttribute... fields) {
         Layout.Builder layout = new Layout.Builder();
         layout.append(Arrays.asList(fields));
@@ -366,10 +404,24 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
+            case "to_lower" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
+            case "to_upper" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
             default -> throw new UnsupportedOperationException(operation);
         }
     }
 
+    private static void checkBytes(String operation, Page actual, BytesRef[] expectedVals) {
+        BytesRef scratch = new BytesRef();
+        BytesRefVector v = actual.<BytesRefBlock>getBlock(1).asVector();
+        for (int i = 0; i < BLOCK_LENGTH; i++) {
+            BytesRef expected = expectedVals[i % 2];
+            BytesRef b = v.getBytesRef(i, scratch);
+            if (b.equals(expected) == false) {
+                throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + b + "]");
+            }
+        }
+    }
+
     private static Page page(String operation) {
         return switch (operation) {
             case "abs", "add", "date_trunc", "equal_to_const" -> {
@@ -440,7 +492,7 @@ private static Page page(String operation) {
                 }
                 yield new Page(builder.build());
             }
-            case "rlike" -> {
+            case "rlike", "to_lower", "to_upper" -> {
                 var builder = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH);
                 BytesRef[] values = new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") };
                 for (int i = 0; i < BLOCK_LENGTH; i++) {

docs/changelog/121920.yaml

@@ -0,0 +1,6 @@
+pr: 121920
+summary: Account for the `SearchHit` source in circuit breaker
+area: Search
+type: enhancement
+issues:
+ - 89656

docs/changelog/122821.yaml

@@ -0,0 +1,6 @@
+pr: 122821
+summary: Fix functions emitting warnings with no source
+area: ES|QL
+type: bug
+issues:
+ - 122588

docs/changelog/122860.yaml

@@ -0,0 +1,5 @@
+pr: 122860
+summary: Improved error message when index field type is invalid
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/123105.yaml

@@ -0,0 +1,6 @@
+pr: 123105
+summary: fix stale data in synthetic source for string stored field
+area: Mapping
+type: bug
+issues:
+  - 123110

docs/changelog/123197.yaml

@@ -0,0 +1,5 @@
+pr: 123197
+summary: Fix early termination in `LuceneSourceOperator`
+area: ES|QL
+type: bug
+issues: []

docs/changelog/123246.yaml

@@ -0,0 +1,5 @@
+pr: 123246
+summary: Deduplicate allocation stats calls
+area: Allocation
+type: bug
+issues: []

docs/changelog/123272.yaml

@@ -0,0 +1,5 @@
+pr: 123272
+summary: Set Connect Timeout to 5s
+area: Machine Learning
+type: bug
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -21,6 +21,7 @@
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 import java.util.function.BiConsumer;
@@ -82,8 +83,8 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         Path jdk = Paths.get(System.getProperty("java.home"));
         addPathAndMaybeLink.accept(jdk.resolve("conf"), Mode.READ);
 
-        readPaths.sort(String::compareTo);
-        writePaths.sort(String::compareTo);
+        readPaths.sort(PATH_ORDER);
+        writePaths.sort(PATH_ORDER);
 
         this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]);
         this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]);
@@ -96,7 +97,7 @@ private static List<String> pruneSortedPaths(List<String> paths) {
             prunedReadPaths.add(currentPath);
             for (int i = 1; i < paths.size(); ++i) {
                 String nextPath = paths.get(i);
-                if (nextPath.startsWith(currentPath) == false) {
+                if (isParent(currentPath, nextPath) == false) {
                     prunedReadPaths.add(nextPath);
                     currentPath = nextPath;
                 }
@@ -124,25 +125,28 @@ static String normalizePath(Path path) {
         // Note that toAbsolutePath produces paths separated by the default file separator,
         // so on Windows, if the given path uses forward slashes, this consistently
         // converts it to backslashes.
-        return path.toAbsolutePath().normalize().toString();
+        String result = path.toAbsolutePath().normalize().toString();
+        while (result.endsWith(FILE_SEPARATOR)) {
+            result = result.substring(0, result.length() - FILE_SEPARATOR.length());
+        }
+        return result;
     }
 
     private static boolean checkPath(String path, String[] paths) {
         if (paths.length == 0) {
             return false;
         }
-        int ndx = Arrays.binarySearch(paths, path);
+        int ndx = Arrays.binarySearch(paths, path, PATH_ORDER);
         if (ndx < -1) {
-            // logger.warn("Couldn't find path [{}] in paths:\n{}", path, Arrays.asList(paths));
-            String maybeParent = paths[-ndx - 2];
-            // logger.warn("Possible parent path: [{}]", maybeParent);
-            // Normalization on Windows does not allways remove trailing backslashes, we need to check both patterns
-            return path.startsWith(maybeParent)
-                && (maybeParent.endsWith(FILE_SEPARATOR) || path.startsWith(FILE_SEPARATOR, maybeParent.length()));
+            return isParent(paths[-ndx - 2], path);
         }
         return ndx >= 0;
     }
 
+    private static boolean isParent(String maybeParent, String path) {
+        return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length());
+    }
+
     @Override
     public boolean equals(Object o) {
         if (o == null || getClass() != o.getClass()) return false;
@@ -154,4 +158,30 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(Arrays.hashCode(readPaths), Arrays.hashCode(writePaths));
     }
+
+    /**
+     * For our lexicographic sort trick to work correctly, we must have path separators sort before
+     * any other character so that files in a directory appear immediately after that directory.
+     * For example, we require [/a, /a/b, /a.xml] rather than the natural order [/a, /a.xml, /a/b].
+     */
+    private static final Comparator<String> PATH_ORDER = (s1, s2) -> {
+        Path p1 = Path.of(s1);
+        Path p2 = Path.of(s2);
+        var i1 = p1.iterator();
+        var i2 = p2.iterator();
+        while (i1.hasNext() && i2.hasNext()) {
+            int cmp = i1.next().compareTo(i2.next());
+            if (cmp != 0) {
+                return cmp;
+            }
+        }
+        if (i1.hasNext()) {
+            return 1;
+        } else if (i2.hasNext()) {
+            return -1;
+        } else {
+            assert p1.equals(p2);
+            return 0;
+        }
+    };
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java

@@ -21,6 +21,8 @@
 import java.util.Objects;
 import java.util.stream.Stream;
 
+import static java.lang.Character.isLetter;
+
 /**
  * Describes a file entitlement with a path and mode.
  */
@@ -90,6 +92,51 @@ static FileData ofPathSetting(String setting, Mode mode) {
         static FileData ofRelativePathSetting(String setting, BaseDir baseDir, Mode mode) {
             return new RelativePathSettingFileData(setting, baseDir, mode, null);
         }
+
+        /**
+         * Tests if a path is absolute or relative, taking into consideration both Unix and Windows conventions.
+         * Note that this leads to a conflict, resolved in favor of Unix rules: `/foo` can be either a Unix absolute path, or a Windows
+         * relative path with "wrong" directory separator (using non-canonical slash in Windows).
+         */
+        static boolean isAbsolutePath(String path) {
+            if (path.isEmpty()) {
+                return false;
+            }
+            if (path.charAt(0) == '/') {
+                // Unix/BSD absolute
+                return true;
+            }
+
+            return isWindowsAbsolutePath(path);
+        }
+
+        private static boolean isSlash(char c) {
+            return (c == '\\') || (c == '/');
+        }
+
+        private static boolean isWindowsAbsolutePath(String input) {
+            // if a prefix is present, we expected (long) UNC or (long) absolute
+            if (input.startsWith("\\\\?\\")) {
+                return true;
+            }
+
+            if (input.length() > 1) {
+                char c0 = input.charAt(0);
+                char c1 = input.charAt(1);
+                char c = 0;
+                int next = 2;
+                if (isSlash(c0) && isSlash(c1)) {
+                    // Two slashes or more: UNC
+                    return true;
+                }
+                if (isLetter(c0) && c1 == ':') {
+                    // A drive: absolute
+                    return true;
+                }
+            }
+            // Otherwise relative
+            return false;
+        }
     }
 
     private sealed interface RelativeFileData extends FileData {
@@ -284,13 +331,13 @@ public static FilesEntitlement build(List<Object> paths) {
                 }
 
                 Path relativePath = Path.of(relativePathAsString);
-                if (platform != null && platform.isCurrent() && relativePath.isAbsolute()) {
+                if (platform != null && platform.isCurrent() && FileData.isAbsolutePath(relativePathAsString)) {
                     throw new PolicyValidationException("'relative_path' [" + relativePathAsString + "] must be relative");
                 }
                 fileData = FileData.ofRelativePath(relativePath, baseDir, mode);
             } else if (pathAsString != null) {
                 Path path = Path.of(pathAsString);
-                if (platform != null && platform.isCurrent() && path.isAbsolute() == false) {
+                if (platform != null && platform.isCurrent() && FileData.isAbsolutePath(pathAsString) == false) {
                     throw new PolicyValidationException("'path' [" + pathAsString + "] must be absolute");
                 }
                 fileData = FileData.ofPath(path, mode);