Merge remote-tracking branch 'origin/main' into engine-shard-field-stats

Author: arteam

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.compute.data.DoubleVector;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
@@ -127,7 +128,9 @@ static void selfTest() {
             "mv_min_ascending",
             "rlike",
             "to_lower",
-            "to_upper" }
+            "to_lower_ords",
+            "to_upper",
+            "to_upper_ords" }
     )
     public String operation;
 
@@ -235,12 +238,12 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
-            case "to_lower" -> {
+            case "to_lower", "to_lower_ords" -> {
                 FieldAttribute keywordField = keywordField();
                 ToLower toLower = new ToLower(Source.EMPTY, keywordField, configuration());
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, toLower, layout(keywordField)).get(driverContext);
             }
-            case "to_upper" -> {
+            case "to_upper", "to_upper_ords" -> {
                 FieldAttribute keywordField = keywordField();
                 ToUpper toUpper = new ToUpper(Source.EMPTY, keywordField, configuration());
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, toUpper, layout(keywordField)).get(driverContext);
@@ -414,13 +417,15 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
-            case "to_lower" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
-            case "to_upper" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
+            case "to_lower" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
+            case "to_lower_ords" -> checkBytes(operation, actual, true, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
+            case "to_upper" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
+            case "to_upper_ords" -> checkBytes(operation, actual, true, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
             default -> throw new UnsupportedOperationException(operation);
         }
     }
 
-    private static void checkBytes(String operation, Page actual, BytesRef[] expectedVals) {
+    private static void checkBytes(String operation, Page actual, boolean expectOrds, BytesRef[] expectedVals) {
         BytesRef scratch = new BytesRef();
         BytesRefVector v = actual.<BytesRefBlock>getBlock(1).asVector();
         for (int i = 0; i < BLOCK_LENGTH; i++) {
@@ -430,6 +435,15 @@ private static void checkBytes(String operation, Page actual, BytesRef[] expecte
                 throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + b + "]");
             }
         }
+        if (expectOrds) {
+            if (v.asOrdinals() == null) {
+                throw new IllegalArgumentException("expected ords but got " + v);
+            }
+        } else {
+            if (v.asOrdinals() != null) {
+                throw new IllegalArgumentException("expected non-ords but got " + v);
+            }
+        }
     }
 
     private static Page page(String operation) {
@@ -510,6 +524,16 @@ private static Page page(String operation) {
                 }
                 yield new Page(builder.build().asBlock());
             }
+            case "to_lower_ords", "to_upper_ords" -> {
+                var bytes = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH);
+                bytes.appendBytesRef(new BytesRef("foo"));
+                bytes.appendBytesRef(new BytesRef("bar"));
+                var ordinals = blockFactory.newIntVectorFixedBuilder(BLOCK_LENGTH);
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    ordinals.appendInt(i % 2);
+                }
+                yield new Page(new OrdinalBytesRefVector(ordinals.build(), bytes.build()).asBlock());
+            }
             default -> throw new UnsupportedOperationException();
         };
     }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy

@@ -20,6 +20,22 @@ abstract class AbstractGradleInternalPluginFuncTest extends AbstractJavaGradleFu
         plugins {
             id 'elasticsearch.java-toolchain'
         }
+
+        toolchainManagement {
+          jvm {
+            javaRepositories {
+              repository('bundledOracleOpendJdk') {
+                resolverClass = org.elasticsearch.gradle.internal.toolchain.OracleOpenJdkToolchainResolver
+              }
+              repository('adoptiumJdks') {
+                resolverClass = org.elasticsearch.gradle.internal.toolchain.AdoptiumJdkToolchainResolver
+              }
+              repository('archivedOracleJdks') {
+                resolverClass = org.elasticsearch.gradle.internal.toolchain.ArchivedOracleJdkToolchainResolver
+              }
+            }
+          }
+        }
         """ + settingsFile.text
 
         buildFile << """

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ConcatFilesTask.java

@@ -8,6 +8,8 @@
  */
 package org.elasticsearch.gradle.internal;
 
+import com.google.common.collect.Iterables;
+
 import org.gradle.api.DefaultTask;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.tasks.Input;
@@ -85,7 +87,7 @@ public void setAdditionalLines(List<String> additionalLines) {
     public void concatFiles() throws IOException {
         if (getHeaderLine() != null) {
             getTarget().getParentFile().mkdirs();
-            Files.write(getTarget().toPath(), (getHeaderLine() + '\n').getBytes(StandardCharsets.UTF_8));
+            Files.writeString(getTarget().toPath(), getHeaderLine() + '\n');
         }
 
         // To remove duplicate lines
@@ -95,11 +97,12 @@ public void concatFiles() throws IOException {
                 uniqueLines.addAll(Files.readAllLines(f.toPath(), StandardCharsets.UTF_8));
             }
         }
-        Files.write(getTarget().toPath(), uniqueLines, StandardCharsets.UTF_8, StandardOpenOption.APPEND);
-
-        for (String additionalLine : additionalLines) {
-            Files.write(getTarget().toPath(), (additionalLine + '\n').getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND);
-        }
+        Files.write(
+            getTarget().toPath(),
+            Iterables.concat(uniqueLines, additionalLines),
+            StandardCharsets.UTF_8,
+            StandardOpenOption.APPEND
+        );
     }
 
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java

@@ -155,7 +155,7 @@ public void generateDependenciesInfo() throws IOException {
             output.append(dep.getGroup() + ":" + dep.getName() + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
         }
 
-        Files.write(outputFile.toPath(), output.toString().getBytes("UTF-8"), StandardOpenOption.CREATE);
+        Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
     }
 
     @Input

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java

@@ -24,6 +24,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -440,7 +441,7 @@ private PrintWriter setupCurrent(Snippet test) {
             // Now setup the writer
             try {
                 Files.createDirectories(dest.getParent());
-                current = new PrintWriter(dest.toFile(), "UTF-8");
+                current = new PrintWriter(dest.toFile(), StandardCharsets.UTF_8);
                 return current;
             } catch (IOException e) {
                 throw new RuntimeException(e);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java

@@ -106,7 +106,7 @@ public void checkInvalidPermissions() throws IOException {
         }
 
         outputMarker.getParentFile().mkdirs();
-        Files.write(outputMarker.toPath(), "done".getBytes("UTF-8"));
+        Files.writeString(outputMarker.toPath(), "done");
     }
 
     @OutputFile

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java

@@ -135,7 +135,7 @@ public void checkInvalidPatterns() throws IOException {
 
         File outputMarker = getOutputMarker();
         outputMarker.getParentFile().mkdirs();
-        Files.write(outputMarker.toPath(), "done".getBytes(StandardCharsets.UTF_8));
+        Files.writeString(outputMarker.toPath(), "done");
     }
 
     @OutputFile

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java

@@ -364,7 +364,7 @@ private String runForbiddenAPIsCli() throws IOException {
         }
         final String forbiddenApisOutput;
         try (ByteArrayOutputStream outputStream = errorOut) {
-            forbiddenApisOutput = outputStream.toString(StandardCharsets.UTF_8.name());
+            forbiddenApisOutput = outputStream.toString(StandardCharsets.UTF_8);
         }
         if (EXPECTED_EXIT_CODES.contains(result.getExitValue()) == false) {
             throw new IllegalStateException("Forbidden APIs cli failed: " + forbiddenApisOutput);
@@ -397,7 +397,7 @@ private Set<String> runJdkJarHellCheck() throws IOException {
         }
         final String jdkJarHellCheckList;
         try (ByteArrayOutputStream outputStream = standardOut) {
-            jdkJarHellCheckList = outputStream.toString(StandardCharsets.UTF_8.name());
+            jdkJarHellCheckList = outputStream.toString(StandardCharsets.UTF_8);
         }
         return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n")));
     }

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java

@@ -60,8 +60,8 @@ public void testConcatenationWithUnique() throws IOException {
         file2.getParentFile().mkdirs();
         file1.createNewFile();
         file2.createNewFile();
-        Files.write(file1.toPath(), ("Hello" + System.lineSeparator() + "Hello").getBytes(StandardCharsets.UTF_8));
-        Files.write(file2.toPath(), ("Hello" + System.lineSeparator() + "नमस्ते").getBytes(StandardCharsets.UTF_8));
+        Files.writeString(file1.toPath(), "Hello" + System.lineSeparator() + "Hello");
+        Files.writeString(file2.toPath(), "Hello" + System.lineSeparator() + "नमस्ते");
 
         concatFilesTask.setFiles(project.fileTree(file1.getParentFile().getParentFile()));
 

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java

@@ -244,7 +244,7 @@ private void createFileIn(File parent, String name, String content) throws IOExc
         Path file = parent.toPath().resolve(name);
         file.toFile().createNewFile();
 
-        Files.write(file, content.getBytes(StandardCharsets.UTF_8));
+        Files.writeString(file, content);
     }
 
     private TaskProvider<DependencyLicensesTask> createDependencyLicensesTask(Project project) {