Merge remote-tracking branch 'upstream/main' into prototype/pattern-text

Author: eyalkoren

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.6", "8.17.4", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -287,8 +287,8 @@ steps:
         env:
           BWC_VERSION: 8.15.5
 
-      - label: "{{matrix.image}} / 8.16.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.6
+      - label: "{{matrix.image}} / 8.16.7 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.7
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -301,10 +301,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.6
+          BWC_VERSION: 8.16.7
 
-      - label: "{{matrix.image}} / 8.17.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.4
+      - label: "{{matrix.image}} / 8.17.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.5
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -317,7 +317,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.4
+          BWC_VERSION: 8.17.5
 
       - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0

.buildkite/pipelines/periodic.yml

@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.6#bwcTest
+      - label: 8.16.7 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.7#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.6
+          BWC_VERSION: 8.16.7
         retry:
           automatic:
             - exit_status: "-1"
@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.4#bwcTest
+      - label: 8.17.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.5#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.4
+          BWC_VERSION: 8.17.5
         retry:
           automatic:
             - exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.6", "8.17.4", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -533,7 +533,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.6", "8.17.4", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/scripts/dra-update-staging.sh

@@ -36,9 +36,6 @@ for BRANCH in "${BRANCHES[@]}"; do
   fi
 
   if [[ "$SHOULD_TRIGGER" == "true" ]]; then
-    if [[ "$BRANCH" == "9.0" ]]; then
-      export VERSION_QUALIFIER="rc1"
-    fi
     echo "Triggering DRA staging workflow for $BRANCH"
     cat << EOF | buildkite-agent pipeline upload
 steps:
@@ -49,7 +46,6 @@ steps:
       branch: "$BRANCH"
       env:
         DRA_WORKFLOW: staging
-        VERSION_QUALIFIER: ${VERSION_QUALIFIER:-}
 EOF
   fi
 done

.buildkite/scripts/dra-workflow.trigger.sh

@@ -8,7 +8,7 @@ source .buildkite/scripts/branches.sh
 
 for BRANCH in "${BRANCHES[@]}"; do
   if [[ "$BRANCH" == "9.0" ]]; then
-    export VERSION_QUALIFIER="rc1"
+    continue
   fi
 
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"

.ci/bwcVersions

@@ -15,8 +15,8 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.5"
-  - "8.16.6"
-  - "8.17.4"
+  - "8.16.7"
+  - "8.17.5"
   - "8.18.0"
   - "8.19.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,6 +1,6 @@
 BWC_VERSION:
-  - "8.16.6"
-  - "8.17.4"
+  - "8.16.7"
+  - "8.17.5"
   - "8.18.0"
   - "8.19.0"
   - "9.0.0"

CONTRIBUTING.md

@@ -467,7 +467,7 @@ expensive messages that will usually be discarded:
 
 Logging is an important behaviour of the system and sometimes deserves its own
 unit tests, especially if there is complex logic for computing what is logged
-and when to log it. You can use a `org.elasticsearch.test.MockLogAppender` to
+and when to log it. You can use a `org.elasticsearch.test.MockLog` to
 make assertions about the logs that are being emitted.
 
 Logging is a powerful diagnostic technique, but it is not the only possibility.

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.compute.data.DoubleVector;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
@@ -127,7 +128,9 @@ static void selfTest() {
             "mv_min_ascending",
             "rlike",
             "to_lower",
-            "to_upper" }
+            "to_lower_ords",
+            "to_upper",
+            "to_upper_ords" }
     )
     public String operation;
 
@@ -235,12 +238,12 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
-            case "to_lower" -> {
+            case "to_lower", "to_lower_ords" -> {
                 FieldAttribute keywordField = keywordField();
                 ToLower toLower = new ToLower(Source.EMPTY, keywordField, configuration());
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, toLower, layout(keywordField)).get(driverContext);
             }
-            case "to_upper" -> {
+            case "to_upper", "to_upper_ords" -> {
                 FieldAttribute keywordField = keywordField();
                 ToUpper toUpper = new ToUpper(Source.EMPTY, keywordField, configuration());
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, toUpper, layout(keywordField)).get(driverContext);
@@ -414,13 +417,15 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
-            case "to_lower" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
-            case "to_upper" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
+            case "to_lower" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
+            case "to_lower_ords" -> checkBytes(operation, actual, true, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
+            case "to_upper" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
+            case "to_upper_ords" -> checkBytes(operation, actual, true, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
             default -> throw new UnsupportedOperationException(operation);
         }
     }
 
-    private static void checkBytes(String operation, Page actual, BytesRef[] expectedVals) {
+    private static void checkBytes(String operation, Page actual, boolean expectOrds, BytesRef[] expectedVals) {
         BytesRef scratch = new BytesRef();
         BytesRefVector v = actual.<BytesRefBlock>getBlock(1).asVector();
         for (int i = 0; i < BLOCK_LENGTH; i++) {
@@ -430,6 +435,15 @@ private static void checkBytes(String operation, Page actual, BytesRef[] expecte
                 throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + b + "]");
             }
         }
+        if (expectOrds) {
+            if (v.asOrdinals() == null) {
+                throw new IllegalArgumentException("expected ords but got " + v);
+            }
+        } else {
+            if (v.asOrdinals() != null) {
+                throw new IllegalArgumentException("expected non-ords but got " + v);
+            }
+        }
     }
 
     private static Page page(String operation) {
@@ -510,6 +524,16 @@ private static Page page(String operation) {
                 }
                 yield new Page(builder.build().asBlock());
             }
+            case "to_lower_ords", "to_upper_ords" -> {
+                var bytes = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH);
+                bytes.appendBytesRef(new BytesRef("foo"));
+                bytes.appendBytesRef(new BytesRef("bar"));
+                var ordinals = blockFactory.newIntVectorFixedBuilder(BLOCK_LENGTH);
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    ordinals.appendInt(i % 2);
+                }
+                yield new Page(new OrdinalBytesRefVector(ordinals.build(), bytes.build()).asBlock());
+            }
             default -> throw new UnsupportedOperationException();
         };
     }

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -178,7 +179,12 @@ public void setUp() throws Exception {
             new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet())
         );
         clusterService.getClusterApplierService().setInitialState(initialClusterState);
-        indicatorService = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, new SystemIndices(List.of()));
+        indicatorService = new ShardsAvailabilityHealthIndicatorService(
+            clusterService,
+            allocationService,
+            new SystemIndices(List.of()),
+            DefaultProjectResolver.INSTANCE
+        );
     }
 
     private int toInt(String v) {