Merge branch 'main' into StatementParserTests

Author: fang-xing-esql

.backportrc.json

@@ -1,10 +1,10 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.x", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.0.0$" : "main",
-    "^v8.18.0$" : "8.x",
+    "^v8.19.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/periodic-packaging.yml

@@ -335,6 +335,22 @@ steps:
         env:
           BWC_VERSION: 8.18.0
 
+      - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.19.0
+
       - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -363,6 +363,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.19.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 8.19.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
       - label: 9.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
         timeout_in_minutes: 300

.ci/bwcVersions

@@ -18,4 +18,5 @@ BWC_VERSION:
   - "8.16.4"
   - "8.17.2"
   - "8.18.0"
+  - "8.19.0"
   - "9.0.0"

branches.json

@@ -7,6 +7,12 @@
     {
       "branch": "8.16"
     },
+    {
+      "branch": "9.0"
+    },
+    {
+      "branch": "8.18"
+    },
     {
       "branch": "8.17"
     },

build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle

@@ -14,6 +14,7 @@ include ":distribution:bwc:bugfix2"
 include ":distribution:bwc:minor"
 include ":distribution:bwc:major"
 include ":distribution:bwc:staged"
+include ":distribution:bwc:staged2"
 include ":distribution:bwc:maintenance"
 include ":distribution:archives:darwin-tar"
 include ":distribution:archives:oss-darwin-tar"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java

@@ -166,6 +166,7 @@ private Map<Version, UnreleasedVersionInfo> computeUnreleased(List<String> devel
             .toList();
 
         boolean existingBugfix = false;
+        boolean existingStaged = false;
         for (int i = 0; i < featureFreezeBranches.size(); i++) {
             String branch = featureFreezeBranches.get(i);
             Version version = versions.stream()
@@ -193,7 +194,9 @@ private Map<Version, UnreleasedVersionInfo> computeUnreleased(List<String> devel
             if (i == featureFreezeBranches.size() - 1) {
                 result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:maintenance"));
             } else if (version.getRevision() == 0) { // This is the next staged minor
-                result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:staged"));
+                String project = existingStaged ? "staged2" : "staged";
+                result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));
+                existingStaged = true;
             } else { // This is a bugfix
                 String project = existingBugfix ? "bugfix2" : "bugfix";
                 result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy

@@ -77,6 +77,39 @@ class BwcVersionsSpec extends Specification {
         bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0')]
     }
 
+    def "current version is next major with two staged minors"() {
+        given:
+        addVersion('7.17.10', '8.9.0')
+        addVersion('8.15.0', '9.9.0')
+        addVersion('8.15.1', '9.9.0')
+        addVersion('8.15.2', '9.9.0')
+        addVersion('8.16.0', '9.10.0')
+        addVersion('8.16.1', '9.10.0')
+        addVersion('8.16.2', '9.10.0')
+        addVersion('8.17.0', '9.10.0')
+        addVersion('8.17.1', '9.10.0')
+        addVersion('8.18.0', '9.10.0')
+        addVersion('8.19.0', '9.10.0')
+        addVersion('9.0.0', '10.0.0')
+        addVersion('9.1.0', '10.1.0')
+
+        when:
+        def bwc = new BwcVersions(versionLines, v('9.1.0'), ['main', '9.0', '8.x', '8.18', '8.17', '8.16', '7.17'])
+        def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] }
+
+        then:
+        unreleased == [
+            (v('8.16.2')): new UnreleasedVersionInfo(v('8.16.2'), '8.16', ':distribution:bwc:bugfix2'),
+            (v('8.17.1')): new UnreleasedVersionInfo(v('8.17.1'), '8.17', ':distribution:bwc:bugfix'),
+            (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.18', ':distribution:bwc:staged2'),
+            (v('8.19.0')): new UnreleasedVersionInfo(v('8.19.0'), '8.x', ':distribution:bwc:minor'),
+            (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), '9.0', ':distribution:bwc:staged'),
+            (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), 'main', ':distribution'),
+        ]
+        bwc.wireCompatible == [v('8.19.0'), v('9.0.0'), v('9.1.0')]
+        bwc.indexCompatible == [v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.16.2'), v('8.17.0'), v('8.17.1'), v('8.18.0'), v('8.19.0'), v('9.0.0'), v('9.1.0')]
+    }
+
     def "current version is first new minor in major series"() {
         given:
         addVersion('7.17.10', '8.9.0')

distribution/bwc/staged2/build.gradle

@@ -0,0 +1,18 @@
+pr: 121193
+summary: Enable LOOKUP JOIN in non-snapshot builds
+area: ES|QL
+type: enhancement
+issues:
+ - 121185
+highlight:
+  title: Enable LOOKUP JOIN in non-snapshot builds
+  body: |-
+    This effectively releases LOOKUP JOIN into tech preview. Docs will
+    follow in a separate PR.
+
+    - Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds.
+    - Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages.
+    - Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries.
+
+    Also fix https://github.com/elastic/elasticsearch/issues/121185
+  notable: true