Merge branch 'main' into failures-lifecycle-config

Author: gmarouli

.backportrc.json

@@ -1,10 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "9.0", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.1.0$" : "main",
-    "^v8.19.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/periodic.template.yml

@@ -237,7 +237,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -656,7 +656,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/scripts/resolve-dra-manifest.sh

@@ -24,11 +24,6 @@ if [ "$LATEST_VERSION" != "$ES_VERSION" ]; then
   echo "Latest build for '$ARTIFACT' is version $LATEST_VERSION but expected version $ES_VERSION." 1>&2
   NEW_BRANCH=$(echo $ES_VERSION | sed -E "s/([0-9]+\.[0-9]+)\.[0-9]/\1/g")
 
-  # Temporary
-  if [[ "$ES_VERSION" == "8.16.0" ]]; then
-    NEW_BRANCH="8.x"
-  fi
-
   echo "Using branch $NEW_BRANCH instead of $BRANCH." 1>&2
   LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $NEW_BRANCH)
 fi

branches.json

@@ -8,13 +8,13 @@
       "branch": "9.0"
     },
     {
-      "branch": "8.18"
+      "branch": "8.19"
     },
     {
-      "branch": "8.17"
+      "branch": "8.18"
     },
     {
-      "branch": "8.x"
+      "branch": "8.17"
     },
     {
       "branch": "7.17"

build-tools-internal/version.properties

@@ -17,7 +17,6 @@ jna               = 5.12.1
 netty             = 4.1.118.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
-awsv1sdk            = 1.12.746
 awsv2sdk            = 2.30.38
 reactive_streams    = 1.0.4
 

docs/changelog/126843.yaml

@@ -0,0 +1,90 @@
+pr: 126843
+summary: Upgrade `repository-s3` to AWS SDK v2
+area: Snapshot/Restore
+type: breaking
+issues:
+ - 120993
+highlight:
+  title: Upgrade `repository-s3` to AWS SDK v2
+  body: >-
+    In earlier versions of {es} the `repository-s3` plugin was based on the AWS
+    SDK v1. AWS will withdraw support for this SDK before the end of the life
+    of {es} {minor-version} so we have migrated this plugin to the newer AWS SDK v2.
+
+    The two SDKs are not quite compatible, so please check the breaking changes
+    documentation and test the new version thoroughly before upgrading any
+    production workloads.
+  notable: true
+breaking:
+  title: Upgrade `repository-s3` to AWS SDK v2
+  area: Cluster and node setting
+  details: >-
+    In earlier versions of {es} the `repository-s3` plugin was based on the AWS
+    SDK v1. AWS will withdraw support for this SDK before the end of the life
+    of {es} {minor-version} so we must migrate to the newer AWS SDK v2.
+
+    Unfortunately there are several differences between the two AWS SDK
+    versions which may require you to adjust your system configuration when
+    upgrading to {es} {minor-version} or later. These differences include, but
+    may not be limited to, the following items.
+
+    * AWS SDK v2 requires users to specify the region to use for signing
+      requests, or else to run in an environment in which it can determine the
+      correct region automatically. The older SDK would try to determine the
+      region based on the endpoint URL as specified with the
+      `s3.client.${CLIENT_NAME}.endpoint` setting, together with other data
+      drawn from the operating environment, and would ultimately fall back to
+      `us-east-1` if no better value could be found.
+
+    * AWS SDK v2 does not support the EC2 IMDSv1 protocol.
+
+    * AWS SDK v2 does not support the
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
+
+    * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
+      the `s3.client.${CLIENT_NAME}.protocol` setting is deprecated and no longer
+      has any effect.
+
+    * AWS SDK v2 does not permit control over throttling for retries, so the
+      the `s3.client.${CLIENT_NAME}.use_throttle_retries` setting is deprecated
+      and no longer has any effect.
+
+    * AWS SDK v2 requires the use of the V4 signature algorithm, so the
+      `s3.client.${CLIENT_NAME}.signer_override` setting is deprecated and no
+      longer has any effect.
+
+    * AWS SDK v2 does not support the `log-delivery-write` canned ACL.
+
+    * AWS SDK v2 counts 4xx responses differently in its metrics reporting.
+
+    * AWS SDK v2 always uses the regional STS endpoint, whereas AWS SDK v2
+      could use either a regional endpoint or the global
+      `https://sts.amazonaws.com` one.
+
+  impact: >-
+    If you use the `repository-s3` module, test your upgrade thoroughly before
+    upgrading any production workloads.
+
+    Adapt your configuration to the new SDK functionality. This includes, but
+    may not be limited to, the following items.
+
+    * Specify the correct signing region using the
+      `s3.client.${CLIENT_NAME}.region` setting on each node. {es} will try and
+      determine the correct region based on the endpoint URL and other data
+      drawn from the operating environment but cannot guarantee to do so
+      correctly in all cases.
+
+    * If you use IMDS to determine the availability zone of a node or to obtain
+      credentials for accessing the EC2 API, ensure that it supports the IMDSv2
+      protocol.
+
+    * If applicable, discontinue use of the
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
+
+    * If applicable, specify that you wish to use the insecure HTTP protocol to
+      access the S3 API by setting `s3.client.${CLIENT_NAME}.endpoint` to a URL
+      which starts with `http://`.
+
+    * If applicable, discontinue use of the `log-delivery-write` canned ACL.
+
+  notable: true

docs/changelog/126884.yaml

@@ -0,0 +1,5 @@
+pr: 126884
+summary: Rare terms aggregation false **positive** fix
+area: Aggregations
+type: bug
+issues: []

docs/changelog/127250.yaml

@@ -0,0 +1,6 @@
+pr: 127250
+summary: Do not apply further shard snapshot status updates after shard snapshot is
+  complete
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/127353.yaml

@@ -0,0 +1,5 @@
+pr: 127353
+summary: Updating tika to 2.9.3
+area: Ingest Node
+type: upgrade
+issues: []