Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

.buildkite/scripts/dra-workflow.trigger.sh

@@ -2,10 +2,34 @@
 
 set -euo pipefail
 
-echo "steps:"
-
 source .buildkite/scripts/branches.sh
 
+# We use that filtering to keep different schedule for different branches
+if [ -n "${INCLUDED_BRANCHES:-}" ]; then
+  # If set, only trigger the pipeline for the specified branches
+  IFS=',' read -r -a BRANCHES <<< "${INCLUDED_BRANCHES}"
+elif [ -n "${EXCLUDED_BRANCHES:-}" ]; then
+  # If set, listed branches will be excluded from the list of branches in branches.json
+  IFS=',' read -r -a EXCLUDED_BRANCHES <<< "${EXCLUDED_BRANCHES}"
+  FILTERED_BRANCHES=()
+    for BRANCH in "${BRANCHES[@]}"; do
+      EXCLUDE=false
+      for EXCLUDED_BRANCH in "${EXCLUDED_BRANCHES[@]}"; do
+        if [ "$BRANCH" == "$EXCLUDED_BRANCH" ]; then
+          EXCLUDE=true
+          break
+        fi
+      done
+      if [ "$EXCLUDE" = false ]; then
+        FILTERED_BRANCHES+=("$BRANCH")
+      fi
+    done
+    BRANCHES=("${FILTERED_BRANCHES[@]}")
+fi
+
+
+echo "steps:"
+
 for BRANCH in "${BRANCHES[@]}"; do
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
   BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')

build-tools-internal/src/main/resources/checkstyle.xml

@@ -68,6 +68,11 @@
 
     <!-- Unused imports are forbidden -->
     <module name="UnusedImports" />
+    <module name="SuppressionCommentFilter">
+      <property name="offCommentFormat" value="begin generated imports"/>
+      <property name="onCommentFormat" value="end generated imports"/>
+      <property name="checkFormat" value="UnusedImports"/>
+    </module>
 
     <!-- Non-inner classes must be in files that match their names. -->
     <module name="OuterTypeFilename" />

distribution/docker/src/docker/Dockerfile.ess-fips

@@ -96,15 +96,6 @@ COPY fips/resources/fips_java.policy /usr/share/elasticsearch/config/fips_java.p
 
 WORKDIR /usr/share/elasticsearch/config
 
-## Add fips specific JVM options
-RUN cat <<EOF > /usr/share/elasticsearch/config/jvm.options.d/fips.options
--Djavax.net.ssl.keyStoreType=BCFKS
--Dorg.bouncycastle.fips.approved_only=true
--Djava.security.properties=config/fips_java.security
--Djava.security.policy=config/fips_java.policy
-EOF
-
-
 ################################################################################
 # Build stage 2 (the actual Elasticsearch image):
 #
@@ -136,6 +127,10 @@ ENV ELASTIC_CONTAINER=true
 WORKDIR /usr/share/elasticsearch
 
 COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch
+COPY --from=builder --chown=0:0 /fips/libs/*.jar /usr/share/elasticsearch/lib/
+COPY --from=builder --chown=0:0 /opt /opt
+
+ENV ES_PLUGIN_ARCHIVE_DIR=/opt/plugins/archive
 ENV PATH=/usr/share/elasticsearch/bin:\$PATH
 ENV SHELL=/bin/bash
 COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
@@ -159,6 +154,28 @@ RUN chmod g=u /etc/passwd && \\
 
 RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
 
+# Convert cacerts (PKCS12) to BCFKS format using POSIX-compatible shell syntax
+RUN printf "\\n" | jdk/bin/keytool -importkeystore \
+  -srckeystore /usr/share/elasticsearch/jdk/lib/security/cacerts \
+  -srcstoretype PKCS12 \
+  -destkeystore config/cacerts.bcfks \
+  -deststorepass passwordcacert \
+  -deststoretype BCFKS \
+  -providerclass org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \
+  -providerpath lib/bc-fips-1.0.2.5.jar \
+  -destprovidername BCFIPS
+
+
+## Add fips specific JVM options
+RUN cat <<EOF > /usr/share/elasticsearch/config/jvm.options.d/fips.options
+-Djavax.net.ssl.keyStoreType=BCFKS
+-Dorg.bouncycastle.fips.approved_only=true
+-Djava.security.properties=config/fips_java.security
+-Djava.security.policy=config/fips_java.policy
+-Djavax.net.ssl.trustStore=config/cacerts.bcfks
+-Djavax.net.ssl.trustStorePassword=passwordcacert
+EOF
+
 EXPOSE 9200 9300
 
 LABEL org.label-schema.build-date="${build_date}" \\
@@ -196,11 +213,6 @@ CMD ["/app/elasticsearch.sh"]
 
 USER 1000:0
 
-COPY --from=builder --chown=0:0 /opt /opt
-ENV ES_PLUGIN_ARCHIVE_DIR=/opt/plugins/archive
-WORKDIR /usr/share/elasticsearch
-COPY --from=builder --chown=0:0 /fips/libs/*.jar /usr/share/elasticsearch/lib/
-
 ################################################################################
 # End of multi-stage Dockerfile
 ################################################################################

docs/changelog/120488.yaml

@@ -0,0 +1,5 @@
+pr: 120488
+summary: Publish queue latency metrics from tracked thread pools
+area: "Infra/Metrics"
+type: enhancement
+issues: []

docs/changelog/127383.yaml

@@ -0,0 +1,5 @@
+pr: 127383
+summary: Don't push down filters on the right hand side of an inlinejoin
+area: ES|QL
+type: bug
+issues: []

docs/changelog/127658.yaml

@@ -0,0 +1,5 @@
+pr: 127658
+summary: Append all data to Chat Completion buffer
+area: Machine Learning
+type: bug
+issues: []

docs/release-notes/breaking-changes.md

@@ -12,10 +12,12 @@ If you are migrating from a version prior to version 9.0, you must first upgrade
 
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
-## 9.1.0 [elasticsearch-910-breaking-changes]
+## 9.0.1 [elasticsearch-9.0.1-breaking-changes]
+```{applies_to}
+stack: coming 9.0.1
+```
 
-ES|QL
-:   * Allow partial results by default in ES|QL [#125060](https://github.com/elastic/elasticsearch/pull/125060)
+No breaking changes in this version.
 
 ## 9.0.0 [elasticsearch-900-breaking-changes]
 

docs/release-notes/deprecations.md

@@ -16,6 +16,13 @@ To give you insight into what deprecated features you’re using, {{es}}:
 
 % ## Next version [elasticsearch-nextversion-deprecations]
 
+## 9.0.1 [elasticsearch-9.0.1-deprecations]
+```{applies_to}
+stack: coming 9.0.1
+```
+
+No deprecations in this version.
+
 ## 9.0.0 [elasticsearch-900-deprecations]
 
 ES|QL:

docs/release-notes/index.md

@@ -20,19 +20,90 @@ To check for security updates, go to [Security announcements for the Elastic sta
 % ### Fixes [elasticsearch-next-fixes]
 % *
 
+## 9.0.1 [elasticsearch-9.0.1-release-notes]
+```{applies_to}
+stack: coming 9.0.1
+```
+
+### Features and enhancements [elasticsearch-9.0.1-features-enhancements]
+
+Infra/Core:
+* Validation checks on paths allowed for 'files' entitlements. Restrict the paths we allow access to, forbidding plugins to specify/request entitlements for reading or writing to specific protected directories. [#126852](https://github.com/elastic/elasticsearch/pull/126852)
+
+Ingest Node:
+* Updating tika to 2.9.3 [#127353](https://github.com/elastic/elasticsearch/pull/127353)
+
+Search:
+* Enable sort optimization on float and `half_float` [#126342](https://github.com/elastic/elasticsearch/pull/126342)
+
+Security:
+* Add Issuer to failed SAML Signature validation logs when available [#126310](https://github.com/elastic/elasticsearch/pull/126310) (issue: [#111022](https://github.com/elastic/elasticsearch/issues/111022))
+
+### Fixes [elasticsearch-9.0.1-fixes]
+
+Aggregations:
+* Rare terms aggregation false **positive** fix [#126884](https://github.com/elastic/elasticsearch/pull/126884)
+
+Allocation:
+* Fix shard size of initializing restored shard [#126783](https://github.com/elastic/elasticsearch/pull/126783) (issue: [#105331](https://github.com/elastic/elasticsearch/issues/105331))
+
+CCS:
+* Cancel expired async search task when a remote returns its results [#126583](https://github.com/elastic/elasticsearch/pull/126583)
+
+Data streams:
+* [otel-data] Bump plugin version to release _metric_names_hash changes [#126850](https://github.com/elastic/elasticsearch/pull/126850)
+
+ES|QL:
+* Fix count optimization with pushable union types [#127225](https://github.com/elastic/elasticsearch/pull/127225) (issue: [#127200](https://github.com/elastic/elasticsearch/issues/127200))
+* Fix join masking eval [#126614](https://github.com/elastic/elasticsearch/pull/126614)
+* Fix sneaky bug in single value query [#127146](https://github.com/elastic/elasticsearch/pull/127146)
+* No, line noise isn't a valid ip [#127527](https://github.com/elastic/elasticsearch/pull/127527)
+
+ILM+SLM:
+* Fix equality bug in `WaitForIndexColorStep` [#126605](https://github.com/elastic/elasticsearch/pull/126605)
+
+Infra/CLI:
+* Use terminal reader in keystore add command [#126729](https://github.com/elastic/elasticsearch/pull/126729) (issue: [#98115](https://github.com/elastic/elasticsearch/issues/98115))
+
+Infra/Core:
+* Fix: consider case sensitiveness differences in Windows/Unix-like filesystems for files entitlements [#126990](https://github.com/elastic/elasticsearch/pull/126990) (issue: [#127047](https://github.com/elastic/elasticsearch/issues/127047))
+* Rework uniquify to not use iterators [#126889](https://github.com/elastic/elasticsearch/pull/126889) (issue: [#126883](https://github.com/elastic/elasticsearch/issues/126883))
+* Workaround max name limit imposed by Jackson 2.17 [#126806](https://github.com/elastic/elasticsearch/pull/126806)
+
+Machine Learning:
+* Adding missing `onFailure` call for Inference API start model request [#126930](https://github.com/elastic/elasticsearch/pull/126930)
+* Fix text structure NPE when fields in list have null value [#125922](https://github.com/elastic/elasticsearch/pull/125922)
+* Leverage threadpool schedule for inference api to avoid long running thread [#126858](https://github.com/elastic/elasticsearch/pull/126858) (issue: [#126853](https://github.com/elastic/elasticsearch/issues/126853))
+
+Ranking:
+* Fix LTR rescorer with model alias [#126273](https://github.com/elastic/elasticsearch/pull/126273)
+* LTR score bounding [#125694](https://github.com/elastic/elasticsearch/pull/125694)
+
+Search:
+* Fix npe when using source confirmed text query against missing field [#127414](https://github.com/elastic/elasticsearch/pull/127414)
+
+TSDB:
+* Improve resiliency of `UpdateTimeSeriesRangeService` [#126637](https://github.com/elastic/elasticsearch/pull/126637)
+
+Task Management:
+* Fix race condition in `RestCancellableNodeClient` [#126686](https://github.com/elastic/elasticsearch/pull/126686) (issue: [#88201](https://github.com/elastic/elasticsearch/issues/88201))
+
+Vector Search:
+* Fix `vec_caps` to test for OS support too (on x64) [#126911](https://github.com/elastic/elasticsearch/pull/126911) (issue: [#126809](https://github.com/elastic/elasticsearch/issues/126809))
+* Fix bbq quantization algorithm but for differently distributed components [#126778](https://github.com/elastic/elasticsearch/pull/126778)
+
+
 ## 9.0.0 [elasticsearch-900-release-notes]
 
 ### Highlights [elasticsearch-900-highlights]
 
 ::::{dropdown} rank_vectors field type is now available for late-interaction ranking
-
 [`rank_vectors`](../reference/elasticsearch/mapping-reference/rank-vectors.md) is a new field type released as an experimental feature in Elasticsearch 9.0. It is designed to be used with dense vectors and allows for late-interaction second order ranking.
 
 Late-interaction models are powerful rerankers. While their size and overall cost doesn’t lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali.
 ::::
 
 ::::{dropdown} ES|QL LOOKUP JOIN is now available in technical preview
-
 [LOOKUP JOIN](../reference/query-languages/esql/esql-commands.md) is now available in technical preview. LOOKUP JOIN combines data from your ES|QL queries with matching records from a lookup index, enabling you to:
 
 - Enrich your search results with reference data