Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

.github/CODEOWNERS

@@ -34,6 +34,11 @@ x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/obs-ds-intake-service
 x-pack/plugin/otel-data/src/main/resources @elastic/obs-ds-intake-services
 x-pack/plugin/otel-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services
 
+# Storage Engine
+x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
+x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
+x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
+
 # Delivery
 gradle @elastic/es-delivery
 build-conventions @elastic/es-delivery

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
             CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
             CompressedExponentialHistogram result = new CompressedExponentialHistogram();
             BytesRef data = histoBytes.bytes().toBytesRef();
-            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), histogram.max(), data);
             return result;
         } catch (IOException e) {
             throw new RuntimeException(e);

docs/changelog/133952.yaml

@@ -0,0 +1,6 @@
+pr: 133952
+summary: Fix warning when creating an OTel data stream
+area: TSDB
+type: bug
+issues:
+ - 132918

docs/reference/elasticsearch/elasticsearch-audit-events.md

@@ -31,7 +31,7 @@ Certain audit events require the `security_config_change` event type to log the
 $$$event-access-denied$$$
 
 `access_denied`
-:   Logged when an authenticated user attempts to execute an action they do not have the necessary [privilege](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/elasticsearch-privileges.md) to perform.
+:   Logged when an authenticated user attempts to execute an action they do not have the necessary [privilege](/reference/elasticsearch/security-privileges.md) to perform.
 
     ::::{dropdown} Example
     ```js
@@ -510,7 +510,7 @@ $$$event-realm-auth-failed$$$
 $$$event-runas-denied$$$
 
 `run_as_denied`
-:   Logged when an authenticated user attempts to [run as](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/submitting-requests-on-behalf-of-other-users.md) another user that they do not have the necessary [privileges](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/elasticsearch-privileges.md) to do so.
+:   Logged when an authenticated user attempts to [run as](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/submitting-requests-on-behalf-of-other-users.md) another user that they do not have the necessary [privileges](/reference/elasticsearch/security-privileges.md) to do so.
 
     ::::{dropdown} Example
     ```js

docs/reference/elasticsearch/rest-apis/create-index-from-source.md

@@ -26,7 +26,7 @@ For the most up-to-date API details, refer to [Index APIs](https://www.elastic.c
 
 ## {{api-prereq-title}} [indices-create-index-from-source-api-prereqs]
 
-* If the {{es}} {{security-features}} are enabled, you must have the `manage` [index privilege](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/elasticsearch-privileges.md#privileges-list-indices) for the index.
+* If the {{es}} {{security-features}} are enabled, you must have the `manage` [index privilege](/reference/elasticsearch/security-privileges.md#privileges-list-indices) for the index.
 
 
 ## {{api-description-title}} [indices-create-index-from-source-api-desc]

docs/reference/elasticsearch/rest-apis/reindex-data-stream.md

@@ -33,7 +33,7 @@ This api runs in the background because reindexing all indices in a large data s
 
 ## {{api-prereq-title}} [data-stream-reindex-api-prereqs]
 
-* If the {{es}} {{security-features}} are enabled, you must have the `manage` [index privilege](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/elasticsearch-privileges.md#privileges-list-indices) for the data stream.
+* If the {{es}} {{security-features}} are enabled, you must have the `manage` [index privilege](/reference/elasticsearch/security-privileges.md#privileges-list-indices) for the data stream.
 
 
 ## {{api-request-body-title}} [data-stream-reindex-body]

docs/reference/search-connectors/release-notes.md

@@ -13,6 +13,9 @@ If you are an Enterprise Search user and want to upgrade to Elastic 9.0, refer t
 It includes detailed steps, tooling, and resources to help you transition to supported alternatives in 9.x, such as Elasticsearch, the Open Web Crawler, and self-managed connectors.
 :::
 
+## 9.1.3 [connectors-9.1.3-release-notes]
+There are no new features, enhancements, fixes, known issues, or deprecations associated with this release.
+
 ## 9.1.2 [connectors-9.1.2-release-notes]
 There are no new features, enhancements, fixes, known issues, or deprecations associated with this release.
 
@@ -29,6 +32,9 @@ Permissions granted to the `Everyone Except External Users` group were previousl
 ## 9.1.0 [connectors-9.1.0-release-notes]
 There are no new features, enhancements, fixes, known issues, or deprecations associated with this release.
 
+## 9.0.6 [connectors-9.0.6-release-notes]
+No changes since 9.0.5
+
 ## 9.0.5 [connectors-9.0.5-release-notes]
 
 ### Fixes [connectors-9.0.5-fixes]

libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java

@@ -24,12 +24,22 @@
 
 public class Utils {
 
+    // TODO Currently ServerProcessBuilder is using --add-modules=ALL-MODULE-PATH, should this rather
+    // reflect below excludes (except for java.desktop which requires a special handling)?
+    // internal and incubator modules are also excluded
     private static final Set<String> EXCLUDED_MODULES = Set.of(
         "java.desktop",
         "jdk.jartool",
         "jdk.jdi",
         "java.security.jgss",
-        "jdk.jshell"
+        "jdk.jshell",
+        "jdk.jcmd",
+        "jdk.hotspot.agent",
+        "jdk.jfr",
+        "jdk.javadoc",
+        // "jdk.jpackage", // Do we want to include this?
+        // "jdk.jlink", // Do we want to include this?
+        "jdk.localedata" // noise, change here are not interesting
     );
 
     private static Map<String, Set<String>> findModuleExports(FileSystem fs) throws IOException {
@@ -70,7 +80,9 @@ public static void walkJdkModules(JdkModuleConsumer c) throws IOException {
 
             for (var kv : modules.entrySet()) {
                 var moduleName = kv.getKey();
-                if (Utils.EXCLUDED_MODULES.contains(moduleName) == false) {
+                if (Utils.EXCLUDED_MODULES.contains(moduleName) == false
+                    && moduleName.contains(".internal.") == false
+                    && moduleName.contains(".incubator.") == false) {
                     var thisModuleExports = moduleExports.get(moduleName);
                     c.accept(moduleName, kv.getValue(), thisModuleExports);
                 }

libs/entitlement/tools/jdk-api-extractor/README.md

@@ -0,0 +1,20 @@
+This tool scans the JDK on which it is running to extract its public accessible API.
+That is:
+- public methods (including constructors) of public, exported classes as well as protected methods of these if not final.
+- internal implementations (overwrites) of above.
+
+The output of this tool is meant to be diffed against the output for another JDK
+version to identify changes that need to be reviewed for entitlements.
+
+Usage example:
+```bash
+./gradlew :libs:entitlement:tools:jdk-api-extractor:run -Druntime.java=24 --args="api-jdk24.tsv"
+./gradlew :libs:entitlement:tools:jdk-api-extractor:run -Druntime.java=25 --args="api-jdk25.tsv"
+diff libs/entitlement/tools/jdk-api-extractor/api-jdk24.tsv libs/entitlement/tools/jdk-api-extractor/api-jdk25.tsv
+```
+
+To review the diff of deprecations (by means of `@Deprecated`), use `--deprecations-only` as 2nd argument.
+
+```bash
+./gradlew :libs:entitlement:tools:jdk-api-extractor:run -Druntime.java=24 --args="deprecations-jdk24.tsv --deprecations-only"
+```

libs/entitlement/tools/jdk-api-extractor/build.gradle

@@ -0,0 +1,66 @@
+import org.elasticsearch.gradle.OS
+
+plugins {
+  id 'application'
+}
+
+apply plugin: 'elasticsearch.build'
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /asm-.*/, to: 'asm'
+}
+
+group = 'org.elasticsearch.entitlement.tools'
+
+ext {
+  javaMainClass = "org.elasticsearch.entitlement.tools.jdkapi.JdkApiExtractor"
+}
+
+application {
+  mainClass.set(javaMainClass)
+  applicationDefaultJvmArgs = [
+    '--add-exports', 'java.base/sun.security.util=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.net=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.net.spi=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+    '--add-opens', 'java.base/javax.crypto=ALL-UNNAMED',
+    '--add-opens', 'java.base/javax.security.auth=ALL-UNNAMED',
+    '--add-opens', 'java.base/jdk.internal.logger=ALL-UNNAMED',
+    '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+    '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED',
+    '--add-opens', 'java.logging/java.util.logging=ALL-UNNAMED',
+    '--add-opens', 'java.logging/sun.util.logging.internal=ALL-UNNAMED',
+    '--add-opens', 'java.naming/javax.naming.ldap.spi=ALL-UNNAMED',
+    '--add-opens', 'java.rmi/sun.rmi.runtime=ALL-UNNAMED',
+    '--add-opens', 'jdk.dynalink/jdk.dynalink=ALL-UNNAMED',
+    '--add-opens', 'jdk.dynalink/jdk.dynalink.linker=ALL-UNNAMED',
+    '--add-opens', 'java.desktop/sun.awt=ALL-UNNAMED',
+    '--add-opens', 'java.sql.rowset/javax.sql.rowset.spi=ALL-UNNAMED',
+    '--add-opens', 'java.sql/java.sql=ALL-UNNAMED',
+    '--add-opens', 'java.xml.crypto/com.sun.org.apache.xml.internal.security.utils=ALL-UNNAMED'
+  ]
+}
+
+tasks.named("run").configure {
+  executable = "${buildParams.runtimeJavaHome.get()}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '')
+}
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  compileOnly(project(':libs:core'))
+  implementation 'org.ow2.asm:asm:9.8'
+  implementation 'org.ow2.asm:asm-util:9.8'
+  implementation(project(':libs:entitlement:tools:common'))
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+
+tasks.named("thirdPartyAudit").configure {
+  ignoreMissingClasses()
+}