Merge branch '8.x' into backport/8.x/pr-119592

prdoyle · web-flow · commit d298ca235bd9 · 2025-01-08T08:46:31.000-05:00
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
@@ -70,7 +70,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
             maybeSetActiveProcessorCount(nodeSettings),
             maybeSetReplayFile(distroType, isHotspot),
             maybeWorkaroundG1Bug(),
-            maybeAllowSecurityManager(),
+            maybeAllowSecurityManager(useEntitlements),
             maybeAttachEntitlementAgent(useEntitlements)
         ).flatMap(s -> s).toList();
     }
@@ -148,9 +148,12 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         return Stream.of();
     }
 
-    private static Stream<String> maybeAllowSecurityManager() {
-        // Will become conditional on useEntitlements once entitlements can run without SM
-        return Stream.of("-Djava.security.manager=allow");
+    private static Stream<String> maybeAllowSecurityManager(boolean useEntitlements) {
+        if (useEntitlements == false) {
+            // Will become conditional on useEntitlements once entitlements can run without SM
+            return Stream.of("-Djava.security.manager=allow");
+        }
+        return Stream.of();
     }
 
     private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlements) {
diff --git a/docs/changelog/118919.yaml b/docs/changelog/118919.yaml
@@ -0,0 +1,5 @@
+pr: 118919
+summary: Remove unsupported timeout from rest-api-spec license API
+area: License
+type: bug
+issues: []
diff --git a/docs/changelog/118921.yaml b/docs/changelog/118921.yaml
@@ -0,0 +1,5 @@
+pr: 118921
+summary: Add missing timeouts to rest-api-spec shutdown APIs
+area: Infra/Node Lifecycle
+type: bug
+issues: []
diff --git a/docs/changelog/118954.yaml b/docs/changelog/118954.yaml
@@ -0,0 +1,5 @@
+pr: 118954
+summary: Add missing parameter to `xpack.info` rest-api-spec
+area: Infra/REST API
+type: bug
+issues: []
diff --git a/docs/changelog/119504.yaml b/docs/changelog/119504.yaml
@@ -0,0 +1,5 @@
+pr: 119504
+summary: Optimized index sorting for OTel logs
+area: Data streams
+type: enhancement
+issues: []
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java
@@ -27,7 +27,7 @@
 import java.util.Set;
 import java.util.function.Function;
 
-import static org.elasticsearch.common.settings.Setting.Property.DeprecatedWarning;
+import static org.elasticsearch.common.settings.Setting.Property.Deprecated;
 import static org.elasticsearch.common.settings.Setting.Property.NodeScope;
 import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic;
 
@@ -250,7 +250,7 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         TELEMETRY_SETTING_PREFIX + "agent.",
         LEGACY_TRACING_APM_SETTING_PREFIX + "agent.",
         (namespace, qualifiedKey) -> qualifiedKey.startsWith(LEGACY_TRACING_APM_SETTING_PREFIX)
-            ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, DeprecatedWarning)
+            ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, Deprecated)
             : concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic)
     );
 
@@ -262,7 +262,7 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         LEGACY_TRACING_APM_SETTING_PREFIX + "names.include",
         OperatorDynamic,
         NodeScope,
-        DeprecatedWarning
+        Deprecated
     );
 
     public static final Setting<List<String>> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.listSetting(
@@ -281,7 +281,7 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         LEGACY_TRACING_APM_SETTING_PREFIX + "names.exclude",
         OperatorDynamic,
         NodeScope,
-        DeprecatedWarning
+        Deprecated
     );
 
     public static final Setting<List<String>> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.listSetting(
@@ -314,7 +314,7 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         ),
         OperatorDynamic,
         NodeScope,
-        DeprecatedWarning
+        Deprecated
     );
 
     public static final Setting<List<String>> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting(
@@ -334,7 +334,7 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         false,
         OperatorDynamic,
         NodeScope,
-        DeprecatedWarning
+        Deprecated
     );
 
     public static final Setting<Boolean> TELEMETRY_TRACING_ENABLED_SETTING = Setting.boolSetting(
@@ -358,7 +358,7 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
     public static final Setting<SecureString> TRACING_APM_SECRET_TOKEN_SETTING = SecureSetting.secureString(
         LEGACY_TRACING_APM_SETTING_PREFIX + "secret_token",
         null,
-        DeprecatedWarning
+        Deprecated
     );
 
     public static final Setting<SecureString> TELEMETRY_SECRET_TOKEN_SETTING = SecureSetting.secureString(
@@ -373,7 +373,7 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
     public static final Setting<SecureString> TRACING_APM_API_KEY_SETTING = SecureSetting.secureString(
         LEGACY_TRACING_APM_SETTING_PREFIX + "api_key",
         null,
-        DeprecatedWarning
+        Deprecated
     );
 
     public static final Setting<SecureString> TELEMETRY_API_KEY_SETTING = SecureSetting.secureString(
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json b/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json
@@ -31,10 +31,6 @@
       "master_timeout": {
         "type": "time",
         "description": "Timeout for processing on master node"
-      },
-      "timeout": {
-        "type": "time",
-        "description": "Timeout for acknowledgement of update from all nodes in cluster"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json
@@ -26,6 +26,15 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json
@@ -26,7 +26,16 @@
         }
       ]
     },
-    "params":{},
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      }
+    },
     "body":{
       "description":"The shutdown type definition to register",
       "required": true
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json
@@ -20,6 +20,12 @@
       ]
     },
     "params":{
+      "human":{
+        "type":"boolean",
+        "required":false,
+        "description":"Defines whether additional human-readable information is included in the response. In particular, it adds descriptions and a tag line. The default value is true.",
+        "default":true
+      },
       "categories":{
         "type":"list",
         "description":"Comma-separated list of info categories. Can be any of: build, license, features"
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -33,6 +33,7 @@ class Bootstrap {
 
     // arguments from the CLI process
     private final ServerArgs args;
+    private final boolean useEntitlements;
 
     // controller for spawning component subprocesses
     private final Spawner spawner = new Spawner();
@@ -46,10 +47,11 @@ class Bootstrap {
     // loads information about plugins required for entitlements in phase 2, used by plugins service in phase 3
     private final SetOnce<PluginsLoader> pluginsLoader = new SetOnce<>();
 
-    Bootstrap(PrintStream out, PrintStream err, ServerArgs args) {
+    Bootstrap(PrintStream out, PrintStream err, ServerArgs args, boolean useEntitlements) {
         this.out = out;
         this.err = err;
         this.args = args;
+        this.useEntitlements = useEntitlements;
     }
 
     ServerArgs args() {
@@ -60,6 +62,10 @@ Spawner spawner() {
         return spawner;
     }
 
+    public boolean useEntitlements() {
+        return useEntitlements;
+    }
+
     void setSecureSettings(SecureSettings secureSettings) {
         this.secureSettings.set(secureSettings);
     }
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -211,7 +211,6 @@ static List<BootstrapCheck> checks() {
         checks.add(new OnErrorCheck());
         checks.add(new OnOutOfMemoryErrorCheck());
         checks.add(new EarlyAccessCheck());
-        checks.add(new AllPermissionCheck());
         checks.add(new DiscoveryConfiguredCheck());
         checks.add(new ByteOrderCheck());
         return Collections.unmodifiableList(checks);
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -51,6 +51,7 @@
 import java.nio.file.Path;
 import java.security.Permission;
 import java.security.Security;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
@@ -105,6 +106,7 @@ private static Bootstrap initPhase1() {
         final PrintStream out = getStdout();
         final PrintStream err = getStderr();
         final ServerArgs args;
+        final boolean useEntitlements = Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"));
         try {
             initSecurityProperties();
 
@@ -113,12 +115,14 @@ private static Bootstrap initPhase1() {
              * the presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy).
              * This forces such policies to take effect immediately.
              */
-            org.elasticsearch.bootstrap.Security.setSecurityManager(new SecurityManager() {
-                @Override
-                public void checkPermission(Permission perm) {
-                    // grant all permissions so that we can later set the security manager to the one that we want
-                }
-            });
+            if (useEntitlements == false) {
+                org.elasticsearch.bootstrap.Security.setSecurityManager(new SecurityManager() {
+                    @Override
+                    public void checkPermission(Permission perm) {
+                        // grant all permissions so that we can later set the security manager to the one that we want
+                    }
+                });
+            }
             LogConfigurator.registerErrorListener();
 
             BootstrapInfo.init();
@@ -144,7 +148,7 @@ public void checkPermission(Permission perm) {
             return null; // unreachable, to satisfy compiler
         }
 
-        return new Bootstrap(out, err, args);
+        return new Bootstrap(out, err, args, useEntitlements);
     }
 
     /**
@@ -205,7 +209,7 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
         var pluginsLoader = PluginsLoader.createPluginsLoader(nodeEnv.modulesFile(), nodeEnv.pluginsFile());
         bootstrap.setPluginsLoader(pluginsLoader);
 
-        if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) {
+        if (bootstrap.useEntitlements()) {
             LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements");
 
             List<EntitlementBootstrap.PluginData> pluginData = Stream.concat(
@@ -269,7 +273,11 @@ protected void validateNodeBeforeAcceptingRequests(
                 final BoundTransportAddress boundTransportAddress,
                 List<BootstrapCheck> checks
             ) throws NodeValidationException {
-                BootstrapChecks.check(context, boundTransportAddress, checks);
+                var additionalChecks = new ArrayList<>(checks);
+                if (bootstrap.useEntitlements() == false) {
+                    additionalChecks.add(new BootstrapChecks.AllPermissionCheck());
+                }
+                BootstrapChecks.check(context, boundTransportAddress, additionalChecks);
             }
         };
         INSTANCE = new Elasticsearch(bootstrap.spawner(), node);
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec
@@ -92,26 +92,26 @@ testMultiValuedFieldWithConjunctionWithScore
 required_capability: match_function
 required_capability: metadata_score
 
-from employees metadata _score
-| where match(job_positions, "Data Scientist") and match(job_positions, "Support Engineer")
-| keep emp_no, first_name, last_name, job_positions, _score;
+from books metadata _score
+| where match(author, "Keith Faulkner") and match(author, "Rory Tyger")
+| keep book_no, title, author, _score;
 
-emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword | _score:double
-10043          | Yishay             | Tzvieli           | [Data Scientist, Python Developer, Support Engineer] | 5.233309745788574
+book_no:keyword | title:text                                                                            | author:text                  | _score:double
+6151            | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 8.5822172164917
 ;
 
 testMatchAndQueryStringFunctionsWithScore
 required_capability: match_function
 required_capability: metadata_score
 
-from employees metadata _score
-| where match(job_positions, "Data Scientist") and qstr("job_positions: (Support Engineer) and gender: F")
-| keep emp_no, first_name, last_name, job_positions, _score;
+from books metadata _score
+| where match(author, "Keith Faulkner") and qstr("author:Rory or author:Beverlie")
+| keep book_no, title, author, _score;
 ignoreOrder:true
 
-emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword | _score:double
-10041          | Uri                 | Lenart           | [Data Scientist, Head Human Resources, Internship, Senior Team Lead] | 3.509873867034912
-10043          | Yishay              | Tzvieli          | [Data Scientist, Python Developer, Support Engineer] | 5.233309745788574
+book_no:keyword | title:text                                                                            | author:text                       | _score:double
+3535            | Rainbow's End: A Magical Story and Moneybox                                           | [Beverlie Manson, Keith Faulkner] | 6.5579609870910645
+6151            | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger]      | 5.975414276123047 
 ;
 
 multipleWhereWithMatchScoringNoSort
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
@@ -858,7 +858,7 @@ private static NestedObjectMapper.Builder createChunksField(
         );
         chunksField.dynamic(ObjectMapper.Dynamic.FALSE);
         if (modelSettings != null) {
-            chunksField.add(createEmbeddingsField(indexSettings.getIndexVersionCreated(), modelSettings));
+            chunksField.add(createEmbeddingsField(indexSettings.getIndexVersionCreated(), modelSettings, useLegacyFormat));
         }
         if (useLegacyFormat) {
             var chunkTextField = new KeywordFieldMapper.Builder(TEXT_FIELD, indexVersionCreated).indexed(false).docValues(false);
@@ -869,9 +869,13 @@ private static NestedObjectMapper.Builder createChunksField(
         return chunksField;
     }
 
-    private static Mapper.Builder createEmbeddingsField(IndexVersion indexVersionCreated, SemanticTextField.ModelSettings modelSettings) {
+    private static Mapper.Builder createEmbeddingsField(
+        IndexVersion indexVersionCreated,
+        SemanticTextField.ModelSettings modelSettings,
+        boolean useLegacyFormat
+    ) {
         return switch (modelSettings.taskType()) {
-            case SPARSE_EMBEDDING -> new SparseVectorFieldMapper.Builder(CHUNKED_EMBEDDINGS_FIELD).setStored(true);
+            case SPARSE_EMBEDDING -> new SparseVectorFieldMapper.Builder(CHUNKED_EMBEDDINGS_FIELD).setStored(useLegacyFormat == false);
             case TEXT_EMBEDDING -> {
                 DenseVectorFieldMapper.Builder denseVectorMapperBuilder = new DenseVectorFieldMapper.Builder(
                     CHUNKED_EMBEDDINGS_FIELD,
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
@@ -531,7 +531,11 @@ private static void assertSemanticTextField(MapperService mapperService, String
             assertTrue(embeddingsFieldMapper.fieldType() == mapperService.mappingLookup().getFieldType(getEmbeddingsFieldName(fieldName)));
             assertThat(embeddingsMapper.fullPath(), equalTo(getEmbeddingsFieldName(fieldName)));
             switch (semanticFieldMapper.fieldType().getModelSettings().taskType()) {
-                case SPARSE_EMBEDDING -> assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class));
+                case SPARSE_EMBEDDING -> {
+                    assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class));
+                    SparseVectorFieldMapper sparseMapper = (SparseVectorFieldMapper) embeddingsMapper;
+                    assertEquals(sparseMapper.fieldType().isStored(), semanticTextFieldType.useLegacyFormat() == false);
+                }
                 case TEXT_EMBEDDING -> assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class));
                 default -> throw new AssertionError("Invalid task type");
             }
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml
@@ -9,6 +9,7 @@ template:
       mode: logsdb
       sort:
         field: [ "resource.attributes.host.name", "@timestamp" ]
+        order: [ "asc", "desc" ]
   mappings:
     properties:
       attributes:

Original file line number	Diff line number	Diff line change
`@@ -31,10 +31,6 @@`
`31`	`31`	`"master_timeout": {`
`32`	`32`	`"type": "time",`
`33`	`33`	`"description": "Timeout for processing on master node"`
`34`		`- },`
`35`		`- "timeout": {`
`36`		`- "type": "time",`
`37`		`- "description": "Timeout for acknowledgement of update from all nodes in cluster"`
`38`	`34`	`}`
`39`	`35`	`}`
`40`	`36`	`}`
Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,15 @@`
`26`	`26`	`}`
`27`	`27`	`]`
`28`	`28`	`},`
`29`		`- "params":{}`
	`29`	`+ "params":{`
	`30`	`+ "master_timeout":{`
	`31`	`+ "type":"time",`
	`32`	`+ "description":"Explicit operation timeout for connection to master node"`
	`33`	`+ },`
	`34`	`+ "timeout":{`
	`35`	`+ "type":"time",`
	`36`	`+ "description":"Explicit operation timeout"`
	`37`	`+ }`
	`38`	`+ }`
`30`	`39`	`}`
`31`	`40`	`}`