Merge branch 'main' into ml-auth-task-node-feature

Author: jonathan-buttner

.gitattributes

@@ -4,12 +4,13 @@ CHANGELOG.asciidoc  merge=union
 # Windows
 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/*.asciidoc text eol=lf
 
+# ESQL parsing and source generated related assets
 x-pack/plugin/esql/compute/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/compute/src/main/generated-src/** linguist-generated=true
 x-pack/plugin/esql/src/main/antlr/*.tokens linguist-generated=true
 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*.interp linguist-generated=true
-x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java linguist-generated=true
-x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*BaseLexer*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*BaseParser*.java linguist-generated=true
 x-pack/plugin/esql/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/src/main/generated-src/** linguist-generated=true
 

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -12,7 +12,7 @@
   <suppress files="modules[/\\]lang-painless[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]SuggestLexer\.java" checks="." />
   <suppress files="plugin[/\\]sql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]sql[/\\]parser[/\\]SqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
   <suppress files="plugin[/\\]eql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]eql[/\\]parser[/\\]EqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
-  <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\]EsqlBase(Parser|Lexer).*.java" checks="." />
+  <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\](EsqlBase|PromqlBase)(Parser|Lexer).*.java" checks="." />
   <suppress files="x-pack[/\\]plugin[/\\]otel-data[/\\]build[/\\]generated[/\\]sources[/\\]" checks="." />
 
   <!-- JNA requires the no-argument constructor on JNAKernel32Library.SizeT to be public-->

docs/changelog/137230.yaml

@@ -0,0 +1,5 @@
+pr: 137230
+summary: Principal Extraction from Certificate RDN Attribute Value in PKI Realm
+area: Security
+type: bug
+issues: []

docs/reference/elasticsearch/configuration-reference/security-settings.md

@@ -769,6 +769,18 @@ In addition to the [settings that are valid for all realms](#ref-realm-settings)
 `username_pattern`
 :   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) The regular expression pattern used to extract the username from the certificate DN. The username is used for auditing and logging. The username can also be used with the [role mapping API](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/mapping-users-groups-to-roles.md) and [authorization delegation](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/authorization-delegation.md). The first match group is the used as the username. Defaults to `CN=(.*?)(?:,|$)`.
 
+    This setting is ignored if either `username_rdn_oid` or `username_rdn_name` is set.
+
+`username_rdn_oid`
+:   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) The relative distinguished name (RDN) attribute OID used to extract the username from the certificate DN. The username is used for auditing and logging. The username can also be used with the [role mapping API](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/mapping-users-groups-to-roles.md) and [authorization delegation](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/authorization-delegation.md). The value of the most specific RDN matching this attribute OID is used as the username.
+
+    This setting takes precedent over `username_pattern`. You cannot use this setting and `username_rdn_name` at the same time.
+
+`username_rdn_name`
+:   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) The relative distinguished name (RDN) attribute name used to extract the username from the certificate DN. The username is used for auditing and logging. The username can also be used with the [role mapping API](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/mapping-users-groups-to-roles.md) and [authorization delegation](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/authorization-delegation.md). The value of the most specific RDN matching this attribute name is used as the username.
+
+    This setting takes precedent over `username_pattern`. You cannot use this setting and `username_rdn_oid` at the same time.
+
 `certificate_authorities`
 :   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) List of paths to the PEM certificate files that should be used to authenticate a user’s certificate as trusted. Defaults to the trusted certificates configured for SSL. This setting cannot be used with `truststore.path`.
 

libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DerParser.java

@@ -36,21 +36,22 @@ public final class DerParser {
     private static final int CONSTRUCTED = 0x20;
 
     // Tag and data types
-    static final class Type {
-        static final int INTEGER = 0x02;
-        static final int OCTET_STRING = 0x04;
-        static final int OBJECT_OID = 0x06;
-        static final int SEQUENCE = 0x10;
-        static final int NUMERIC_STRING = 0x12;
-        static final int PRINTABLE_STRING = 0x13;
-        static final int VIDEOTEX_STRING = 0x15;
-        static final int IA5_STRING = 0x16;
-        static final int GRAPHIC_STRING = 0x19;
-        static final int ISO646_STRING = 0x1A;
-        static final int GENERAL_STRING = 0x1B;
-        static final int UTF8_STRING = 0x0C;
-        static final int UNIVERSAL_STRING = 0x1C;
-        static final int BMP_STRING = 0x1E;
+    public static final class Type {
+        public static final int INTEGER = 0x02;
+        public static final int OCTET_STRING = 0x04;
+        public static final int OBJECT_OID = 0x06;
+        public static final int SEQUENCE = 0x10;
+        public static final int SET = 0x11;
+        public static final int NUMERIC_STRING = 0x12;
+        public static final int PRINTABLE_STRING = 0x13;
+        public static final int VIDEOTEX_STRING = 0x15;
+        public static final int IA5_STRING = 0x16;
+        public static final int GRAPHIC_STRING = 0x19;
+        public static final int ISO646_STRING = 0x1A;
+        public static final int GENERAL_STRING = 0x1B;
+        public static final int UTF8_STRING = 0x0C;
+        public static final int UNIVERSAL_STRING = 0x1C;
+        public static final int BMP_STRING = 0x1E;
     }
 
     private InputStream derInputStream;

muted-tests.yml

@@ -411,9 +411,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/138319
-- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests
-  method: testKnnQuantizedFlatVectorsFormat
-  issue: https://github.com/elastic/elasticsearch/issues/138368
 - class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
   method: test {yaml=termvectors/30_realtime/Realtime Term Vectors}
   issue: https://github.com/elastic/elasticsearch/issues/138370

server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java

@@ -2003,7 +2003,7 @@ public void testKnnQuantizedFlatVectorsFormat() throws IOException {
                     + (quantizedFlatFormat.equals("int4_flat") ? 4 : 7)
                     + ", compressed="
                     + quantizedFlatFormat.equals("int4_flat")
-                    + ", flatVectorScorer=ESQuantizedFlatVectorsScorer("
+                    + ", flatVectorScorer=ESFlatVectorsScorer("
                     + "delegate=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())"
                     + ", factory="
                     + (factory != null ? factory : "null")

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/pki/PkiRealmSettings.java

@@ -6,6 +6,10 @@
  */
 package org.elasticsearch.xpack.core.security.authc.pki;
 
+import com.unboundid.ldap.sdk.LDAPException;
+import com.unboundid.ldap.sdk.schema.AttributeTypeDefinition;
+import com.unboundid.ldap.sdk.schema.Schema;
+
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.TimeValue;
@@ -29,6 +33,33 @@ public final class PkiRealmSettings {
         key -> new Setting<>(key, DEFAULT_USERNAME_PATTERN, s -> Pattern.compile(s, Pattern.CASE_INSENSITIVE), Setting.Property.NodeScope)
     );
 
+    public static final Setting.AffixSetting<String> USERNAME_RDN_OID_SETTING = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "username_rdn_oid",
+        key -> Setting.simpleString(key, Setting.Property.NodeScope)
+    );
+
+    public static final Setting.AffixSetting<String> USERNAME_RDN_NAME_SETTING = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "username_rdn_name",
+        key -> new Setting<>(key, (String) null, s -> {
+            if (s == null) {
+                return "";
+            }
+            Schema schema;
+            try {
+                schema = Schema.getDefaultStandardSchema();
+            } catch (LDAPException e) {
+                throw new IllegalStateException("Unexpected error occurred obtaining default LDAP schema", e);
+            }
+            AttributeTypeDefinition atd = schema.getAttributeType(s);
+            if (atd == null) {
+                throw new IllegalArgumentException("Unknown RDN name [" + s + "] for setting [" + key + "]");
+            }
+            return atd.getOID();
+        }, Setting.Property.NodeScope)
+    );
+
     private static final TimeValue DEFAULT_TTL = TimeValue.timeValueMinutes(20);
     public static final Setting.AffixSetting<TimeValue> CACHE_TTL_SETTING = Setting.affixKeySetting(
         RealmSettings.realmSettingPrefix(TYPE),
@@ -75,6 +106,8 @@ private PkiRealmSettings() {}
     public static Set<Setting.AffixSetting<?>> getSettings() {
         Set<Setting.AffixSetting<?>> settings = new HashSet<>();
         settings.add(USERNAME_PATTERN_SETTING);
+        settings.add(USERNAME_RDN_OID_SETTING);
+        settings.add(USERNAME_RDN_NAME_SETTING);
         settings.add(CACHE_TTL_SETTING);
         settings.add(CACHE_MAX_USERS_SETTING);
         settings.add(DELEGATION_ENABLED_SETTING);

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java

@@ -22,6 +22,7 @@
 
 import java.io.IOException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Collection;
 import java.util.Objects;
 
@@ -217,6 +218,10 @@ public static Literal timeDuration(Source source, Duration literal) {
         return new Literal(source, literal, DataType.TIME_DURATION);
     }
 
+    public static Literal dateTime(Source source, Instant literal) {
+        return new Literal(source, literal, DataType.DATETIME);
+    }
+
     public static Literal integer(Source source, Integer literal) {
         return new Literal(source, literal, INTEGER);
     }
@@ -229,6 +234,10 @@ public static Literal fromLong(Source source, Long literal) {
         return new Literal(source, literal, LONG);
     }
 
+    public static Expression fromBoolean(Source source, Boolean literal) {
+        return new Literal(source, literal, DataType.BOOLEAN);
+    }
+
     private static BytesRef longAsWKB(DataType dataType, long encoded) {
         return dataType == GEO_POINT ? GEO.longAsWkb(encoded) : CARTESIAN.longAsWkb(encoded);
     }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Arithmetics.java

@@ -148,7 +148,7 @@ public static Number mod(Number l, Number r) {
         return l.intValue() % r.intValue();
     }
 
-    static Number negate(Number n) {
+    public static Number negate(Number n) {
         if (n == null) {
             return null;
         }