Adds certificate identity field to cross-cluster API keys (#134604)

* Add integration testing for CrossCluster API Key certificate_identity field

* [CI] Auto commit changes from spotless

* Update docs/changelog/134604.yaml

* Add 'certificate_identity' field to API Key, modify create/update CrossClusterAPIKey methods to account for new field

* fix merge conflicts

* [CI] Auto commit changes from spotless

* Fix constructors, spotlessApply

* Fix createCrossClusterApiKey unit test

* [CI] Auto commit changes from spotless

* Fix updateCrossClusterApiKeyRequestTest

* [CI] Auto commit changes from spotless

* Fix CC API Key Update Message and Test Assertion

# Conflicts:
#	x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateCrossClusterApiKeyRequestTests.java

* spotlessApply

* [CI] Auto commit changes from spotless

* Fix ApiKeyBackwardsCompatibilityIT, fix update error message check

* Fix ApiKeyBackwardsCompatibilityIT.testCertificateIdentityBackwardsCompatibility()

* [CI] Update transport version definitions

* Remove authenticateWithApiKey from ApiKeyBackwardsCompatibilityIT.testCertificateIdentityBackwardsCompatibility UPGRADED Case

* Add minimum version check to ApiKeyBackwardsCompatibilityIT.testCertificateIdentityBackwardsCompatibility()

* [CI] Update transport version definitions

* Add validation for certificate_identity to update path

* Fix validation logic, add capability to delete certificate_identity from API key

* Fix NPE

* [CI] Update transport version definitions

* Fix testing bugs that resulted from new CertificateIdentity record

* Fix ApiKeyIntegTest assertion bug

* Remove certificate_identity field from UpdateApiKeyRequestTranslator, UpdateApiKeyRequest class. Add testing

* Clean up certificate_identity testing in ApiKeyServiceTests

* Delete redundant integ tests

* Consolidate redundant code in ApiKeyBackwardsCompatibilityIT

* Change cert_identity validation failure message

* Update cert_identity version to 9.3.0 in APIKey BWC Test

* [CI] Auto commit changes from spotless

* CertID parser no longer differentiates between explicit vs implicit null

* [CI] Auto commit changes from spotless

* Move shared bwc test logic to AbstractUpgradeTestCase

* [CI] Auto commit changes from spotless

* Rename cleanUp method in TokenBackwwardsCompatbilityIT

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: gmjehovich

docs/changelog/134604.yaml

@@ -0,0 +1,5 @@
+pr: 134604
+summary: Adds certificate identity field to cross-cluster API keys
+area: Security
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java

@@ -121,6 +121,8 @@ public VersionId<?> versionNumber() {
     private final List<RoleDescriptor> roleDescriptors;
     @Nullable
     private final RoleDescriptorsIntersection limitedBy;
+    @Nullable
+    private final String certificateIdentity;
 
     public ApiKey(
         String name,
@@ -135,7 +137,8 @@ public ApiKey(
         @Nullable String realmType,
         @Nullable Map<String, Object> metadata,
         @Nullable List<RoleDescriptor> roleDescriptors,
-        @Nullable List<RoleDescriptor> limitedByRoleDescriptors
+        @Nullable List<RoleDescriptor> limitedByRoleDescriptors,
+        @Nullable String certificateIdentity
     ) {
         this(
             name,
@@ -150,7 +153,8 @@ public ApiKey(
             realmType,
             metadata,
             roleDescriptors,
-            limitedByRoleDescriptors == null ? null : new RoleDescriptorsIntersection(List.of(Set.copyOf(limitedByRoleDescriptors)))
+            limitedByRoleDescriptors == null ? null : new RoleDescriptorsIntersection(List.of(Set.copyOf(limitedByRoleDescriptors))),
+            certificateIdentity
         );
     }
 
@@ -167,7 +171,8 @@ private ApiKey(
         @Nullable String realmType,
         @Nullable Map<String, Object> metadata,
         @Nullable List<RoleDescriptor> roleDescriptors,
-        @Nullable RoleDescriptorsIntersection limitedBy
+        @Nullable RoleDescriptorsIntersection limitedBy,
+        @Nullable String certificateIdentity
     ) {
         this.name = name;
         this.id = id;
@@ -187,6 +192,7 @@ private ApiKey(
         // This assertion will need to be changed (or removed) when derived keys are properly supported
         assert limitedBy == null || limitedBy.roleDescriptorsList().size() == 1 : "can only have one set of limited-by role descriptors";
         this.limitedBy = limitedBy;
+        this.certificateIdentity = certificateIdentity;
     }
 
     // Should only be used by XContent parsers
@@ -205,7 +211,8 @@ private ApiKey(
             (String) parsed[9],
             (parsed[10] == null) ? null : (Map<String, Object>) parsed[10],
             (List<RoleDescriptor>) parsed[11],
-            (RoleDescriptorsIntersection) parsed[12]
+            (RoleDescriptorsIntersection) parsed[12],
+            (String) parsed[13]
         );
     }
 
@@ -268,6 +275,10 @@ public RoleDescriptorsIntersection getLimitedBy() {
         return limitedBy;
     }
 
+    public @Nullable String getCertificateIdentity() {
+        return certificateIdentity;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -306,6 +317,11 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
             assert type != Type.CROSS_CLUSTER;
             builder.field("limited_by", limitedBy);
         }
+
+        if (certificateIdentity != null) {
+            builder.field("certificate_identity", certificateIdentity);
+        }
+
         return builder;
     }
 
@@ -357,7 +373,8 @@ public int hashCode() {
             realmType,
             metadata,
             roleDescriptors,
-            limitedBy
+            limitedBy,
+            certificateIdentity
         );
     }
 
@@ -385,7 +402,9 @@ public boolean equals(Object obj) {
             && Objects.equals(realmType, other.realmType)
             && Objects.equals(metadata, other.metadata)
             && Objects.equals(roleDescriptors, other.roleDescriptors)
-            && Objects.equals(limitedBy, other.limitedBy);
+            && Objects.equals(limitedBy, other.limitedBy)
+            && Objects.equals(certificateIdentity, other.certificateIdentity);
+
     }
 
     @Override
@@ -416,6 +435,8 @@ public String toString() {
             + roleDescriptors
             + ", limited_by="
             + limitedBy
+            + ", certificate_identity="
+            + certificateIdentity
             + "]";
     }
 
@@ -452,6 +473,8 @@ static int initializeParser(AbstractObjectParser<?, Void> parser) {
             new ParseField("limited_by"),
             ObjectParser.ValueType.OBJECT_ARRAY
         );
-        return 13; // the number of fields to parse
+        parser.declareStringOrNull(optionalConstructorArg(), new ParseField("certificate_identity"));
+
+        return 14; // the number of fields to parse
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BaseBulkUpdateApiKeyRequest.java

@@ -26,9 +26,10 @@ public BaseBulkUpdateApiKeyRequest(
         final List<String> ids,
         @Nullable final List<RoleDescriptor> roleDescriptors,
         @Nullable final Map<String, Object> metadata,
-        @Nullable final TimeValue expiration
+        @Nullable final TimeValue expiration,
+        @Nullable final CertificateIdentity certificateIdentity
     ) {
-        super(roleDescriptors, metadata, expiration);
+        super(roleDescriptors, metadata, expiration, certificateIdentity);
         this.ids = Objects.requireNonNull(ids, "API key IDs must not be null");
     }
 
@@ -38,6 +39,16 @@ public ActionRequestValidationException validate() {
         if (ids.isEmpty()) {
             validationException = addValidationError("Field [ids] cannot be empty", validationException);
         }
+
+        if (getCertificateIdentity() != null && ids.size() > 1) {
+            validationException = addValidationError(
+                "Certificate identity can only be updated for a single API key at a time. Found ["
+                    + ids.size()
+                    + "] API key IDs in the request.",
+                validationException
+            );
+        }
+
         return validationException;
     }
 
@@ -54,11 +65,12 @@ public boolean equals(Object o) {
         return Objects.equals(getIds(), that.getIds())
             && Objects.equals(metadata, that.metadata)
             && Objects.equals(expiration, that.expiration)
-            && Objects.equals(roleDescriptors, that.roleDescriptors);
+            && Objects.equals(roleDescriptors, that.roleDescriptors)
+            && Objects.equals(certificateIdentity, that.certificateIdentity);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(getIds(), expiration, metadata, roleDescriptors);
+        return Objects.hash(getIds(), expiration, metadata, roleDescriptors, certificateIdentity);
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BaseSingleUpdateApiKeyRequest.java

@@ -23,9 +23,10 @@ public BaseSingleUpdateApiKeyRequest(
         @Nullable final List<RoleDescriptor> roleDescriptors,
         @Nullable final Map<String, Object> metadata,
         @Nullable final TimeValue expiration,
-        String id
+        String id,
+        @Nullable CertificateIdentity certificateIdentity
     ) {
-        super(roleDescriptors, metadata, expiration);
+        super(roleDescriptors, metadata, expiration, certificateIdentity);
         this.id = Objects.requireNonNull(id, "API key ID must not be null");
     }
 
@@ -42,11 +43,12 @@ public boolean equals(Object o) {
         return Objects.equals(getId(), that.getId())
             && Objects.equals(metadata, that.metadata)
             && Objects.equals(expiration, that.expiration)
-            && Objects.equals(roleDescriptors, that.roleDescriptors);
+            && Objects.equals(roleDescriptors, that.roleDescriptors)
+            && Objects.equals(certificateIdentity, that.certificateIdentity);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(getId(), expiration, metadata, roleDescriptors);
+        return Objects.hash(getId(), expiration, metadata, roleDescriptors, certificateIdentity);
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BaseUpdateApiKeyRequest.java

@@ -31,15 +31,19 @@ public abstract class BaseUpdateApiKeyRequest extends LegacyActionRequest {
     protected final Map<String, Object> metadata;
     @Nullable
     protected final TimeValue expiration;
+    @Nullable
+    protected final CertificateIdentity certificateIdentity;
 
     public BaseUpdateApiKeyRequest(
         @Nullable final List<RoleDescriptor> roleDescriptors,
         @Nullable final Map<String, Object> metadata,
-        @Nullable final TimeValue expiration
+        @Nullable final TimeValue expiration,
+        @Nullable final CertificateIdentity certificateIdentity
     ) {
         this.roleDescriptors = roleDescriptors;
         this.metadata = metadata;
         this.expiration = expiration;
+        this.certificateIdentity = certificateIdentity;
     }
 
     public Map<String, Object> getMetadata() {
@@ -54,6 +58,10 @@ public TimeValue getExpiration() {
         return expiration;
     }
 
+    public CertificateIdentity getCertificateIdentity() {
+        return certificateIdentity;
+    }
+
     public abstract ApiKey.Type getType();
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BulkUpdateApiKeyRequest.java

@@ -15,7 +15,7 @@
 import java.util.List;
 import java.util.Map;
 
-public final class BulkUpdateApiKeyRequest extends BaseBulkUpdateApiKeyRequest {
+public class BulkUpdateApiKeyRequest extends BaseBulkUpdateApiKeyRequest {
 
     public static BulkUpdateApiKeyRequest usingApiKeyIds(String... ids) {
         return new BulkUpdateApiKeyRequest(Arrays.stream(ids).toList(), null, null, null);
@@ -36,7 +36,7 @@ public BulkUpdateApiKeyRequest(
         @Nullable final Map<String, Object> metadata,
         @Nullable final TimeValue expiration
     ) {
-        super(ids, roleDescriptors, metadata, expiration);
+        super(ids, roleDescriptors, metadata, expiration, null);
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BulkUpdateApiKeyRequestTranslator.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
@@ -51,6 +52,14 @@ protected static ConstructingObjectParser<BulkUpdateApiKeyRequest, Void> createP
             }, new ParseField("role_descriptors"));
             parser.declareObject(optionalConstructorArg(), (p, c) -> p.map(), new ParseField("metadata"));
             parser.declareString(optionalConstructorArg(), new ParseField("expiration"));
+            parser.declareField(
+                optionalConstructorArg(),
+                (p) -> p.currentToken() == XContentParser.Token.VALUE_NULL
+                    ? new CertificateIdentity(null)
+                    : new CertificateIdentity(p.text()),
+                new ParseField("certificate_identity"),
+                ObjectParser.ValueType.STRING_OR_NULL
+            );
             return parser;
         }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CertificateIdentity.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.apikey;
+
+import org.elasticsearch.core.Nullable;
+
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public record CertificateIdentity(@Nullable String value) {
+
+    public CertificateIdentity {
+        if (value != null) {
+            try {
+                Pattern.compile(value);
+            } catch (PatternSyntaxException e) {
+                throw new IllegalArgumentException("Invalid certificate_identity format: [" + value + "]. Must be a valid regex.", e);
+            }
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateCrossClusterApiKeyRequest.java

@@ -21,17 +21,21 @@
 
 public final class CreateCrossClusterApiKeyRequest extends AbstractCreateApiKeyRequest {
 
+    private final CertificateIdentity certificateIdentity;
+
     public CreateCrossClusterApiKeyRequest(
         String name,
         CrossClusterApiKeyRoleDescriptorBuilder roleDescriptorBuilder,
         @Nullable TimeValue expiration,
-        @Nullable Map<String, Object> metadata
+        @Nullable Map<String, Object> metadata,
+        @Nullable CertificateIdentity certificateIdentity
     ) {
         super();
         this.name = Objects.requireNonNull(name);
         this.roleDescriptors = List.of(roleDescriptorBuilder.build());
         this.expiration = expiration;
         this.metadata = metadata;
+        this.certificateIdentity = certificateIdentity;
     }
 
     @Override
@@ -60,15 +64,21 @@ public boolean equals(Object o) {
             && Objects.equals(expiration, that.expiration)
             && Objects.equals(metadata, that.metadata)
             && Objects.equals(roleDescriptors, that.roleDescriptors)
-            && refreshPolicy == that.refreshPolicy;
+            && refreshPolicy == that.refreshPolicy
+            && Objects.equals(certificateIdentity, that.certificateIdentity);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, name, expiration, metadata, roleDescriptors, refreshPolicy);
+        return Objects.hash(id, name, expiration, metadata, roleDescriptors, refreshPolicy, certificateIdentity);
     }
 
     public static CreateCrossClusterApiKeyRequest withNameAndAccess(String name, String access) throws IOException {
-        return new CreateCrossClusterApiKeyRequest(name, CrossClusterApiKeyRoleDescriptorBuilder.parse(access), null, null);
+        return new CreateCrossClusterApiKeyRequest(name, CrossClusterApiKeyRoleDescriptorBuilder.parse(access), null, null, null);
+    }
+
+    public CertificateIdentity getCertificateIdentity() {
+        return certificateIdentity;
     }
+
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyResponse.java

@@ -126,7 +126,7 @@ public String toString() {
 
     static final ConstructingObjectParser<GetApiKeyResponse, Void> RESPONSE_PARSER;
     static {
-        int nFieldsForParsingApiKeyInfo = 13; // this must be changed whenever ApiKey#initializeParser is changed for the number of parsers
+        int nFieldsForParsingApiKeyInfo = 14; // this must be changed whenever ApiKey#initializeParser is changed for the number of parsers
         ConstructingObjectParser<Item, Void> keyInfoParser = new ConstructingObjectParser<>(
             "api_key_with_profile_uid",
             true,