Certificate audit logging

docs/changelog/136299.yaml

@@ -0,0 +1,5 @@
+pr: 136299
+summary: Validate certificate identity from cross cluster creds
+area: Security
+type: enhancement
+issues: []

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCrossClusterApiKeySigningIT.java

@@ -25,6 +25,7 @@
 import org.junit.rules.TestRule;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -38,7 +39,7 @@
 
 public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRemoteClusterSecurityTestCase {
 
-    private static final AtomicReference<Map<String, Object>> API_KEY_MAP_REF = new AtomicReference<>();
+    private static final AtomicReference<Map<String, Object>> MY_REMOTE_API_KEY_MAP_REF = new AtomicReference<>();
 
     static {
         fulfillingCluster = ElasticsearchCluster.local()
@@ -49,8 +50,12 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
             .setting("xpack.security.remote_cluster_server.ssl.enabled", "true")
             .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
             .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
+            .setting("xpack.security.audit.enabled", "true")
+            .setting(
+                "xpack.security.audit.logfile.events.include",
+                "[authentication_success, authentication_failed, access_denied, access_granted]"
+            )
             .configFile("signing_ca.crt", Resource.fromClasspath("signing/root.crt"))
-            .setting("cluster.remote.signing.certificate_authorities", "signing_ca.crt")
             .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password")
             .build();
 
@@ -60,22 +65,25 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
             .setting("xpack.security.remote_cluster_client.ssl.enabled", "true")
             .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
             .configFile("signing.crt", Resource.fromClasspath("signing/signing.crt"))
-            .setting("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
             .configFile("signing.key", Resource.fromClasspath("signing/signing.key"))
-            .setting("cluster.remote.my_remote_cluster.signing.key", "signing.key")
             .keystore("cluster.remote.my_remote_cluster.credentials", () -> {
-                if (API_KEY_MAP_REF.get() == null) {
-                    final Map<String, Object> apiKeyMap = createCrossClusterAccessApiKey("""
+                if (MY_REMOTE_API_KEY_MAP_REF.get() == null) {
+                    final var accessJson = """
                         {
                             "search": [
                               {
                                 "names": ["index*", "not_found_index"]
                               }
                             ]
-                        }""");
-                    API_KEY_MAP_REF.set(apiKeyMap);
+                        }""";
+                    MY_REMOTE_API_KEY_MAP_REF.set(
+                        createCrossClusterAccessApiKey(
+                            accessJson,
+                            randomFrom("CN=instance", "^CN=instance$", "(?i)^CN=instance$", "^CN=[A-Za-z0-9_]+$")
+                        )
+                    );
                 }
-                return (String) API_KEY_MAP_REF.get().get("encoded");
+                return (String) MY_REMOTE_API_KEY_MAP_REF.get().get("encoded");
             })
             .keystore("cluster.remote.invalid_remote.credentials", randomEncodedApiKey())
             .build();
@@ -86,33 +94,102 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
     public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster);
 
     public void testCrossClusterSearchWithCrossClusterApiKeySigning() throws Exception {
-        indexTestData();
-        assertCrossClusterSearchSuccessfulWithResult();
+        updateClusterSettings(
+            Settings.builder()
+                .put("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
+                .put("cluster.remote.my_remote_cluster.signing.key", "signing.key")
+                .build()
+        );
 
-        // Change the CA to something that doesn't trust the signing cert
         updateClusterSettingsFulfillingCluster(
-            Settings.builder().put("cluster.remote.signing.certificate_authorities", "transport-ca.crt").build()
+            Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
         );
-        assertCrossClusterAuthFail();
 
-        // Update settings on query cluster to ignore unavailable remotes
-        updateClusterSettings(Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(true)).build());
+        indexTestData();
+
+        // Make sure we can search if cert trusted
+        {
+            assertCrossClusterSearchSuccessfulWithResult();
+        }
+
+        // Test CA that does not trust cert
+        {
+            // Change the CA to something that doesn't trust the signing cert
+            updateClusterSettingsFulfillingCluster(
+                Settings.builder().put("cluster.remote.signing.certificate_authorities", "transport-ca.crt").build()
+            );
+            assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");
+
+            // Change the CA to the default trust store
+            updateClusterSettingsFulfillingCluster(Settings.builder().putNull("cluster.remote.signing.certificate_authorities").build());
+            assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");
+
+            // Update settings on query cluster to ignore unavailable remotes
+            updateClusterSettings(
+                Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(true)).build()
+            );
+            assertCrossClusterSearchSuccessfulWithoutResult();
+
+            // Reset skip_unavailable
+            updateClusterSettings(
+                Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(false)).build()
+            );
 
-        assertCrossClusterSearchSuccessfulWithoutResult();
+            // Reset ca cert
+            updateClusterSettingsFulfillingCluster(
+                Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
+            );
+            // Confirm reset was successful
+            assertCrossClusterSearchSuccessfulWithResult();
+        }
+
+        // Test no signature provided
+        {
+            updateClusterSettings(
+                Settings.builder()
+                    .putNull("cluster.remote.my_remote_cluster.signing.certificate")
+                    .putNull("cluster.remote.my_remote_cluster.signing.key")
+                    .build()
+            );
+            assertCrossClusterAuthFail("Expected signature for cross cluster API key, but no signature was provided");
 
-        // TODO add test for certificate identity configured for API key but no signature provided (should 401)
+            // Reset
+            updateClusterSettings(
+                Settings.builder()
+                    .put("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
+                    .put("cluster.remote.my_remote_cluster.signing.key", "signing.key")
+                    .build()
+            );
+        }
 
-        // TODO add test for certificate identity not configured for API key but signature provided (should 200)
+        // Test API key without certificate identity and send signature anyway
+        {
+            final var accessJson = """
+                {
+                    "search": [
+                        {
+                        "names": ["index*", "not_found_index"]
+                        }
+                    ]
+                }""";
+            MY_REMOTE_API_KEY_MAP_REF.set(createCrossClusterAccessApiKey(accessJson));
+            assertCrossClusterSearchSuccessfulWithResult();
 
-        // TODO add test for certificate identity not configured for API key but wrong signature provided (should 401)
+            // Change the CA to the default trust store to make sure untrusted signature fails auth even if it's not required
+            updateClusterSettingsFulfillingCluster(Settings.builder().putNull("cluster.remote.signing.certificate_authorities").build());
+            assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");
 
-        // TODO add test for certificate identity regex matching (should 200)
+            // Reset
+            updateClusterSettingsFulfillingCluster(
+                Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
+            );
+        }
     }
 
-    private void assertCrossClusterAuthFail() {
+    private void assertCrossClusterAuthFail(String expectedMessage) {
         var responseException = assertThrows(ResponseException.class, () -> simpleCrossClusterSearch(randomBoolean()));
         assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(401));
-        assertThat(responseException.getMessage(), containsString("Failed to verify cross cluster api key signature certificate from [("));
+        assertThat(responseException.getMessage(), containsString(expectedMessage));
     }
 
     private void assertCrossClusterSearchSuccessfulWithoutResult() throws IOException {
@@ -227,4 +304,25 @@ private Response performRequestWithRemoteAccessUser(final Request request) throw
         request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue(REMOTE_SEARCH_USER, PASS)));
         return client().performRequest(request);
     }
+
+    protected static Map<String, Object> createCrossClusterAccessApiKey(String accessJson, String certificateIdentity) {
+        initFulfillingClusterClient();
+        final var createCrossClusterApiKeyRequest = new Request("POST", "/_security/cross_cluster/api_key");
+        createCrossClusterApiKeyRequest.setJsonEntity(Strings.format("""
+            {
+              "name": "cross_cluster_access_key",
+              "certificate_identity": "%s",
+              "access": %s
+            }""", certificateIdentity, accessJson));
+        try {
+            final Response createCrossClusterApiKeyResponse = performRequestWithAdminUser(
+                fulfillingClusterClient,
+                createCrossClusterApiKeyRequest
+            );
+            assertOK(createCrossClusterApiKeyResponse);
+            return responseAsMap(createCrossClusterApiKeyResponse);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
 }

x-pack/plugin/security/qa/rcs-extension/src/main/java/module-info.java

@@ -7,6 +7,7 @@
     requires org.elasticsearch.xcore;
     requires org.elasticsearch.security;
     requires org.elasticsearch.sslconfig;
+    requires io.netty.transport;
 
     provides RemoteClusterSecurityExtension.Provider with TestRemoteClusterSecurityExtension.Provider;
 }

x-pack/plugin/security/qa/rcs-extension/src/main/java/org/elasticsearch/xpack/security/rcs/extension/TestRemoteClusterSecurityExtension.java

@@ -7,9 +7,12 @@
 
 package org.elasticsearch.xpack.security.rcs.extension;
 
+import io.netty.channel.Channel;
+
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.transport.Header;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
@@ -78,7 +81,7 @@ public void authenticate(String action, TransportRequest request, ActionListener
             }
 
             @Override
-            public void authenticateHeaders(Map<String, String> headers, ActionListener<Void> listener) {
+            public void authenticateHeaders(Map<String, String> headers, Channel channel, Header header, ActionListener<Void> listener) {
                 listener.onResponse(null);
             }
         };

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java

@@ -182,6 +182,8 @@ protected void headerReceived(Header header) {
                         try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().newStoredContext()) {
                             remoteClusterAuthenticationService.authenticateHeaders(
                                 header.getRequestHeaders(),
+                                channel,
+                                header,
                                 ActionListener.runAfter(ActionListener.wrap(aVoid -> {
                                     // authn is successful -> NOOP (the complete request will be subsequently authn & authz & audited)
                                     // Header#toString does not print credentials (which are stored in request headers)

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -1198,7 +1198,8 @@ Collection<Object> createComponents(
             environment,
             threadPool,
             settings,
-            client
+            client,
+            auditTrailService
         );
         remoteClusterSecurityExtension.set(this.getRemoteClusterSecurityExtension(rcsComponents));
         remoteClusterAuthenticationService.set(remoteClusterSecurityExtension.get().getAuthenticationService());

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 
 import java.net.InetSocketAddress;
+import java.net.SocketAddress;
 
 public interface AuditTrail {
 
@@ -43,6 +44,8 @@ public interface AuditTrail {
 
     void authenticationFailed(String requestId, String realm, AuthenticationToken token, HttpPreRequest request);
 
+    void authenticationFailed(String requestId, AuthenticationToken token, String action, SocketAddress remoteAddress);
+
     void accessGranted(
         String requestId,
         Authentication authentication,

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 
 import java.net.InetSocketAddress;
+import java.net.SocketAddress;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.concurrent.atomic.AtomicReference;
@@ -118,6 +119,9 @@ public void authenticationFailed(String requestId, AuthenticationToken token, St
         @Override
         public void authenticationFailed(String requestId, AuthenticationToken token, HttpPreRequest request) {}
 
+        @Override
+        public void authenticationFailed(String requestId, AuthenticationToken token, String action, SocketAddress remoteAddress) {};
+
         @Override
         public void authenticationFailed(
             String requestId,

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -117,6 +117,7 @@
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.net.SocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -649,6 +650,23 @@ public void authenticationFailed(String requestId, String realm, AuthenticationT
         }
     }
 
+    @Override
+    public void authenticationFailed(String requestId, AuthenticationToken token, String action, SocketAddress remoteAddress) {
+        if (events.contains(AUTHENTICATION_FAILED)) {
+            final LogEntryBuilder logEntryBuilder = new LogEntryBuilder().with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
+                .with(EVENT_ACTION_FIELD_NAME, "authentication_failed")
+                .with(ACTION_FIELD_NAME, action)
+                .with(PRINCIPAL_FIELD_NAME, token.principal())
+                .withRequestId(requestId)
+                .withRestOrTransportOrigin(remoteAddress, threadContext)  // Use remoteAddress here somehow
+                .withThreadContext(threadContext);
+            if (token instanceof ServiceAccountToken) {
+                logEntryBuilder.with(SERVICE_TOKEN_NAME_FIELD_NAME, ((ServiceAccountToken) token).getTokenName());
+            }
+            logEntryBuilder.build();
+        }
+    }
+
     @Override
     public void accessGranted(
         String requestId,
@@ -1600,6 +1618,21 @@ LogEntryBuilder withRestOrTransportOrigin(TransportRequest transportRequest, Thr
             return this;
         }
 
+        LogEntryBuilder withRestOrTransportOrigin(SocketAddress remoteAddress, ThreadContext threadContext) {
+            assert LOCAL_ORIGIN_FIELD_VALUE.equals(logEntry.get(ORIGIN_TYPE_FIELD_NAME));
+            final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext);
+            if (restAddress != null) {
+                logEntry.with(ORIGIN_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE)
+                    .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(restAddress));
+            } else if (remoteAddress instanceof InetSocketAddress inetSocketAddress) {
+                logEntry.with(ORIGIN_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
+                    .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetSocketAddress));
+            }
+
+            // fall through to local_node default
+            return this;
+        }
+
         LogEntryBuilder withRequestBody(RestRequest request) {
             if (includeRequestBody) {
                 final String requestContent = restRequestContent(request);