[Backport 7.11] Add a smoke test for security realms (#68951)

This changes adds a new QA test that runs a smoke test on a node that
has been configured with one realm of each type.

Not all of the realms work, because some of them would depend on
external fixtures (LDAP, SAML, etc) and this particularly test suite
is intended to be as stable as possible and have no external
dependencies.

The primary purpose of this test is to catch any issues that prevent
a node from starting with particular realms configurd (e.g. security
manager or classpath issues). We don't depend on external fixtures
becaused we want this to be a smoke test that clearly indicates when a
(seemingly unrelated) change in Elasticsearch has unintended
consequences on realms. The use of external dependencies would
increase the number of things that could go wrong and move this from a
smoke test to a potentially noisy integration test.

Backport of: #68881

Author: tvernum

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -16,6 +16,7 @@
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -68,10 +69,12 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.GeneralSecurityException;
 import java.security.KeyManagementException;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
 import java.util.ArrayList;
@@ -81,6 +84,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
@@ -106,7 +110,13 @@
 public abstract class ESRestTestCase extends ESTestCase {
     public static final String TRUSTSTORE_PATH = "truststore.path";
     public static final String TRUSTSTORE_PASSWORD = "truststore.password";
+
     public static final String CERTIFICATE_AUTHORITIES = "certificate_authorities";
+
+    public static final String CLIENT_CERT_PATH = "client.cert.path";
+    public static final String CLIENT_KEY_PATH = "client.key.path";
+    public static final String CLIENT_KEY_PASSWORD = "client.key.password";
+
     public static final String CLIENT_SOCKET_TIMEOUT = "client.socket.timeout";
     public static final String CLIENT_PATH_PREFIX = "client.path.prefix";
 
@@ -1013,29 +1023,30 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE
     }
 
     protected static void configureClient(RestClientBuilder builder, Settings settings) throws IOException {
+        String truststorePath = settings.get(TRUSTSTORE_PATH);
         String certificateAuthorities = settings.get(CERTIFICATE_AUTHORITIES);
-        String keystorePath = settings.get(TRUSTSTORE_PATH);
+        String clientCertificatePath = settings.get(CLIENT_CERT_PATH);
 
-        if (certificateAuthorities != null && keystorePath != null) {
+        if (certificateAuthorities != null && truststorePath != null) {
             throw new IllegalStateException("Cannot set both " + CERTIFICATE_AUTHORITIES + " and " + TRUSTSTORE_PATH
                 + ". Please configure one of these.");
 
         }
-        if (keystorePath != null) {
+        if (truststorePath != null) {
             if (inFipsJvm()) {
-                throw new IllegalStateException("Keystore " + keystorePath + "cannot be used in FIPS 140 mode. Please configure "
+                throw new IllegalStateException("Keystore " + truststorePath + "cannot be used in FIPS 140 mode. Please configure "
                     + CERTIFICATE_AUTHORITIES + " with a PEM encoded trusted CA/certificate instead");
             }
             final String keystorePass = settings.get(TRUSTSTORE_PASSWORD);
             if (keystorePass == null) {
                 throw new IllegalStateException(TRUSTSTORE_PATH + " is provided but not " + TRUSTSTORE_PASSWORD);
             }
-            Path path = PathUtils.get(keystorePath);
-            if (!Files.exists(path)) {
+            Path path = PathUtils.get(truststorePath);
+            if (Files.exists(path) == false) {
                 throw new IllegalStateException(TRUSTSTORE_PATH + " is set but points to a non-existing file");
             }
             try {
-                final String keyStoreType = keystorePath.endsWith(".p12") ? "PKCS12" : "jks";
+                final String keyStoreType = truststorePath.endsWith(".p12") ? "PKCS12" : "jks";
                 KeyStore keyStore = KeyStore.getInstance(keyStoreType);
                 try (InputStream is = Files.newInputStream(path)) {
                     keyStore.load(is, keystorePass.toCharArray());
@@ -1048,21 +1059,35 @@ protected static void configureClient(RestClientBuilder builder, Settings settin
             }
         }
         if (certificateAuthorities != null) {
-            Path path = PathUtils.get(certificateAuthorities);
-            if (!Files.exists(path)) {
+            Path caPath = PathUtils.get(certificateAuthorities);
+            if (Files.exists(caPath) == false) {
                 throw new IllegalStateException(CERTIFICATE_AUTHORITIES + " is set but points to a non-existing file");
             }
             try {
                 KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
                 keyStore.load(null, null);
-                Certificate cert = PemUtils.readCertificates(Collections.singletonList(path)).get(0);
-                keyStore.setCertificateEntry(cert.toString(), cert);
-                SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(keyStore, null).build();
+                Certificate caCert = PemUtils.readCertificates(Collections.singletonList(caPath)).get(0);
+                keyStore.setCertificateEntry(caCert.toString(), caCert);
+                final SSLContextBuilder sslContextBuilder = SSLContexts.custom();
+                if (clientCertificatePath != null) {
+                    final Path certPath = PathUtils.get(clientCertificatePath);
+                    final Path keyPath = PathUtils.get(Objects.requireNonNull(settings.get(CLIENT_KEY_PATH), "No key provided"));
+                    final String password = settings.get(CLIENT_KEY_PASSWORD);
+                    final char[] passwordChars = password == null ? null : password.toCharArray();
+                    final PrivateKey key = PemUtils.readPrivateKey(keyPath, () -> passwordChars);
+                    final Certificate[] clientCertChain
+                        = PemUtils.readCertificates(Collections.singletonList(certPath)).toArray(new Certificate[1]);
+                    keyStore.setKeyEntry("client", key, passwordChars, clientCertChain);
+                    sslContextBuilder.loadKeyMaterial(keyStore, passwordChars);
+                }
+                SSLContext sslcontext = sslContextBuilder.loadTrustMaterial(keyStore, null).build();
                 SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslcontext);
                 builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy));
-            } catch (KeyStoreException | NoSuchAlgorithmException | KeyManagementException | CertificateException e) {
+            } catch (GeneralSecurityException e) {
                 throw new RuntimeException("Error setting up ssl", e);
             }
+        } else if (clientCertificatePath != null) {
+            throw new IllegalStateException("Client certificates are currently only supported when using a custom CA");
         }
         Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings);
         Header[] defaultHeaders = new Header[headers.size()];

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java

@@ -258,7 +258,7 @@ public static X509ExtendedTrustManager trustManager(Certificate[] certificates)
         return trustManager(store, TrustManagerFactory.getDefaultAlgorithm());
     }
 
-    static KeyStore trustStore(Certificate[] certificates)
+    public static KeyStore trustStore(Certificate[] certificates)
         throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
         assert certificates != null : "Cannot create trust store with null certificates";
         KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());

x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle

@@ -0,0 +1,85 @@
+/*
+ * This QA test is intended to smoke test all security realms with minimal dependencies.
+ * That is, it makes sure a node that has every realm configured can start, and tests those realms that can be tested without needing external services.
+ * This tradeoff is intentional because we want this set of tests to be very stable - failures in this QA suite should be an indicator that
+ *    something is broken in Elasticsearch (and not that an external docker fixture broke)
+ * This test is also intended to work correctly on FIPS mode because we also want to know if a realm breaks on FIPS.
+ */
+
+apply plugin: 'elasticsearch.java-rest-test'
+
+dependencies {
+  javaRestTestImplementation project(path: xpackModule('core'))
+  javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+testClusters.javaRestTest {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 2
+
+  extraConfigFile 'http-server.key', file('src/javaRestTest/resources/ssl/http-server.key')
+  extraConfigFile 'http-server.crt', file('src/javaRestTest/resources/ssl/http-server.crt')
+  extraConfigFile 'http-client-ca.crt', file('src/javaRestTest/resources/ssl/http-client-ca.crt')
+  extraConfigFile 'saml-metadata.xml', file('src/javaRestTest/resources/saml-metadata.xml')
+  extraConfigFile 'kerberos.keytab', file('src/javaRestTest/resources/kerberos.keytab')
+  extraConfigFile 'oidc-jwkset.json', file('src/javaRestTest/resources/oidc-jwkset.json')
+
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
+
+  // Need a trial license (not basic) to enable all realms
+  setting 'xpack.license.self_generated.type', 'trial'
+  // Need SSL to enable PKI realms
+  setting 'xpack.security.http.ssl.enabled', 'true'
+  setting 'xpack.security.http.ssl.certificate', 'http-server.crt'
+  setting 'xpack.security.http.ssl.key', 'http-server.key'
+  setting 'xpack.security.http.ssl.key_passphrase', 'http-password'
+  setting 'xpack.security.http.ssl.client_authentication', 'optional'
+  setting 'xpack.security.http.ssl.certificate_authorities', 'http-client-ca.crt'
+
+  // Don't need transport SSL, so leave it out
+  setting 'xpack.security.transport.ssl.enabled', 'false'
+
+  // Configure every realm type
+  //  - File
+  setting 'xpack.security.authc.realms.file.file0.order', '0'
+  //  - Native
+  setting 'xpack.security.authc.realms.native.native1.order', '1'
+  //  - LDAP (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.ldap.ldap2.order', '2'
+  setting 'xpack.security.authc.realms.ldap.ldap2.url', 'ldap://localhost:7777'
+  setting 'xpack.security.authc.realms.ldap.ldap2.user_search.base_dn', 'OU=users,DC=example,DC=com'
+  //  - AD (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.active_directory.ad3.order', '3'
+  setting 'xpack.security.authc.realms.active_directory.ad3.domain_name', 'localhost'
+  //  - PKI (works)
+  setting 'xpack.security.authc.realms.pki.pki4.order', '4'
+  //  - SAML (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.saml.saml5.order', '5'
+  setting 'xpack.security.authc.realms.saml.saml5.idp.metadata.path', 'saml-metadata.xml'
+  setting 'xpack.security.authc.realms.saml.saml5.idp.entity_id', 'http://idp.example.com/'
+  setting 'xpack.security.authc.realms.saml.saml5.sp.entity_id', 'http://kibana.example.net/'
+  setting 'xpack.security.authc.realms.saml.saml5.sp.acs', 'http://kibana.example.net/api/security/v1/saml'
+  setting 'xpack.security.authc.realms.saml.saml5.attributes.principal', 'uid'
+  //  - Kerberos (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.kerberos.kerb6.order', '6'
+  setting 'xpack.security.authc.realms.kerberos.kerb6.keytab.path', 'kerberos.keytab'
+  //  - OIDC (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.oidc.openid7.order', '7'
+  setting 'xpack.security.authc.realms.oidc.openid7.rp.client_id', 'http://rp.example.net'
+  setting 'xpack.security.authc.realms.oidc.openid7.rp.response_type', 'id_token'
+  setting 'xpack.security.authc.realms.oidc.openid7.rp.redirect_uri', 'https://kibana.example.net/api/security/v1/oidc'
+  setting 'xpack.security.authc.realms.oidc.openid7.op.issuer', 'https://op.example.com/'
+  setting 'xpack.security.authc.realms.oidc.openid7.op.authorization_endpoint', 'https://op.example.com/auth'
+  setting 'xpack.security.authc.realms.oidc.openid7.op.jwkset_path', 'oidc-jwkset.json'
+  setting 'xpack.security.authc.realms.oidc.openid7.claims.principal', 'sub'
+  keystore 'xpack.security.authc.realms.oidc.openid7.rp.client_secret', 'this-is-my-secret'
+
+  extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')
+  user username: "admin_user", password: "admin-password"
+  user username: "security_test_user", password: "security-test-password", role: "security_test_role"
+}
+

x-pack/plugin/security/qa/smoke-test-all-realms/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/FileRealmAuthIT.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Integration Rest Test for testing authentication when all possible realms are configured
+ */
+public class FileRealmAuthIT extends SecurityRealmSmokeTestCase {
+
+    // Declared in build.gradle
+    private static final String USERNAME = "security_test_user";
+    private static final SecureString PASSWORD = new SecureString("security-test-password".toCharArray());
+    private static final String ROLE_NAME = "security_test_role";
+
+    public void testAuthenticationUsingFileRealm() throws IOException {
+        Map<String, Object> authenticate = super.authenticate(
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+                UsernamePasswordToken.basicAuthHeaderValue(USERNAME, PASSWORD))
+        );
+
+        assertUsername(authenticate, USERNAME);
+        assertRealm(authenticate, "file", "file0");
+        assertRoles(authenticate, ROLE_NAME);
+    }
+
+}

x-pack/plugin/security/qa/smoke-test-all-realms/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/NativeRealmAuthIT.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Integration Rest Test for testing authentication when all possible realms are configured
+ */
+public class NativeRealmAuthIT extends SecurityRealmSmokeTestCase {
+
+    private static final String USERNAME = "test_native_user";
+    private static final SecureString PASSWORD = new SecureString("native-user-password".toCharArray());
+    private static final String ROLE_NAME = "native_role";
+
+    @Before
+    public void createUsersAndRoles() throws IOException {
+        createUser(USERNAME, PASSWORD, Collections.singletonList(ROLE_NAME));
+        createRole("native_role", Collections.singleton("monitor"));
+    }
+
+    @After
+    public void cleanUp() throws IOException {
+        deleteUser(USERNAME);
+        deleteRole(ROLE_NAME);
+    }
+
+    public void testAuthenticationUsingNativeRealm() throws IOException {
+        Map<String, Object> authenticate = super.authenticate(
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+                UsernamePasswordToken.basicAuthHeaderValue(USERNAME, PASSWORD))
+        );
+
+        assertUsername(authenticate, USERNAME);
+        assertRealm(authenticate, "native", "native1");
+        assertRoles(authenticate, ROLE_NAME);
+    }
+
+}

x-pack/plugin/security/qa/smoke-test-all-realms/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/PkiRealmAuthIT.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Integration Rest Test for testing authentication when all possible realms are configured
+ */
+public class PkiRealmAuthIT extends SecurityRealmSmokeTestCase {
+
+    // Derived from certificate attributes (pki-auth.crt)
+    private static final String USERNAME = "pki-auth";
+
+    @Override
+    protected Settings restClientSettings() {
+        Settings.Builder builder = Settings.builder()
+            .put(super.restClientSettings())
+            .put(CLIENT_CERT_PATH, getDataPath("/ssl/pki-auth.crt"))
+            .put(CLIENT_KEY_PATH, getDataPath("/ssl/pki-auth.key"))
+            .put(CLIENT_KEY_PASSWORD, "http-password");
+        builder.remove(ThreadContext.PREFIX + ".Authorization");
+        return builder.build();
+    }
+
+    public void testAuthenticationUsingFileRealm() throws IOException {
+        Map<String, Object> authenticate = super.authenticate(RequestOptions.DEFAULT.toBuilder());
+
+        assertUsername(authenticate, USERNAME);
+        assertRealm(authenticate, "pki", "pki4");
+        assertRoles(authenticate, new String[0]);
+    }
+
+}

x-pack/plugin/security/qa/smoke-test-all-realms/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/RealmInfoIT.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Integration Rest Test for testing authentication when all possible realms are configured
+ */
+public class RealmInfoIT extends SecurityRealmSmokeTestCase {
+
+    public void testThatAllRealmTypesAreEnabled() throws IOException {
+        final Request request = new Request("GET", "_xpack/usage");
+        final Response response = client().performRequest(request);
+        Map<String, Object> usage = entityAsMap(response);
+
+        Map<String, Object> realms = ObjectPath.evaluate(usage, "security.realms");
+        realms.forEach((type, config) -> {
+            assertThat(config, Matchers.instanceOf(Map.class));
+            assertThat("Realm type [" + type + "] is not enabled",
+                ((Map<?, ?>) config).get("enabled"), Matchers.equalTo(true));
+        });
+    }
+
+}