Enable queryable built-in roles feature by default

docs/build.gradle

@@ -120,6 +120,8 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
+
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml

@@ -240,7 +240,8 @@
             test: {}
 
   - do:
-      indices.get_alias: { }
+      indices.get_alias:
+        index: test*
   - match: { test1.aliases.test: { } }
   - match: { test2.aliases.test: { } }
   - match: { test3.aliases.test: { } }
@@ -255,7 +256,8 @@
   - is_true: acknowledged
 
   - do:
-      indices.get_alias: {}
+      indices.get_alias:
+        index: test*
   - match: {test1.aliases: {}}
   - match: {test2.aliases: {}}
   - match: {test3.aliases: {}}

modules/dot-prefix-validation/build.gradle

@@ -26,4 +26,5 @@ tasks.named('yamlRestTest') {
 
 tasks.named('yamlRestCompatTest') {
   usesDefaultDistribution()
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }

modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java

@@ -21,6 +21,8 @@
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.ClassRule;
 
+import java.util.Objects;
+
 import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED;
 
 public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -55,6 +57,10 @@ private static ElasticsearchCluster createCluster() {
         if (setNodes) {
             clusterBuilder.nodes(2);
         }
+        clusterBuilder.systemProperty("es.queryable_built_in_roles_enabled", () -> {
+            final String enabled = System.getProperty("es.queryable_built_in_roles_enabled");
+            return Objects.requireNonNullElse(enabled, "");
+        });
         return clusterBuilder.build();
     }
 

modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml

@@ -2,7 +2,7 @@
 teardown:
   - do:
       indices.delete:
-        index: .*
+        index: .*,-.security-*
 
 ---
 "Index creation with a dot-prefix is deprecated unless x-elastic-product-origin set":

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java

@@ -20,6 +20,7 @@
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
@@ -47,7 +48,9 @@ public void test010Install() throws Exception {
     public void test20GeneratePasswords() throws Exception {
         assertWhileRunning(() -> {
             ServerUtils.waitForElasticsearch(installation);
-            Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+            Shell.Result result = retryOnAuthenticationErrors(
+                () -> installation.executables().setupPasswordsTool.run("auto --batch", null)
+            );
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout());
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
                 String response = ServerUtils.makeRequest(
@@ -102,20 +105,26 @@ public void test30AddBootstrapPassword() throws Exception {
         installation.executables().keystoreTool.run("add --stdin bootstrap.password", BOOTSTRAP_PASSWORD);
 
         assertWhileRunning(() -> {
-            String response = ServerUtils.makeRequest(
-                Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
-                "elastic",
-                BOOTSTRAP_PASSWORD,
-                null
+            ServerUtils.waitForElasticsearch("green", null, installation, "elastic", BOOTSTRAP_PASSWORD, null);
+            final String response = retryOnAuthenticationErrors(
+                () -> ServerUtils.makeRequest(
+                    Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
+                    "elastic",
+                    BOOTSTRAP_PASSWORD,
+                    null
+                )
             );
             assertThat(response, containsString("\"status\":\"green\""));
         });
+
     }
 
     public void test40GeneratePasswordsBootstrapAlreadySet() throws Exception {
         assertWhileRunning(() -> {
-
-            Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+            ServerUtils.waitForElasticsearch("green", null, installation, "elastic", BOOTSTRAP_PASSWORD, null);
+            Shell.Result result = retryOnAuthenticationErrors(
+                () -> installation.executables().setupPasswordsTool.run("auto --batch", null)
+            );
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout());
             assertThat(userpasses, hasKey("elastic"));
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
@@ -130,6 +139,48 @@ public void test40GeneratePasswordsBootstrapAlreadySet() throws Exception {
         });
     }
 
+    /**
+     * The security index is created on startup.
+     * It can happen that even when the security index exists, we get an authentication failure as `elastic`
+     * user because the reserved realm checks the security index first.
+     * This is because we check the security index too early (just after the creation) when all shards did not get allocated yet.
+     * Hence, the call can result in an `UnavailableShardsException` and cause the authentication to fail.
+     * We retry here on authentication errors for a couple of seconds just to verify that this is not the case.
+     */
+    private <R> R retryOnAuthenticationErrors(final Callable<R> callable) throws Exception {
+        Exception failure = null;
+        int retries = 5;
+        while (retries-- > 0) {
+            try {
+                return callable.call();
+            } catch (Exception e) {
+                if (e.getMessage() != null
+                    && (e.getMessage().contains("401 Unauthorized") || e.getMessage().contains("Failed to authenticate user"))) {
+                    logger.info(
+                        "Authentication failed (possibly due to UnavailableShardsException for the security index), retrying [{}].",
+                        retries,
+                        e
+                    );
+                    if (failure == null) {
+                        failure = e;
+                    } else {
+                        failure.addSuppressed(e);
+                    }
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException interrupted) {
+                        Thread.currentThread().interrupt();
+                        failure.addSuppressed(interrupted);
+                        throw failure;
+                    }
+                } else {
+                    throw e;
+                }
+            }
+        }
+        throw failure;
+    }
+
     private Map<String, String> parseUsersAndPasswords(String output) {
         Matcher matcher = USERPASS_REGEX.matcher(output);
         assertNotNull(matcher);

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -66,7 +66,7 @@ public class ServerUtils {
     private static final long waitTime = TimeUnit.MINUTES.toMillis(3);
     private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30);
     private static final long requestInterval = TimeUnit.SECONDS.toMillis(5);
-    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(25);
+    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(60);
 
     public static void waitForElasticsearch(Installation installation) throws Exception {
         final boolean securityEnabled;
@@ -260,9 +260,7 @@ public static void waitForElasticsearch(
                         // `elastic` , the reserved realm checks the security index first. It can happen that we check the security index
                         // too early after the security index creation in DockerTests causing an UnavailableShardsException. We retry
                         // authentication errors for a couple of seconds just to verify this is not the case.
-                        if (installation.distribution.isDocker()
-                            && timeElapsed < dockerWaitForSecurityIndex
-                            && response.getStatusLine().getStatusCode() == 401) {
+                        if (timeElapsed < dockerWaitForSecurityIndex && response.getStatusLine().getStatusCode() == 401) {
                             logger.info(
                                 "Authentication against docker failed (possibly due to UnavailableShardsException for the security index)"
                                     + ", retrying..."

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -18,6 +18,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
@@ -146,6 +147,8 @@
 import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING;
 import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT;
 import static org.elasticsearch.test.ESTestCase.assertBusy;
+import static org.elasticsearch.test.ESTestCase.assertFalse;
+import static org.elasticsearch.test.ESTestCase.assertTrue;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.ESTestCase.runInParallel;
 import static org.elasticsearch.test.ESTestCase.safeAwait;
@@ -160,9 +163,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -1240,16 +1241,29 @@ public synchronized void validateClusterFormed() {
         }
         logger.trace("validating cluster formed, expecting {}", expectedNodes);
 
-        assertFalse(
-            client().admin()
-                .cluster()
-                .prepareHealth(TEST_REQUEST_TIMEOUT)
-                .setWaitForEvents(Priority.LANGUID)
-                .setWaitForNodes(Integer.toString(expectedNodes.size()))
-                .get(TimeValue.timeValueSeconds(40))
-                .isTimedOut()
-        );
         try {
+            assertBusy(() -> {
+                try {
+                    final boolean timeout = client().admin()
+                        .cluster()
+                        .prepareHealth(TEST_REQUEST_TIMEOUT)
+                        .setWaitForEvents(Priority.LANGUID)
+                        .setWaitForNodes(Integer.toString(expectedNodes.size()))
+                        .get(TimeValue.timeValueSeconds(40))
+                        .isTimedOut();
+                    if (timeout) {
+                        throw new IllegalStateException("timed out waiting for cluster to form");
+                    }
+                } catch (UnavailableShardsException e) {
+                    if (e.getMessage() != null && e.getMessage().contains(".security")) {
+                        // security index may not be ready yet, throwing assertion error to retry
+                        throw new AssertionError(e);
+                    } else {
+                        throw e;
+                    }
+                }
+            }, 30, TimeUnit.SECONDS);
+
             final Object[] previousStates = new Object[1];
             assertBusy(() -> {
                 final List<ClusterState> states = nodes.values()

x-pack/plugin/build.gradle

@@ -104,3 +104,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
 
 })
 
+tasks.named('yamlRestCompatTest').configure {
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
+}

x-pack/plugin/core/build.gradle

@@ -158,6 +158,7 @@ testClusters.configureEach {
   keystore 'bootstrap.password', 'x-pack-test-password'
   user username: "x_pack_rest_user", password: "x-pack-test-password"
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.15.0")
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }
 
 if (buildParams.isSnapshotBuild() == false) {

x-pack/plugin/fleet/build.gradle

@@ -29,4 +29,5 @@ testClusters.configureEach {
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.autoconfiguration.enabled', 'false'
   user username: 'x_pack_rest_user', password: 'x-pack-test-password'
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }

x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java

@@ -48,6 +48,11 @@ protected Settings restAdminSettings() {
             .build();
     }
 
+    @Override
+    protected boolean preserveSecurityIndicesUponCompletion() {
+        return true;
+    }
+
     public void testAliasWithSystemDataStream() throws Exception {
         // Create a system data stream
         Request initialDocResponse = new Request("POST", ".fleet-actions-results/_doc");

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySpecialUserIT.java

@@ -218,7 +218,7 @@ public void testAnonymousUserFromQueryClusterWorks() throws Exception {
                 { "password": "%s" }""", PASS));
             assertOK(client().performRequest(changePasswordRequest));
 
-            final Request elasticUserSearchRequest = new Request("GET", "/*:.security*/_search");
+            final Request elasticUserSearchRequest = new Request("GET", "/*:.security*/_search?size=1");
             elasticUserSearchRequest.setOptions(
                 RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue("elastic", PASS))
             );

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java

@@ -132,7 +132,8 @@ public void testQueryDLSFLSRolesShowAsDisabled() throws Exception {
                     .build() };
             createRoleWithIndicesPrivileges(adminClient(), "role_with_FLS_and_DLS", indicesPrivileges);
         }
-        assertQuery(client(), "", 4, roles -> {
+        assertQuery(client(), """
+            {"query":{"bool":{"must_not":{"term":{"metadata._reserved":true}}}}}""", 4, roles -> {
             roles.sort(Comparator.comparing(o -> ((String) o.get("name"))));
             assertThat(roles, iterableWithSize(4));
             assertThat(roles.get(0).get("name"), equalTo("role_with_DLS"));
@@ -152,7 +153,8 @@ public void testQueryDLSFLSRolesShowAsDisabled() throws Exception {
         assertTrue(((Boolean) responseMap.get("basic_was_started")));
         assertTrue(((Boolean) responseMap.get("acknowledged")));
         // now the same roles show up as disabled ("enabled" is "false")
-        assertQuery(client(), "", 4, roles -> {
+        assertQuery(client(), """
+            {"query":{"bool":{"must_not":{"term":{"metadata._reserved":true}}}}}""", 4, roles -> {
             roles.sort(Comparator.comparing(o -> ((String) o.get("name"))));
             assertThat(roles, iterableWithSize(4));
             assertThat(roles.get(0).get("name"), equalTo("role_with_DLS"));

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java

@@ -16,8 +16,10 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityMigrations;
 import org.hamcrest.Matchers;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -49,15 +51,23 @@ public final class QueryRoleIT extends SecurityInBasicRestTestCase {
 
     private static final String READ_SECURITY_USER_AUTH_HEADER = "Basic cmVhZF9zZWN1cml0eV91c2VyOnJlYWQtc2VjdXJpdHktcGFzc3dvcmQ=";
 
-    public void testSimpleQueryAllRoles() throws IOException {
-        assertQuery("", 0, roles -> assertThat(roles, emptyIterable()));
-        RoleDescriptor createdRole = createRandomRole();
-        assertQuery("", 1, roles -> {
-            assertThat(roles, iterableWithSize(1));
-            assertRoleMap(roles.get(0), createdRole);
+    @Before
+    public void initialize() {
+        new ReservedRolesStore();
+    }
+
+    public void testSimpleQueryAllRoles() throws Exception {
+        createRandomRole();
+        assertQuery("", 1 + ReservedRolesStore.names().size(), roles -> {
+            // default size is 10
+            assertThat(roles, iterableWithSize(10));
         });
-        assertQuery("""
-            {"query":{"match_all":{}},"from":1}""", 1, roles -> assertThat(roles, emptyIterable()));
+        assertQuery(
+            Strings.format("""
+                {"query":{"match_all":{}},"from":%d}""", 1 + ReservedRolesStore.names().size()),
+            1 + ReservedRolesStore.names().size(),
+            roles -> assertThat(roles, emptyIterable())
+        );
     }
 
     public void testDisallowedFields() throws Exception {