Enable queryable built-in roles feature by default (#120323)

Making the `es.queryable_built_in_roles_enabled` feature flag enabled by default.
This feature makes the built-in roles automatically indexed in `.security` index and available
for querying via Query Role API. The consequence of this is that `.security` index is now 
created eagerly (if it's not existing) on cluster formation.

In order to keep the scope of this PR small, the feature is disabled for some of the tests, 
because they are either non-trivial to adjust or the gain is not worthy the effort to do it now. 
The tests will be adjusted in a follow-up PR and later the flag will be removed completely.

Relates to #117581

Author: slobodanadamovic

docs/build.gradle

@@ -120,6 +120,8 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
+
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml

@@ -240,7 +240,8 @@
             test: {}
 
   - do:
-      indices.get_alias: { }
+      indices.get_alias:
+        index: test*
   - match: { test1.aliases.test: { } }
   - match: { test2.aliases.test: { } }
   - match: { test3.aliases.test: { } }
@@ -255,7 +256,8 @@
   - is_true: acknowledged
 
   - do:
-      indices.get_alias: {}
+      indices.get_alias:
+        index: test*
   - match: {test1.aliases: {}}
   - match: {test2.aliases: {}}
   - match: {test3.aliases: {}}

modules/dot-prefix-validation/build.gradle

@@ -26,4 +26,5 @@ tasks.named('yamlRestTest') {
 
 tasks.named('yamlRestCompatTest') {
   usesDefaultDistribution()
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }

modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java

@@ -21,6 +21,8 @@
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.ClassRule;
 
+import java.util.Objects;
+
 import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED;
 
 public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -55,6 +57,10 @@ private static ElasticsearchCluster createCluster() {
         if (setNodes) {
             clusterBuilder.nodes(2);
         }
+        clusterBuilder.systemProperty("es.queryable_built_in_roles_enabled", () -> {
+            final String enabled = System.getProperty("es.queryable_built_in_roles_enabled");
+            return Objects.requireNonNullElse(enabled, "");
+        });
         return clusterBuilder.build();
     }
 

modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml

@@ -2,7 +2,7 @@
 teardown:
   - do:
       indices.delete:
-        index: .*
+        index: .*,-.security-*
 
 ---
 "Index creation with a dot-prefix is deprecated unless x-elastic-product-origin set":

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java

@@ -20,6 +20,7 @@
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
@@ -47,7 +48,9 @@ public void test010Install() throws Exception {
     public void test20GeneratePasswords() throws Exception {
         assertWhileRunning(() -> {
             ServerUtils.waitForElasticsearch(installation);
-            Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+            Shell.Result result = retryOnAuthenticationErrors(
+                () -> installation.executables().setupPasswordsTool.run("auto --batch", null)
+            );
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout());
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
                 String response = ServerUtils.makeRequest(
@@ -102,20 +105,26 @@ public void test30AddBootstrapPassword() throws Exception {
         installation.executables().keystoreTool.run("add --stdin bootstrap.password", BOOTSTRAP_PASSWORD);
 
         assertWhileRunning(() -> {
-            String response = ServerUtils.makeRequest(
-                Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
-                "elastic",
-                BOOTSTRAP_PASSWORD,
-                null
+            ServerUtils.waitForElasticsearch("green", null, installation, "elastic", BOOTSTRAP_PASSWORD, null);
+            final String response = retryOnAuthenticationErrors(
+                () -> ServerUtils.makeRequest(
+                    Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
+                    "elastic",
+                    BOOTSTRAP_PASSWORD,
+                    null
+                )
             );
             assertThat(response, containsString("\"status\":\"green\""));
         });
+
     }
 
     public void test40GeneratePasswordsBootstrapAlreadySet() throws Exception {
         assertWhileRunning(() -> {
-
-            Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+            ServerUtils.waitForElasticsearch("green", null, installation, "elastic", BOOTSTRAP_PASSWORD, null);
+            Shell.Result result = retryOnAuthenticationErrors(
+                () -> installation.executables().setupPasswordsTool.run("auto --batch", null)
+            );
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout());
             assertThat(userpasses, hasKey("elastic"));
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
@@ -130,6 +139,48 @@ public void test40GeneratePasswordsBootstrapAlreadySet() throws Exception {
         });
     }
 
+    /**
+     * The security index is created on startup.
+     * It can happen that even when the security index exists, we get an authentication failure as `elastic`
+     * user because the reserved realm checks the security index first.
+     * This is because we check the security index too early (just after the creation) when all shards did not get allocated yet.
+     * Hence, the call can result in an `UnavailableShardsException` and cause the authentication to fail.
+     * We retry here on authentication errors for a couple of seconds just to verify that this is not the case.
+     */
+    private <R> R retryOnAuthenticationErrors(final Callable<R> callable) throws Exception {
+        Exception failure = null;
+        int retries = 5;
+        while (retries-- > 0) {
+            try {
+                return callable.call();
+            } catch (Exception e) {
+                if (e.getMessage() != null
+                    && (e.getMessage().contains("401 Unauthorized") || e.getMessage().contains("Failed to authenticate user"))) {
+                    logger.info(
+                        "Authentication failed (possibly due to UnavailableShardsException for the security index), retrying [{}].",
+                        retries,
+                        e
+                    );
+                    if (failure == null) {
+                        failure = e;
+                    } else {
+                        failure.addSuppressed(e);
+                    }
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException interrupted) {
+                        Thread.currentThread().interrupt();
+                        failure.addSuppressed(interrupted);
+                        throw failure;
+                    }
+                } else {
+                    throw e;
+                }
+            }
+        }
+        throw failure;
+    }
+
     private Map<String, String> parseUsersAndPasswords(String output) {
         Matcher matcher = USERPASS_REGEX.matcher(output);
         assertNotNull(matcher);

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -66,7 +66,7 @@ public class ServerUtils {
     private static final long waitTime = TimeUnit.MINUTES.toMillis(3);
     private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30);
     private static final long requestInterval = TimeUnit.SECONDS.toMillis(5);
-    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(25);
+    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(60);
 
     public static void waitForElasticsearch(Installation installation) throws Exception {
         final boolean securityEnabled;
@@ -260,9 +260,7 @@ public static void waitForElasticsearch(
                         // `elastic` , the reserved realm checks the security index first. It can happen that we check the security index
                         // too early after the security index creation in DockerTests causing an UnavailableShardsException. We retry
                         // authentication errors for a couple of seconds just to verify this is not the case.
-                        if (installation.distribution.isDocker()
-                            && timeElapsed < dockerWaitForSecurityIndex
-                            && response.getStatusLine().getStatusCode() == 401) {
+                        if (timeElapsed < dockerWaitForSecurityIndex && response.getStatusLine().getStatusCode() == 401) {
                             logger.info(
                                 "Authentication against docker failed (possibly due to UnavailableShardsException for the security index)"
                                     + ", retrying..."

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -18,6 +18,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
@@ -146,6 +147,8 @@
 import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING;
 import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT;
 import static org.elasticsearch.test.ESTestCase.assertBusy;
+import static org.elasticsearch.test.ESTestCase.assertFalse;
+import static org.elasticsearch.test.ESTestCase.assertTrue;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.ESTestCase.runInParallel;
 import static org.elasticsearch.test.ESTestCase.safeAwait;
@@ -160,9 +163,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -1240,16 +1241,29 @@ public synchronized void validateClusterFormed() {
         }
         logger.trace("validating cluster formed, expecting {}", expectedNodes);
 
-        assertFalse(
-            client().admin()
-                .cluster()
-                .prepareHealth(TEST_REQUEST_TIMEOUT)
-                .setWaitForEvents(Priority.LANGUID)
-                .setWaitForNodes(Integer.toString(expectedNodes.size()))
-                .get(TimeValue.timeValueSeconds(40))
-                .isTimedOut()
-        );
         try {
+            assertBusy(() -> {
+                try {
+                    final boolean timeout = client().admin()
+                        .cluster()
+                        .prepareHealth(TEST_REQUEST_TIMEOUT)
+                        .setWaitForEvents(Priority.LANGUID)
+                        .setWaitForNodes(Integer.toString(expectedNodes.size()))
+                        .get(TimeValue.timeValueSeconds(40))
+                        .isTimedOut();
+                    if (timeout) {
+                        throw new IllegalStateException("timed out waiting for cluster to form");
+                    }
+                } catch (UnavailableShardsException e) {
+                    if (e.getMessage() != null && e.getMessage().contains(".security")) {
+                        // security index may not be ready yet, throwing assertion error to retry
+                        throw new AssertionError(e);
+                    } else {
+                        throw e;
+                    }
+                }
+            }, 30, TimeUnit.SECONDS);
+
             final Object[] previousStates = new Object[1];
             assertBusy(() -> {
                 final List<ClusterState> states = nodes.values()

x-pack/plugin/build.gradle

@@ -104,3 +104,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
 
 })
 
+tasks.named('yamlRestCompatTest').configure {
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
+}

x-pack/plugin/core/build.gradle

@@ -158,6 +158,7 @@ testClusters.configureEach {
   keystore 'bootstrap.password', 'x-pack-test-password'
   user username: "x_pack_rest_user", password: "x-pack-test-password"
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.15.0")
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }
 
 if (buildParams.isSnapshotBuild() == false) {