Revert "Use index setting to track migration version index created on"

This reverts commit 9533f6f.

Author: jfreden

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -118,7 +118,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0);
-
+    public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_00_0, Version.LUCENE_9_12_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java

@@ -8,9 +8,6 @@
 package org.elasticsearch.xpack.security.support;
 
 import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.TimeValue;
@@ -34,9 +31,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListener;
@@ -48,7 +43,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class CleanupRoleMappingDuplicatesMigrationIT extends SecurityIntegTestCase {
@@ -261,20 +255,6 @@ public void testSkipMigrationNoFileBasedMappings() throws Exception {
         assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone");
     }
 
-    public void testNewIndexSkipMigration() {
-        internalCluster().setBootstrapMasterNodeIndex(0);
-        final String masterNode = internalCluster().getMasterName();
-        ensureGreen();
-        CountDownLatch awaitMigrations = awaitMigrationVersionUpdates(
-            masterNode,
-            SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION
-        );
-        // Create a native role mapping to create security index and trigger migration
-        createNativeRoleMapping("everyone_kibana_alone");
-        // Make sure no migration ran (set to current version without applying prior migrations)
-        safeAwait(awaitMigrations);
-    }
-
     private void assertAllRoleMappings(String... roleMappingNames) {
         GetRoleMappingsResponse response = client().execute(GetRoleMappingsAction.INSTANCE, new GetRoleMappingsRequest()).actionGet();
 
@@ -290,34 +270,6 @@ private void assertAllRoleMappings(String... roleMappingNames) {
         );
     }
 
-    /**
-     * Make sure all versions are applied to cluster state sequentially
-     */
-    private CountDownLatch awaitMigrationVersionUpdates(String node, final int... versions) {
-        final ClusterService clusterService = internalCluster().clusterService(node);
-        final CountDownLatch allVersionsCountDown = new CountDownLatch(1);
-        final AtomicInteger currentVersionIdx = new AtomicInteger(0);
-        clusterService.addListener(new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                int currentMigrationVersion = getCurrentMigrationVersion(event.state());
-                if (currentMigrationVersion > 0) {
-                    assertThat(versions[currentVersionIdx.get()], lessThanOrEqualTo(currentMigrationVersion));
-                    if (versions[currentVersionIdx.get()] == currentMigrationVersion) {
-                        currentVersionIdx.incrementAndGet();
-                    }
-
-                    if (currentVersionIdx.get() >= versions.length) {
-                        clusterService.removeListener(this);
-                        allVersionsCountDown.countDown();
-                    }
-                }
-            }
-        });
-
-        return allVersionsCountDown;
-    }
-
     private void awaitFileSettingsWatcher() throws Exception {
         final String masterNode = internalCluster().getMasterName();
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
@@ -359,11 +311,8 @@ private void assertMigrationLessThan(int expectedVersion) {
     }
 
     private int getCurrentMigrationVersion() {
-        return getCurrentMigrationVersion(internalCluster().getInstance(ClusterService.class).state());
-    }
-
-    private int getCurrentMigrationVersion(ClusterState state) {
-        IndexMetadata indexMetadata = state.metadata().getIndices().get(INTERNAL_SECURITY_MAIN_INDEX_7);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
+        IndexMetadata indexMetadata = clusterService.state().metadata().getIndices().get(INTERNAL_SECURITY_MAIN_INDEX_7);
         if (indexMetadata == null || indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) == null) {
             return 0;
         }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -1444,7 +1444,6 @@ public static List<Setting<?>> getSettings(List<SecurityExtension> securityExten
         settingsList.add(CachingServiceAccountTokenStore.CACHE_MAX_TOKENS_SETTING);
         settingsList.add(SimpleRole.CACHE_SIZE_SETTING);
         settingsList.add(NativeRoleMappingStore.LAST_LOAD_CACHE_ENABLED_SETTING);
-        settingsList.add(SecuritySystemIndices.MAIN_INDEX_CREATED_ON_MIGRATION_VERSION);
 
         // hide settings
         settingsList.add(Setting.stringListSetting(SecurityField.setting("hide_settings"), Property.NodeScope, Property.Filtered));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java

@@ -271,19 +271,10 @@ private SystemIndexDescriptor.MappingsVersion getMinSecurityIndexMappingVersion(
      * Check if the index was created on the latest index version available in the cluster
      */
     private static boolean isCreatedOnLatestVersion(IndexMetadata indexMetadata) {
-        if (indexMetadata == null) {
-            return false;
-        }
-
-        final Integer indexCreatedOnMigrationVersion = SecuritySystemIndices.MAIN_INDEX_CREATED_ON_MIGRATION_VERSION.get(
-            indexMetadata.getSettings()
-        );
-
-        // 1 is the first migration, before the MAIN_INDEX_CREATED_ON_MIGRATION_VERSION was introduced
-        if (indexCreatedOnMigrationVersion > 1) {
-            return indexCreatedOnMigrationVersion.equals(SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey());
-        }
-        return SETTING_INDEX_VERSION_CREATED.get(indexMetadata.getSettings()).onOrAfter(IndexVersion.current());
+        final IndexVersion indexVersionCreated = indexMetadata != null
+            ? SETTING_INDEX_VERSION_CREATED.get(indexMetadata.getSettings())
+            : null;
+        return indexVersionCreated != null && indexVersionCreated.onOrAfter(IndexVersion.current());
     }
 
     /**

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java

@@ -45,9 +45,6 @@
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_MANAGE_ROLES_PRIVILEGE;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS;
 
-/**
- * Interface for creating SecurityMigrations that will be automatically applied once to existing .security indices
- */
 public class SecurityMigrations {
 
     /**

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java

@@ -16,7 +16,6 @@
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.VersionId;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
@@ -64,16 +63,6 @@ public class SecuritySystemIndices {
     public static final NodeFeature SECURITY_ROLES_METADATA_FLATTENED = new NodeFeature("security.roles_metadata_flattened");
     public static final NodeFeature SECURITY_ROLE_MAPPING_CLEANUP = new NodeFeature("security.role_mapping_cleanup");
 
-    /**
-     * The latest version of the security migration when the main security index was created
-     */
-    public static final Setting<Integer> MAIN_INDEX_CREATED_ON_MIGRATION_VERSION = Setting.intSetting(
-        "index.security.migration.version",
-        0,
-        Setting.Property.IndexScope,
-        Setting.Property.Final
-    );
-
     /**
      * Security managed index mappings used to be updated based on the product version. They are now updated based on per-index mappings
      * versions. However, older nodes will still look for a product version in the mappings metadata, so we have to put <em>something</em>
@@ -170,7 +159,6 @@ private static Settings getMainIndexSettings() {
             .put(DataTier.TIER_PREFERENCE, "data_hot,data_content")
             .put(IndexMetadata.SETTING_PRIORITY, 1000)
             .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_MAIN_INDEX_FORMAT)
-            .put(MAIN_INDEX_CREATED_ON_MIGRATION_VERSION.getKey(), SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey())
             .put("analysis.filter.email.type", "pattern_capture")
             .put("analysis.filter.email.preserve_original", true)
             .putList("analysis.filter.email.patterns", List.of("([^@]+)", "(\\p{L}+)", "(\\d+)", "@(.+)"))

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java

@@ -209,10 +209,7 @@ private Collection<Object> createComponentsUtil(Settings settings) throws Except
         settings = Security.additionalSettings(settings, true);
         Set<Setting<?>> allowedSettings = new HashSet<>(Security.getSettings(null));
         allowedSettings.addAll(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterSettings clusterSettings = new ClusterSettings(
-            settings,
-            allowedSettings.stream().filter(Setting::hasNodeScope).collect(Collectors.toSet())
-        );
+        ClusterSettings clusterSettings = new ClusterSettings(settings, allowedSettings);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         when(threadPool.relativeTimeInMillis()).thenReturn(1L);
         threadContext = new ThreadContext(Settings.EMPTY);