Use index setting to track migration version index created on

Author: jfreden

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -118,7 +118,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0);
-    public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_00_0, Version.LUCENE_9_12_0);
+
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java

@@ -8,6 +8,9 @@
 package org.elasticsearch.xpack.security.support;
 
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.TimeValue;
@@ -31,7 +34,9 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListener;
@@ -43,6 +48,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class CleanupRoleMappingDuplicatesMigrationIT extends SecurityIntegTestCase {
@@ -255,6 +261,20 @@ public void testSkipMigrationNoFileBasedMappings() throws Exception {
         assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone");
     }
 
+    public void testNewIndexSkipMigration() {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        final String masterNode = internalCluster().getMasterName();
+        ensureGreen();
+        CountDownLatch awaitMigrations = awaitMigrationVersionUpdates(
+            masterNode,
+            SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION
+        );
+        // Create a native role mapping to create security index and trigger migration
+        createNativeRoleMapping("everyone_kibana_alone");
+        // Make sure no migration ran (set to current version without applying prior migrations)
+        safeAwait(awaitMigrations);
+    }
+
     private void assertAllRoleMappings(String... roleMappingNames) {
         GetRoleMappingsResponse response = client().execute(GetRoleMappingsAction.INSTANCE, new GetRoleMappingsRequest()).actionGet();
 
@@ -270,6 +290,34 @@ private void assertAllRoleMappings(String... roleMappingNames) {
         );
     }
 
+    /**
+     * Make sure all versions are applied to cluster state sequentially
+     */
+    private CountDownLatch awaitMigrationVersionUpdates(String node, final int... versions) {
+        final ClusterService clusterService = internalCluster().clusterService(node);
+        final CountDownLatch allVersionsCountDown = new CountDownLatch(1);
+        final AtomicInteger currentVersionIdx = new AtomicInteger(0);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                int currentMigrationVersion = getCurrentMigrationVersion(event.state());
+                if (currentMigrationVersion > 0) {
+                    assertThat(versions[currentVersionIdx.get()], lessThanOrEqualTo(currentMigrationVersion));
+                    if (versions[currentVersionIdx.get()] == currentMigrationVersion) {
+                        currentVersionIdx.incrementAndGet();
+                    }
+
+                    if (currentVersionIdx.get() >= versions.length) {
+                        clusterService.removeListener(this);
+                        allVersionsCountDown.countDown();
+                    }
+                }
+            }
+        });
+
+        return allVersionsCountDown;
+    }
+
     private void awaitFileSettingsWatcher() throws Exception {
         final String masterNode = internalCluster().getMasterName();
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
@@ -311,8 +359,11 @@ private void assertMigrationLessThan(int expectedVersion) {
     }
 
     private int getCurrentMigrationVersion() {
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-        IndexMetadata indexMetadata = clusterService.state().metadata().getIndices().get(INTERNAL_SECURITY_MAIN_INDEX_7);
+        return getCurrentMigrationVersion(internalCluster().getInstance(ClusterService.class).state());
+    }
+
+    private int getCurrentMigrationVersion(ClusterState state) {
+        IndexMetadata indexMetadata = state.metadata().getIndices().get(INTERNAL_SECURITY_MAIN_INDEX_7);
         if (indexMetadata == null || indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) == null) {
             return 0;
         }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -1444,6 +1444,7 @@ public static List<Setting<?>> getSettings(List<SecurityExtension> securityExten
         settingsList.add(CachingServiceAccountTokenStore.CACHE_MAX_TOKENS_SETTING);
         settingsList.add(SimpleRole.CACHE_SIZE_SETTING);
         settingsList.add(NativeRoleMappingStore.LAST_LOAD_CACHE_ENABLED_SETTING);
+        settingsList.add(SecuritySystemIndices.MAIN_INDEX_CREATED_ON_MIGRATION_VERSION);
 
         // hide settings
         settingsList.add(Setting.stringListSetting(SecurityField.setting("hide_settings"), Property.NodeScope, Property.Filtered));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java

@@ -271,10 +271,19 @@ private SystemIndexDescriptor.MappingsVersion getMinSecurityIndexMappingVersion(
      * Check if the index was created on the latest index version available in the cluster
      */
     private static boolean isCreatedOnLatestVersion(IndexMetadata indexMetadata) {
-        final IndexVersion indexVersionCreated = indexMetadata != null
-            ? SETTING_INDEX_VERSION_CREATED.get(indexMetadata.getSettings())
-            : null;
-        return indexVersionCreated != null && indexVersionCreated.onOrAfter(IndexVersion.current());
+        if (indexMetadata == null) {
+            return false;
+        }
+
+        final Integer indexCreatedOnMigrationVersion = SecuritySystemIndices.MAIN_INDEX_CREATED_ON_MIGRATION_VERSION.get(
+            indexMetadata.getSettings()
+        );
+
+        // 1 is the first migration, before the MAIN_INDEX_CREATED_ON_MIGRATION_VERSION was introduced
+        if (indexCreatedOnMigrationVersion > 1) {
+            return indexCreatedOnMigrationVersion.equals(SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey());
+        }
+        return SETTING_INDEX_VERSION_CREATED.get(indexMetadata.getSettings()).onOrAfter(IndexVersion.current());
     }
 
     /**

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java

@@ -45,6 +45,9 @@
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_MANAGE_ROLES_PRIVILEGE;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS;
 
+/**
+ * Interface for creating SecurityMigrations that will be automatically applied once to existing .security indices
+ */
 public class SecurityMigrations {
 
     /**

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.VersionId;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
@@ -63,6 +64,16 @@ public class SecuritySystemIndices {
     public static final NodeFeature SECURITY_ROLES_METADATA_FLATTENED = new NodeFeature("security.roles_metadata_flattened");
     public static final NodeFeature SECURITY_ROLE_MAPPING_CLEANUP = new NodeFeature("security.role_mapping_cleanup");
 
+    /**
+     * The latest version of the security migration when the main security index was created
+     */
+    public static final Setting<Integer> MAIN_INDEX_CREATED_ON_MIGRATION_VERSION = Setting.intSetting(
+        "index.security.migration.version",
+        0,
+        Setting.Property.IndexScope,
+        Setting.Property.Final
+    );
+
     /**
      * Security managed index mappings used to be updated based on the product version. They are now updated based on per-index mappings
      * versions. However, older nodes will still look for a product version in the mappings metadata, so we have to put <em>something</em>
@@ -159,6 +170,7 @@ private static Settings getMainIndexSettings() {
             .put(DataTier.TIER_PREFERENCE, "data_hot,data_content")
             .put(IndexMetadata.SETTING_PRIORITY, 1000)
             .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_MAIN_INDEX_FORMAT)
+            .put(MAIN_INDEX_CREATED_ON_MIGRATION_VERSION.getKey(), SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey())
             .put("analysis.filter.email.type", "pattern_capture")
             .put("analysis.filter.email.preserve_original", true)
             .putList("analysis.filter.email.patterns", List.of("([^@]+)", "(\\p{L}+)", "(\\d+)", "@(.+)"))

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java

@@ -209,7 +209,10 @@ private Collection<Object> createComponentsUtil(Settings settings) throws Except
         settings = Security.additionalSettings(settings, true);
         Set<Setting<?>> allowedSettings = new HashSet<>(Security.getSettings(null));
         allowedSettings.addAll(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        ClusterSettings clusterSettings = new ClusterSettings(settings, allowedSettings);
+        ClusterSettings clusterSettings = new ClusterSettings(
+            settings,
+            allowedSettings.stream().filter(Setting::hasNodeScope).collect(Collectors.toSet())
+        );
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         when(threadPool.relativeTimeInMillis()).thenReturn(1L);
         threadContext = new ThreadContext(Settings.EMPTY);