Skip to content

Commit c23b783

Browse files
jfredenjfreden
committed
Add security migration for cleaning up ECK role mappings
1 parent b69d39c commit c23b783

File tree

17 files changed

+420
-27
lines changed

17 files changed

+420
-27
lines changed

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
117117
public static final IndexVersion ENABLE_IGNORE_MALFORMED_LOGSDB = def(8_514_00_0, Version.LUCENE_9_11_1);
118118
public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1);
119119
public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0);
120+
public static final IndexVersion ADD_ROLE_MAPPING_MIGRATION = def(8_517_00_0, Version.LUCENE_9_12_0);
120121

121122
/*
122123
* STOP! READ THIS FIRST! No, really,

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,14 @@
1717
import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MIGRATION_FRAMEWORK;
1818
import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_PROFILE_ORIGIN_FEATURE;
1919
import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED;
20+
import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP;
2021
import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.VERSION_SECURITY_PROFILE_ORIGIN;
2122

2223
public class SecurityFeatures implements FeatureSpecification {
2324

2425
@Override
2526
public Set<NodeFeature> getFeatures() {
26-
return Set.of(SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK);
27+
return Set.of(SECURITY_ROLE_MAPPING_CLEANUP, SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK);
2728
}
2829

2930
@Override

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java‎

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.cluster.metadata.IndexMetadata;
3232
import org.elasticsearch.cluster.metadata.MappingMetadata;
3333
import org.elasticsearch.cluster.metadata.Metadata;
34+
import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
3435
import org.elasticsearch.cluster.routing.IndexRoutingTable;
3536
import org.elasticsearch.cluster.service.ClusterService;
3637
import org.elasticsearch.core.TimeValue;
@@ -46,6 +47,8 @@
4647
import org.elasticsearch.rest.RestStatus;
4748
import org.elasticsearch.threadpool.Scheduler;
4849
import org.elasticsearch.xcontent.XContentType;
50+
import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
51+
import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
4952
import org.elasticsearch.xpack.security.SecurityFeatures;
5053

5154
import java.time.Instant;
@@ -74,7 +77,8 @@
7477
public class SecurityIndexManager implements ClusterStateListener {
7578

7679
public static final String SECURITY_VERSION_STRING = "security-version";
77-
80+
private static final String FILE_SETTINGS_METADATA_NAMESPACE = "file_settings";
81+
private static final String HANDLER_ROLE_MAPPINGS_NAME = "role_mappings";
7882
private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class);
7983

8084
/**
@@ -267,6 +271,22 @@ private static boolean isCreatedOnLatestVersion(IndexMetadata indexMetadata) {
267271
return indexVersionCreated != null && indexVersionCreated.onOrAfter(IndexVersion.current());
268272
}
269273

274+
private static Set<String> getFileSettingsMetadataHandlerRoleMappingKeys(ClusterState clusterState) {
275+
ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(FILE_SETTINGS_METADATA_NAMESPACE);
276+
if (fileSettingsMetadata != null && fileSettingsMetadata.handlers().containsKey(HANDLER_ROLE_MAPPINGS_NAME)) {
277+
return fileSettingsMetadata.handlers().get(HANDLER_ROLE_MAPPINGS_NAME).keys();
278+
}
279+
return Set.of();
280+
}
281+
282+
private static Set<ExpressionRoleMapping> getRoleMappingMetadataMappings(ClusterState clusterState) {
283+
RoleMappingMetadata roleMappingMetadata = RoleMappingMetadata.getFromClusterState(clusterState);
284+
if (roleMappingMetadata != null) {
285+
return roleMappingMetadata.getRoleMappings();
286+
}
287+
return Set.of();
288+
}
289+
270290
@Override
271291
public void clusterChanged(ClusterChangedEvent event) {
272292
if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
@@ -284,6 +304,9 @@ public void clusterChanged(ClusterChangedEvent event) {
284304
Tuple<Boolean, Boolean> available = checkIndexAvailable(event.state());
285305
final boolean indexAvailableForWrite = available.v1();
286306
final boolean indexAvailableForSearch = available.v2();
307+
final Set<String> reservedStateRoleMappingNames = getFileSettingsMetadataHandlerRoleMappingKeys(event.state());
308+
final boolean reservedRoleMappingsSynced = reservedStateRoleMappingNames.size() == getRoleMappingMetadataMappings(event.state())
309+
.size();
287310
final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state());
288311
final int migrationsVersion = getMigrationVersionFromIndexMetadata(indexMetadata);
289312
final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion = getMinSecurityIndexMappingVersion(event.state());
@@ -314,6 +337,7 @@ public void clusterChanged(ClusterChangedEvent event) {
314337
indexAvailableForWrite,
315338
mappingIsUpToDate,
316339
createdOnLatestVersion,
340+
reservedRoleMappingsSynced,
317341
migrationsVersion,
318342
minClusterMappingVersion,
319343
indexMappingVersion,
@@ -323,7 +347,8 @@ public void clusterChanged(ClusterChangedEvent event) {
323347
indexUUID,
324348
allSecurityFeatures.stream()
325349
.filter(feature -> featureService.clusterHasFeature(event.state(), feature))
326-
.collect(Collectors.toSet())
350+
.collect(Collectors.toSet()),
351+
reservedStateRoleMappingNames
327352
);
328353
this.state = newState;
329354

@@ -334,6 +359,10 @@ public void clusterChanged(ClusterChangedEvent event) {
334359
}
335360
}
336361

362+
public Set<String> getReservedStateRoleMappingNames() {
363+
return state.reservedStateRoleMappingNames;
364+
}
365+
337366
public static int getMigrationVersionFromIndexMetadata(IndexMetadata indexMetadata) {
338367
Map<String, String> customMetadata = indexMetadata == null ? null : indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY);
339368
if (customMetadata == null) {
@@ -438,7 +467,8 @@ private Tuple<Boolean, Boolean> checkIndexAvailable(ClusterState state) {
438467

439468
public boolean isEligibleSecurityMigration(SecurityMigrations.SecurityMigration securityMigration) {
440469
return state.securityFeatures.containsAll(securityMigration.nodeFeaturesRequired())
441-
&& state.indexMappingVersion >= securityMigration.minMappingVersion();
470+
&& state.indexMappingVersion >= securityMigration.minMappingVersion()
471+
&& securityMigration.checkPreConditions(state);
442472
}
443473

444474
public boolean isReadyForSecurityMigration(SecurityMigrations.SecurityMigration securityMigration) {
@@ -671,13 +701,15 @@ public static class State {
671701
false,
672702
false,
673703
false,
704+
false,
674705
null,
675706
null,
676707
null,
677708
null,
678709
null,
679710
null,
680711
null,
712+
Set.of(),
681713
Set.of()
682714
);
683715
public final Instant creationTime;
@@ -686,6 +718,7 @@ public static class State {
686718
public final boolean indexAvailableForWrite;
687719
public final boolean mappingUpToDate;
688720
public final boolean createdOnLatestVersion;
721+
public final boolean reservedRoleMappingsSynced;
689722
public final Integer migrationsVersion;
690723
// Min mapping version supported by the descriptors in the cluster
691724
public final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion;
@@ -696,6 +729,7 @@ public static class State {
696729
public final IndexMetadata.State indexState;
697730
public final String indexUUID;
698731
public final Set<NodeFeature> securityFeatures;
732+
public final Set<String> reservedStateRoleMappingNames;
699733

700734
public State(
701735
Instant creationTime,
@@ -704,14 +738,16 @@ public State(
704738
boolean indexAvailableForWrite,
705739
boolean mappingUpToDate,
706740
boolean createdOnLatestVersion,
741+
boolean reservedRoleMappingsSynced,
707742
Integer migrationsVersion,
708743
SystemIndexDescriptor.MappingsVersion minClusterMappingVersion,
709744
Integer indexMappingVersion,
710745
String concreteIndexName,
711746
ClusterHealthStatus indexHealth,
712747
IndexMetadata.State indexState,
713748
String indexUUID,
714-
Set<NodeFeature> securityFeatures
749+
Set<NodeFeature> securityFeatures,
750+
Set<String> reservedStateRoleMappingNames
715751
) {
716752
this.creationTime = creationTime;
717753
this.isIndexUpToDate = isIndexUpToDate;
@@ -720,13 +756,15 @@ public State(
720756
this.mappingUpToDate = mappingUpToDate;
721757
this.migrationsVersion = migrationsVersion;
722758
this.createdOnLatestVersion = createdOnLatestVersion;
759+
this.reservedRoleMappingsSynced = reservedRoleMappingsSynced;
723760
this.minClusterMappingVersion = minClusterMappingVersion;
724761
this.indexMappingVersion = indexMappingVersion;
725762
this.concreteIndexName = concreteIndexName;
726763
this.indexHealth = indexHealth;
727764
this.indexState = indexState;
728765
this.indexUUID = indexUUID;
729766
this.securityFeatures = securityFeatures;
767+
this.reservedStateRoleMappingNames = reservedStateRoleMappingNames;
730768
}
731769

732770
@Override
@@ -740,13 +778,15 @@ public boolean equals(Object o) {
740778
&& indexAvailableForWrite == state.indexAvailableForWrite
741779
&& mappingUpToDate == state.mappingUpToDate
742780
&& createdOnLatestVersion == state.createdOnLatestVersion
781+
&& reservedRoleMappingsSynced == state.reservedRoleMappingsSynced
743782
&& Objects.equals(indexMappingVersion, state.indexMappingVersion)
744783
&& Objects.equals(migrationsVersion, state.migrationsVersion)
745784
&& Objects.equals(minClusterMappingVersion, state.minClusterMappingVersion)
746785
&& Objects.equals(concreteIndexName, state.concreteIndexName)
747786
&& indexHealth == state.indexHealth
748787
&& indexState == state.indexState
749-
&& Objects.equals(securityFeatures, state.securityFeatures);
788+
&& Objects.equals(securityFeatures, state.securityFeatures)
789+
&& Objects.equals(reservedStateRoleMappingNames, state.reservedStateRoleMappingNames);
750790
}
751791

752792
public boolean indexExists() {
@@ -762,12 +802,14 @@ public int hashCode() {
762802
indexAvailableForWrite,
763803
mappingUpToDate,
764804
createdOnLatestVersion,
805+
reservedRoleMappingsSynced,
765806
migrationsVersion,
766807
minClusterMappingVersion,
767808
indexMappingVersion,
768809
concreteIndexName,
769810
indexHealth,
770-
securityFeatures
811+
securityFeatures,
812+
reservedStateRoleMappingNames
771813
);
772814
}
773815

@@ -786,6 +828,8 @@ public String toString() {
786828
+ mappingUpToDate
787829
+ ", createdOnLatestVersion="
788830
+ createdOnLatestVersion
831+
+ ", reservedRoleMappingsSynced="
832+
+ reservedRoleMappingsSynced
789833
+ ", migrationsVersion="
790834
+ migrationsVersion
791835
+ ", minClusterMappingVersion="
@@ -804,6 +848,8 @@ public String toString() {
804848
+ '\''
805849
+ ", securityFeatures="
806850
+ securityFeatures
851+
+ ", reservedStateRoleMappingNames="
852+
+ reservedStateRoleMappingNames
807853
+ '}';
808854
}
809855
}

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java‎

Lines changed: 91 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.apache.logging.log4j.Logger;
1212
import org.elasticsearch.action.ActionListener;
1313
import org.elasticsearch.action.search.SearchRequest;
14+
import org.elasticsearch.action.support.WriteRequest;
1415
import org.elasticsearch.client.internal.Client;
1516
import org.elasticsearch.features.NodeFeature;
1617
import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -20,12 +21,23 @@
2021
import org.elasticsearch.script.Script;
2122
import org.elasticsearch.script.ScriptType;
2223
import org.elasticsearch.search.builder.SearchSourceBuilder;
24+
import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
25+
import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequestBuilder;
26+
import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
27+
import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequestBuilder;
28+
import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
2329

30+
import java.util.Arrays;
2431
import java.util.Collections;
32+
import java.util.Iterator;
33+
import java.util.List;
2534
import java.util.Map;
2635
import java.util.Set;
2736
import java.util.TreeMap;
2837

38+
import static org.elasticsearch.TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE;
39+
import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
40+
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
2941
import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS;
3042

3143
/**
@@ -52,6 +64,16 @@ public interface SecurityMigration {
5264
*/
5365
Set<NodeFeature> nodeFeaturesRequired();
5466

67+
/**
68+
* Check that any pre-conditions are met before launching migration
69+
*
70+
* @param securityIndexManagerState current state of the security index
71+
* @return true if pre-conditions met, otherwise false
72+
*/
73+
default boolean checkPreConditions(SecurityIndexManager.State securityIndexManagerState) {
74+
return true;
75+
}
76+
5577
/**
5678
* The min mapping version required to support this migration. This makes sure that the index has at least the min mapping that is
5779
* required to support the migration.
@@ -62,11 +84,11 @@ public interface SecurityMigration {
6284
}
6385

6486
public static final Integer ROLE_METADATA_FLATTENED_MIGRATION_VERSION = 1;
87+
public static final Integer ROLE_MAPPING_CLEANUP_DUPLICATES = 2;
88+
private static final Logger logger = LogManager.getLogger(SecurityMigration.class);
6589

6690
public static final TreeMap<Integer, SecurityMigration> MIGRATIONS_BY_VERSION = new TreeMap<>(
6791
Map.of(ROLE_METADATA_FLATTENED_MIGRATION_VERSION, new SecurityMigration() {
68-
private static final Logger logger = LogManager.getLogger(SecurityMigration.class);
69-
7092
@Override
7193
public void migrate(SecurityIndexManager indexManager, Client client, ActionListener<Void> listener) {
7294
BoolQueryBuilder filterQuery = new BoolQueryBuilder().filter(QueryBuilders.termQuery("type", "role"))
@@ -119,6 +141,73 @@ public Set<NodeFeature> nodeFeaturesRequired() {
119141
public int minMappingVersion() {
120142
return ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS.id();
121143
}
144+
}, ROLE_MAPPING_CLEANUP_DUPLICATES, new SecurityMigration() {
145+
@Override
146+
public void migrate(SecurityIndexManager indexManager, Client client, ActionListener<Void> listener) {
147+
Set<String> clusterStateRoleMappingNames = indexManager.getReservedStateRoleMappingNames();
148+
149+
// No role mappings in cluster state -> no cleanup needed
150+
if (clusterStateRoleMappingNames.isEmpty()) {
151+
listener.onResponse(null);
152+
return;
153+
}
154+
155+
getNativeRoleMappings(client, ActionListener.wrap(roleMappings -> {
156+
logger.info("Found [" + roleMappings.size() + "] role mappings to cleanup in .security index.");
157+
deleteNativeRoleMappings(client, roleMappings.iterator(), listener);
158+
}, listener::onFailure), clusterStateRoleMappingNames.toArray(String[]::new));
159+
}
160+
161+
private void getNativeRoleMappings(Client client, ActionListener<List<String>> listener, String... mappingNames) {
162+
executeAsyncWithOrigin(
163+
client,
164+
SECURITY_ORIGIN,
165+
GetRoleMappingsAction.INSTANCE,
166+
new GetRoleMappingsRequestBuilder(client).names(mappingNames).request(),
167+
ActionListener.wrap(
168+
response -> listener.onResponse(Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList()),
169+
listener::onFailure
170+
)
171+
);
172+
}
173+
174+
private void deleteNativeRoleMappings(Client client, Iterator<String> namesIterator, ActionListener<Void> listener) {
175+
String name = namesIterator.next();
176+
executeAsyncWithOrigin(
177+
client,
178+
SECURITY_ORIGIN,
179+
DeleteRoleMappingAction.INSTANCE,
180+
new DeleteRoleMappingRequestBuilder(client).name(name).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).request(),
181+
ActionListener.wrap(response -> {
182+
if (response.isFound() == false) {
183+
logger.warn("Expected role mapping [" + name + "] not found during role mapping clean up.");
184+
} else {
185+
logger.info("Deleted duplicated role mapping [" + name + "] from .security index");
186+
}
187+
if (namesIterator.hasNext()) {
188+
deleteNativeRoleMappings(client, namesIterator, listener);
189+
} else {
190+
listener.onResponse(null);
191+
}
192+
}, listener::onFailure)
193+
);
194+
}
195+
196+
@Override
197+
public boolean checkPreConditions(SecurityIndexManager.State securityIndexManagerState) {
198+
// If there are operator defined role mappings, make sure they've been loaded in to cluster state before launching migration
199+
return securityIndexManagerState.reservedRoleMappingsSynced;
200+
}
201+
202+
@Override
203+
public Set<NodeFeature> nodeFeaturesRequired() {
204+
return Set.of(SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP);
205+
}
206+
207+
@Override
208+
public int minMappingVersion() {
209+
return ADD_MANAGE_ROLES_PRIVILEGE.id();
210+
}
122211
})
123212
);
124213
}

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ public class SecuritySystemIndices {
6161
public static final NodeFeature SECURITY_PROFILE_ORIGIN_FEATURE = new NodeFeature("security.security_profile_origin");
6262
public static final NodeFeature SECURITY_MIGRATION_FRAMEWORK = new NodeFeature("security.migration_framework");
6363
public static final NodeFeature SECURITY_ROLES_METADATA_FLATTENED = new NodeFeature("security.roles_metadata_flattened");
64+
public static final NodeFeature SECURITY_ROLE_MAPPING_CLEANUP = new NodeFeature("security.role_mapping_cleanup");
6465

6566
/**
6667
* Security managed index mappings used to be updated based on the product version. They are now updated based on per-index mappings

0 commit comments

Comments
 (0)