[RemoteStore]: Add support for repository with Server side encryption enabled and client side encryption as well based on a flag.

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use Lucene `pack` method for `half_float` and `usigned_long` when using `ApproximatePointRangeQuery`.
 - Add a mapper for context aware segments grouping criteria ([#19233](https://github.com/opensearch-project/OpenSearch/pull/19233))
 - Return full error for GRPC error response ([#19568](https://github.com/opensearch-project/OpenSearch/pull/19568))
+- Add support for repository with Server side encryption enabled and client side encryption as well based on a flag. ([#19630)](https://github.com/opensearch-project/OpenSearch/pull/19630))
 - Add pluggable gRPC interceptors with explicit ordering([#19005](https://github.com/opensearch-project/OpenSearch/pull/19005))
 
 ### Changed

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java

@@ -230,7 +230,7 @@ public boolean serverSideEncryptionBucketKey() {
      * null as the S3 client ignores null header values
      */
     public String serverSideEncryptionEncryptionContext() {
-        return serverSideEncryptionEncryptionContext.isEmpty()
+        return serverSideEncryptionEncryptionContext == null || serverSideEncryptionEncryptionContext.isEmpty()
             ? null
             : Base64.getEncoder().encodeToString(serverSideEncryptionEncryptionContext.getBytes(StandardCharsets.UTF_8));
     }
@@ -239,7 +239,7 @@ public String serverSideEncryptionEncryptionContext() {
      * Returns the expected bucket owner if set, else null as the S3 client ignores null header values
      */
     public String expectedBucketOwner() {
-        return expectedBucketOwner.isEmpty() ? null : expectedBucketOwner;
+        return expectedBucketOwner == null || expectedBucketOwner.isEmpty() ? null : expectedBucketOwner;
     }
 
     public long bufferSizeInBytes() {

server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/TransportRemoteStoreMetadataAction.java

@@ -198,7 +198,9 @@ private Map<String, Map<String, Object>> getSegmentMetadata(
             IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexMetadata.getSettings()),
             index.getUUID(),
             shardId,
-            indexSettings.getRemoteStorePathStrategy()
+            indexSettings.getRemoteStorePathStrategy(),
+            null,
+            RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata())
         );
 
         Map<String, RemoteSegmentMetadata> segmentMetadataMapWithFilenames = remoteDirectory.readLatestNMetadataFiles(5);
@@ -257,7 +259,8 @@ private Map<String, Map<String, Object>> getTranslogMetadataFiles(
             tracker,
             indexSettings.getRemoteStorePathStrategy(),
             new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings()),
-            RemoteStoreUtils.determineTranslogMetadataEnabled(indexMetadata)
+            RemoteStoreUtils.determineTranslogMetadataEnabled(indexMetadata),
+            RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata())
         );
 
         Map<String, TranslogTransferMetadata> metadataMap = manager.readLatestNMetadataFiles(5);

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -371,6 +371,7 @@ public Iterator<Setting<?>> settings() {
     );
 
     public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled";
+    // public static final String SETTING_REMOTE_STORE_SSE_ENABLED = "index.remote_store.sse.enabled";
     public static final String SETTING_INDEX_APPEND_ONLY_ENABLED = "index.append_only.enabled";
 
     public static final String SETTING_REMOTE_SEGMENT_STORE_REPOSITORY = "index.remote_store.segment.repository";
@@ -992,6 +993,7 @@ public Iterator<Setting<?>> settings() {
     public static final String KEY_PRIMARY_TERMS = "primary_terms";
     public static final String REMOTE_STORE_CUSTOM_KEY = "remote_store";
     public static final String TRANSLOG_METADATA_KEY = "translog_metadata";
+    public static final String REMOTE_STORE_SSE_ENABLED_INDEX_KEY = "sse_enabled_index";
     public static final String CONTEXT_KEY = "context";
     public static final String INGESTION_SOURCE_KEY = "ingestion_source";
     public static final String INGESTION_STATUS_KEY = "ingestion_status";

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -632,7 +632,8 @@ static Optional<String> validateOverlap(Set<String> requestSettings, Settings co
     IndexMetadata buildAndValidateTemporaryIndexMetadata(
         final Settings aggregatedIndexSettings,
         final CreateIndexClusterStateUpdateRequest request,
-        final int routingNumShards
+        final int routingNumShards,
+        final ClusterState clusterState
     ) {
 
         final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings);
@@ -642,7 +643,7 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
         tmpImdBuilder.setRoutingNumShards(routingNumShards);
         tmpImdBuilder.settings(aggregatedIndexSettings);
         tmpImdBuilder.system(isSystem);
-        addRemoteStoreCustomMetadata(tmpImdBuilder, true);
+        addRemoteStoreCustomMetadata(tmpImdBuilder, true, clusterState);
 
         if (request.context() != null) {
             tmpImdBuilder.context(request.context());
@@ -661,7 +662,9 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
      * @param tmpImdBuilder     index metadata builder.
      * @param assertNullOldType flag to verify that the old remote store path type is null
      */
-    public void addRemoteStoreCustomMetadata(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) {
+    public void addRemoteStoreCustomMetadata(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType, ClusterState clusterState) {
+
+        boolean isRestoreFromSnapshot = !assertNullOldType;
         if (remoteStoreCustomMetadataResolver == null) {
             return;
         }
@@ -676,6 +679,27 @@ public void addRemoteStoreCustomMetadata(IndexMetadata.Builder tmpImdBuilder, bo
         boolean isTranslogMetadataEnabled = remoteStoreCustomMetadataResolver.isTranslogMetadataEnabled();
         remoteCustomData.put(IndexMetadata.TRANSLOG_METADATA_KEY, Boolean.toString(isTranslogMetadataEnabled));
 
+        Optional<DiscoveryNode> remoteNode = clusterState.nodes()
+            .getNodes()
+            .values()
+            .stream()
+            .filter(DiscoveryNode::isRemoteStoreNode)
+            .findFirst();
+
+        String sseEnabledIndex = existingCustomData == null
+            ? null
+            : existingCustomData.get(IndexMetadata.REMOTE_STORE_SSE_ENABLED_INDEX_KEY);
+        if (isRestoreFromSnapshot && sseEnabledIndex != null) {
+            remoteCustomData.put(IndexMetadata.REMOTE_STORE_SSE_ENABLED_INDEX_KEY, sseEnabledIndex);
+        }
+
+        if (remoteNode.isPresent()
+            && !isRestoreFromSnapshot
+            && RemoteStoreSettings.isServerSideEncryptionRepoEnabled(clusterState.nodes().getMinNodeVersion())
+            && RemoteStoreNodeAttribute.isRemoteStoreServerSideEncryptionEnabled(remoteNode.get().getAttributes())) {
+            remoteCustomData.put(IndexMetadata.REMOTE_STORE_SSE_ENABLED_INDEX_KEY, Boolean.toString(true));
+        }
+
         // Determine the path type for use using the remoteStorePathResolver.
         RemoteStorePathStrategy newPathStrategy = remoteStoreCustomMetadataResolver.getPathStrategy();
         remoteCustomData.put(PathType.NAME, newPathStrategy.getType().name());
@@ -730,7 +754,9 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
             clusterService.getClusterSettings()
         );
         int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
-        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
+        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards, currentState);
+
+        // buildServerSideEncryptionSetting(tmpImd, clusterService.getClusterSettings(), false);
 
         return applyCreateIndexWithTemporaryService(
             currentState,
@@ -755,6 +781,23 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
         );
     }
 
+    // private void buildServerSideEncryptionSetting(IndexMetadata tmpImd, ClusterSettings clusterSettings, boolean isRestore) {
+    //
+    // if (isRestore)
+    // if (remoteStoreCustomMetadataResolver == null) {
+    // return;
+    // }
+    //
+    // Map<String, String> remoteCustomMetadata = tmpImd.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
+    //
+    // String sseEnabledIndex = existingCustomData == null ? null : existingCustomData.get(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED);
+    // if (assertNullOldType || sseEnabledIndex != null) {
+    //
+    // }
+    // remoteCustomData.put(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, Boolean.toString(true));
+    //
+    // }
+
     private ClusterState applyCreateIndexRequestWithV2Template(
         final ClusterState currentState,
         final CreateIndexClusterStateUpdateRequest request,
@@ -795,7 +838,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(
             clusterService.getClusterSettings()
         );
         int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
-        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
+        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards, currentState);
 
         return applyCreateIndexWithTemporaryService(
             currentState,
@@ -879,7 +922,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(
             clusterService.getClusterSettings()
         );
         final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata);
-        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
+        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards, currentState);
 
         return applyCreateIndexWithTemporaryService(
             currentState,
@@ -1177,8 +1220,8 @@ public static void updateRemoteStoreSettings(
                 .findFirst();
 
             if (remoteNode.isPresent()) {
-                translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes());
                 segmentRepo = RemoteStoreNodeAttribute.getSegmentRepoName(remoteNode.get().getAttributes());
+                translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes());
                 if (segmentRepo != null) {
                     settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true).put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo);
                     if (translogRepo != null) {

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -810,6 +810,8 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED,
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX,
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX,
+                // Server Side encryption enabled
+                RemoteStoreSettings.CLUSTER_SERVER_SIDE_ENCRYPTION_REPO_ENABLED,
 
                 // Snapshot related Settings
                 BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING,

server/src/main/java/org/opensearch/index/IndexService.java

@@ -84,6 +84,7 @@
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.query.SearchIndexNameMatcher;
 import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
+import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.seqno.RetentionLeaseSyncer;
 import org.opensearch.index.shard.IndexEventListener;
 import org.opensearch.index.shard.IndexShard;
@@ -701,7 +702,8 @@ public synchronized IndexShard createShard(
                         this.indexSettings.getUUID(),
                         shardId,
                         this.indexSettings.getRemoteStorePathStrategy(),
-                        this.indexSettings.getRemoteStoreSegmentPathPrefix()
+                        this.indexSettings.getRemoteStoreSegmentPathPrefix(),
+                        RemoteStoreUtils.isServerSideEncryptionEnabledIndex(this.indexSettings.getIndexMetadata())
                     );
                 }
                 // When an instance of Store is created, a shardlock is created which is released on closing the instance of store.

server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java

@@ -248,6 +248,11 @@ public static Map<String, String> determineRemoteStoreCustomMetadataDuringMigrat
         return remoteCustomData;
     }
 
+    public static boolean isServerSideEncryptionEnabledIndex(IndexMetadata indexMetadata) {
+        Map<String, String> remoteCustomData = indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
+        return remoteCustomData != null && "true".equalsIgnoreCase(remoteCustomData.get(IndexMetadata.REMOTE_STORE_SSE_ENABLED_INDEX_KEY));
+    }
+
     /**
      * Fetches segment and translog repository names from remote store node attributes.
      * Returns a blank {@link HashMap} if the cluster does not contain any remote nodes.

server/src/main/java/org/opensearch/index/shard/IndexShard.java

@@ -162,6 +162,7 @@
 import org.opensearch.index.remote.RemoteSegmentStats;
 import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
+import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.search.stats.SearchStats;
 import org.opensearch.index.search.stats.ShardSearchStats;
 import org.opensearch.index.seqno.ReplicationTracker;
@@ -5289,7 +5290,8 @@ public void deleteTranslogFilesFromRemoteTranslog() throws IOException {
             getThreadPool(),
             indexSettings.getRemoteStorePathStrategy(),
             remoteStoreSettings,
-            indexSettings().isTranslogMetadataEnabled()
+            indexSettings().isTranslogMetadataEnabled(),
+            RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata())
         );
     }
 
@@ -5312,7 +5314,8 @@ public void syncTranslogFilesFromRemoteTranslog() throws IOException {
             shardId,
             indexSettings.getRemoteStorePathStrategy(),
             indexSettings().isTranslogMetadataEnabled(),
-            0
+            0,
+            RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata())
         );
     }
 
@@ -5322,6 +5325,24 @@ public void syncTranslogFilesFromGivenRemoteTranslog(
         RemoteStorePathStrategy remoteStorePathStrategy,
         boolean isTranslogMetadataEnabled,
         long timestamp
+    ) throws IOException {
+        this.syncTranslogFilesFromGivenRemoteTranslog(
+            repository,
+            shardId,
+            remoteStorePathStrategy,
+            isTranslogMetadataEnabled,
+            timestamp,
+            false
+        );
+    }
+
+    public void syncTranslogFilesFromGivenRemoteTranslog(
+        Repository repository,
+        ShardId shardId,
+        RemoteStorePathStrategy remoteStorePathStrategy,
+        boolean isTranslogMetadataEnabled,
+        long timestamp,
+        boolean isServerSideEncryptionEnabled
     ) throws IOException {
         RemoteFsTranslog.download(
             repository,
@@ -5333,7 +5354,8 @@ public void syncTranslogFilesFromGivenRemoteTranslog(
             logger,
             shouldSeedRemoteStore(),
             isTranslogMetadataEnabled,
-            timestamp
+            timestamp,
+            isServerSideEncryptionEnabled
         );
     }
 

server/src/main/java/org/opensearch/index/shard/StoreRecovery.java

@@ -56,6 +56,7 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.engine.Engine;
 import org.opensearch.index.engine.EngineException;
 import org.opensearch.index.mapper.MapperService;
@@ -422,7 +423,9 @@ void recoverFromSnapshotAndRemoteStore(
                     remoteStoreRepository,
                     indexUUID,
                     shardId,
-                    shallowCopyShardMetadata.getRemoteStorePathStrategy()
+                    shallowCopyShardMetadata.getRemoteStorePathStrategy(),
+                    null,
+                    RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexShard.indexSettings.getIndexMetadata())
                 );
                 RemoteSegmentMetadata remoteSegmentMetadata = sourceRemoteDirectory.initializeToSpecificCommit(
                     primaryTerm,
@@ -498,12 +501,15 @@ void recoverShallowSnapshotV2(
                             prevIndexMetadata.getSettings()
                         );
                     }
+                    IndexSettings indexSettings = new IndexSettings(prevIndexMetadata, prevIndexMetadata.getSettings());
                     RemoteStorePathStrategy remoteStorePathStrategy = RemoteStoreUtils.determineRemoteStorePathStrategy(prevIndexMetadata);
                     RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory(
                         remoteSegmentStoreRepository,
                         prevIndexMetadata.getIndexUUID(),
                         shardId,
-                        remoteStorePathStrategy
+                        remoteStorePathStrategy,
+                        null,
+                        RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexShard.indexSettings.getIndexMetadata())
                     );
                     RemoteSegmentMetadata remoteSegmentMetadata = sourceRemoteDirectory.initializeToSpecificTimestamp(
                         recoverySource.pinnedTimestamp()
@@ -523,7 +529,8 @@ void recoverShallowSnapshotV2(
                         new ShardId(prevIndexMetadata.getIndex(), shardId.id()),
                         remoteStorePathStrategy,
                         RemoteStoreUtils.determineTranslogMetadataEnabled(prevIndexMetadata),
-                        recoverySource.pinnedTimestamp()
+                        recoverySource.pinnedTimestamp(),
+                        RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexShard.indexSettings.getIndexMetadata())
                     );
 
                     assert indexShard.shardRouting.primary() : "only primary shards can recover from store";