diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cf0e78220f4a..b67c4f7b64ae0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Use Lucene `pack` method for `half_float` and `usigned_long` when using `ApproximatePointRangeQuery`. - Add a mapper for context aware segments grouping criteria ([#19233](https://github.com/opensearch-project/OpenSearch/pull/19233)) - Return full error for GRPC error response ([#19568](https://github.com/opensearch-project/OpenSearch/pull/19568)) +- Add support for repository with Server side encryption enabled and client side encryption as well based on a flag. ([#19630)](https://github.com/opensearch-project/OpenSearch/pull/19630)) - Add pluggable gRPC interceptors with explicit ordering([#19005](https://github.com/opensearch-project/OpenSearch/pull/19005)) ### Changed diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java index 5b677a49694a2..677b3d3fdfd5d 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java @@ -230,7 +230,7 @@ public boolean serverSideEncryptionBucketKey() { * null as the S3 client ignores null header values */ public String serverSideEncryptionEncryptionContext() { - return serverSideEncryptionEncryptionContext.isEmpty() + return serverSideEncryptionEncryptionContext == null || serverSideEncryptionEncryptionContext.isEmpty() ? null : Base64.getEncoder().encodeToString(serverSideEncryptionEncryptionContext.getBytes(StandardCharsets.UTF_8)); } @@ -239,7 +239,7 @@ public String serverSideEncryptionEncryptionContext() { * Returns the expected bucket owner if set, else null as the S3 client ignores null header values */ public String expectedBucketOwner() { - return expectedBucketOwner.isEmpty() ? null : expectedBucketOwner; + return expectedBucketOwner == null || expectedBucketOwner.isEmpty() ? null : expectedBucketOwner; } public long bufferSizeInBytes() { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/TransportRemoteStoreMetadataAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/TransportRemoteStoreMetadataAction.java index 16c29d7586a98..c72820d8df19a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/TransportRemoteStoreMetadataAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/TransportRemoteStoreMetadataAction.java @@ -198,7 +198,9 @@ private Map> getSegmentMetadata( IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexMetadata.getSettings()), index.getUUID(), shardId, - indexSettings.getRemoteStorePathStrategy() + indexSettings.getRemoteStorePathStrategy(), + null, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata()) ); Map segmentMetadataMapWithFilenames = remoteDirectory.readLatestNMetadataFiles(5); @@ -257,7 +259,8 @@ private Map> getTranslogMetadataFiles( tracker, indexSettings.getRemoteStorePathStrategy(), new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings()), - RemoteStoreUtils.determineTranslogMetadataEnabled(indexMetadata) + RemoteStoreUtils.determineTranslogMetadataEnabled(indexMetadata), + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata()) ); Map metadataMap = manager.readLatestNMetadataFiles(5); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 4d53a547db714..a5f84d7101855 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -371,6 +371,7 @@ public Iterator> settings() { ); public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled"; + // public static final String SETTING_REMOTE_STORE_SSE_ENABLED = "index.remote_store.sse.enabled"; public static final String SETTING_INDEX_APPEND_ONLY_ENABLED = "index.append_only.enabled"; public static final String SETTING_REMOTE_SEGMENT_STORE_REPOSITORY = "index.remote_store.segment.repository"; @@ -992,6 +993,7 @@ public Iterator> settings() { public static final String KEY_PRIMARY_TERMS = "primary_terms"; public static final String REMOTE_STORE_CUSTOM_KEY = "remote_store"; public static final String TRANSLOG_METADATA_KEY = "translog_metadata"; + public static final String REMOTE_STORE_SSE_ENABLED_INDEX_KEY = "sse_enabled_index"; public static final String CONTEXT_KEY = "context"; public static final String INGESTION_SOURCE_KEY = "ingestion_source"; public static final String INGESTION_STATUS_KEY = "ingestion_status"; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index a889091140d12..3a25fc8a66c33 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -632,7 +632,8 @@ static Optional validateOverlap(Set requestSettings, Settings co IndexMetadata buildAndValidateTemporaryIndexMetadata( final Settings aggregatedIndexSettings, final CreateIndexClusterStateUpdateRequest request, - final int routingNumShards + final int routingNumShards, + final ClusterState clusterState ) { final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings); @@ -642,7 +643,7 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata( tmpImdBuilder.setRoutingNumShards(routingNumShards); tmpImdBuilder.settings(aggregatedIndexSettings); tmpImdBuilder.system(isSystem); - addRemoteStoreCustomMetadata(tmpImdBuilder, true); + addRemoteStoreCustomMetadata(tmpImdBuilder, true, clusterState); if (request.context() != null) { tmpImdBuilder.context(request.context()); @@ -661,7 +662,9 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata( * @param tmpImdBuilder index metadata builder. * @param assertNullOldType flag to verify that the old remote store path type is null */ - public void addRemoteStoreCustomMetadata(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) { + public void addRemoteStoreCustomMetadata(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType, ClusterState clusterState) { + + boolean isRestoreFromSnapshot = !assertNullOldType; if (remoteStoreCustomMetadataResolver == null) { return; } @@ -676,6 +679,27 @@ public void addRemoteStoreCustomMetadata(IndexMetadata.Builder tmpImdBuilder, bo boolean isTranslogMetadataEnabled = remoteStoreCustomMetadataResolver.isTranslogMetadataEnabled(); remoteCustomData.put(IndexMetadata.TRANSLOG_METADATA_KEY, Boolean.toString(isTranslogMetadataEnabled)); + Optional remoteNode = clusterState.nodes() + .getNodes() + .values() + .stream() + .filter(DiscoveryNode::isRemoteStoreNode) + .findFirst(); + + String sseEnabledIndex = existingCustomData == null + ? null + : existingCustomData.get(IndexMetadata.REMOTE_STORE_SSE_ENABLED_INDEX_KEY); + if (isRestoreFromSnapshot && sseEnabledIndex != null) { + remoteCustomData.put(IndexMetadata.REMOTE_STORE_SSE_ENABLED_INDEX_KEY, sseEnabledIndex); + } + + if (remoteNode.isPresent() + && !isRestoreFromSnapshot + && RemoteStoreSettings.isServerSideEncryptionRepoEnabled(clusterState.nodes().getMinNodeVersion()) + && RemoteStoreNodeAttribute.isRemoteStoreServerSideEncryptionEnabled(remoteNode.get().getAttributes())) { + remoteCustomData.put(IndexMetadata.REMOTE_STORE_SSE_ENABLED_INDEX_KEY, Boolean.toString(true)); + } + // Determine the path type for use using the remoteStorePathResolver. RemoteStorePathStrategy newPathStrategy = remoteStoreCustomMetadataResolver.getPathStrategy(); remoteCustomData.put(PathType.NAME, newPathStrategy.getType().name()); @@ -730,7 +754,9 @@ private ClusterState applyCreateIndexRequestWithV1Templates( clusterService.getClusterSettings() ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards, currentState); + + // buildServerSideEncryptionSetting(tmpImd, clusterService.getClusterSettings(), false); return applyCreateIndexWithTemporaryService( currentState, @@ -755,6 +781,23 @@ private ClusterState applyCreateIndexRequestWithV1Templates( ); } + // private void buildServerSideEncryptionSetting(IndexMetadata tmpImd, ClusterSettings clusterSettings, boolean isRestore) { + // + // if (isRestore) + // if (remoteStoreCustomMetadataResolver == null) { + // return; + // } + // + // Map remoteCustomMetadata = tmpImd.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY); + // + // String sseEnabledIndex = existingCustomData == null ? null : existingCustomData.get(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED); + // if (assertNullOldType || sseEnabledIndex != null) { + // + // } + // remoteCustomData.put(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, Boolean.toString(true)); + // + // } + private ClusterState applyCreateIndexRequestWithV2Template( final ClusterState currentState, final CreateIndexClusterStateUpdateRequest request, @@ -795,7 +838,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( clusterService.getClusterSettings() ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards, currentState); return applyCreateIndexWithTemporaryService( currentState, @@ -879,7 +922,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( clusterService.getClusterSettings() ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards, currentState); return applyCreateIndexWithTemporaryService( currentState, @@ -1177,8 +1220,8 @@ public static void updateRemoteStoreSettings( .findFirst(); if (remoteNode.isPresent()) { - translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes()); segmentRepo = RemoteStoreNodeAttribute.getSegmentRepoName(remoteNode.get().getAttributes()); + translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes()); if (segmentRepo != null) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true).put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo); if (translogRepo != null) { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 572d8ab2914c7..bf5ba8515ed1e 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -810,6 +810,8 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED, RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX, + // Server Side encryption enabled + RemoteStoreSettings.CLUSTER_SERVER_SIDE_ENCRYPTION_REPO_ENABLED, // Snapshot related Settings BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 6612ea732942d..9c1bcf77f7db2 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -84,6 +84,7 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.SearchIndexNameMatcher; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; @@ -701,7 +702,8 @@ public synchronized IndexShard createShard( this.indexSettings.getUUID(), shardId, this.indexSettings.getRemoteStorePathStrategy(), - this.indexSettings.getRemoteStoreSegmentPathPrefix() + this.indexSettings.getRemoteStoreSegmentPathPrefix(), + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(this.indexSettings.getIndexMetadata()) ); } // When an instance of Store is created, a shardlock is created which is released on closing the instance of store. diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index 32a1ca0e5d5ab..3b3b9b729bc39 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -248,6 +248,11 @@ public static Map determineRemoteStoreCustomMetadataDuringMigrat return remoteCustomData; } + public static boolean isServerSideEncryptionEnabledIndex(IndexMetadata indexMetadata) { + Map remoteCustomData = indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY); + return remoteCustomData != null && "true".equalsIgnoreCase(remoteCustomData.get(IndexMetadata.REMOTE_STORE_SSE_ENABLED_INDEX_KEY)); + } + /** * Fetches segment and translog repository names from remote store node attributes. * Returns a blank {@link HashMap} if the cluster does not contain any remote nodes. diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 0a365e4d756d8..8c33c9e0763ff 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -162,6 +162,7 @@ import org.opensearch.index.remote.RemoteSegmentStats; import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.search.stats.SearchStats; import org.opensearch.index.search.stats.ShardSearchStats; import org.opensearch.index.seqno.ReplicationTracker; @@ -5289,7 +5290,8 @@ public void deleteTranslogFilesFromRemoteTranslog() throws IOException { getThreadPool(), indexSettings.getRemoteStorePathStrategy(), remoteStoreSettings, - indexSettings().isTranslogMetadataEnabled() + indexSettings().isTranslogMetadataEnabled(), + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata()) ); } @@ -5312,7 +5314,8 @@ public void syncTranslogFilesFromRemoteTranslog() throws IOException { shardId, indexSettings.getRemoteStorePathStrategy(), indexSettings().isTranslogMetadataEnabled(), - 0 + 0, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata()) ); } @@ -5322,6 +5325,24 @@ public void syncTranslogFilesFromGivenRemoteTranslog( RemoteStorePathStrategy remoteStorePathStrategy, boolean isTranslogMetadataEnabled, long timestamp + ) throws IOException { + this.syncTranslogFilesFromGivenRemoteTranslog( + repository, + shardId, + remoteStorePathStrategy, + isTranslogMetadataEnabled, + timestamp, + false + ); + } + + public void syncTranslogFilesFromGivenRemoteTranslog( + Repository repository, + ShardId shardId, + RemoteStorePathStrategy remoteStorePathStrategy, + boolean isTranslogMetadataEnabled, + long timestamp, + boolean isServerSideEncryptionEnabled ) throws IOException { RemoteFsTranslog.download( repository, @@ -5333,7 +5354,8 @@ public void syncTranslogFilesFromGivenRemoteTranslog( logger, shouldSeedRemoteStore(), isTranslogMetadataEnabled, - timestamp + timestamp, + isServerSideEncryptionEnabled ); } diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 8f99efe502858..80f91d421db97 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -56,6 +56,7 @@ import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineException; import org.opensearch.index.mapper.MapperService; @@ -422,7 +423,9 @@ void recoverFromSnapshotAndRemoteStore( remoteStoreRepository, indexUUID, shardId, - shallowCopyShardMetadata.getRemoteStorePathStrategy() + shallowCopyShardMetadata.getRemoteStorePathStrategy(), + null, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexShard.indexSettings.getIndexMetadata()) ); RemoteSegmentMetadata remoteSegmentMetadata = sourceRemoteDirectory.initializeToSpecificCommit( primaryTerm, @@ -498,12 +501,15 @@ void recoverShallowSnapshotV2( prevIndexMetadata.getSettings() ); } + IndexSettings indexSettings = new IndexSettings(prevIndexMetadata, prevIndexMetadata.getSettings()); RemoteStorePathStrategy remoteStorePathStrategy = RemoteStoreUtils.determineRemoteStorePathStrategy(prevIndexMetadata); RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( remoteSegmentStoreRepository, prevIndexMetadata.getIndexUUID(), shardId, - remoteStorePathStrategy + remoteStorePathStrategy, + null, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexShard.indexSettings.getIndexMetadata()) ); RemoteSegmentMetadata remoteSegmentMetadata = sourceRemoteDirectory.initializeToSpecificTimestamp( recoverySource.pinnedTimestamp() @@ -523,7 +529,8 @@ void recoverShallowSnapshotV2( new ShardId(prevIndexMetadata.getIndex(), shardId.id()), remoteStorePathStrategy, RemoteStoreUtils.determineTranslogMetadataEnabled(prevIndexMetadata), - recoverySource.pinnedTimestamp() + recoverySource.pinnedTimestamp(), + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexShard.indexSettings.getIndexMetadata()) ); assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index 35aba694729cb..57455d3bbdd7c 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -14,6 +14,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; import org.opensearch.index.remote.RemoteStorePathStrategy; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; @@ -60,12 +61,20 @@ public RemoteSegmentStoreDirectoryFactory( public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throws IOException { String repositoryName = indexSettings.getRemoteStoreRepository(); String indexUUID = indexSettings.getIndex().getUUID(); - return newDirectory(repositoryName, indexUUID, path.getShardId(), indexSettings.getRemoteStorePathStrategy()); + + return newDirectory( + repositoryName, + indexUUID, + path.getShardId(), + indexSettings.getRemoteStorePathStrategy(), + null, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata()) + ); } public Directory newDirectory(String repositoryName, String indexUUID, ShardId shardId, RemoteStorePathStrategy pathStrategy) throws IOException { - return newDirectory(repositoryName, indexUUID, shardId, pathStrategy, null); + return newDirectory(repositoryName, indexUUID, shardId, pathStrategy, null, false); } public Directory newDirectory( @@ -75,9 +84,21 @@ public Directory newDirectory( RemoteStorePathStrategy pathStrategy, String indexFixedPrefix ) throws IOException { - assert Objects.nonNull(pathStrategy); - try (Repository repository = repositoriesService.get().repository(repositoryName)) { + return newDirectory(repositoryName, indexUUID, shardId, pathStrategy, indexFixedPrefix, false); + } + public Directory newDirectory( + String repositoryName, + String indexUUID, + ShardId shardId, + RemoteStorePathStrategy pathStrategy, + String indexFixedPrefix, + boolean isServerSideEncryptionEnabled + ) throws IOException { + assert Objects.nonNull(pathStrategy); + // We should be not calling close for repository. + Repository repository = repositoriesService.get().repository(repositoryName); + try { assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobStoreRepository blobStoreRepository = ((BlobStoreRepository) repository); BlobPath repositoryBasePath = blobStoreRepository.basePath(); @@ -93,10 +114,11 @@ public Directory newDirectory( .fixedPrefix(segmentsPathFixedPrefix) .indexFixedPrefix(indexFixedPrefix) .build(); + // Derive the path for data directory of SEGMENTS BlobPath dataPath = pathStrategy.generatePath(dataPathInput); RemoteDirectory dataDirectory = new RemoteDirectory( - blobStoreRepository.blobStore().blobContainer(dataPath), + blobStoreRepository.blobStore(isServerSideEncryptionEnabled).blobContainer(dataPath), blobStoreRepository::maybeRateLimitRemoteUploadTransfers, blobStoreRepository::maybeRateLimitLowPriorityRemoteUploadTransfers, blobStoreRepository::maybeRateLimitRemoteDownloadTransfers, @@ -115,7 +137,9 @@ public Directory newDirectory( .build(); // Derive the path for metadata directory of SEGMENTS BlobPath mdPath = pathStrategy.generatePath(mdPathInput); - RemoteDirectory metadataDirectory = new RemoteDirectory(blobStoreRepository.blobStore().blobContainer(mdPath)); + RemoteDirectory metadataDirectory = new RemoteDirectory( + blobStoreRepository.blobStore(isServerSideEncryptionEnabled).blobContainer(mdPath) + ); // The path for lock is derived within the RemoteStoreLockManagerFactory RemoteStoreLockManager mdLockManager = RemoteStoreLockManagerFactory.newLockManager( diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java index b1742695b6748..08926c09fa033 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java @@ -67,7 +67,8 @@ public static RemoteStoreMetadataLockManager newLockManager( String segmentsPathFixedPrefix, String indexFixedPrefix ) { - try (Repository repository = repositoriesService.repository(repositoryName)) { + Repository repository = repositoriesService.repository(repositoryName); + try { assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobPath repositoryBasePath = ((BlobStoreRepository) repository).basePath(); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java index 1f2b2c48b471a..501dbe2962d29 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java @@ -37,12 +37,15 @@ public class RemoteBlobStoreInternalTranslogFactory implements TranslogFactory { private final RemoteStoreSettings remoteStoreSettings; + private final boolean isServerSideEncryptionEnabled; + public RemoteBlobStoreInternalTranslogFactory( Supplier repositoriesServiceSupplier, ThreadPool threadPool, String repositoryName, RemoteTranslogTransferTracker remoteTranslogTransferTracker, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + boolean isServerSideEncryptionEnabled ) { Repository repository; try { @@ -54,6 +57,7 @@ public RemoteBlobStoreInternalTranslogFactory( this.threadPool = threadPool; this.remoteTranslogTransferTracker = remoteTranslogTransferTracker; this.remoteStoreSettings = remoteStoreSettings; + this.isServerSideEncryptionEnabled = isServerSideEncryptionEnabled; } @Override @@ -107,7 +111,8 @@ public Translog newTranslog( startedPrimarySupplier, remoteTranslogTransferTracker, remoteStoreSettings, - translogOperationHelper + translogOperationHelper, + isServerSideEncryptionEnabled ); } else { return new RemoteFsTranslog( @@ -123,7 +128,8 @@ public Translog newTranslog( remoteTranslogTransferTracker, remoteStoreSettings, translogOperationHelper, - null + null, + isServerSideEncryptionEnabled ); } } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 7fd915ba2c297..1832d1e7d035a 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -76,7 +76,8 @@ public RemoteFsTimestampAwareTranslog( BooleanSupplier startedPrimarySupplier, RemoteTranslogTransferTracker remoteTranslogTransferTracker, RemoteStoreSettings remoteStoreSettings, - TranslogOperationHelper translogOperationHelper + TranslogOperationHelper translogOperationHelper, + boolean isServerSideEncryptionEnabled ) throws IOException { super( config, @@ -91,7 +92,8 @@ public RemoteFsTimestampAwareTranslog( remoteTranslogTransferTracker, remoteStoreSettings, translogOperationHelper, - null + null, + isServerSideEncryptionEnabled ); logger = Loggers.getLogger(getClass(), shardId); this.metadataFilePinnedTimestampMap = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index bbe8b739e2da4..1e3415f4044a6 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -95,6 +95,7 @@ public class RemoteFsTranslog extends Translog { private final Semaphore syncPermit = new Semaphore(SYNC_PERMIT); protected final AtomicBoolean pauseSync = new AtomicBoolean(false); private final boolean isTranslogMetadataEnabled; + private final boolean isServerSideEncryptionEnabled; public RemoteFsTranslog( TranslogConfig config, @@ -109,7 +110,8 @@ public RemoteFsTranslog( RemoteTranslogTransferTracker remoteTranslogTransferTracker, RemoteStoreSettings remoteStoreSettings, TranslogOperationHelper translogOperationHelper, - ChannelFactory channelFactory + ChannelFactory channelFactory, + boolean isServerSideEncryptionEnabled ) throws IOException { super( config, @@ -126,6 +128,7 @@ public RemoteFsTranslog( this.remoteTranslogTransferTracker = remoteTranslogTransferTracker; fileTransferTracker = new FileTransferTracker(shardId, remoteTranslogTransferTracker); isTranslogMetadataEnabled = indexSettings().isTranslogMetadataEnabled(); + this.isServerSideEncryptionEnabled = isServerSideEncryptionEnabled; this.translogTransferManager = buildTranslogTransferManager( blobStoreRepository, threadPool, @@ -134,7 +137,8 @@ public RemoteFsTranslog( remoteTranslogTransferTracker, indexSettings().getRemoteStorePathStrategy(), remoteStoreSettings, - isTranslogMetadataEnabled + isTranslogMetadataEnabled, + isServerSideEncryptionEnabled ); try { if (config.downloadRemoteTranslogOnInit()) { @@ -193,7 +197,8 @@ public static void download( Logger logger, boolean seedRemote, boolean isTranslogMetadataEnabled, - long timestamp + long timestamp, + boolean isServerSideEncryptionEnabled ) throws IOException { assert repository instanceof BlobStoreRepository : String.format( Locale.ROOT, @@ -213,7 +218,8 @@ public static void download( remoteTranslogTransferTracker, pathStrategy, remoteStoreSettings, - isTranslogMetadataEnabled + isTranslogMetadataEnabled, + isServerSideEncryptionEnabled ); RemoteFsTranslog.download(translogTransferManager, location, logger, seedRemote, timestamp); logger.trace(remoteTranslogTransferTracker.toString()); @@ -325,7 +331,8 @@ public static TranslogTransferManager buildTranslogTransferManager( RemoteTranslogTransferTracker tracker, RemoteStorePathStrategy pathStrategy, RemoteStoreSettings remoteStoreSettings, - boolean isTranslogMetadataEnabled + boolean isTranslogMetadataEnabled, + boolean isServerSideEncryptionEnabled ) { assert Objects.nonNull(pathStrategy); String indexUUID = shardId.getIndex().getUUID(); @@ -348,7 +355,10 @@ public static TranslogTransferManager buildTranslogTransferManager( .fixedPrefix(remoteStoreSettings.getTranslogPathFixedPrefix()) .build(); BlobPath mdPath = pathStrategy.generatePath(mdPathInput); - BlobStoreTransferService transferService = new BlobStoreTransferService(blobStoreRepository.blobStore(), threadPool); + BlobStoreTransferService transferService = new BlobStoreTransferService( + blobStoreRepository.blobStore(isServerSideEncryptionEnabled), + threadPool + ); return new TranslogTransferManager( shardId, transferService, @@ -655,7 +665,8 @@ public static void cleanup( ThreadPool threadPool, RemoteStorePathStrategy pathStrategy, RemoteStoreSettings remoteStoreSettings, - boolean isTranslogMetadataEnabled + boolean isTranslogMetadataEnabled, + boolean isServerSideEncryptionEnabled ) throws IOException { assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; @@ -671,7 +682,8 @@ public static void cleanup( remoteTranslogTransferTracker, pathStrategy, remoteStoreSettings, - isTranslogMetadataEnabled + isTranslogMetadataEnabled, + isServerSideEncryptionEnabled ); // clean up all remote translog files translogTransferManager.deleteTranslogFiles(); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 667eb9edb65f7..417e408f510d1 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -132,6 +132,7 @@ import org.opensearch.index.recovery.RecoveryStats; import org.opensearch.index.refresh.RefreshStats; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.search.stats.SearchStats; import org.opensearch.index.seqno.RetentionLeaseStats; import org.opensearch.index.seqno.RetentionLeaseSyncer; @@ -728,7 +729,8 @@ private static BiFunction getTrans threadPool, indexSettings.getRemoteStoreTranslogRepository(), remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()), - remoteStoreSettings + remoteStoreSettings, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata()) ); } else if (RemoteStoreNodeAttribute.isTranslogRepoConfigured(settings) && shardRouting.primary()) { return new RemoteBlobStoreInternalTranslogFactory( @@ -736,7 +738,8 @@ private static BiFunction getTrans threadPool, RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(indexSettings.getNodeSettings()), remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()), - remoteStoreSettings + remoteStoreSettings, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata()) ); } return new InternalTranslogFactory(); diff --git a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java index 1f09af234ae30..16320f66429e5 100644 --- a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java +++ b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java @@ -8,6 +8,7 @@ package org.opensearch.indices; +import org.opensearch.Version; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.ClusterSettings; @@ -184,6 +185,16 @@ public class RemoteStoreSettings { Property.Final ); + /** + * Controls the ServerSideEncryption Settings. + */ + public static final Setting CLUSTER_SERVER_SIDE_ENCRYPTION_REPO_ENABLED = Setting.boolSetting( + "cluster.server_side_encryption.repo.enabled", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private volatile TimeValue clusterRemoteTranslogBufferInterval; private volatile int minRemoteSegmentMetadataFiles; private volatile TimeValue clusterRemoteTranslogTransferTimeout; @@ -191,6 +202,7 @@ public class RemoteStoreSettings { private volatile RemoteStoreEnums.PathType pathType; private volatile RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm; private volatile int maxRemoteTranslogReaders; + private static volatile boolean isClusterServerSideEncryptionRepoEnabled; private volatile boolean isTranslogMetadataEnabled; private static volatile boolean isPinnedTimestampsEnabled; private static volatile TimeValue pinnedTimestampsSchedulerInterval; @@ -235,6 +247,12 @@ public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) { this::setClusterRemoteSegmentTransferTimeout ); + isClusterServerSideEncryptionRepoEnabled = CLUSTER_SERVER_SIDE_ENCRYPTION_REPO_ENABLED.get(settings); + clusterSettings.addSettingsUpdateConsumer( + CLUSTER_SERVER_SIDE_ENCRYPTION_REPO_ENABLED, + RemoteStoreSettings::setClusterServerSideEncryptionRepoEnabled + ); + pinnedTimestampsSchedulerInterval = CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_SCHEDULER_INTERVAL.get(settings); pinnedTimestampsLookbackInterval = CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_LOOKBACK_INTERVAL.get(settings); isPinnedTimestampsEnabled = CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.get(settings); @@ -309,6 +327,14 @@ private void setMaxRemoteTranslogReaders(int maxRemoteTranslogReaders) { this.maxRemoteTranslogReaders = maxRemoteTranslogReaders; } + public static boolean isClusterServerSideEncryptionRepoEnabled() { + return isClusterServerSideEncryptionRepoEnabled; + } + + private static void setClusterServerSideEncryptionRepoEnabled(boolean clusterServerSideEncryptionRepoEnabled) { + isClusterServerSideEncryptionRepoEnabled = clusterServerSideEncryptionRepoEnabled; + } + public static TimeValue getPinnedTimestampsSchedulerInterval() { return pinnedTimestampsSchedulerInterval; } @@ -333,4 +359,8 @@ public String getTranslogPathFixedPrefix() { public String getSegmentsPathFixedPrefix() { return segmentsPathFixedPrefix; } + + public static boolean isServerSideEncryptionRepoEnabled(Version minNodeVersion) { + return Version.V_3_1_0.compareTo(minNodeVersion) <= 0 && isClusterServerSideEncryptionRepoEnabled(); + } } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 56c3af3410643..23f45d48aff01 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -68,6 +68,10 @@ public class RemoteStoreNodeAttribute { public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; public static final String REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "%s.repository.%s.type"; + + public static final String REPOSITORY_SERVER_SIDE_ENCRYPTION_ATTRIBUTE_KEY_FORMAT = + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX + "server_side_encryption_kms_key_id"; + public static final String REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "%s.repository.%s." + CryptoMetadata.CRYPTO_METADATA_KEY; public static final String REPOSITORY_CRYPTO_SETTINGS_PREFIX = REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT + "." @@ -249,6 +253,15 @@ public static boolean isRemoteStoreAttributePresent(Settings settings) { return false; } + // public static boolean isServerSideEncryptionEnabled(Settings settings) { + // for (String metadataKey : settings.keySet()) { + // if (metadataKey.equals(REPOSITORY_METADATA_SERVER_SIDE_ENCRYPTION_ENABLED_KEY) && settings.getAsBoolean(metadataKey, false)) { + // return true; + // } + // } + // return false; + // } + public static boolean isRemoteDataAttributePresent(Settings settings) { return isSegmentRepoConfigured(settings) || isTranslogRepoConfigured(settings); } @@ -363,6 +376,14 @@ public static String getTranslogRepoName(Map repos) { return getValueFromAnyKey(repos, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS); } + public static boolean isRemoteStoreServerSideEncryptionEnabled(Map repos) { + String segmentRepoName = getSegmentRepoName(repos); + return getValueFromAnyKey( + repos, + List.of(String.format(Locale.getDefault(), REPOSITORY_SERVER_SIDE_ENCRYPTION_ATTRIBUTE_KEY_FORMAT, segmentRepoName)) + ) != null; + } + private static String getValueFromAnyKey(Map repos, List keys) { for (String key : keys) { if (repos.get(key) != null) { diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreProvider.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreProvider.java new file mode 100644 index 0000000000000..463d573a63d22 --- /dev/null +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreProvider.java @@ -0,0 +1,103 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.blobstore; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.SetOnce; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.EncryptedBlobStore; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.repositories.RepositoryException; + +/** + * Provide for the BlobStore class + * + * @opensearch.internal + */ +public class BlobStoreProvider { + private static final Logger logger = LogManager.getLogger(BlobStoreProvider.class); + protected final Lifecycle lifecycle; + protected final RepositoryMetadata metadata; + protected final Object lock; + protected final BlobStoreRepository repository; + private final SetOnce blobStore = new SetOnce<>(); + private final SetOnce serverSideEncryptedBlobStore = new SetOnce<>(); + + public BlobStoreProvider(BlobStoreRepository repository, RepositoryMetadata metadata, Lifecycle lifecycle, Object lock) { + this.lifecycle = lifecycle; + this.metadata = metadata; + this.lock = lock; + this.repository = repository; + } + + protected BlobStore blobStore(boolean serverSideEncryptionEnabled) { + if (serverSideEncryptionEnabled) { + logger.info("[BlobStoreProvider]: Using serverSideEncryptedBlobStore"); + return createBlobStore(serverSideEncryptedBlobStore, true); + } + logger.info("[BlobStoreProvider]: Using blobStore"); + return createBlobStore(blobStore, false); + } + + public BlobStore blobStore() { + // Assertion not true as Kraken threads use blobStore + return blobStore(false); + } + + protected BlobStore createBlobStore(SetOnce blobStore, boolean serverSideEncryption) { + // assertSnapshotOrGenericThread(); + BlobStore store = blobStore.get(); + logger.debug("blob store fetched = " + store); + if (store == null) { + synchronized (lock) { + store = blobStore.get(); + if (store == null) { + store = initBlobStore(); + if (!serverSideEncryption && metadata.cryptoMetadata() != null) { + store = new EncryptedBlobStore(store, metadata.cryptoMetadata()); + } + blobStore.set(store); + } + } + } + return store; + } + + public BlobStore getBlobStore(boolean serverSideEncryptionEnabled) { + return blobStore(serverSideEncryptionEnabled); + } + + protected BlobStore initBlobStore() { + if (lifecycle.started() == false) { + throw new RepositoryException(metadata.name(), "repository is not in started state" + lifecycle.state()); + } + try { + return repository.createBlobStore(); + } catch (RepositoryException e) { + throw e; + } catch (Exception e) { + throw new RepositoryException(metadata.name(), "cannot create blob store", e); + } + } + + public void close() { + try { + if (blobStore.get() != null) { + blobStore.get().close(); + } + if (serverSideEncryptedBlobStore.get() != null) { + serverSideEncryptedBlobStore.get().close(); + } + } catch (Exception t) { + logger.warn("cannot close blob store", t); + } + } +} diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 1b6aa3df2bc8a..605cf52ed0879 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -74,7 +74,6 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.DeleteResult; -import org.opensearch.common.blobstore.EncryptedBlobStore; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeInputStream; import org.opensearch.common.blobstore.transfer.stream.RateLimitingOffsetRangeInputStream; @@ -567,7 +566,7 @@ protected static long calculateMaxWithinIntLimit(long defaultThresholdOfHeap, lo private final SetOnce snapshotShardPathBlobContainer = new SetOnce<>(); - private final SetOnce blobStore = new SetOnce<>(); + private final SetOnce blobStoreProvider = new SetOnce<>(); protected final ClusterService clusterService; @@ -683,19 +682,21 @@ protected void doStop() {} @Override protected void doClose() { - BlobStore store; + BlobStoreProvider provider = null; // to close blobStore if blobStore initialization is started during close synchronized (lock) { - store = blobStore.get(); - } - if (store != null) { - try { - closed = true; - store.close(); - } catch (Exception t) { - logger.warn("cannot close blob store", t); + provider = blobStoreProvider.get(); + // Moving this inside synchronized block + if (provider != null) { + try { + provider.close(); + closed = true; + } catch (Exception t) { + logger.warn("cannot close blob store", t); + } } } + } @Override @@ -983,7 +984,7 @@ public SetOnce getSnapshotShardPathBlobContainer() { // for test purposes only protected BlobStore getBlobStore() { - return blobStore.get(); + return blobStoreProvider.get() != null ? blobStoreProvider.get().getBlobStore(false) : null; } boolean getPrefixModeVerification() { @@ -1052,29 +1053,26 @@ protected BlobContainer snapshotShardPathBlobContainer() { * Public for testing. */ public BlobStore blobStore() { - BlobStore store = blobStore.get(); - if (store == null) { + return blobStore(false); + } + + /** + * Calls the existing blobStore() method. Specific repositories can implement the support for + * Server side encryption + * @param serverSideEncryptionEnabled ServerSideEncryptionEnabled Value. + * @return BlobStore `Blobstore` for the repository + */ + public BlobStore blobStore(boolean serverSideEncryptionEnabled) { + BlobStoreProvider provider = this.blobStoreProvider.get(); + logger.info("1. provider = " + provider); + if (provider == null) { synchronized (lock) { - store = blobStore.get(); - if (store == null) { - if (lifecycle.started() == false) { - throw new RepositoryException(metadata.name(), "repository is not in started state"); - } - try { - store = createBlobStore(); - if (metadata.cryptoMetadata() != null) { - store = new EncryptedBlobStore(store, metadata.cryptoMetadata()); - } - } catch (RepositoryException e) { - throw e; - } catch (Exception e) { - throw new RepositoryException(metadata.name(), "cannot create blob store", e); - } - blobStore.set(store); - } + provider = new BlobStoreProvider(this, metadata, lifecycle, lock); + this.blobStoreProvider.set(provider); } } - return store; + logger.info("provider = " + provider); + return provider.blobStore(serverSideEncryptionEnabled); } /** @@ -1122,7 +1120,12 @@ public Compressor getCompressor() { @Override public RepositoryStats stats() { - final BlobStore store = blobStore.get(); + BlobStoreProvider provider = blobStoreProvider.get(); + BlobStore store = null; + if (provider == null) { + return RepositoryStats.EMPTY_STATS; + } + store = provider.getBlobStore(false); if (store == null) { return RepositoryStats.EMPTY_STATS; } else if (store.extendedStats() != null && store.extendedStats().isEmpty() == false) { @@ -2394,7 +2397,8 @@ private void remoteTranslogCleanupAsync( remoteTranslogTransferTracker, remoteStorePathStrategy, remoteStoreSettings, - indexMetadataEnabled + indexMetadataEnabled, + false ); try { RemoteFsTimestampAwareTranslog.cleanupOfDeletedIndex(translogTransferManager, forceClean); @@ -4518,7 +4522,7 @@ public void verify(String seed, DiscoveryNode localNode) { @Override public String toString() { - return "BlobStoreRepository[" + "[" + metadata.name() + "], [" + blobStore.get() + ']' + ']'; + return "BlobStoreRepository[" + "[" + metadata.name() + "], [" + blobStoreProvider.get() + ']' + ']'; } /** diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 3a333abe5b59f..ef4d0df2e9636 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -469,7 +469,7 @@ public ClusterState execute(ClusterState currentState) { .put(snapshotIndexMetadata.getSettings()) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) ); - createIndexService.addRemoteStoreCustomMetadata(indexMdBuilder, false); + createIndexService.addRemoteStoreCustomMetadata(indexMdBuilder, false, currentState); shardLimitValidator.validateShardLimit( renamedIndexName, snapshotIndexMetadata.getSettings(), diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 785ba35afaf4d..a75ada3caab9c 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -93,6 +93,7 @@ import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.SystemIndices; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -1821,7 +1822,12 @@ private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, PathType .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) .build(); - IndexMetadata indexMetadata = metadataCreateIndexService.buildAndValidateTemporaryIndexMetadata(indexSettings, request, 0); + IndexMetadata indexMetadata = metadataCreateIndexService.buildAndValidateTemporaryIndexMetadata( + indexSettings, + request, + 0, + clusterService.state() + ); threadPool.shutdown(); return indexMetadata; } @@ -1861,7 +1867,8 @@ public void testNumberOfRoutingShardsShowsInIndexSettings() { IndexMetadata indexMetadata = checkerService.buildAndValidateTemporaryIndexMetadata( indexSettings, request, - routingNumberOfShards + routingNumberOfShards, + clusterService.state() ); assertEquals(INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(indexMetadata.getSettings()).intValue(), routingNumberOfShards); })); @@ -2615,6 +2622,112 @@ public void testIndexTotalPrimaryShardsPerNodeSettingValidationWithoutRemoteStor ); } + // public void testAddRemoteStoreCustomMetadata() { + // + // MetadataCreateIndexService checkerService = new MetadataCreateIndexService( + // Settings.EMPTY, + // clusterService, + // indicesServices, + // null, + // null, + // createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService), + // null, + // null, + // null, + // null, + // new SystemIndices(Collections.emptyMap()), + // false, + // new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()), + // DefaultRemoteStoreSettings.INSTANCE, + // repositoriesServiceSupplier + // ); + // + // + // Map attributes = getNodeAttributes(); + // DiscoveryNode remoteNode = new DiscoveryNode( + // UUIDs.base64UUID(), + // buildNewFakeTransportAddress(), + // attributes, + // DiscoveryNodeRole.BUILT_IN_ROLES, + // Version.CURRENT + // ); + // ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + // .nodes(DiscoveryNodes.builder().add(remoteNode).build()) + // .build(); + // + // Settings indexSettings = Settings.builder() + // .put(SETTING_VERSION_CREATED, Version.CURRENT) + // .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + // .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + // .build(); + // + // IndexMetadata.Builder imdBuilder = IndexMetadata.builder("test").settings(indexSettings); + // + // Settings settings = Settings.builder().put("node.attr.remote_store.segment.repository", "my-segment-repo-1").build(); + // final Settings.Builder requestSettings = Settings.builder() + // .put(INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey(), -1) + // .put(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT.toString()); + // + // Settings clusterSettingsSetting = Settings.builder() + // .put(RemoteStoreSettings.CLUSTER_SERVER_SIDE_ENCRYPTION_REPO_ENABLED.getKey(), true) + // .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.STRICT) + // .build(); + // clusterSettings = new ClusterSettings(clusterSettingsSetting, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + // + // new RemoteStoreSettings(clusterSettingsSetting, clusterSettings); + // + // MetadataCreateIndexService.add(imdBuilder, ); + // + // assertTrue(requestSettings.build().getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, false)); + // } + // + // public void testUpdateRemoteStoreSettings_For_Snapshot_restore() { + // Map attributes = getNodeAttributes(); + // DiscoveryNode remoteNode = new DiscoveryNode( + // UUIDs.base64UUID(), + // buildNewFakeTransportAddress(), + // attributes, + // DiscoveryNodeRole.BUILT_IN_ROLES, + // Version.CURRENT + // ); + // ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + // .nodes(DiscoveryNodes.builder().add(remoteNode).build()) + // .build(); + // + // Settings settings = Settings.builder().put("node.attr.remote_store.segment.repository", "my-segment-repo-1").build(); + // final Settings.Builder requestSettings = Settings.builder() + // .put(INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey(), -1) + // .put(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT.toString()); + // + // Settings clusterSettingsSetting = Settings.builder() + // .put(RemoteStoreSettings.CLUSTER_SERVER_SIDE_ENCRYPTION_REPO_ENABLED.getKey(), true) + // .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.STRICT) + // .build(); + // clusterSettings = new ClusterSettings(clusterSettingsSetting, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + // + // new RemoteStoreSettings(clusterSettingsSetting, clusterSettings); + // + // MetadataCreateIndexService.updateRemoteStoreSettings(requestSettings, clusterState, clusterSettings, settings, "test-index", true); + // + // assertFalse(requestSettings.build().getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, false)); + // } + + private static Map getNodeAttributes() { + String segmentRepositoryName = "my-segment-repo-1"; + String serverSideEncryptionAttribute = String.format( + Locale.getDefault(), + RemoteStoreNodeAttribute.REPOSITORY_SERVER_SIDE_ENCRYPTION_ATTRIBUTE_KEY_FORMAT, + segmentRepositoryName + ); + Map attributes = new HashMap<>(); + + attributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-cluster-rep-1"); + attributes.put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepositoryName); + attributes.put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-translog-repo-1"); + attributes.put(serverSideEncryptionAttribute, "true"); + return attributes; + } + public void testIndexTotalPrimaryShardsPerNodeSettingValidationWithDefaultValue() { // Test case with default value (-1) without remote store (should succeed) Settings settings = Settings.builder().build(); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 29dd60c3e638f..8b78796b4679b 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -87,6 +87,7 @@ import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.mapper.Uid; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexingOperationListener; @@ -239,7 +240,8 @@ private IndexService newIndexService(IndexModule module) throws IOException { threadPool, indexSettings.getRemoteStoreTranslogRepository(), new RemoteTranslogTransferTracker(shardRouting.shardId(), 10), - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(indexSettings.getIndexMetadata()) ); } return new InternalTranslogFactory(); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index 95b5d4456baf6..b8ae85a5af2b3 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -71,6 +71,7 @@ public void testNewDirectory() throws IOException { BlobStoreRepository repository = mock(BlobStoreRepository.class); BlobStore blobStore = mock(BlobStore.class); BlobContainer blobContainer = mock(BlobContainer.class); + when(repository.blobStore(false)).thenReturn(blobStore); when(repository.blobStore()).thenReturn(blobStore); when(repository.basePath()).thenReturn(new BlobPath().add("base_path")); when(blobStore.blobContainer(any())).thenReturn(blobContainer); @@ -117,7 +118,7 @@ public void testNewDirectoryRepositoryDoesNotExist() { when(repositoriesService.repository("remote_store_repository")).thenThrow(new RepositoryMissingException("Missing")); - assertThrows(IllegalArgumentException.class, () -> remoteSegmentStoreDirectoryFactory.newDirectory(indexSettings, shardPath)); + assertThrows(RepositoryMissingException.class, () -> remoteSegmentStoreDirectoryFactory.newDirectory(indexSettings, shardPath)); } } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 6c89cf2adf988..5ab26084dcfd2 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -137,7 +137,8 @@ public void setUp() throws Exception { protected RemoteFsTranslog createTranslogInstance( TranslogConfig translogConfig, String translogUUID, - TranslogDeletionPolicy deletionPolicy + TranslogDeletionPolicy deletionPolicy, + boolean isServerSideEncryptionEnabled ) throws IOException { return new RemoteFsTimestampAwareTranslog( translogConfig, @@ -151,7 +152,8 @@ protected RemoteFsTranslog createTranslogInstance( primaryMode::get, new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + isServerSideEncryptionEnabled ); } @@ -622,7 +624,8 @@ public void testExtraGenToKeep() throws Exception { new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, TranslogOperationHelper.DEFAULT, - channelFactory + channelFactory, + false ) ) { addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index edcdca3f7b3de..7b20c7e22f2f4 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -171,23 +171,30 @@ public void tearDown() throws Exception { protected RemoteFsTranslog create(Path path) throws IOException { final String translogUUID = Translog.createEmptyTranslog(path, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); - return create(path, createRepository(), translogUUID, 0); + return create(path, createRepository(), translogUUID, 0, false); } - private RemoteFsTranslog create(Path path, BlobStoreRepository repository, String translogUUID, int extraGenToKeep) throws IOException { + private RemoteFsTranslog create( + Path path, + BlobStoreRepository repository, + String translogUUID, + int extraGenToKeep, + boolean isServerSideEncryptionEnabled + ) throws IOException { this.repository = repository; globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final TranslogConfig translogConfig = getTranslogConfig(path, extraGenToKeep); final TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(translogConfig.getIndexSettings()); threadPool = new TestThreadPool(getClass().getName()); blobStoreTransferService = new BlobStoreTransferService(repository.blobStore(), threadPool); - return createTranslogInstance(translogConfig, translogUUID, deletionPolicy); + return createTranslogInstance(translogConfig, translogUUID, deletionPolicy, isServerSideEncryptionEnabled); } protected RemoteFsTranslog createTranslogInstance( TranslogConfig translogConfig, String translogUUID, - TranslogDeletionPolicy deletionPolicy + TranslogDeletionPolicy deletionPolicy, + boolean isServerSideEncryptionEnabled ) throws IOException { return new RemoteFsTranslog( translogConfig, @@ -202,12 +209,14 @@ protected RemoteFsTranslog createTranslogInstance( new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, TranslogOperationHelper.DEFAULT, - null + null, + isServerSideEncryptionEnabled ); } - private RemoteFsTranslog create(Path path, BlobStoreRepository repository, String translogUUID) throws IOException { - return create(path, repository, translogUUID, 0); + private RemoteFsTranslog create(Path path, BlobStoreRepository repository, String translogUUID, boolean isServerSideEncryptionEnabled) + throws IOException { + return create(path, repository, translogUUID, 0, isServerSideEncryptionEnabled); } private TranslogConfig getTranslogConfig(final Path path) { @@ -477,7 +486,8 @@ public void testExtraGenToKeep() throws Exception { new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, TranslogOperationHelper.DEFAULT, - null + null, + false ) ) { addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); @@ -554,7 +564,7 @@ public void testReadLocationDownload() throws IOException { } // Creating RemoteFsTranslog with the same location - RemoteFsTranslog newTranslog = create(translogDir, repository, translogUUID); + RemoteFsTranslog newTranslog = create(translogDir, repository, translogUUID, false); i = 0; for (Translog.Operation op : ops) { assertEquals(op, newTranslog.readOperation(locs.get(i++))); @@ -825,7 +835,7 @@ public void testMetadataFileDeletion() throws Exception { long newPrimaryTerm = primaryTerm.incrementAndGet(); // Creating RemoteFsTranslog with the same location - Translog newTranslog = create(translogDir, repository, translogUUID); + Translog newTranslog = create(translogDir, repository, translogUUID, false); int newPrimaryTermDocs = randomIntBetween(5, 10); for (int i = totalDocs + 1; i <= totalDocs + newPrimaryTermDocs; i++) { addToTranslogAndListAndUpload(newTranslog, ops, new Translog.Index(String.valueOf(i), i, primaryTerm.get(), new byte[] { 1 })); @@ -1523,7 +1533,8 @@ public void testTranslogWriterCanFlushInAddOrReadCall() throws IOException { new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, TranslogOperationHelper.DEFAULT, - channelFactory + channelFactory, + false ) ) { TranslogWriter writer = translog.getCurrent(); @@ -1630,7 +1641,8 @@ public void force(boolean metaData) throws IOException { new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, TranslogOperationHelper.DEFAULT, - channelFactory + channelFactory, + false ) ) { TranslogWriter writer = translog.getCurrent(); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreProviderTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreProviderTests.java new file mode 100644 index 0000000000000..1bd9bf16ee357 --- /dev/null +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreProviderTests.java @@ -0,0 +1,124 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.blobstore; + +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.repositories.RepositoryException; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Test class for {@link BlobStoreProvider}. + */ +public class BlobStoreProviderTests extends OpenSearchTestCase { + @Mock + private BlobStoreRepository mockRepository; + + @Mock + private RepositoryMetadata mockMetadata; + + @Mock + private Lifecycle mockLifecycle; + + @Mock + private BlobStore mockBlobStore; + + @Mock + private BlobStore mockServerSideEncryptionBlobStore; + + private Object lock; + private BlobStoreProvider provider; + + @Before + public void setUp() throws Exception { + super.setUp(); + MockitoAnnotations.openMocks(this); + lock = new Object(); + when(mockMetadata.name()).thenReturn("test-repository"); + provider = new BlobStoreProvider(mockRepository, mockMetadata, mockLifecycle, lock); + } + + public void testGetBlobStore() throws Exception { + // Setup: Mock the serverSideEncryptedBlobStore to return a value + // Note: Since SetOnce is used internally, we need to first call blobStore() to initialize it + when(mockLifecycle.started()).thenReturn(true); + when(mockRepository.createBlobStore()).thenReturn(mockBlobStore); + + // Initialize the server-side encrypted blob store + provider.blobStore(false); + + // Test + BlobStore result = provider.getBlobStore(false); + + // Verify + assertEquals(mockBlobStore, result); + } + + public void testGetBlobStoreWithServerSideEncryption() throws Exception { + // Setup: Mock the serverSideEncryptedBlobStore to return a value + // Note: Since SetOnce is used internally, we need to first call blobStore() to initialize it + when(mockLifecycle.started()).thenReturn(true); + when(mockRepository.createBlobStore()).thenReturn(mockServerSideEncryptionBlobStore); + + BlobStore result = provider.getBlobStore(true); + + // Verify + assertEquals(mockServerSideEncryptionBlobStore, result); + } + + public void testBlobStoreWithClientSideEncryptionFirstTime() throws Exception { + // Setup + when(mockLifecycle.started()).thenReturn(true); + when(mockRepository.createBlobStore()).thenReturn(mockBlobStore); + + // Test + BlobStore result = provider.blobStore(false); + + // Verify + assertEquals(mockBlobStore, result); + verify(mockRepository).createBlobStore(); + } + + public void testBlobStoreWithClientSideEncryptionEnabledSubsequentCalls() throws Exception { + // Setup + when(mockLifecycle.started()).thenReturn(true); + when(mockRepository.createBlobStore()).thenReturn(mockBlobStore); + + // First call + BlobStore firstResult = provider.blobStore(false); + + // Second call + BlobStore secondResult = provider.blobStore(false); + + // Verify + assertEquals(mockBlobStore, firstResult); + assertEquals(mockBlobStore, secondResult); + assertSame(firstResult, secondResult); + // Verify createServerSideEncryptedBlobStore is called only once + verify(mockRepository, times(1)).createBlobStore(); + } + + public void testInitBlobStoreWhenLifecycleNotStarted() { + // Setup + when(mockLifecycle.started()).thenReturn(false); + when(mockLifecycle.state()).thenReturn(Lifecycle.State.STOPPED); + + // Test - should throw RepositoryException + expectThrows(RepositoryException.class, () -> provider.initBlobStore()); + } +} diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 6d2bdbc8510f6..36f4e77228325 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -102,6 +102,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.SourceToParse; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.replication.TestReplicationSource; import org.opensearch.index.seqno.ReplicationTracker; @@ -742,7 +743,8 @@ protected IndexShard newShard( threadPool, settings.getRemoteStoreTranslogRepository(), new RemoteTranslogTransferTracker(shardRouting.shardId(), 20), - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + RemoteStoreUtils.isServerSideEncryptionEnabledIndex(settings.getIndexMetadata()) ); } return new InternalTranslogFactory();