diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 493f4d10eea6d..f852284d21b2a 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -17,6 +17,8 @@ import com.sun.net.httpserver.HttpHandler; import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.blobstore.BlobContainer; @@ -172,8 +174,13 @@ public TestAzureRepositoryPlugin(Settings settings) { } @Override - AzureStorageService createAzureStorageService(Settings settingsToUse, AzureClientProvider azureClientProvider) { - return new AzureStorageService(settingsToUse, azureClientProvider) { + AzureStorageService createAzureStorageService( + Settings settingsToUse, + AzureClientProvider azureClientProvider, + ClusterService clusterService, + ProjectResolver projectResolver + ) { + return new AzureStorageService(settingsToUse, azureClientProvider, clusterService, projectResolver) { @Override RequestRetryOptions getRetryOptions(LocationMode locationMode, AzureStorageSettings azureStorageSettings) { return new RequestRetryOptions( diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java index 2f63fb36613ee..3087c9b2a920a 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java @@ -21,6 +21,9 @@ import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.blobstore.BlobContainer; @@ -76,9 +79,14 @@ public TestAzureRepositoryPlugin(Settings settings) { } @Override - AzureStorageService createAzureStorageService(Settings settings, AzureClientProvider azureClientProvider) { + AzureStorageService createAzureStorageService( + Settings settings, + AzureClientProvider azureClientProvider, + ClusterService clusterService, + ProjectResolver projectResolver + ) { final long blockSize = ByteSizeValue.ofKb(64L).getBytes() * randomIntBetween(1, 15); - return new AzureStorageService(settings, azureClientProvider) { + return new AzureStorageService(settings, azureClientProvider, clusterService, projectResolver) { @Override long getUploadBlockSize() { return blockSize; @@ -163,7 +171,7 @@ private void ensureSasTokenPermissions() { repository.threadPool().generic().execute(ActionRunnable.wrap(future, l -> { final AzureBlobStore blobStore = (AzureBlobStore) repository.blobStore(); final AzureBlobServiceClient azureBlobServiceClient = blobStore.getService() - .client("default", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values())); + .client(ProjectId.DEFAULT, "default", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values())); final BlobServiceClient client = azureBlobServiceClient.getSyncClient(); try { final BlobContainerClient blobContainer = client.getBlobContainerClient(blobStore.toString()); diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java index f027492393aff..bb330be6266d4 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java @@ -49,6 +49,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.util.Throwables; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.blobstore.BlobContainer; @@ -117,6 +118,8 @@ public class AzureBlobStore implements BlobStore { private static final long DEFAULT_READ_CHUNK_SIZE = ByteSizeValue.of(32, ByteSizeUnit.MB).getBytes(); private static final int DEFAULT_UPLOAD_BUFFERS_SIZE = (int) ByteSizeValue.of(64, ByteSizeUnit.KB).getBytes(); + @Nullable // for cluster level object store in MP + private final ProjectId projectId; private final AzureStorageService service; private final BigArrays bigArrays; private final RepositoryMetadata repositoryMetadata; @@ -133,11 +136,13 @@ public class AzureBlobStore implements BlobStore { private final AzureClientProvider.RequestMetricsHandler requestMetricsHandler; public AzureBlobStore( + @Nullable ProjectId projectId, RepositoryMetadata metadata, AzureStorageService service, BigArrays bigArrays, RepositoriesMetrics repositoriesMetrics ) { + this.projectId = projectId; this.container = Repository.CONTAINER_SETTING.get(metadata.settings()); this.clientName = Repository.CLIENT_NAME.get(metadata.settings()); this.service = service; @@ -355,7 +360,7 @@ public InputStream getInputStream(OperationPurpose purpose, String blob, long po totalSize = position + length; } BlobAsyncClient blobAsyncClient = asyncClient.getBlobContainerAsyncClient(container).getBlobAsyncClient(blob); - int maxReadRetries = service.getMaxReadRetries(clientName); + int maxReadRetries = service.getMaxReadRetries(projectId, clientName); try { return new AzureInputStream( blobAsyncClient, @@ -941,7 +946,7 @@ private BlobServiceAsyncClient asyncClient(OperationPurpose purpose) { } private AzureBlobServiceClient getAzureBlobServiceClientClient(OperationPurpose purpose) { - return service.client(clientName, locationMode, purpose, requestMetricsHandler); + return service.client(projectId, clientName, locationMode, purpose, requestMetricsHandler); } @Override diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index c3d46b10d92d3..1b05f9873be74 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -178,7 +178,7 @@ protected BlobStore getBlobStore() { @Override protected AzureBlobStore createBlobStore() { - final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService, bigArrays, repositoriesMetrics); + final AzureBlobStore blobStore = new AzureBlobStore(getProjectId(), metadata, storageService, bigArrays, repositoriesMetrics); logger.debug( () -> format( @@ -204,6 +204,6 @@ public boolean isReadOnly() { @Override protected Set getExtraUsageFeatures() { - return storageService.getExtraUsageFeatures(Repository.CLIENT_NAME.get(getMetadata().settings())); + return storageService.getExtraUsageFeatures(getProjectId(), Repository.CLIENT_NAME.get(getMetadata().settings())); } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index 8a3194b23d907..e419cb1169cf1 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -11,6 +11,7 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -84,13 +85,20 @@ public Map getRepositories( @Override public Collection createComponents(PluginServices services) { AzureClientProvider azureClientProvider = AzureClientProvider.create(services.threadPool(), settings); - azureStoreService.set(createAzureStorageService(settings, azureClientProvider)); + azureStoreService.set( + createAzureStorageService(settings, azureClientProvider, services.clusterService(), services.projectResolver()) + ); assert assertRepositoryAzureMaxThreads(settings, services.threadPool()); return List.of(azureClientProvider); } - AzureStorageService createAzureStorageService(Settings settingsToUse, AzureClientProvider azureClientProvider) { - return new AzureStorageService(settingsToUse, azureClientProvider); + AzureStorageService createAzureStorageService( + Settings settingsToUse, + AzureClientProvider azureClientProvider, + ClusterService clusterService, + ProjectResolver projectResolver + ) { + return new AzureStorageService(settingsToUse, azureClientProvider, clusterService, projectResolver); } @Override @@ -140,7 +148,7 @@ public void reload(Settings settingsToLoad) { final Map clientsSettings = AzureStorageSettings.load(settingsToLoad); AzureStorageService storageService = azureStoreService.get(); assert storageService != null; - storageService.refreshSettings(clientsSettings); + storageService.refreshClusterClientSettings(clientsSettings); } private static boolean assertRepositoryAzureMaxThreads(Settings settings, ThreadPool threadPool) { diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java index dc8faa7d9e1e6..e736d627e6fe9 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java @@ -14,12 +14,24 @@ import com.azure.storage.common.policy.RequestRetryOptions; import com.azure.storage.common.policy.RetryPolicyType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateApplier; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.OperationPurpose; +import org.elasticsearch.common.settings.ProjectSecrets; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.monitor.jvm.JvmInfo; @@ -27,10 +39,11 @@ import java.net.Proxy; import java.util.Map; import java.util.Set; - -import static java.util.Collections.emptyMap; +import java.util.stream.Collectors; public class AzureStorageService { + private static final Logger logger = LogManager.getLogger(AzureStorageService.class); + public static final ByteSizeValue MIN_CHUNK_SIZE = ByteSizeValue.ofBytes(1); /** @@ -70,33 +83,51 @@ public class AzureStorageService { private static final long DEFAULT_UPLOAD_BLOCK_SIZE = DEFAULT_BLOCK_SIZE.getBytes(); private final int multipartUploadMaxConcurrency; - // 'package' for testing - volatile Map storageSettings = emptyMap(); private final AzureClientProvider azureClientProvider; private final ClientLogger clientLogger = new ClientLogger(AzureStorageService.class); + private final AzureStorageClientsManager clientsManager; private final boolean stateless; - public AzureStorageService(Settings settings, AzureClientProvider azureClientProvider) { - // eagerly load client settings so that secure settings are read - final Map clientsSettings = AzureStorageSettings.load(settings); - refreshSettings(clientsSettings); + public AzureStorageService( + Settings settings, + AzureClientProvider azureClientProvider, + ClusterService clusterService, + ProjectResolver projectResolver + ) { + this.clientsManager = new AzureStorageClientsManager(settings, projectResolver.supportsMultipleProjects()); this.azureClientProvider = azureClientProvider; this.stateless = DiscoveryNode.isStateless(settings); this.multipartUploadMaxConcurrency = azureClientProvider.getMultipartUploadMaxConcurrency(); + if (projectResolver.supportsMultipleProjects()) { + clusterService.addHighPriorityApplier(this.clientsManager); + } } - public AzureBlobServiceClient client(String clientName, LocationMode locationMode, OperationPurpose purpose) { - return client(clientName, locationMode, purpose, null); + public AzureBlobServiceClient client( + @Nullable ProjectId projectId, + String clientName, + LocationMode locationMode, + OperationPurpose purpose + ) { + return client(projectId, clientName, locationMode, purpose, null); } public AzureBlobServiceClient client( + @Nullable ProjectId projectId, String clientName, LocationMode locationMode, OperationPurpose purpose, AzureClientProvider.RequestMetricsHandler requestMetricsHandler ) { - final AzureStorageSettings azureStorageSettings = getClientSettings(clientName); + return clientsManager.client(projectId, clientName, locationMode, purpose, requestMetricsHandler); + } + private AzureBlobServiceClient buildClient( + LocationMode locationMode, + OperationPurpose purpose, + AzureClientProvider.RequestMetricsHandler requestMetricsHandler, + AzureStorageSettings azureStorageSettings + ) { RequestRetryOptions retryOptions = getRetryOptions(locationMode, azureStorageSettings); ProxyOptions proxyOptions = getProxyOptions(azureStorageSettings); return azureClientProvider.createClient( @@ -109,14 +140,6 @@ public AzureBlobServiceClient client( ); } - private AzureStorageSettings getClientSettings(String clientName) { - final AzureStorageSettings azureStorageSettings = this.storageSettings.get(clientName); - if (azureStorageSettings == null) { - throw new SettingsException("Unable to find client with name [" + clientName + "]"); - } - return azureStorageSettings; - } - private static ProxyOptions getProxyOptions(AzureStorageSettings settings) { Proxy proxy = settings.getProxy(); if (proxy == null) { @@ -135,8 +158,8 @@ long getUploadBlockSize() { return DEFAULT_UPLOAD_BLOCK_SIZE; } - int getMaxReadRetries(String clientName) { - AzureStorageSettings azureStorageSettings = getClientSettings(clientName); + int getMaxReadRetries(@Nullable ProjectId projectId, String clientName) { + AzureStorageSettings azureStorageSettings = clientsManager.getClientSettings(projectId, clientName); return azureStorageSettings.getMaxRetries(); } @@ -178,22 +201,21 @@ RequestRetryOptions getRetryOptions(LocationMode locationMode, AzureStorageSetti } /** - * Updates settings for building clients. Any client cache is cleared. Future + * Updates settings for building cluster level clients. Any client cache is cleared. Future * client requests will use the new refreshed settings. * * @param clientsSettings the settings for new clients */ - public void refreshSettings(Map clientsSettings) { - this.storageSettings = Map.copyOf(clientsSettings); - // clients are built lazily by {@link client(String, LocationMode)} + public void refreshClusterClientSettings(Map clientsSettings) { + clientsManager.refreshClusterClientSettings(clientsSettings); } /** * For Azure repositories, we report the different kinds of credentials in use in the telemetry. */ - public Set getExtraUsageFeatures(String clientName) { + public Set getExtraUsageFeatures(@Nullable ProjectId projectId, String clientName) { try { - return getClientSettings(clientName).credentialsUsageFeatures(); + return clientsManager.getClientSettings(projectId, clientName).credentialsUsageFeatures(); } catch (Exception e) { return Set.of(); } @@ -202,4 +224,155 @@ public Set getExtraUsageFeatures(String clientName) { public int getMultipartUploadMaxConcurrency() { return multipartUploadMaxConcurrency; } + + // Package private for testing + Map getStorageSettings() { + return clientsManager.clusterStorageSettings; + } + + // Package private for testing + AzureStorageClientsManager getClientsManager() { + return clientsManager; + } + + class AzureStorageClientsManager implements ClusterStateApplier { + private static final String AZURE_SETTING_PREFIX = "azure."; + + private final Settings nodeAzureSettings; + private volatile Map clusterStorageSettings; + private final Map> perProjectStorageSettings; + + AzureStorageClientsManager(Settings nodeSettings, boolean supportsMultipleProjects) { + // eagerly load client settings so that secure settings are read + final Map clientsSettings = AzureStorageSettings.load(nodeSettings); + refreshClusterClientSettings(clientsSettings); + + this.nodeAzureSettings = Settings.builder() + .put(nodeSettings.getByPrefix(AZURE_SETTING_PREFIX), false) // not rely on any cluster scoped secrets + .normalizePrefix(AZURE_SETTING_PREFIX) + .build(); + if (supportsMultipleProjects) { + this.perProjectStorageSettings = ConcurrentCollections.newConcurrentMap(); + } else { + this.perProjectStorageSettings = null; + } + } + + @Override + public void applyClusterState(ClusterChangedEvent event) { + assert perProjectStorageSettings != null; + final Map currentProjects = event.state().metadata().projects(); + + for (var project : currentProjects.values()) { + // Skip the default project, it is tracked separately with clusterStorageSettings and + // updated differently with the ReloadablePlugin interface + if (ProjectId.DEFAULT.equals(project.id())) { + continue; + } + final ProjectSecrets projectSecrets = project.custom(ProjectSecrets.TYPE); + // Project secrets can be null when node restarts. It may not have any azure credentials if azure is not in use. + if (projectSecrets == null + || projectSecrets.getSettingNames().stream().noneMatch(key -> key.startsWith(AZURE_SETTING_PREFIX))) { + // Most likely there won't be any existing client settings, but attempt to remove it anyway just in case + perProjectStorageSettings.remove(project.id()); + continue; + } + + final Settings currentSettings = Settings.builder() + // merge with static settings such as max retries etc + // TODO: https://elasticco.atlassian.net/browse/ES-11716 Consider change this to use per-project settings + .put(nodeAzureSettings) + .setSecureSettings(projectSecrets.getSettings()) + .build(); + final var allClientSettings = AzureStorageSettings.load(currentSettings); + // Skip project client settings that have no credentials configured. This should not happen in serverless. + // But it is safer to skip them and is also a more consistent behaviour with the cases when + // project secrets are not present. + final Map clientSettingsWithCredentials = allClientSettings.entrySet() + .stream() + .filter(entry -> entry.getValue().hasCredentials()) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); + + // TODO: If performance is an issue, we may consider comparing just the relevant project secrets for new or updated clients + // and avoid building the clientSettings + if (newOrUpdated(project.id(), clientSettingsWithCredentials)) { + if (allClientSettings.size() != clientSettingsWithCredentials.size()) { + logger.warn( + "Project [{}] has [{}] azure client settings, but [{}] is usable due to missing credentials for clients {}", + project.id(), + allClientSettings.size(), + clientSettingsWithCredentials.size(), + Sets.difference(allClientSettings.keySet(), clientSettingsWithCredentials.keySet()) + ); + } + perProjectStorageSettings.put(project.id(), clientSettingsWithCredentials); + } + } + + // Removed projects + for (var projectId : perProjectStorageSettings.keySet()) { + if (currentProjects.containsKey(projectId) == false) { + assert ProjectId.DEFAULT.equals(projectId) == false; + perProjectStorageSettings.remove(projectId); + } + } + } + + public AzureBlobServiceClient client( + @Nullable ProjectId projectId, + String clientName, + LocationMode locationMode, + OperationPurpose purpose, + AzureClientProvider.RequestMetricsHandler requestMetricsHandler + ) { + final var azureStorageSettings = getClientSettings(projectId, clientName); // ensure the client exists + return buildClient(locationMode, purpose, requestMetricsHandler, azureStorageSettings); + } + + public AzureStorageSettings getClientSettings(@Nullable ProjectId projectId, String clientName) { + final var allClientSettings = getAllClientSettings(projectId); + final var azureStorageSettings = allClientSettings.get(clientName); + if (azureStorageSettings == null) { + throw new SettingsException( + "Unable to find client with name [" + + clientName + + "]" + + (isDefaultProjectIdOrNull(projectId) ? "" : " for project [" + projectId + "]") + ); + } + return azureStorageSettings; + } + + Map getAllClientSettings(@Nullable ProjectId projectId) { + if (projectId == null || ProjectId.DEFAULT.equals(projectId)) { + return clusterStorageSettings; + } + final var projectClientSettings = perProjectStorageSettings.get(projectId); + if (projectClientSettings == null) { + throw new SettingsException("Unable to find any client for project [" + projectId + "]"); + } + return projectClientSettings; + } + + Map> getPerProjectStorageSettings() { + return perProjectStorageSettings == null ? null : Map.copyOf(perProjectStorageSettings); + } + + private void refreshClusterClientSettings(Map clientsSettings) { + this.clusterStorageSettings = Map.copyOf(clientsSettings); + // clients are built lazily by {@link client(String, LocationMode)} + } + + private boolean newOrUpdated(ProjectId projectId, Map currentClientSettings) { + final var old = perProjectStorageSettings.get(projectId); + if (old == null) { + return true; + } + return currentClientSettings.equals(old) == false; + } + + private static boolean isDefaultProjectIdOrNull(@Nullable ProjectId projectId) { + return projectId == null || ProjectId.DEFAULT.equals(projectId); + } + } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java index f86c4d661cece..e1fa466ea2e44 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; final class AzureStorageSettings { @@ -378,4 +379,23 @@ private String deriveURIFromSettings(boolean isPrimary) { public Set credentialsUsageFeatures() { return credentialsUsageFeatures; } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + AzureStorageSettings that = (AzureStorageSettings) o; + return maxRetries == that.maxRetries + && hasCredentials == that.hasCredentials + && Objects.equals(account, that.account) + && Objects.equals(connectString, that.connectString) + && Objects.equals(endpointSuffix, that.endpointSuffix) + && Objects.equals(timeout, that.timeout) + && Objects.equals(proxy, that.proxy) + && Objects.equals(credentialsUsageFeatures, that.credentialsUsageFeatures); + } + + @Override + public int hashCode() { + return Objects.hash(account, connectString, endpointSuffix, timeout, maxRetries, proxy, hasCredentials, credentialsUsageFeatures); + } } diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java index 39fe222d47a04..04566a1874b3c 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java @@ -14,8 +14,11 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.project.TestProjectResolvers; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.network.InetAddresses; @@ -30,6 +33,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.mocksocket.MockHttpServer; import org.elasticsearch.repositories.RepositoriesMetrics; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -70,6 +74,7 @@ public abstract class AbstractAzureServerTestCase extends ESTestCase { protected boolean serverlessMode; private ThreadPool threadPool; private AzureClientProvider clientProvider; + private ClusterService clusterService; @Before public void setUp() throws Exception { @@ -85,6 +90,7 @@ public void setUp() throws Exception { secondaryHttpServer.start(); clientProvider = AzureClientProvider.create(threadPool, Settings.EMPTY); clientProvider.start(); + clusterService = ClusterServiceUtils.createClusterService(threadPool); super.setUp(); } @@ -131,7 +137,12 @@ protected BlobContainer createBlobContainer( clientSettings.setSecureSettings(secureSettings); clientSettings.put(DiscoveryNode.STATELESS_ENABLED_SETTING_NAME, serverlessMode); - final AzureStorageService service = new AzureStorageService(clientSettings.build(), clientProvider) { + final AzureStorageService service = new AzureStorageService( + clientSettings.build(), + clientProvider, + clusterService, + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ) { @Override RequestRetryOptions getRetryOptions(LocationMode locationMode, AzureStorageSettings azureStorageSettings) { return new RequestRetryOptions( @@ -153,7 +164,7 @@ long getUploadBlockSize() { } @Override - int getMaxReadRetries(String clientName) { + int getMaxReadRetries(ProjectId projectId, String clientName) { return maxRetries; } }; @@ -171,7 +182,7 @@ int getMaxReadRetries(String clientName) { return new AzureBlobContainer( BlobPath.EMPTY, - new AzureBlobStore(repositoryMetadata, service, BigArrays.NON_RECYCLING_INSTANCE, RepositoriesMetrics.NOOP) + new AzureBlobStore(ProjectId.DEFAULT, repositoryMetadata, service, BigArrays.NON_RECYCLING_INSTANCE, RepositoriesMetrics.NOOP) ); } diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageClientsManagerTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageClientsManagerTests.java new file mode 100644 index 0000000000000..3a02eb97b3119 --- /dev/null +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageClientsManagerTests.java @@ -0,0 +1,439 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.azure; + +import joptsimple.internal.Strings; + +import com.azure.core.http.ProxyOptions; +import com.azure.storage.common.policy.RequestRetryOptions; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.project.TestProjectResolvers; +import org.elasticsearch.cluster.routing.GlobalRoutingTable; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.blobstore.OperationPurpose; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.ProjectSecrets; +import org.elasticsearch.common.settings.SecureClusterStateSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsException; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.repositories.azure.AzureStorageService.AzureStorageClientsManager; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.elasticsearch.repositories.azure.AzureStorageServiceTests.encodeKey; +import static org.elasticsearch.repositories.azure.AzureStorageSettings.ACCOUNT_SETTING; +import static org.elasticsearch.repositories.azure.AzureStorageSettings.KEY_SETTING; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +public class AzureStorageClientsManagerTests extends ESTestCase { + + private Map idGenerators; + private List clientNames; + private Map clusterClientsSettings; + private TestThreadPool threadPool; + private ClusterService clusterService; + private TestAzureClientProvider azureClientProvider; + private AzureStorageService azureStorageService; + private AzureStorageClientsManager azureClientsManager; + + @Override + public void setUp() throws Exception { + super.setUp(); + idGenerators = ConcurrentCollections.newConcurrentMap(); + clientNames = IntStream.range(0, between(2, 5)).mapToObj(i -> randomIdentifier() + "_" + i).toList(); + + final Settings.Builder builder = Settings.builder(); + final var mockSecureSettings = new MockSecureSettings(); + clientNames.forEach(clientName -> { + mockSecureSettings.setString( + ACCOUNT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), + clientName + "_cluster_account" + ); + mockSecureSettings.setString( + KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), + encodeKey(clientName + "_cluster_key") + ); + if (randomBoolean()) { + builder.put("azure.client." + clientName + ".max_retries", between(1, 10)); + } + if (randomBoolean()) { + builder.put("azure.client." + clientName + ".timeout", between(1, 99) + "s"); + } + }); + + final Settings settings = builder.setSecureSettings(mockSecureSettings).build(); + clusterClientsSettings = AzureStorageSettings.load(settings); + threadPool = new TestThreadPool(getTestName()); + clusterService = ClusterServiceUtils.createClusterService(threadPool, settings); + + azureClientProvider = new TestAzureClientProvider(between(1, 10)); + azureStorageService = new AzureStorageService( + settings, + azureClientProvider, + clusterService, + TestProjectResolvers.allProjects() // with multiple projects support + ); + azureClientsManager = azureStorageService.getClientsManager(); + assertThat(azureClientsManager.getAllClientSettings(randomFrom(ProjectId.DEFAULT, null)), equalTo(clusterClientsSettings)); + assertNotNull(azureClientsManager.getPerProjectStorageSettings()); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + clusterService.close(); + threadPool.close(); + } + + public void testDoesNotCreateClientWhenSecretsAreNotConfigured() { + assertThat(azureClientsManager.getPerProjectStorageSettings(), anEmptyMap()); + final ProjectId projectId = randomUniqueProjectId(); + + // No project secrets at all + ClusterServiceUtils.setState( + clusterService, + ClusterState.builder(clusterService.state()).putProjectMetadata(ProjectMetadata.builder(projectId)).build() + ); + assertThat(azureClientsManager.getPerProjectStorageSettings(), anEmptyMap()); + + // Project secrets but no azure credentials + final var mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setFile( + Strings.join(randomList(1, 5, ESTestCase::randomIdentifier), "."), + randomByteArrayOfLength(between(8, 20)) + ); + ClusterServiceUtils.setState( + clusterService, + ClusterState.builder(clusterService.state()) + .putProjectMetadata( + ProjectMetadata.builder(projectId) + .putCustom(ProjectSecrets.TYPE, new ProjectSecrets(new SecureClusterStateSettings(mockSecureSettings))) + ) + .build() + ); + assertThat(azureClientsManager.getPerProjectStorageSettings(), anEmptyMap()); + } + + public void testClientsLifeCycleForSingleProject() throws Exception { + final ProjectId projectId = randomUniqueProjectId(); + final String clientName = randomFrom(clientNames); + final String anotherClientName = randomValueOtherThan(clientName, () -> randomFrom(clientNames)); + + // Configure project secrets for one client + assertClientNotFound(projectId, clientName); + updateProjectInClusterState(projectId, newProjectClientsSecrets(projectId, clientName)); + { + assertProjectClientSettings(projectId, clientName); + + // Retrieve client for the 1st time + final var initialClient = getClientFromManager(projectId, clientName); + assertClientCredentials(projectId, clientName, initialClient); + + // Client not configured cannot be accessed, + assertClientNotFound(projectId, anotherClientName); + + // Update client secrets to enable another client + updateProjectInClusterState(projectId, newProjectClientsSecrets(projectId, clientName, anotherClientName)); + assertProjectClientSettings(projectId, clientName, anotherClientName); + final var anotherClient = getClientFromManager(projectId, anotherClientName); + assertClientCredentials(projectId, anotherClientName, anotherClient); + } + + // Remove project secrets or the entire project + if (randomBoolean()) { + updateProjectInClusterState(projectId, Map.of()); + } else { + removeProjectFromClusterState(projectId); + } + assertClientNotFound(projectId, clientName); + + assertThat(azureClientsManager.getPerProjectStorageSettings(), not(hasKey(projectId))); + } + + public void testClientsWithNoCredentialsAreFilteredOut() throws IOException { + final ProjectId projectId = randomUniqueProjectId(); + updateProjectInClusterState(projectId, newProjectClientsSecrets(projectId, clientNames.toArray(String[]::new))); + for (var clientName : clientNames) { + assertNotNull(getClientFromManager(projectId, clientName)); + } + + final List clientsWithIncorrectSecretsConfig = randomNonEmptySubsetOf(clientNames); + + final Map projectClientSecrets = new HashMap<>(); + for (var clientName : clientNames) { + if (clientsWithIncorrectSecretsConfig.contains(clientName)) { + projectClientSecrets.put("azure.client." + clientName + ".some_non_existing_setting", randomAlphaOfLength(12)); + } else { + projectClientSecrets.put( + ACCOUNT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), + projectClientAccount(projectId, clientName) + ); + projectClientSecrets.put( + KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), + projectClientKey(projectId, clientName) + ); + } + } + updateProjectInClusterState(projectId, projectClientSecrets); + for (var clientName : clientNames) { + if (clientsWithIncorrectSecretsConfig.contains(clientName)) { + assertClientNotFound(projectId, clientName); + } else { + assertNotNull(getClientFromManager(projectId, clientName)); + } + } + } + + public void testClientsForMultipleProjects() throws InterruptedException { + final List projectIds = randomList(2, 8, ESTestCase::randomUniqueProjectId); + + final List threads = projectIds.stream().map(projectId -> new Thread(() -> { + final int iterations = between(1, 3); + for (var i = 0; i < iterations; i++) { + final List clientNames = randomNonEmptySubsetOf(this.clientNames); + updateProjectInClusterState(projectId, newProjectClientsSecrets(projectId, clientNames.toArray(String[]::new))); + + assertProjectClientSettings(projectId, clientNames.toArray(String[]::new)); + for (var clientName : shuffledList(clientNames)) { + try { + final var azureBlobServiceClient = getClientFromManager(projectId, clientName); + assertClientCredentials(projectId, clientName, azureBlobServiceClient); + } catch (IOException e) { + fail(e); + } + } + + if (randomBoolean()) { + updateProjectInClusterState(projectId, Map.of()); + } else { + removeProjectFromClusterState(projectId); + } + assertThat(azureClientsManager.getPerProjectStorageSettings(), not(hasKey(projectId))); + clientNames.forEach(clientName -> assertClientNotFound(projectId, clientName)); + } + })).toList(); + + threads.forEach(Thread::start); + for (var thread : threads) { + assertTrue(thread.join(Duration.ofSeconds(10))); + } + } + + public void testClusterAndProjectClients() throws IOException { + final ProjectId projectId = randomUniqueProjectId(); + final String clientName = randomFrom(clientNames); + final boolean configureProjectClientsFirst = randomBoolean(); + if (configureProjectClientsFirst) { + updateProjectInClusterState(projectId, newProjectClientsSecrets(projectId, clientName)); + } + + final var clusterClient = getClientFromService(projectIdForClusterClient(), clientName); + if (configureProjectClientsFirst == false) { + assertThat(azureClientsManager.getPerProjectStorageSettings(), anEmptyMap()); + updateProjectInClusterState(projectId, newProjectClientsSecrets(projectId, clientName)); + } + final var projectClient = getClientFromService(projectId, clientName); + assertThat(projectClient, not(sameInstance(clusterClient))); + } + + public void testProjectClientsDisabled() throws IOException { + final var clusterService = spy(this.clusterService); + final var serviceWithNoProjectSupport = new AzureStorageService( + clusterService.getSettings(), + azureClientProvider, + clusterService, + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ); + verify(clusterService, never()).addHighPriorityApplier(any()); + assertNull(serviceWithNoProjectSupport.getClientsManager().getPerProjectStorageSettings()); + + // Cluster client still works + final String clientName = randomFrom(clientNames); + assertNotNull(getClientFromService(projectIdForClusterClient(), clientName)); + } + + private TestAzureBlobServiceClient getClientFromManager(ProjectId projectId, String clientName) throws IOException { + return asInstanceOf( + TestAzureBlobServiceClient.class, + azureClientsManager.client(projectId, clientName, LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()), null) + ); + } + + private TestAzureBlobServiceClient getClientFromService(ProjectId projectId, String clientName) throws IOException { + return asInstanceOf( + TestAzureBlobServiceClient.class, + azureStorageService.client(projectId, clientName, LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values())) + ); + } + + private ProjectId projectIdForClusterClient() { + return randomBoolean() ? ProjectId.DEFAULT : null; + } + + private void assertProjectClientSettings(ProjectId projectId, String... clientNames) { + final var allClientSettingsMap = azureClientsManager.getPerProjectStorageSettings().get(projectId); + assertNotNull(allClientSettingsMap); + // The default client is always present when there is any other client configured + assertThat( + allClientSettingsMap.keySet(), + equalTo(Stream.concat(Stream.of("default"), Stream.of(clientNames)).collect(Collectors.toSet())) + ); + + for (var clientName : clientNames) { + final var projectClientSettings = allClientSettingsMap.get(clientName); + final var clusterClientSettings = clusterClientsSettings.get(clientName); + assertNotNull(clusterClientSettings); + + // Picks up the correct project scoped credentials + final var projectClientCredentials = projectClientSettings.getConnectString(); + assertThat(projectClientCredentials, not(equalTo(clusterClientSettings.getConnectString()))); + assertThat(projectClientCredentials, containsString(";AccountName=" + projectClientAccount(projectId, clientName))); + assertThat(projectClientCredentials, containsString(";AccountKey=" + projectClientKey(projectId, clientName))); + // Inherit setting override from the cluster client of the same name + assertThat(projectClientSettings.getMaxRetries(), equalTo(clusterClientSettings.getMaxRetries())); + assertThat(projectClientSettings.getTimeout(), equalTo(clusterClientSettings.getTimeout())); + } + } + + private void assertClientCredentials(ProjectId projectId, String clientName, TestAzureBlobServiceClient azureBlobServiceClient) { + final String connectString = azureBlobServiceClient.settings.getConnectString(); + assertThat(connectString, containsString(";AccountName=" + projectClientAccount(projectId, clientName))); + assertThat(connectString, containsString(";AccountKey=" + projectClientKey(projectId, clientName))); + } + + private void assertClientNotFound(ProjectId projectId, String clientName) { + final SettingsException e = expectThrows(SettingsException.class, () -> getClientFromManager(projectId, clientName)); + assertThat( + e.getMessage(), + anyOf( + containsString("Unable to find client with name [" + clientName + "]"), + containsString("Unable to find any client for project [" + projectId + "]") + ) + ); + } + + private void updateProjectInClusterState(ProjectId projectId, Map projectClientSecrets) { + final var mockSecureSettings = new MockSecureSettings(); + projectClientSecrets.forEach((k, v) -> mockSecureSettings.setFile(k, v.getBytes(StandardCharsets.UTF_8))); + // Sometimes add an unrelated project secret + if (randomBoolean() && randomBoolean()) { + mockSecureSettings.setFile( + Strings.join(randomList(1, 5, ESTestCase::randomIdentifier), "."), + randomByteArrayOfLength(between(18, 20)) + ); + } + final var secureClusterStateSettings = new SecureClusterStateSettings(mockSecureSettings); + + synchronized (this) { + final ClusterState initialState = clusterService.state(); + final ProjectMetadata.Builder projectBuilder = initialState.metadata().hasProject(projectId) + ? ProjectMetadata.builder(initialState.metadata().getProject(projectId)) + : ProjectMetadata.builder(projectId); + + if (secureClusterStateSettings.getSettingNames().isEmpty() == false + || projectBuilder.getCustom(ProjectSecrets.TYPE) != null + || randomBoolean()) { + projectBuilder.putCustom(ProjectSecrets.TYPE, new ProjectSecrets(secureClusterStateSettings)); + } + + final ClusterState stateWithProject = ClusterState.builder(initialState).putProjectMetadata(projectBuilder).build(); + ClusterServiceUtils.setState(clusterService, stateWithProject); + } + } + + private void removeProjectFromClusterState(ProjectId projectId) { + synchronized (this) { + final ClusterState initialState = clusterService.state(); + final ClusterState stateWithoutProject = ClusterState.builder(initialState) + .metadata(Metadata.builder(initialState.metadata()).removeProject(projectId)) + .routingTable(GlobalRoutingTable.builder(initialState.globalRoutingTable()).removeProject(projectId).build()) + .build(); + ClusterServiceUtils.setState(clusterService, stateWithoutProject); + } + } + + private Map newProjectClientsSecrets(ProjectId projectId, String... clientNames) { + idGenerators.computeIfAbsent(projectId, ignored -> new AtomicInteger(0)).incrementAndGet(); + final Map m = new HashMap<>(); + Arrays.stream(clientNames).forEach(clientName -> { + m.put(ACCOUNT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectClientAccount(projectId, clientName)); + m.put(KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectClientKey(projectId, clientName)); + }); + return Map.copyOf(m); + } + + private String projectClientAccount(ProjectId projectId, String clientName) { + return projectId + "_" + clientName + "_account_" + idGenerators.get(projectId).get(); + } + + private String projectClientKey(ProjectId projectId, String clientName) { + return encodeKey(projectId + "_" + clientName + "_key_" + idGenerators.get(projectId).get()); + } + + private String repoNameForClient(String clientName) { + return "repo_for_" + clientName; + } + + static class TestAzureClientProvider extends AzureClientProvider { + + TestAzureClientProvider(int multipartUploadMaxConcurrency) { + super(null, null, null, null, null, multipartUploadMaxConcurrency); + } + + @Override + TestAzureBlobServiceClient createClient( + AzureStorageSettings settings, + LocationMode locationMode, + RequestRetryOptions retryOptions, + ProxyOptions proxyOptions, + RequestMetricsHandler requestMetricsHandler, + OperationPurpose purpose + ) { + return new TestAzureBlobServiceClient(settings); + } + } + + static class TestAzureBlobServiceClient extends AzureBlobServiceClient { + final AzureStorageSettings settings; + + TestAzureBlobServiceClient(AzureStorageSettings settings) { + super(null, null, settings.getMaxRetries(), null); + this.settings = settings; + } + } +} diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java index f4cb86cb5431f..9eda4c4666aa5 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java @@ -11,6 +11,8 @@ import com.azure.storage.common.policy.RequestRetryOptions; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; @@ -79,6 +81,7 @@ private AzureRepositoryPlugin pluginWithSettingsValidation(Settings settings) { new SettingsModule(settings, plugin.getSettings(), Collections.emptyList()); Plugin.PluginServices services = mock(Plugin.PluginServices.class); when(services.threadPool()).thenReturn(threadPool); + when(services.projectResolver()).thenReturn(TestProjectResolvers.DEFAULT_PROJECT_ONLY); plugin.createComponents(services); return plugin; } @@ -99,6 +102,7 @@ public void testCreateClientWithEndpointSuffix() throws IOException { try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); AzureBlobServiceClient client1 = azureStorageService.client( + ProjectId.DEFAULT, "azure1", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -106,6 +110,7 @@ public void testCreateClientWithEndpointSuffix() throws IOException { assertThat(client1.getSyncClient().getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); AzureBlobServiceClient client2 = azureStorageService.client( + ProjectId.DEFAULT, "azure2", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -131,6 +136,7 @@ public void testReinitClientSettings() throws IOException { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); AzureBlobServiceClient client11 = azureStorageService.client( + ProjectId.DEFAULT, "azure1", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -138,6 +144,7 @@ public void testReinitClientSettings() throws IOException { assertThat(client11.getSyncClient().getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net")); AzureBlobServiceClient client12 = azureStorageService.client( + ProjectId.DEFAULT, "azure2", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -147,7 +154,12 @@ public void testReinitClientSettings() throws IOException { // client 3 is missing final SettingsException e1 = expectThrows( SettingsException.class, - () -> azureStorageService.client("azure3", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values())) + () -> azureStorageService.client( + ProjectId.DEFAULT, + "azure3", + LocationMode.PRIMARY_ONLY, + randomFrom(OperationPurpose.values()) + ) ); assertThat(e1.getMessage(), is("Unable to find client with name [azure3]")); @@ -159,6 +171,7 @@ public void testReinitClientSettings() throws IOException { // new client 1 is changed AzureBlobServiceClient client21 = azureStorageService.client( + ProjectId.DEFAULT, "azure1", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -171,12 +184,18 @@ public void testReinitClientSettings() throws IOException { // new client2 is gone final SettingsException e2 = expectThrows( SettingsException.class, - () -> azureStorageService.client("azure2", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values())) + () -> azureStorageService.client( + ProjectId.DEFAULT, + "azure2", + LocationMode.PRIMARY_ONLY, + randomFrom(OperationPurpose.values()) + ) ); assertThat(e2.getMessage(), is("Unable to find client with name [azure2]")); // client 3 emerged AzureBlobServiceClient client23 = azureStorageService.client( + ProjectId.DEFAULT, "azure3", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -193,6 +212,7 @@ public void testReinitClientEmptySettings() throws IOException { try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); AzureBlobServiceClient client11 = azureStorageService.client( + ProjectId.DEFAULT, "azure1", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -205,7 +225,12 @@ public void testReinitClientEmptySettings() throws IOException { // client is no longer registered final SettingsException e = expectThrows( SettingsException.class, - () -> azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values())) + () -> azureStorageService.client( + ProjectId.DEFAULT, + "azure1", + LocationMode.PRIMARY_ONLY, + randomFrom(OperationPurpose.values()) + ) ); assertThat(e.getMessage(), equalTo("Unable to find client with name [azure1]")); } @@ -225,6 +250,7 @@ public void testReinitClientWrongSettings() throws IOException { try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); AzureBlobServiceClient client11 = azureStorageService.client( + ProjectId.DEFAULT, "azure1", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -242,9 +268,9 @@ public void testReinitClientWrongSettings() throws IOException { public void testNoProxy() { final Settings settings = Settings.builder().setSecureSettings(buildSecureSettings()).build(); final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - assertThat(mock.storageSettings.get("azure1").getProxy(), nullValue()); - assertThat(mock.storageSettings.get("azure2").getProxy(), nullValue()); - assertThat(mock.storageSettings.get("azure3").getProxy(), nullValue()); + assertThat(mock.getStorageSettings().get("azure1").getProxy(), nullValue()); + assertThat(mock.getStorageSettings().get("azure2").getProxy(), nullValue()); + assertThat(mock.getStorageSettings().get("azure3").getProxy(), nullValue()); } public void testProxyHttp() throws UnknownHostException { @@ -255,13 +281,13 @@ public void testProxyHttp() throws UnknownHostException { .put("azure.client.azure1.proxy.type", "http") .build(); final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final Proxy azure1Proxy = mock.storageSettings.get("azure1").getProxy(); + final Proxy azure1Proxy = mock.getStorageSettings().get("azure1").getProxy(); assertThat(azure1Proxy, notNullValue()); assertThat(azure1Proxy.type(), is(Proxy.Type.HTTP)); assertThat(azure1Proxy.address(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); - assertThat(mock.storageSettings.get("azure2").getProxy(), nullValue()); - assertThat(mock.storageSettings.get("azure3").getProxy(), nullValue()); + assertThat(mock.getStorageSettings().get("azure2").getProxy(), nullValue()); + assertThat(mock.getStorageSettings().get("azure3").getProxy(), nullValue()); } public void testMultipleProxies() throws UnknownHostException { @@ -275,15 +301,15 @@ public void testMultipleProxies() throws UnknownHostException { .put("azure.client.azure2.proxy.type", "http") .build(); final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final Proxy azure1Proxy = mock.storageSettings.get("azure1").getProxy(); + final Proxy azure1Proxy = mock.getStorageSettings().get("azure1").getProxy(); assertThat(azure1Proxy, notNullValue()); assertThat(azure1Proxy.type(), is(Proxy.Type.HTTP)); assertThat(azure1Proxy.address(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); - final Proxy azure2Proxy = mock.storageSettings.get("azure2").getProxy(); + final Proxy azure2Proxy = mock.getStorageSettings().get("azure2").getProxy(); assertThat(azure2Proxy, notNullValue()); assertThat(azure2Proxy.type(), is(Proxy.Type.HTTP)); assertThat(azure2Proxy.address(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8081))); - assertThat(mock.storageSettings.get("azure3").getProxy(), nullValue()); + assertThat(mock.getStorageSettings().get("azure3").getProxy(), nullValue()); } public void testProxySocks() throws UnknownHostException { @@ -294,12 +320,12 @@ public void testProxySocks() throws UnknownHostException { .put("azure.client.azure1.proxy.type", "socks") .build(); final AzureStorageService mock = storageServiceWithSettingsValidation(settings); - final Proxy azure1Proxy = mock.storageSettings.get("azure1").getProxy(); + final Proxy azure1Proxy = mock.getStorageSettings().get("azure1").getProxy(); assertThat(azure1Proxy, notNullValue()); assertThat(azure1Proxy.type(), is(Proxy.Type.SOCKS)); assertThat(azure1Proxy.address(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080))); - assertThat(mock.storageSettings.get("azure2").getProxy(), nullValue()); - assertThat(mock.storageSettings.get("azure3").getProxy(), nullValue()); + assertThat(mock.getStorageSettings().get("azure2").getProxy(), nullValue()); + assertThat(mock.getStorageSettings().get("azure3").getProxy(), nullValue()); } public void testProxyNoHost() { @@ -351,7 +377,7 @@ public void testDefaultTimeOut() throws Exception { try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); - AzureStorageSettings azureStorageSettings = azureStorageService.storageSettings.get("azure1"); + AzureStorageSettings azureStorageSettings = azureStorageService.getStorageSettings().get("azure1"); RequestRetryOptions retryOptions = azureStorageService.getRetryOptions(LocationMode.PRIMARY_ONLY, azureStorageSettings); assertThat(retryOptions.getTryTimeout(), equalTo(Integer.MAX_VALUE)); } @@ -365,7 +391,7 @@ public void testMillisecondsTimeOutIsRoundedUp() throws Exception { try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); - AzureStorageSettings azureStorageSettings = azureStorageService.storageSettings.get("azure1"); + AzureStorageSettings azureStorageSettings = azureStorageService.getStorageSettings().get("azure1"); RequestRetryOptions retryOptions = azureStorageService.getRetryOptions(LocationMode.PRIMARY_ONLY, azureStorageSettings); assertThat(retryOptions.getTryTimeout(), equalTo(1)); } @@ -379,7 +405,7 @@ public void testTimeoutConfiguration() throws Exception { try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); - AzureStorageSettings azureStorageSettings = azureStorageService.storageSettings.get("azure1"); + AzureStorageSettings azureStorageSettings = azureStorageService.getStorageSettings().get("azure1"); RequestRetryOptions retryOptions = azureStorageService.getRetryOptions(LocationMode.PRIMARY_ONLY, azureStorageSettings); assertThat(retryOptions.getTryTimeout(), equalTo(200)); } @@ -409,7 +435,7 @@ public void testRetryConfigurationForSecondaryFallbackLocationModeImpl(boolean e try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); - AzureStorageSettings azureStorageSettings = azureStorageService.storageSettings.get("azure1"); + AzureStorageSettings azureStorageSettings = azureStorageService.getStorageSettings().get("azure1"); RequestRetryOptions retryOptions = azureStorageService.getRetryOptions( LocationMode.PRIMARY_THEN_SECONDARY, azureStorageSettings @@ -442,7 +468,7 @@ private void testRetryConfigurationForPrimaryFallbackLocationModeImpl(boolean en try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); - AzureStorageSettings azureStorageSettings = azureStorageService.storageSettings.get("azure1"); + AzureStorageSettings azureStorageSettings = azureStorageService.getStorageSettings().get("azure1"); RequestRetryOptions retryOptions = azureStorageService.getRetryOptions( LocationMode.SECONDARY_THEN_PRIMARY, azureStorageSettings @@ -459,7 +485,7 @@ public void testRetryConfigurationForLocationModeWithoutFallback() throws Except try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); - AzureStorageSettings azureStorageSettings = azureStorageService.storageSettings.get("azure1"); + AzureStorageSettings azureStorageSettings = azureStorageService.getStorageSettings().get("azure1"); LocationMode locationMode = randomFrom(LocationMode.PRIMARY_ONLY, LocationMode.SECONDARY_ONLY); RequestRetryOptions retryOptions = azureStorageService.getRetryOptions(locationMode, azureStorageSettings); @@ -476,7 +502,7 @@ public void testInvalidSettingsRetryConfigurationForLocationModeWithSecondaryFal try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) { final AzureStorageService azureStorageService = plugin.azureStoreService.get(); - AzureStorageSettings azureStorageSettings = azureStorageService.storageSettings.get("azure1"); + AzureStorageSettings azureStorageSettings = azureStorageService.getStorageSettings().get("azure1"); expectThrows( IllegalArgumentException.class, @@ -498,18 +524,34 @@ public void testCreateClientWithEndpoints() throws IOException { expectThrows( IllegalArgumentException.class, - () -> azureStorageService.client("azure1", LocationMode.PRIMARY_THEN_SECONDARY, randomFrom(OperationPurpose.values())) + () -> azureStorageService.client( + ProjectId.DEFAULT, + "azure1", + LocationMode.PRIMARY_THEN_SECONDARY, + randomFrom(OperationPurpose.values()) + ) ); expectThrows( IllegalArgumentException.class, - () -> azureStorageService.client("azure1", LocationMode.SECONDARY_ONLY, randomFrom(OperationPurpose.values())) + () -> azureStorageService.client( + ProjectId.DEFAULT, + "azure1", + LocationMode.SECONDARY_ONLY, + randomFrom(OperationPurpose.values()) + ) ); expectThrows( IllegalArgumentException.class, - () -> azureStorageService.client("azure1", LocationMode.SECONDARY_THEN_PRIMARY, randomFrom(OperationPurpose.values())) + () -> azureStorageService.client( + ProjectId.DEFAULT, + "azure1", + LocationMode.SECONDARY_THEN_PRIMARY, + randomFrom(OperationPurpose.values()) + ) ); AzureBlobServiceClient client1 = azureStorageService.client( + ProjectId.DEFAULT, "azure1", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()) @@ -518,6 +560,7 @@ public void testCreateClientWithEndpoints() throws IOException { assertThat( azureStorageService.client( + ProjectId.DEFAULT, "azure2", randomBoolean() ? LocationMode.PRIMARY_ONLY : LocationMode.PRIMARY_THEN_SECONDARY, randomFrom(OperationPurpose.values()) @@ -527,6 +570,7 @@ public void testCreateClientWithEndpoints() throws IOException { assertThat( azureStorageService.client( + ProjectId.DEFAULT, "azure2", randomBoolean() ? LocationMode.SECONDARY_ONLY : LocationMode.SECONDARY_THEN_PRIMARY, randomFrom(OperationPurpose.values()) @@ -576,7 +620,7 @@ private static MockSecureSettings buildSecureSettings() { return secureSettings; } - private static String encodeKey(final String value) { + public static String encodeKey(final String value) { return Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8)); } }