Skip to content

Commit 0813c44

Browse files
original-brownbearoriginal-brownbear
authored
Remove GCS Bucket Exists Check (#60899) (#60917)
Same as #43288 for GCS. We don't need to do the bucket exists check before using the repo, that just needlessly increases the necessary permissions for using the GCS repository.
1 parent d52d003 commit 0813c44

File tree

6 files changed

+12
-63
lines changed

6 files changed

+12
-63
lines changed

‎plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import com.google.cloud.storage.Blob;
2626
import com.google.cloud.storage.BlobId;
2727
import com.google.cloud.storage.BlobInfo;
28-
import com.google.cloud.storage.Bucket;
2928
import com.google.cloud.storage.Storage;
3029
import com.google.cloud.storage.Storage.BlobListOption;
3130
import com.google.cloud.storage.StorageBatch;
@@ -39,7 +38,6 @@
3938
import org.elasticsearch.common.blobstore.BlobMetadata;
4039
import org.elasticsearch.common.blobstore.BlobPath;
4140
import org.elasticsearch.common.blobstore.BlobStore;
42-
import org.elasticsearch.common.blobstore.BlobStoreException;
4341
import org.elasticsearch.common.blobstore.DeleteResult;
4442
import org.elasticsearch.common.blobstore.support.PlainBlobMetadata;
4543
import org.elasticsearch.common.collect.MapBuilder;
@@ -114,9 +112,6 @@ class GoogleCloudStorageBlobStore implements BlobStore {
114112
this.storageService = storageService;
115113
this.stats = new GoogleCloudStorageOperationsStats(bucketName);
116114
this.bufferSize = bufferSize;
117-
if (doesBucketExist(bucketName) == false) {
118-
throw new BlobStoreException("Bucket [" + bucketName + "] does not exist");
119-
}
120115
}
121116

122117
private Storage client() throws IOException {
@@ -133,21 +128,6 @@ public void close() {
133128
storageService.closeRepositoryClient(repositoryName);
134129
}
135130

136-
/**
137-
* Return true iff the given bucket exists
138-
*
139-
* @param bucketName name of the bucket
140-
* @return true iff the bucket exists
141-
*/
142-
private boolean doesBucketExist(String bucketName) {
143-
try {
144-
final Bucket bucket = SocketAccess.doPrivilegedIOException(() -> client().get(bucketName));
145-
return bucket != null;
146-
} catch (final Exception e) {
147-
throw new BlobStoreException("Unable to check if bucket [" + bucketName + "] exists", e);
148-
}
149-
}
150-
151131
/**
152132
* List blobs in the specific bucket under the specified path. The path root is removed.
153133
*
@@ -171,7 +151,7 @@ Map<String, BlobMetadata> listBlobsByPrefix(String path, String prefix) throws I
171151
final String pathPrefix = buildKey(path, prefix);
172152
final MapBuilder<String, BlobMetadata> mapBuilder = MapBuilder.newMapBuilder();
173153
SocketAccess.doPrivilegedVoidIOException(
174-
() -> client().get(bucketName).list(BlobListOption.currentDirectory(), BlobListOption.prefix(pathPrefix)).iterateAll().forEach(
154+
() -> client().list(bucketName, BlobListOption.currentDirectory(), BlobListOption.prefix(pathPrefix)).iterateAll().forEach(
175155
blob -> {
176156
assert blob.getName().startsWith(path);
177157
if (blob.isDirectory() == false) {
@@ -186,7 +166,7 @@ Map<String, BlobContainer> listChildren(BlobPath path) throws IOException {
186166
final String pathStr = path.buildAsString();
187167
final MapBuilder<String, BlobContainer> mapBuilder = MapBuilder.newMapBuilder();
188168
SocketAccess.doPrivilegedVoidIOException
189-
(() -> client().get(bucketName).list(BlobListOption.currentDirectory(), BlobListOption.prefix(pathStr)).iterateAll().forEach(
169+
(() -> client().list(bucketName, BlobListOption.currentDirectory(), BlobListOption.prefix(pathStr)).iterateAll().forEach(
190170
blob -> {
191171
if (blob.isDirectory()) {
192172
assert blob.getName().startsWith(pathStr);
@@ -378,7 +358,7 @@ private void writeBlobMultipart(BlobInfo blobInfo, InputStream inputStream, long
378358
DeleteResult deleteDirectory(String pathStr) throws IOException {
379359
return SocketAccess.doPrivilegedIOException(() -> {
380360
DeleteResult deleteResult = DeleteResult.ZERO;
381-
Page<Blob> page = client().get(bucketName).list(BlobListOption.prefix(pathStr));
361+
Page<Blob> page = client().list(bucketName, BlobListOption.prefix(pathStr));
382362
do {
383363
final Collection<String> blobsToDelete = new ArrayList<>();
384364
final AtomicLong blobsDeleted = new AtomicLong(0L);

‎plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import com.google.cloud.http.HttpTransportOptions;
2323
import com.google.cloud.storage.StorageException;
2424
import com.google.cloud.storage.StorageOptions;
25-
import com.sun.net.httpserver.HttpContext;
2625
import com.sun.net.httpserver.HttpHandler;
2726
import fixture.gcs.FakeOAuth2HttpHandler;
2827
import org.apache.http.HttpStatus;
@@ -57,7 +56,6 @@
5756
import java.net.SocketTimeoutException;
5857
import java.util.Arrays;
5958
import java.util.HashMap;
60-
import java.util.List;
6159
import java.util.Locale;
6260
import java.util.Map;
6361
import java.util.Objects;
@@ -157,21 +155,9 @@ StorageOptions createStorageOptions(final GoogleCloudStorageClientSettings clien
157155
};
158156
service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(clientSettings.build()));
159157

160-
final List<HttpContext> httpContexts = Arrays.asList(
161-
// Auth
162-
httpServer.createContext("/token", new FakeOAuth2HttpHandler()),
163-
// Does bucket exists?
164-
httpServer.createContext("/storage/v1/b/bucket", safeHandler(exchange -> {
165-
byte[] response = ("{\"kind\":\"storage#bucket\",\"name\":\"bucket\",\"id\":\"0\"}").getBytes(UTF_8);
166-
exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8");
167-
exchange.sendResponseHeaders(HttpStatus.SC_OK, response.length);
168-
exchange.getResponseBody().write(response);
169-
}))
170-
);
171-
158+
httpServer.createContext("/token", new FakeOAuth2HttpHandler());
172159
final GoogleCloudStorageBlobStore blobStore = new GoogleCloudStorageBlobStore("bucket", client, "repo", service,
173160
randomIntBetween(1, 8) * 1024);
174-
httpContexts.forEach(httpContext -> httpServer.removeContext(httpContext));
175161

176162
return new GoogleCloudStorageBlobContainer(BlobPath.cleanPath(), blobStore);
177163
}

‎plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838
import org.elasticsearch.common.blobstore.BlobContainer;
3939
import org.elasticsearch.common.blobstore.BlobPath;
4040
import org.elasticsearch.common.blobstore.BlobStore;
41-
import org.elasticsearch.common.blobstore.BlobStoreException;
4241
import org.elasticsearch.common.bytes.BytesArray;
4342
import org.elasticsearch.common.io.Streams;
4443
import org.elasticsearch.common.regex.Regex;
@@ -52,7 +51,6 @@
5251
import org.elasticsearch.plugins.Plugin;
5352
import org.elasticsearch.repositories.RepositoriesService;
5453
import org.elasticsearch.repositories.Repository;
55-
import org.elasticsearch.repositories.RepositoryException;
5654
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
5755
import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase;
5856
import org.junit.BeforeClass;
@@ -74,8 +72,6 @@
7472
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.TOKEN_URI_SETTING;
7573
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.BUCKET;
7674
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.CLIENT_NAME;
77-
import static org.hamcrest.Matchers.instanceOf;
78-
import static org.hamcrest.Matchers.is;
7975

8076
@SuppressForbidden(reason = "this test uses a HttpServer to emulate a Google Cloud Storage endpoint")
8177
public class GoogleCloudStorageBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTestCase {
@@ -215,16 +211,6 @@ public void testWriteReadLarge() throws IOException {
215211
}
216212
}
217213

218-
public void testBucketDoesNotExist() {
219-
RepositoryException ex = expectThrows(RepositoryException.class, () ->
220-
client().admin().cluster().preparePutRepository("invalid")
221-
.setType(repositoryType())
222-
.setVerify(true)
223-
.setSettings(Settings.builder().put(repositorySettings()).put("bucket", "missing")).get());
224-
assertThat(ex.getCause(), instanceOf(BlobStoreException.class));
225-
assertThat(ex.getCause().getMessage(), is("Bucket [missing] does not exist"));
226-
}
227-
228214
public static class TestGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin {
229215

230216
public TestGoogleCloudStoragePlugin(Settings settings) {

‎plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/20_repository.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ setup:
192192
"Register a repository with a non existing bucket":
193193

194194
- do:
195-
catch: /repository_exception/
195+
catch: /repository_verification_exception/
196196
snapshot.create_repository:
197197
repository: repository
198198
body:
@@ -205,7 +205,7 @@ setup:
205205
"Register a repository with a non existing client":
206206

207207
- do:
208-
catch: /repository_exception/
208+
catch: /repository_verification_exception/
209209
snapshot.create_repository:
210210
repository: repository
211211
body:

‎server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1203,7 +1203,7 @@ public String startVerification() {
12031203
}
12041204
return seed;
12051205
}
1206-
} catch (IOException exp) {
1206+
} catch (Exception exp) {
12071207
throw new RepositoryVerificationException(metadata.name(), "path " + basePath() + " is not accessible on master node", exp);
12081208
}
12091209
}
@@ -1214,7 +1214,7 @@ public void endVerification(String seed) {
12141214
try {
12151215
final String testPrefix = testBlobPrefix(seed);
12161216
blobStore().blobContainer(basePath().add(testPrefix)).delete();
1217-
} catch (IOException exp) {
1217+
} catch (Exception exp) {
12181218
throw new RepositoryVerificationException(metadata.name(), "cannot delete test data at " + basePath(), exp);
12191219
}
12201220
}
@@ -2152,7 +2152,7 @@ public void verify(String seed, DiscoveryNode localNode) {
21522152
if (isReadOnly()) {
21532153
try {
21542154
latestIndexBlobId();
2155-
} catch (IOException e) {
2155+
} catch (Exception e) {
21562156
throw new RepositoryVerificationException(metadata.name(), "path " + basePath() +
21572157
" is not accessible on node " + localNode, e);
21582158
}
@@ -2163,7 +2163,7 @@ public void verify(String seed, DiscoveryNode localNode) {
21632163
try (InputStream stream = bytes.streamInput()) {
21642164
testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length(), true);
21652165
}
2166-
} catch (IOException exp) {
2166+
} catch (Exception exp) {
21672167
throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() +
21682168
"] is not accessible on the node [" + localNode + "]", exp);
21692169
}
@@ -2178,7 +2178,7 @@ public void verify(String seed, DiscoveryNode localNode) {
21782178
"] cannot be accessed on the node [" + localNode + "]. " +
21792179
"This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or " +
21802180
"that permissions on the store don't allow reading files written by the master node", e);
2181-
} catch (IOException e) {
2181+
} catch (Exception e) {
21822182
throw new RepositoryVerificationException(metadata.name(), "Failed to verify repository", e);
21832183
}
21842184
}

‎test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,7 @@ public void handle(final HttpExchange exchange) throws IOException {
132132

133133
} else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "*", request)) {
134134
// GET Bucket https://cloud.google.com/storage/docs/json_api/v1/buckets/get
135-
byte[] response = ("{\"kind\":\"storage#bucket\",\"name\":\""+ bucket + "\",\"id\":\"0\"}").getBytes(UTF_8);
136-
exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8");
137-
exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
138-
exchange.getResponseBody().write(response);
135+
throw new AssertionError("Should not call get bucket API");
139136

140137
} else if (Regex.simpleMatch("GET /download/storage/v1/b/" + bucket + "/o/*", request)) {
141138
// Download Object https://cloud.google.com/storage/docs/request-body

0 commit comments

Comments
 (0)