Skip to content

Commit fa29dca

Browse files
psainicspsainics
authored
Merge pull request #1503 from cloudsufi/cherry-pick/083b4308a9a8624519ebe012659c5ede9ffd749b
[🍒]Wrap error thrown in Storage Client
2 parents a53e16d + e458e88 commit fa29dca

File tree

1 file changed

+103
-31
lines changed

1 file changed

+103
-31
lines changed

‎src/main/java/io/cdap/plugin/gcp/gcs/StorageClient.java‎

Lines changed: 103 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@
2828
import com.google.cloud.storage.StorageException;
2929
import com.google.cloud.storage.StorageOptions;
3030
import com.google.common.annotations.VisibleForTesting;
31+
import io.cdap.cdap.api.exception.ErrorCategory;
32+
import io.cdap.cdap.api.exception.ErrorType;
33+
import io.cdap.cdap.api.exception.ErrorUtils;
3134
import io.cdap.plugin.gcp.common.GCPConnectorConfig;
35+
import io.cdap.plugin.gcp.common.GCPErrorDetailsProviderUtil;
3236
import io.cdap.plugin.gcp.common.GCPUtils;
3337
import org.slf4j.Logger;
3438
import org.slf4j.LoggerFactory;
@@ -68,7 +72,14 @@ public Blob pickABlob(String path) {
6872
return null;
6973
}
7074
GCSPath gcsPath = GCSPath.from(path);
71-
Page<Blob> blobPage = storage.list(gcsPath.getBucket(), Storage.BlobListOption.prefix(gcsPath.getName()));
75+
Page<Blob> blobPage;
76+
try {
77+
blobPage = storage.list(gcsPath.getBucket(), Storage.BlobListOption.prefix(gcsPath.getName()));
78+
} catch (Exception e) {
79+
String errorReason = String.format("Unable to list objects in bucket %s.", gcsPath.getBucket());
80+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
81+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
82+
}
7283
Iterator<Blob> iterator = blobPage.getValues().iterator();
7384
while (iterator.hasNext()) {
7485
Blob blob = iterator.next();
@@ -89,7 +100,13 @@ public void setMetaData(Blob blob, Map<String, String> metaData) {
89100
if (blob == null || metaData == null || metaData.isEmpty()) {
90101
return;
91102
}
92-
storage.update(BlobInfo.newBuilder(blob.getBlobId()).setMetadata(metaData).build());
103+
try {
104+
storage.update(BlobInfo.newBuilder(blob.getBlobId()).setMetadata(metaData).build());
105+
} catch (Exception e) {
106+
String errorReason = String.format("Unable to update metadata for blob %s.", blob.getName());
107+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
108+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
109+
}
93110
}
94111

95112
/**
@@ -102,7 +119,14 @@ public void mapMetaDataForAllBlobs(String path, Consumer<Map<String, String>> fu
102119
return;
103120
}
104121
GCSPath gcsPath = GCSPath.from(path);
105-
Page<Blob> blobPage = storage.list(gcsPath.getBucket(), Storage.BlobListOption.prefix(gcsPath.getName()));
122+
Page<Blob> blobPage;
123+
try {
124+
blobPage = storage.list(gcsPath.getBucket(), Storage.BlobListOption.prefix(gcsPath.getName()));
125+
} catch (Exception e) {
126+
String errorReason = String.format("Unable to list objects in bucket %s.", gcsPath.getBucket());
127+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
128+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
129+
}
106130
Iterator<Blob> blobIterator = blobPage.iterateAll().iterator();
107131
while (blobIterator.hasNext()) {
108132
Blob blob = blobIterator.next();
@@ -132,9 +156,11 @@ public void createBucketIfNotExists(GCSPath path, @Nullable String location, @Nu
132156
LOG.warn("Getting 409 Conflict: {} Bucket at destination path {} may already exist.",
133157
e.getMessage(), path.getUri());
134158
} else {
135-
throw new RuntimeException(
159+
String errorReason =
136160
String.format("Unable to create bucket %s. Ensure you entered the correct bucket path and " +
137-
"have permissions for it.", path.getBucket()), e);
161+
"have permissions for it.", path.getBucket());
162+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
163+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
138164
}
139165
}
140166
}
@@ -173,9 +199,16 @@ public void move(GCSPath sourcePath, GCSPath destPath, boolean recursive, boolea
173199
* Get all the matching wildcard paths given the regex input.
174200
*/
175201
public List<GCSPath> getMatchedPaths(GCSPath sourcePath, boolean recursive, Pattern wildcardRegex) {
176-
Page<Blob> blobPage = storage.list(sourcePath.getBucket(), Storage.BlobListOption.prefix(
202+
Page<Blob> blobPage;
203+
try {
204+
blobPage = storage.list(sourcePath.getBucket(), Storage.BlobListOption.prefix(
177205
getWildcardPathPrefix(sourcePath, wildcardRegex)
178-
));
206+
));
207+
} catch (Exception e) {
208+
String errorReason = String.format("Unable to list objects in bucket %s.", sourcePath.getBucket());
209+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
210+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
211+
}
179212
List<String> blobPageNames = new ArrayList<>();
180213
blobPage.getValues().forEach(blob -> blobPageNames.add(blob.getName()));
181214
return getFilterMatchedPaths(sourcePath, blobPageNames, recursive);
@@ -212,58 +245,84 @@ static List<GCSPath> getFilterMatchedPaths(GCSPath sourcePath, List<String> blob
212245
private void pairTraverse(GCSPath sourcePath, GCSPath destPath, boolean recursive, boolean overwrite,
213246
Consumer<BlobPair> consumer) {
214247

215-
Bucket sourceBucket = null;
248+
Bucket sourceBucket;
216249
try {
217250
sourceBucket = storage.get(sourcePath.getBucket());
218-
} catch (StorageException e) {
251+
} catch (Exception e) {
219252
// Add more descriptive error message
220-
throw new RuntimeException(
221-
String.format("Unable to access source bucket %s. ", sourcePath.getBucket())
222-
+ "Ensure you entered the correct bucket path.", e);
253+
String errorReason = String.format("Unable to access GCS bucket '%s'", sourcePath.getBucket());
254+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
255+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
223256
}
224257
if (sourceBucket == null) {
225-
throw new IllegalArgumentException(
226-
String.format("Source bucket '%s' does not exist.", sourcePath.getBucket()));
258+
String errorReason = String.format("Source bucket '%s' does not exist.", sourcePath.getBucket());
259+
throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN),
260+
errorReason, errorReason, ErrorType.USER, true, null);
227261
}
228-
Bucket destBucket = null;
262+
Bucket destBucket;
229263
try {
230264
destBucket = storage.get(destPath.getBucket());
231-
} catch (StorageException e) {
265+
} catch (Exception e) {
232266
// Add more descriptive error message
233-
throw new RuntimeException(
234-
String.format("Unable to access destination bucket %s. ", destPath.getBucket())
235-
+ "Ensure you entered the correct bucket path.", e);
267+
String errorReason = String.format("Unable to access GCS bucket '%s'", destPath.getBucket());
268+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
269+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
236270
}
237271
if (destBucket == null) {
238-
throw new IllegalArgumentException(
239-
String.format("Destination bucket '%s' does not exist. Please create it first.", destPath.getBucket()));
272+
String errorReason =
273+
String.format("Destination bucket '%s' does not exist. Please create it first.", destPath.getBucket());
274+
throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN),
275+
errorReason, errorReason, ErrorType.USER, true, null);
240276
}
241277

242278
boolean destinationBaseExists;
243279
String baseDestName = destPath.getName();
244-
if (destPath.isBucket() || storage.get(BlobId.of(destPath.getBucket(), baseDestName)) != null) {
280+
boolean destinationBlobExists;
281+
try {
282+
destinationBlobExists = destPath.isBucket() || storage.get(BlobId.of(destPath.getBucket(), baseDestName)) != null;
283+
} catch (Exception e) {
284+
String errorReason = String.format("Unable to access GCS bucket '%s'", destPath.getBucket());
285+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
286+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
287+
}
288+
if (destinationBlobExists) {
245289
destinationBaseExists = true;
246290
} else {
247291
// if gs://bucket2/subdir doesn't exist, check if gs://bucket2/subdir/ exists
248292
// similarly, if gs://bucket2/subdir/ doesn't exist, check if gs://bucket2/subdir exists
249293
// this is because "cp dir0 subdir" and "cp dir0 subdir/" are equivalent if the 'subdir' directory exists
250294
String modifiedName = baseDestName.endsWith("/") ?
251295
baseDestName.substring(0, baseDestName.length() - 1) : baseDestName + "/";
252-
destinationBaseExists = storage.get(BlobId.of(destPath.getBucket(), modifiedName)) != null;
296+
try {
297+
destinationBaseExists = storage.get(BlobId.of(destPath.getBucket(), modifiedName)) != null;
298+
} catch (Exception e) {
299+
String errorReason = String.format("Unable to access GCS bucket '%s'", destPath.getBucket());
300+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
301+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
302+
}
253303
}
254304

255305
List<BlobPair> copyList = new ArrayList<>();
256306
traverse(BlobId.of(sourcePath.getBucket(), sourcePath.getName()), recursive, sourceBlob -> {
257307
BlobId destBlobID = resolve(sourcePath.getName(), sourceBlob.getBlobId().getName(),
258308
destPath, destinationBaseExists);
259309
if (!overwrite) {
260-
Blob destBlob = storage.get(destBlobID);
310+
Blob destBlob;
311+
try {
312+
destBlob = storage.get(destBlobID);
313+
} catch (Exception e) {
314+
String errorReason = String.format("Unable to access GCS bucket '%s'", destPath.getBucket());
315+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
316+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
317+
}
261318
// we can't just use Blob's isDirectory() because the cloud console will create a 'directory' by creating
262319
// a 0 size placeholder blob that ends with '/'. This placeholder blob's isDirectory() method returns false,
263320
// but we don't want the overwrite check to fail on it. So we explicitly ignore the check for these 0 size
264321
// placeholder blobs.
265322
if (destBlob != null && !destBlob.getName().endsWith("/") && destBlob.getSize() != 0) {
266-
throw new IllegalArgumentException(String.format("%s already exists.", toPath(destBlobID)));
323+
String errorReason = String.format("%s already exists.", toPath(destBlobID));
324+
throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN),
325+
errorReason, errorReason, ErrorType.USER, true, null);
267326
}
268327
}
269328
copyList.add(new BlobPair(sourceBlob, destBlobID));
@@ -347,8 +406,15 @@ static String append(String base, String part) {
347406
* @param consumer the blob consumer
348407
*/
349408
private void traverse(BlobId blobId, boolean recursive, Consumer<Blob> consumer) {
350-
Page<Blob> blobList = storage.list(blobId.getBucket(), Storage.BlobListOption.currentDirectory(),
351-
Storage.BlobListOption.prefix(blobId.getName()));
409+
Page<Blob> blobList;
410+
try {
411+
blobList = storage.list(blobId.getBucket(), Storage.BlobListOption.currentDirectory(),
412+
Storage.BlobListOption.prefix(blobId.getName()));
413+
} catch (Exception e) {
414+
String errorReason = String.format("");
415+
throw GCPErrorDetailsProviderUtil.getHttpResponseExceptionDetailsFromChain(e, errorReason, ErrorType.UNKNOWN,
416+
true, GCPUtils.GCS_SUPPORTED_DOC_URL);
417+
}
352418
for (Blob blob : blobList.iterateAll()) {
353419
if (!blob.isDirectory()) {
354420
consumer.accept(blob);
@@ -363,11 +429,17 @@ private static String toPath(BlobId blobId) {
363429
}
364430

365431
public static StorageClient create(String project, @Nullable String serviceAccount,
366-
Boolean isServiceAccountFilePath, @Nullable Integer readTimeout)
367-
throws IOException {
432+
Boolean isServiceAccountFilePath, @Nullable Integer readTimeout) {
368433
StorageOptions.Builder builder = StorageOptions.newBuilder().setProjectId(project);
369434
if (serviceAccount != null) {
370-
builder.setCredentials(GCPUtils.loadServiceAccountCredentials(serviceAccount, isServiceAccountFilePath));
435+
try {
436+
builder.setCredentials(GCPUtils.loadServiceAccountCredentials(serviceAccount, isServiceAccountFilePath));
437+
} catch (IOException e) {
438+
String errorReason = "Unable to load service account credentials.";
439+
throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN),
440+
errorReason, String.format("%s %s: %s", errorReason, e.getClass().getName(), e.getMessage()),
441+
ErrorType.UNKNOWN, false, null);
442+
}
371443
}
372444
if (readTimeout != null) {
373445
builder.setTransportOptions(HttpTransportOptions.newBuilder().setReadTimeout(readTimeout * 1000).build());
@@ -376,7 +448,7 @@ public static StorageClient create(String project, @Nullable String serviceAccou
376448
return new StorageClient(storage);
377449
}
378450

379-
public static StorageClient create(GCPConnectorConfig config) throws IOException {
451+
public static StorageClient create(GCPConnectorConfig config) {
380452
return create(config.getProject(), config.getServiceAccount(), config.isServiceAccountFilePath(), null);
381453
}
382454

0 commit comments

Comments
 (0)