Pull request #1456: CATS-1057: Verify that S3 bucket exists on startup, create it if not

Merge in CATS/clowder from CATS-1057-verify-that-s3-bucket-exists-on-startup to develop

* commit 'aed0c9995b0e4032db335f21f5d62006271a5b9f':
  Add back Logger warning message on bucket create
  Verify bucket exists on startup, create it if not

Author: lmarini

CHANGELOG.md

@@ -23,6 +23,8 @@ of these changes a reindex of Elasticsearch is required. This can be started by
 - Ability to filter the files and folders in a dataset when sorting is enabled.
 - Visualize existing relations between datasets on the dataset page. This can be extended other resource types.
   [CATS-1000](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1000)
+- S3ByteStorageService should verify bucket existence on startup and create if it does not exist.
+  [CATS-1057](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1057)
 
 ### Fixed
 - When adding tags to a section of an image, show the new tag without having to refresh the page.

app/services/s3/S3ByteStorageService.scala

@@ -6,7 +6,7 @@ import models.UUID
 import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
 import com.amazonaws.client.builder.AwsClientBuilder
 import com.amazonaws.regions.Regions
-import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata}
+import com.amazonaws.services.s3.model.{HeadBucketRequest, CreateBucketRequest, GetObjectRequest, ObjectMetadata}
 import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 import com.amazonaws.{AmazonClientException, ClientConfiguration}
 import com.google.inject.Inject
@@ -46,6 +46,42 @@ object S3ByteStorageService {
   */
 class S3ByteStorageService @Inject()() extends ByteStorageService {
 
+
+  // Only run a single thread at a time when verifying bucket existence
+  synchronized {
+    Play.current.configuration.getString(S3ByteStorageService.BucketName) match {
+      case Some(bucketName) => {
+        try {
+          // Validate configuration by checking for bucket existence on startup
+          this.s3Bucket.headBucket(new HeadBucketRequest(bucketName))
+        } catch {
+          case sdke @ (_: AmazonClientException | _: AmazonServiceException) => {
+            if (sdke.getMessage.contains("Status Code: 404")) {
+              Logger.warn("Configured S3 bucket does not exist, attempting to create it now...")
+              try {
+                // Bucket does not exist - create the bucket
+                this.s3Bucket.createBucket(new CreateBucketRequest(bucketName))
+              } catch {
+                // Bucket could not be created - abort
+                case _: Throwable => throw new RuntimeException("Bad S3 configuration: Bucket does not exist and could not be created.")
+              }
+            } else if (sdke.getMessage.contains("Status Code: 403")) {
+              // Bucket exists, but you do not have permission to access it
+              throw new RuntimeException("Bad S3 configuration: You do not have access to the configured bucket.")
+            } else {
+              // Unknown error - print status code for further investigation
+              val errMsg = sdke.getLocalizedMessage
+              Logger.error(errMsg)
+              throw new RuntimeException("Bad S3 configuration: an unknown error has occurred - " + errMsg)
+            }
+          }
+          case _: Throwable => handleUnknownError(_)
+        }
+      }
+      case _ => throw new RuntimeException("Bad S3 configuration: verify that you have set all configuration options.")
+    }
+  }
+
   /**
     * Grabs config parameters from Clowder to return a
     * AmazonS3 pointing at the configured service endpoint.