Merge pull request #403 from IABTechLab/gdm-UID2-4997-verbose-logging

Added verbose logging for S3 download

Author: gmsdelmundo

src/main/java/com/uid2/shared/Const.java

@@ -44,6 +44,7 @@ public static class Config {
         public static final String S3EndpointProp = "aws_s3_endpoint";
         public static final String CoreS3BucketProp = "core_s3_bucket";
         public static final String OptOutS3BucketProp = "optout_s3_bucket";
+        public static final String S3VerboseLoggingProp = "aws_s3_verbose_logging";
 
         // Config stores
         public static final String CloudRefreshIntervalProp = "cloud_refresh_interval";

src/main/java/com/uid2/shared/cloud/CloudStorageS3.java

@@ -25,9 +25,10 @@ public class CloudStorageS3 implements TaggableCloudStorage {
 
     private final AmazonS3 s3;
     private final String bucket;
+    private final boolean verbose;
     private long preSignedUrlExpiryInSeconds = 3600;
 
-    public CloudStorageS3(String accessKeyId, String secretAccessKey, String region, String bucket, String s3Endpoint) {
+    public CloudStorageS3(String accessKeyId, String secretAccessKey, String region, String bucket, String s3Endpoint, boolean verbose) {
         // Reading https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html
         AWSCredentials creds = new BasicAWSCredentials(
                 accessKeyId,
@@ -45,11 +46,14 @@ public CloudStorageS3(String accessKeyId, String secretAccessKey, String region,
                     .build();
         }
         this.bucket = bucket;
+        this.verbose = verbose;
     }
 
-    public CloudStorageS3(String region, String bucket, String s3Endpoint) {
-        this.bucket = bucket;
+    public CloudStorageS3(String accessKeyId, String secretAccessKey, String region, String bucket, String s3Endpoint) {
+        this(accessKeyId, secretAccessKey, region, bucket, s3Endpoint, false);
+    }
 
+    public CloudStorageS3(String region, String bucket, String s3Endpoint) {
         // In theory `new InstanceProfileCredentialsProvider()` or even omitting credentials provider should work,
         // but for some unknown reason it doesn't. The credential it provides look realistic, but are not valid.
         // After a lot of experimentation and help of Abu Abraham and Isaac Wilson the only working solution we've
@@ -71,8 +75,9 @@ public CloudStorageS3(String region, String bucket, String s3Endpoint) {
                     .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(s3Endpoint, region))
                     .enablePathStyleAccess()
                     .build();
-
         }
+        this.bucket = bucket;
+        this.verbose = false;
     }
 
     @Override
@@ -134,7 +139,7 @@ public InputStream download(String cloudPath) throws CloudStorageException {
             if (e.getErrorCode().equals("NoSuchKey")) {
                 throw new CloudStorageException("The specified key does not exist: " + e.getClass().getSimpleName() + ": " + bucket);
             } else {
-                throw new CloudStorageException("s3 get error: " + e.getClass().getSimpleName() + ": " + bucket);
+                throw new CloudStorageException("s3 get error: " + e.getClass().getSimpleName() + ": " + bucket + (verbose ? " - " + e.getMessage() : ""));
             }
         } catch (Throwable t) {
             // Do not log the message or the original exception as that may contain the pre-signed url

src/main/java/com/uid2/shared/cloud/CloudUtils.java

@@ -7,20 +7,11 @@
 import io.vertx.core.json.JsonObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.kms.KmsClient;
-import software.amazon.awssdk.services.kms.KmsClientBuilder;
 
 import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import java.net.*;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 
 public class CloudUtils {
     private static final Logger LOGGER = LoggerFactory.getLogger(CloudUtils.class);
@@ -31,15 +22,15 @@ public static TaggableCloudStorage createStorage(String cloudBucket, JsonObject
         var accessKeyId = jsonConfig.getString(Const.Config.AccessKeyIdProp);
         var secretAccessKey = jsonConfig.getString(Const.Config.SecretAccessKeyProp);
         var s3Endpoint = jsonConfig.getString(Const.Config.S3EndpointProp, "");
+        var verboseLogging = jsonConfig.getBoolean(Const.Config.S3VerboseLoggingProp, false);
 
         if (accessKeyId == null || secretAccessKey == null) {
             // IAM authentication
             return new CloudStorageS3(region, cloudBucket, s3Endpoint);
         }
 
         // User access key authentication
-        return new CloudStorageS3(accessKeyId, secretAccessKey, region, cloudBucket, s3Endpoint);
-
+        return new CloudStorageS3(accessKeyId, secretAccessKey, region, cloudBucket, s3Endpoint, verboseLogging);
     }
 
     // I think this is not used, Aleksandrs Ulme 26/07/2023
@@ -50,7 +41,8 @@ public static TaggableCloudStorage createStorage(String cloudBucket) {
                 System.getProperty(Const.Config.SecretAccessKeyProp),
                 System.getProperty(Const.Config.AwsRegionProp),
                 cloudBucket,
-                System.getProperty(Const.Config.S3EndpointProp, "")
+                System.getProperty(Const.Config.S3EndpointProp, ""),
+                Boolean.parseBoolean(System.getProperty(Const.Config.S3VerboseLoggingProp, "false"))
         );
     }