Merge pull request #86 from edouarda/master

Win32 fixes + US-EAST-1 fix

Author: hicder

cloud/aws/aws_env.cc

@@ -2,8 +2,6 @@
 //
 #include "cloud/aws/aws_env.h"
 
-#include <unistd.h>
-
 #include <chrono>
 #include <cinttypes>
 #include <fstream>
@@ -15,7 +13,7 @@
 #include "cloud/cloud_scheduler.h"
 #include "cloud/cloud_storage_provider_impl.h"
 #include "cloud/filename.h"
-#include "port/port_posix.h"
+#include "port/port.h"
 #include "rocksdb/cloud/cloud_log_controller.h"
 #include "rocksdb/cloud/cloud_storage_provider.h"
 #include "rocksdb/env.h"
@@ -120,8 +118,8 @@ Status AwsCloudAccessCredentials::GetCredentialsProvider(
   AwsAccessType aws_type = GetAccessType();
   Status status = CheckCredentials(aws_type);
   if (status.ok()) {
-    switch (aws_type) {
 #ifdef USE_AWS
+    switch (aws_type) {
       case AwsAccessType::kSimple: {
         const char* access_key =
             (access_key_id.empty() ? getenv("AWS_ACCESS_KEY_ID")
@@ -155,11 +153,13 @@ Status AwsCloudAccessCredentials::GetCredentialsProvider(
         // Use AWS SDK's default credential chain
         result->reset();
         break;
-#endif
       default:
         status = Status::NotSupported("AWS credentials type not supported");
         break;  // not supported
     }
+#else
+    status = Status::NotSupported("AWS credentials type not supported");
+#endif
   }
   return status;
 }

cloud/aws/aws_env.h

@@ -14,6 +14,8 @@
 
 #ifdef USE_AWS
 
+#include <string.h>
+
 #include <chrono>
 #include <list>
 #include <unordered_map>

cloud/aws/aws_s3.cc

@@ -44,13 +44,17 @@
 #include "cloud/aws/aws_file.h"
 #include "cloud/cloud_storage_provider_impl.h"
 #include "cloud/filename.h"
-#include "port/port_posix.h"
+#include "port/port.h"
 #include "rocksdb/cloud/cloud_env_options.h"
 #include "rocksdb/cloud/cloud_storage_provider.h"
 #include "rocksdb/options.h"
 #include "util/stderr_logger.h"
 #include "util/string_util.h"
 
+#ifdef _WIN32_WINNT
+#undef GetMessage
+#endif
+
 namespace ROCKSDB_NAMESPACE {
 #ifdef USE_AWS
 class CloudRequestCallbackGuard {
@@ -477,8 +481,20 @@ Status S3StorageProvider::CreateBucket(const std::string& bucket) {
   Aws::S3::Model::BucketLocationConstraint bucket_location = Aws::S3::Model::
       BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
           ToAwsString(env_->GetCloudEnvOptions().dest_bucket.GetRegion()));
-  if (bucket_location != Aws::S3::Model::BucketLocationConstraint::NOT_SET) {
-    // only set the location constraint if it's not not set
+  //
+  // If you create a bucket in US-EAST-1, no location constraint should be
+  // specified
+  //
+  // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html
+  //
+  // By default, the bucket is created in the US East (N. Virginia) Region.
+  // You can optionally specify a Region in the request body. You might choose
+  // a Region to optimize latency, minimize costs, or address regulatory
+  // requirements.
+  //
+  if ((bucket_location != Aws::S3::Model::BucketLocationConstraint::NOT_SET) &&
+      (bucket_location !=
+       Aws::S3::Model::BucketLocationConstraint::us_east_1)) {
     conf.SetLocationConstraint(bucket_location);
   }
 

cloud/cloud_env.cc

@@ -1,7 +1,11 @@
 // Copyright (c) 2017 Rockset.
 #ifndef ROCKSDB_LITE
 
+#ifndef _WIN32_WINNT
 #include <unistd.h>
+#else
+#include <windows.h>
+#endif
 
 #include "cloud/aws/aws_env.h"
 #include "cloud/cloud_env_impl.h"
@@ -66,7 +70,19 @@ void BucketOptions::TEST_Initialize(const std::string& bucket,
   if (!CloudEnvOptions::GetNameFromEnvironment("ROCKSDB_CLOUD_TEST_BUCKET_NAME",
                                                "ROCKSDB_CLOUD_BUCKET_NAME",
                                                &bucket_)) {
+#ifdef _WIN32_WINNT
+    char user_name[257];  // UNLEN + 1
+    DWORD dwsize = sizeof(user_name);
+    if (!::GetUserName(user_name, &dwsize)) {
+      bucket_ = bucket_ + "unknown";
+    } else {
+      bucket_ =
+          bucket_ +
+          std::string(user_name, static_cast<std::string::size_type>(dwsize));
+    }
+#else
     bucket_ = bucket + std::to_string(geteuid());
+#endif
   }
   if (CloudEnvOptions::GetNameFromEnvironment(
           "ROCKSDB_CLOUD_TEST_BUCKET_PREFIX", "ROCKSDB_CLOUD_BUCKET_PREFIX",

include/rocksdb/cloud/cloud_env_options.h

@@ -313,7 +313,7 @@ class CloudEnvOptions {
       bool _skip_dbid_verification = false,
       bool _use_aws_transfer_manager = false,
       int _number_objects_listed_in_one_iteration = 5000,
-      bool _constant_sst_file_size_in_sst_file_manager = -1,
+      int _constant_sst_file_size_in_sst_file_manager = -1,
       bool _skip_cloud_files_in_getchildren = false)
       : cloud_type(_cloud_type),
         log_type(_log_type),