More changes for AWS on Windows

Author: mrambacher

cloud/aws/aws_env.cc

@@ -102,7 +102,6 @@ void AwsCloudAccessCredentials::InitializeConfig(
 Status AwsCloudAccessCredentials::HasValid() const {
   AwsAccessType aws_type = GetAccessType();
   Status status = CheckCredentials(aws_type);
-  printf("MJR: Credentials HasValid=%d status=%s\n", (int) aws_type, status.ToString().c_str());
   return status;
 }
 
@@ -117,7 +116,6 @@ Status AwsCloudAccessCredentials::GetCredentialsProvider(
 
   AwsAccessType aws_type = GetAccessType();
   Status status = CheckCredentials(aws_type);
-  printf("MJR: Credentials=%d status=%s\n", (int) aws_type, status.ToString().c_str());
   if (status.ok()) {
 #ifdef USE_AWS
     switch (aws_type) {
@@ -166,6 +164,7 @@ Status AwsCloudAccessCredentials::GetCredentialsProvider(
 }
 
 #ifdef USE_AWS
+static Aws::SDKOptions sdkOptions;
 
 //
 // The AWS credentials are specified to the constructor via
@@ -174,7 +173,8 @@ Status AwsCloudAccessCredentials::GetCredentialsProvider(
 AwsEnv::AwsEnv(Env* underlying_env, const CloudEnvOptions& _cloud_env_options,
                const std::shared_ptr<Logger>& info_log)
     : CloudEnvImpl(_cloud_env_options, underlying_env, info_log) {
-  Aws::InitAPI(Aws::SDKOptions());
+  Aws::InitAPI(sdkOptions);  //**TODO: Move this into PrepareOptions and do it
+                             // conditionally (first time)
   if (cloud_env_options.src_bucket.GetRegion().empty() ||
       cloud_env_options.dest_bucket.GetRegion().empty()) {
     std::string region;
@@ -192,7 +192,11 @@ AwsEnv::AwsEnv(Env* underlying_env, const CloudEnvOptions& _cloud_env_options,
   base_env_ = underlying_env;
 }
 
-void AwsEnv::Shutdown() { Aws::ShutdownAPI(Aws::SDKOptions()); }
+AwsEnv::~AwsEnv() {
+  //**TODO: Conditionally call shutdown (or make shutdown conditional on last...
+}
+
+void AwsEnv::Shutdown() { Aws::ShutdownAPI(sdkOptions); }
 
 //
 // All db in a bucket are stored in path /.rockset/dbid/<dbid>

cloud/aws/aws_env.h

@@ -51,7 +51,7 @@ class AwsEnv : public CloudEnvImpl {
                           const std::shared_ptr<Logger>& info_log,
                           CloudEnv** cenv);
 
-  virtual ~AwsEnv() {}
+  virtual ~AwsEnv();
 
   const char* Name() const override { return "aws"; }
 

cloud/cloud_env.cc

@@ -12,6 +12,7 @@
 #include "cloud/cloud_env_wrapper.h"
 #include "cloud/db_cloud_impl.h"
 #include "cloud/filename.h"
+#include "file/filename.h"
 #include "port/likely.h"
 #include "rocksdb/cloud/cloud_log_controller.h"
 #include "rocksdb/db.h"
@@ -59,6 +60,23 @@ void BucketOptions::SetBucketName(const std::string& bucket,
   }
 }
 
+void BucketOptions::SetObjectPath(const std::string& object) {
+  // Remove the drive if there is one...
+  auto colon = object.find(':');
+  if (colon != std::string::npos) {
+    object_ = object.substr(colon + 1);
+  } else {
+    object_ = object;
+  }
+  // Replace any "\" with "/"
+  for (auto pos = object_.find('\\'); pos != std::string::npos;
+       pos = object_.find('\\', pos)) {
+    object_[pos] = '/';
+  }
+  // Remove any duplicate markers
+  object_ = NormalizePath(object_);
+}
+
 // Initializes the bucket properties
 
 void BucketOptions::TEST_Initialize(const std::string& bucket,
@@ -70,15 +88,13 @@ void BucketOptions::TEST_Initialize(const std::string& bucket,
   if (!CloudEnvOptions::GetNameFromEnvironment("ROCKSDB_CLOUD_TEST_BUCKET_NAME",
                                                "ROCKSDB_CLOUD_BUCKET_NAME",
                                                &bucket_)) {
-#ifdef _WIN32_WINNT
+#ifdef _WIN32
     char user_name[257];  // UNLEN + 1
     DWORD dwsize = sizeof(user_name);
     if (!::GetUserName(user_name, &dwsize)) {
       bucket_ = bucket_ + "unknown";
     } else {
-      bucket_ =
-          bucket_ +
-          std::string(user_name, static_cast<std::string::size_type>(dwsize));
+      bucket_ = bucket_ + user_name;
     }
 #else
     bucket_ = bucket + std::to_string(geteuid());

cloud/db_cloud_test.cc

@@ -64,13 +64,14 @@ class CloudTest : public testing::Test {
 
     CloudEnv* aenv;
     // create a dummy aws env
+
     ASSERT_OK(CloudEnv::NewAwsEnv(base_env_, cloud_env_options_,
                                   options_.info_log, &aenv));
     ASSERT_NE(aenv, nullptr);
     aenv_.reset(aenv);
     // delete all pre-existing contents from the bucket
     Status st = aenv_->GetCloudEnvOptions().storage_provider->EmptyBucket(
-        aenv_->GetSrcBucketName(), dbname_);
+        aenv_->GetSrcBucketName(), aenv_->GetSrcObjectPath());
     ASSERT_TRUE(st.ok() || st.IsNotFound());
     aenv_.reset();
 
@@ -109,7 +110,7 @@ class CloudTest : public testing::Test {
                                       options_.info_log, &aenv);
       if (st.ok()) {
         aenv->GetCloudEnvOptions().storage_provider->EmptyBucket(
-            aenv->GetSrcBucketName(), dbname_);
+            aenv->GetSrcBucketName(), aenv->GetSrcObjectPath());
         delete aenv;
       }
     }
@@ -1539,8 +1540,11 @@ TEST_F(CloudTest, SharedBlockCache) {
 
 }  //  namespace ROCKSDB_NAMESPACE
 
+#include "port/stack_trace.h"
+
 // A black-box test for the cloud wrapper around rocksdb
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

include/rocksdb/cloud/cloud_env_options.h

@@ -125,7 +125,7 @@ class BucketOptions {
   void SetBucketName(const std::string& bucket, const std::string& prefix = "");
   const std::string& GetBucketName() const { return name_; }
   const std::string& GetObjectPath() const { return object_; }
-  void SetObjectPath(const std::string& object) { object_ = object; }
+  void SetObjectPath(const std::string& object);
   const std::string& GetRegion() const { return region_; }
   void SetRegion(const std::string& region) { region_ = region; }