Options to disable GetChildren from cloud

Summary: Add an option that short-circuits the `GetChildren` call in the cloud. This option is useful during DB Opening where `GetChildren` becomes too expensive, since it was called multiple times. Only use this when opening DB in a temporary clone.

Test Plan:
Run in my namespace. This cuts the DB Opening time from 7s to < 800ms, which makes a huge difference during remote compaction.
{F37274}

Reviewers: dhruba, igor, #platform

Reviewed By: dhruba, igor, #platform

Differential Revision: https://rockset.phacility.com/D6035

Author: Duc Hieu Pham

cloud/aws/aws_env.cc

@@ -796,7 +796,7 @@ Status AwsEnv::GetChildren(const std::string& path,
 
   // Fetch the list of children from both buckets in S3
   Status st;
-  if (HasSrcBucket()) {
+  if (HasSrcBucket() && !cloud_env_options.skip_cloud_files_in_getchildren) {
     st = cloud_env_options.storage_provider->ListObjects(
         GetSrcBucketName(), GetSrcObjectPath(), result);
     if (!st.ok()) {
@@ -806,7 +806,8 @@ Status AwsEnv::GetChildren(const std::string& path,
       return st;
     }
   }
-  if (HasDestBucket() && !SrcMatchesDest()) {
+  if (HasDestBucket() && !SrcMatchesDest() &&
+      !cloud_env_options.skip_cloud_files_in_getchildren) {
     st = cloud_env_options.storage_provider->ListObjects(
         GetDestBucketName(), GetDestObjectPath(), result);
     if (!st.ok()) {

include/rocksdb/cloud/cloud_env_options.h

@@ -287,6 +287,19 @@ class CloudEnvOptions {
   // Default: -1, means don't use this option.
   int64_t constant_sst_file_size_in_sst_file_manager;
 
+  // Skip listing files in the cloud in GetChildren. That means GetChildren
+  // will only return files in local directory. During DB opening, RocksDB
+  // makes multiple GetChildren calls, which are very expensive if we list
+  // objects in the cloud.
+  //
+  // This option is used in remote compaction where we open the DB in a
+  // temporary folder, and then the folder is deleted after the RPC is done.
+  // This requires opening DB to be really fast, and it's unnecessary to cleanup
+  // various things, which is what RocksDB calls GetChildren for.
+  //
+  // Default: false.
+  bool skip_cloud_files_in_getchildren;
+
   CloudEnvOptions(
       CloudType _cloud_type = CloudType::kCloudAws,
       LogType _log_type = LogType::kLogKafka,
@@ -300,7 +313,8 @@ class CloudEnvOptions {
       bool _skip_dbid_verification = false,
       bool _use_aws_transfer_manager = false,
       int _number_objects_listed_in_one_iteration = 5000,
-      bool _constant_sst_file_size_in_sst_file_manager = -1)
+      bool _constant_sst_file_size_in_sst_file_manager = -1,
+      bool _skip_cloud_files_in_getchildren = false)
       : cloud_type(_cloud_type),
         log_type(_log_type),
         keep_local_sst_files(_keep_local_sst_files),
@@ -319,7 +333,8 @@ class CloudEnvOptions {
         number_objects_listed_in_one_iteration(
             _number_objects_listed_in_one_iteration),
         constant_sst_file_size_in_sst_file_manager(
-            _constant_sst_file_size_in_sst_file_manager) {}
+            _constant_sst_file_size_in_sst_file_manager),
+        skip_cloud_files_in_getchildren(_skip_cloud_files_in_getchildren) {}
 
   // print out all options to the log
   void Dump(Logger* log) const;