Skip to content

Commit 84c099b

Browse files
mrambachermrambacher
committed
Move static methods to CloudEnvImpl
1 parent 8c05633 commit 84c099b

File tree

10 files changed

+1051
-1066
lines changed

10 files changed

+1051
-1066
lines changed

‎cloud/aws/aws_env.cc‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -258,11 +258,9 @@ Aws::S3::Model::HeadObjectOutcome AwsS3ClientWrapper::HeadObject(
258258
// The AWS credentials are specified to the constructor via
259259
// access_key_id and secret_key.
260260
//
261-
AwsEnv::AwsEnv(Env* underlying_env,
262-
const CloudEnvOptions& _cloud_env_options,
263-
const std::shared_ptr<Logger> & info_log)
264-
: CloudEnvImpl(_cloud_env_options,
265-
underlying_env),
261+
AwsEnv::AwsEnv(Env* underlying_env, const CloudEnvOptions& _cloud_env_options,
262+
const std::shared_ptr<Logger>& info_log)
263+
: CloudEnvImpl(_cloud_env_options, underlying_env),
266264
info_log_(info_log),
267265
running_(true) {
268266
Aws::InitAPI(Aws::SDKOptions());

‎cloud/cloud_env.cc‎

Lines changed: 25 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ bool CloudEnvOptions::GetNameFromEnvironment(const char *name, const char *alt,
3131
return false; // No, return not found
3232
}
3333
}
34-
void CloudEnvOptions::TEST_Initialize(const std::string & bucket,
35-
const std::string & object,
36-
const std::string & region) {
34+
void CloudEnvOptions::TEST_Initialize(const std::string& bucket,
35+
const std::string& object,
36+
const std::string& region) {
3737
src_bucket.TEST_Initialize(bucket, object, region);
3838
dest_bucket = src_bucket;
3939
}
@@ -42,8 +42,8 @@ BucketOptions::BucketOptions() {
4242
prefix_ = "rockset.";
4343
}
4444

45-
void BucketOptions::SetBucketName(const std::string & bucket,
46-
const std::string & prefix) {
45+
void BucketOptions::SetBucketName(const std::string& bucket,
46+
const std::string& prefix) {
4747
if (!prefix.empty()) {
4848
prefix_ = prefix;
4949
}
@@ -56,30 +56,32 @@ void BucketOptions::SetBucketName(const std::string & bucket,
5656
}
5757
}
5858

59-
6059
// Initializes the bucket properties
6160

62-
void BucketOptions::TEST_Initialize(const std::string & bucket,
63-
const std::string & object,
64-
const std::string & region) {
61+
void BucketOptions::TEST_Initialize(const std::string& bucket,
62+
const std::string& object,
63+
const std::string& region) {
6564
std::string prefix;
6665
// If the bucket name is not set, then the bucket name is not set,
6766
// Set it to either the value of the environment variable or geteuid
68-
if (! CloudEnvOptions::GetNameFromEnvironment("ROCKSDB_CLOUD_TEST_BUCKET_NAME",
69-
"ROCKSDB_CLOUD_BUCKET_NAME", &bucket_)) {
67+
if (!CloudEnvOptions::GetNameFromEnvironment("ROCKSDB_CLOUD_TEST_BUCKET_NAME",
68+
"ROCKSDB_CLOUD_BUCKET_NAME",
69+
&bucket_)) {
7070
bucket_ = bucket + std::to_string(geteuid());
7171
}
72-
if (CloudEnvOptions::GetNameFromEnvironment("ROCKSDB_CLOUD_TEST_BUCKET_PREFIX",
73-
"ROCKSDB_CLOUD_BUCKET_PREFIX", &prefix)) {
72+
if (CloudEnvOptions::GetNameFromEnvironment(
73+
"ROCKSDB_CLOUD_TEST_BUCKET_PREFIX", "ROCKSDB_CLOUD_BUCKET_PREFIX",
74+
&prefix)) {
7475
prefix_ = prefix;
7576
}
7677
name_ = prefix_ + bucket_;
77-
if (! CloudEnvOptions::GetNameFromEnvironment("ROCKSDB_CLOUD_TEST_OBECT_PATH",
78-
"ROCKSDB_CLOUD_OBJECT_PATH", &object_)) {
78+
if (!CloudEnvOptions::GetNameFromEnvironment("ROCKSDB_CLOUD_TEST_OBECT_PATH",
79+
"ROCKSDB_CLOUD_OBJECT_PATH",
80+
&object_)) {
7981
object_ = object;
8082
}
81-
if (! CloudEnvOptions::GetNameFromEnvironment("ROCKSDB_CLOUD_TEST_REGION",
82-
"ROCKSDB_CLOUD_REGION", &region_)) {
83+
if (!CloudEnvOptions::GetNameFromEnvironment(
84+
"ROCKSDB_CLOUD_TEST_REGION", "ROCKSDB_CLOUD_REGION", &region_)) {
8385
region_ = region;
8486
}
8587
}
@@ -88,137 +90,12 @@ CloudEnv::~CloudEnv() {}
8890

8991
CloudEnvWrapper::~CloudEnvWrapper() {}
9092

91-
CloudEnvImpl::CloudEnvImpl(const CloudEnvOptions& opts, Env* base_env)
92-
: CloudEnv(opts),
93-
base_env_(base_env), purger_is_running_(true) {}
94-
95-
CloudEnvImpl::~CloudEnvImpl() { StopPurger(); }
96-
97-
void CloudEnvImpl::StopPurger() {
98-
{
99-
std::lock_guard<std::mutex> lk(purger_lock_);
100-
purger_is_running_ = false;
101-
purger_cv_.notify_one();
102-
}
103-
104-
// wait for the purger to stop
105-
if (purge_thread_.joinable()) {
106-
purge_thread_.join();
107-
}
108-
}
109-
110-
Status CloudEnvImpl::LoadLocalCloudManifest(const std::string& dbname) {
111-
if (cloud_manifest_) {
112-
cloud_manifest_.reset();
113-
}
114-
unique_ptr<SequentialFile> file;
115-
auto cloudManifestFile = CloudManifestFile(dbname);
116-
auto s = GetBaseEnv()->NewSequentialFile(cloudManifestFile, &file, EnvOptions());
117-
if (!s.ok()) {
118-
return s;
119-
}
120-
return CloudManifest::LoadFromLog(
121-
unique_ptr<SequentialFileReader>(
122-
new SequentialFileReader(std::move(file), cloudManifestFile)),
123-
&cloud_manifest_);
124-
}
125-
126-
std::string CloudEnvImpl::RemapFilename(const std::string& logical_path) const {
127-
if (UNLIKELY(GetCloudType() == CloudType::kCloudNone) ||
128-
UNLIKELY(test_disable_cloud_manifest_)) {
129-
return logical_path;
130-
}
131-
auto file_name = basename(logical_path);
132-
uint64_t fileNumber;
133-
FileType type;
134-
WalFileType walType;
135-
if (file_name == "MANIFEST") {
136-
type = kDescriptorFile;
137-
} else {
138-
bool ok = ParseFileName(file_name, &fileNumber, &type, &walType);
139-
if (!ok) {
140-
return logical_path;
141-
}
142-
}
143-
Slice epoch;
144-
switch (type) {
145-
case kTableFile:
146-
// We should not be accessing sst files before CLOUDMANIFEST is loaded
147-
assert(cloud_manifest_);
148-
epoch = cloud_manifest_->GetEpoch(fileNumber);
149-
break;
150-
case kDescriptorFile:
151-
// We should not be accessing MANIFEST files before CLOUDMANIFEST is
152-
// loaded
153-
assert(cloud_manifest_);
154-
// Even though logical file might say MANIFEST-000001, we cut the number
155-
// suffix and store MANIFEST-[epoch] in the cloud and locally.
156-
file_name = "MANIFEST";
157-
epoch = cloud_manifest_->GetCurrentEpoch();
158-
break;
159-
default:
160-
return logical_path;
161-
};
162-
auto dir = dirname(logical_path);
163-
return dir + (dir.empty() ? "" : "/") + file_name +
164-
(epoch.empty() ? "" : ("-" + epoch.ToString()));
165-
}
166-
167-
Status CloudEnvImpl::DeleteInvisibleFiles(const std::string& dbname) {
168-
Status s;
169-
if (HasDestBucket()) {
170-
BucketObjectMetadata metadata;
171-
s = ListObjects(GetDestBucketName(), GetDestObjectPath(), &metadata);
172-
if (!s.ok()) {
173-
return s;
174-
}
175-
176-
for (auto& fname : metadata.pathnames) {
177-
auto noepoch = RemoveEpoch(fname);
178-
if (IsSstFile(noepoch) || IsManifestFile(noepoch)) {
179-
if (RemapFilename(noepoch) != fname) {
180-
// Ignore returned status on purpose.
181-
Log(InfoLogLevel::INFO_LEVEL, info_log_,
182-
"DeleteInvisibleFiles deleting %s from destination bucket",
183-
fname.c_str());
184-
DeleteCloudFileFromDest(fname);
185-
}
186-
}
187-
}
188-
}
189-
std::vector<std::string> children;
190-
s = GetBaseEnv()->GetChildren(dbname, &children);
191-
if (!s.ok()) {
192-
return s;
193-
}
194-
for (auto& fname : children) {
195-
auto noepoch = RemoveEpoch(fname);
196-
if (IsSstFile(noepoch) || IsManifestFile(noepoch)) {
197-
if (RemapFilename(RemoveEpoch(fname)) != fname) {
198-
// Ignore returned status on purpose.
199-
Log(InfoLogLevel::INFO_LEVEL, info_log_,
200-
"DeleteInvisibleFiles deleting file %s from local dir",
201-
fname.c_str());
202-
GetBaseEnv()->DeleteFile(dbname + "/" + fname);
203-
}
204-
}
205-
}
206-
return s;
207-
}
208-
209-
void CloudEnvImpl::TEST_InitEmptyCloudManifest() {
210-
CloudManifest::CreateForEmptyDatabase("", &cloud_manifest_);
211-
}
212-
213-
Status CloudEnv::NewAwsEnv(Env* base_env,
214-
const std::string& src_cloud_bucket,
215-
const std::string& src_cloud_object,
216-
const std::string& src_cloud_region,
217-
const std::string& dest_cloud_bucket,
218-
const std::string& dest_cloud_object,
219-
const std::string& dest_cloud_region,
220-
const CloudEnvOptions& cloud_options,
221-
const std::shared_ptr<Logger> & logger, CloudEnv** cenv) {
93+
Status CloudEnv::NewAwsEnv(
94+
Env* base_env, const std::string& src_cloud_bucket,
95+
const std::string& src_cloud_object, const std::string& src_cloud_region,
96+
const std::string& dest_cloud_bucket, const std::string& dest_cloud_object,
97+
const std::string& dest_cloud_region, const CloudEnvOptions& cloud_options,
98+
const std::shared_ptr<Logger>& logger, CloudEnv** cenv) {
22299
CloudEnvOptions options = cloud_options;
223100
if (!src_cloud_bucket.empty()) options.src_bucket.SetBucketName(src_cloud_bucket);
224101
if (!src_cloud_object.empty()) options.src_bucket.SetObjectPath(src_cloud_object);

0 commit comments

Comments
 (0)