Skip to content

Commit ca11ed2

Browse files
bekadavis9bekadavis9
authored
Integrate filesystem unit tests into a single test. (#4635)
Begin work on integrating _all_ `unit-<filesystem>.cc` tests into a _single_ `unit-vfs.cc` test. Eventually, the individual tests will be removed entirely and the single test will exist solely as a _unit_ test in `tiledb/sm/filesystem/test`. This work begins that transition. --- TYPE: NO_HISTORY DESC: Integrate filesystem unit tests into a single test.
1 parent 0df8863 commit ca11ed2

File tree

5 files changed

+298
-616
lines changed

5 files changed

+298
-616
lines changed

‎test/src/unit-azure.cc‎

Lines changed: 1 addition & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
*
66
* The MIT License
77
*
8-
* @copyright Copyright (c) 2017-2021 TileDB, Inc.
8+
* @copyright Copyright (c) 2017-2023 TileDB, Inc.
99
*
1010
* Permission is hereby granted, free of charge, to any person obtaining a copy
1111
* of this software and associated documentation files (the "Software"), to deal
@@ -131,146 +131,11 @@ std::string AzureFx::random_container_name(const std::string& prefix) {
131131
return ss.str();
132132
}
133133

134-
TEST_CASE_METHOD(AzureFx, "Test Azure filesystem, file management", "[azure]") {
135-
Config config;
136-
REQUIRE(config.set("vfs.azure.use_block_list_upload", "true").ok());
137-
138-
auto settings =
139-
GENERATE(from_range(test_settings.begin(), test_settings.end()));
140-
init_azure(std::move(config), settings);
141-
142-
/* Create the following file hierarchy:
143-
*
144-
* TEST_DIR/dir/subdir/file1
145-
* TEST_DIR/dir/subdir/file2
146-
* TEST_DIR/dir/file3
147-
* TEST_DIR/file4
148-
* TEST_DIR/file5
149-
*/
150-
auto dir = TEST_DIR + "dir/";
151-
auto dir2 = TEST_DIR + "dir2/";
152-
auto subdir = dir + "subdir/";
153-
auto file1 = subdir + "file1";
154-
auto file2 = subdir + "file2";
155-
auto file3 = dir + "file3";
156-
auto file4 = TEST_DIR + "file4";
157-
auto file5 = TEST_DIR + "file5";
158-
auto file6 = TEST_DIR + "file6";
159-
160-
// Check that container is empty
161-
bool is_empty;
162-
REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok());
163-
REQUIRE(is_empty);
164-
165-
// Continue building the hierarchy
166-
bool is_blob = false;
167-
REQUIRE(azure_.touch(URI(file1)).ok());
168-
REQUIRE(azure_.is_blob(URI(file1), &is_blob).ok());
169-
REQUIRE(is_blob);
170-
REQUIRE(azure_.touch(URI(file2)).ok());
171-
REQUIRE(azure_.is_blob(URI(file2), &is_blob).ok());
172-
REQUIRE(is_blob);
173-
REQUIRE(azure_.touch(URI(file3)).ok());
174-
REQUIRE(azure_.is_blob(URI(file3), &is_blob).ok());
175-
REQUIRE(is_blob);
176-
REQUIRE(azure_.touch(URI(file4)).ok());
177-
REQUIRE(azure_.is_blob(URI(file4), &is_blob).ok());
178-
REQUIRE(is_blob);
179-
REQUIRE(azure_.touch(URI(file5)).ok());
180-
REQUIRE(azure_.is_blob(URI(file5), &is_blob).ok());
181-
REQUIRE(is_blob);
182-
183-
// Check that container is not empty
184-
REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok());
185-
REQUIRE(!is_empty);
186-
187-
// Check invalid file
188-
REQUIRE(azure_.is_blob(URI(TEST_DIR + "foo"), &is_blob).ok());
189-
REQUIRE(!is_blob);
190-
191-
// List with prefix
192-
std::vector<std::string> paths;
193-
REQUIRE(azure_.ls(URI(TEST_DIR), &paths).ok());
194-
REQUIRE(paths.size() == 3);
195-
paths.clear();
196-
REQUIRE(azure_.ls(URI(dir), &paths).ok());
197-
REQUIRE(paths.size() == 2);
198-
paths.clear();
199-
REQUIRE(azure_.ls(URI(subdir), &paths).ok());
200-
REQUIRE(paths.size() == 2);
201-
paths.clear();
202-
REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter
203-
REQUIRE(paths.size() == 5);
204-
205-
// Check if a directory exists
206-
bool is_dir = false;
207-
REQUIRE(azure_.is_dir(URI(file1), &is_dir).ok());
208-
REQUIRE(!is_dir); // Not a dir
209-
REQUIRE(azure_.is_dir(URI(file4), &is_dir).ok());
210-
REQUIRE(!is_dir); // Not a dir
211-
REQUIRE(azure_.is_dir(URI(dir), &is_dir).ok());
212-
REQUIRE(is_dir); // This is viewed as a dir
213-
REQUIRE(azure_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok());
214-
REQUIRE(is_dir); // This is viewed as a dir
215-
216-
// ls_with_sizes
217-
std::string s = "abcdef";
218-
CHECK(azure_.write(URI(file3), s.data(), s.size()).ok());
219-
REQUIRE(azure_.flush_blob(URI(file3)).ok());
220-
221-
auto&& [status, rv] = azure_.ls_with_sizes(URI(dir));
222-
auto children = *rv;
223-
REQUIRE(status.ok());
224-
225-
REQUIRE(children.size() == 2);
226-
CHECK(children[0].path().native() == file3);
227-
CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1));
228-
229-
CHECK(children[0].file_size() == s.size());
230-
// Directories don't get a size
231-
CHECK(children[1].file_size() == 0);
232-
233-
// Move file
234-
REQUIRE(azure_.move_object(URI(file5), URI(file6)).ok());
235-
REQUIRE(azure_.is_blob(URI(file5), &is_blob).ok());
236-
REQUIRE(!is_blob);
237-
REQUIRE(azure_.is_blob(URI(file6), &is_blob).ok());
238-
REQUIRE(is_blob);
239-
paths.clear();
240-
REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter
241-
REQUIRE(paths.size() == 5);
242-
243-
// Move directory
244-
REQUIRE(azure_.move_dir(URI(dir), URI(dir2)).ok());
245-
REQUIRE(azure_.is_dir(URI(dir), &is_dir).ok());
246-
REQUIRE(!is_dir);
247-
REQUIRE(azure_.is_dir(URI(dir2), &is_dir).ok());
248-
REQUIRE(is_dir);
249-
paths.clear();
250-
REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter
251-
REQUIRE(paths.size() == 5);
252-
253-
// Remove files
254-
REQUIRE(azure_.remove_blob(URI(file4)).ok());
255-
REQUIRE(azure_.is_blob(URI(file4), &is_blob).ok());
256-
REQUIRE(!is_blob);
257-
258-
// Remove directories
259-
REQUIRE(azure_.remove_dir(URI(dir2)).ok());
260-
REQUIRE(azure_.is_blob(URI(file1), &is_blob).ok());
261-
REQUIRE(!is_blob);
262-
REQUIRE(azure_.is_blob(URI(file2), &is_blob).ok());
263-
REQUIRE(!is_blob);
264-
REQUIRE(azure_.is_blob(URI(file3), &is_blob).ok());
265-
REQUIRE(!is_blob);
266-
}
267-
268134
TEST_CASE_METHOD(
269135
AzureFx, "Test Azure filesystem, file I/O", "[azure][multipart]") {
270136
Config config;
271137
const uint64_t max_parallel_ops = 2;
272138
const uint64_t block_list_block_size = 4 * 1024 * 1024;
273-
REQUIRE(config.set("vfs.azure.use_block_list_upload", "true").ok());
274139
REQUIRE(
275140
config.set("vfs.azure.max_parallel_ops", std::to_string(max_parallel_ops))
276141
.ok());

‎test/src/unit-gcs.cc‎

Lines changed: 1 addition & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
*
66
* The MIT License
77
*
8-
* @copyright Copyright (c) 2017-2021 TileDB, Inc.
8+
* @copyright Copyright (c) 2017-2023 TileDB, Inc.
99
*
1010
* Permission is hereby granted, free of charge, to any person obtaining a copy
1111
* of this software and associated documentation files (the "Software"), to deal
@@ -120,137 +120,6 @@ TEST_CASE_METHOD(GCSFx, "Test GCS init", "[gcs]") {
120120
}
121121
}
122122

123-
TEST_CASE_METHOD(GCSFx, "Test GCS filesystem, file management", "[gcs]") {
124-
Config config;
125-
REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok());
126-
init_gcs(std::move(config));
127-
128-
/* Create the following file hierarchy:
129-
*
130-
* TEST_DIR/dir/subdir/file1
131-
* TEST_DIR/dir/subdir/file2
132-
* TEST_DIR/dir/file3
133-
* TEST_DIR/file4
134-
* TEST_DIR/file5
135-
*/
136-
auto dir = TEST_DIR + "dir/";
137-
auto dir2 = TEST_DIR + "dir2/";
138-
auto subdir = dir + "subdir/";
139-
auto file1 = subdir + "file1";
140-
auto file2 = subdir + "file2";
141-
auto file3 = dir + "file3";
142-
auto file4 = TEST_DIR + "file4";
143-
auto file5 = TEST_DIR + "file5";
144-
auto file6 = TEST_DIR + "file6";
145-
146-
// Check that bucket is empty
147-
bool is_empty;
148-
REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok());
149-
REQUIRE(is_empty);
150-
151-
// Continue building the hierarchy
152-
bool is_object = false;
153-
REQUIRE(gcs_.touch(URI(file1)).ok());
154-
REQUIRE(gcs_.is_object(URI(file1), &is_object).ok());
155-
REQUIRE(is_object);
156-
REQUIRE(gcs_.touch(URI(file2)).ok());
157-
REQUIRE(gcs_.is_object(URI(file2), &is_object).ok());
158-
REQUIRE(is_object);
159-
REQUIRE(gcs_.touch(URI(file3)).ok());
160-
REQUIRE(gcs_.is_object(URI(file3), &is_object).ok());
161-
REQUIRE(is_object);
162-
REQUIRE(gcs_.touch(URI(file4)).ok());
163-
REQUIRE(gcs_.is_object(URI(file4), &is_object).ok());
164-
REQUIRE(is_object);
165-
REQUIRE(gcs_.touch(URI(file5)).ok());
166-
REQUIRE(gcs_.is_object(URI(file5), &is_object).ok());
167-
REQUIRE(is_object);
168-
169-
// Check that bucket is not empty
170-
REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok());
171-
REQUIRE(!is_empty);
172-
173-
// Check invalid file
174-
REQUIRE(gcs_.is_object(URI(TEST_DIR + "foo"), &is_object).ok());
175-
REQUIRE(!is_object);
176-
177-
// List with prefix
178-
std::vector<std::string> paths;
179-
REQUIRE(gcs_.ls(URI(TEST_DIR), &paths).ok());
180-
REQUIRE(paths.size() == 3);
181-
paths.clear();
182-
REQUIRE(gcs_.ls(URI(dir), &paths).ok());
183-
REQUIRE(paths.size() == 2);
184-
paths.clear();
185-
REQUIRE(gcs_.ls(URI(subdir), &paths).ok());
186-
REQUIRE(paths.size() == 2);
187-
paths.clear();
188-
REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter
189-
REQUIRE(paths.size() == 5);
190-
191-
// Check if a directory exists
192-
bool is_dir = false;
193-
REQUIRE(gcs_.is_dir(URI(file1), &is_dir).ok());
194-
REQUIRE(!is_dir); // Not a dir
195-
REQUIRE(gcs_.is_dir(URI(file4), &is_dir).ok());
196-
REQUIRE(!is_dir); // Not a dir
197-
REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok());
198-
REQUIRE(is_dir); // This is viewed as a dir
199-
REQUIRE(gcs_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok());
200-
REQUIRE(is_dir); // This is viewed as a dir
201-
202-
// ls_with_sizes
203-
std::string s = "abcdef";
204-
CHECK(gcs_.write(URI(file3), s.data(), s.size()).ok());
205-
REQUIRE(gcs_.flush_object(URI(file3)).ok());
206-
207-
auto&& [status, rv] = gcs_.ls_with_sizes(URI(dir));
208-
auto children = *rv;
209-
REQUIRE(status.ok());
210-
211-
REQUIRE(children.size() == 2);
212-
CHECK(children[0].path().native() == file3);
213-
CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1));
214-
215-
CHECK(children[0].file_size() == s.size());
216-
// Directories don't get a size
217-
CHECK(children[1].file_size() == 0);
218-
219-
// Move file
220-
REQUIRE(gcs_.move_object(URI(file5), URI(file6)).ok());
221-
REQUIRE(gcs_.is_object(URI(file5), &is_object).ok());
222-
REQUIRE(!is_object);
223-
REQUIRE(gcs_.is_object(URI(file6), &is_object).ok());
224-
REQUIRE(is_object);
225-
paths.clear();
226-
REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter
227-
REQUIRE(paths.size() == 5);
228-
229-
// Move directory
230-
REQUIRE(gcs_.move_dir(URI(dir), URI(dir2)).ok());
231-
REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok());
232-
REQUIRE(!is_dir);
233-
REQUIRE(gcs_.is_dir(URI(dir2), &is_dir).ok());
234-
REQUIRE(is_dir);
235-
paths.clear();
236-
REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter
237-
REQUIRE(paths.size() == 5);
238-
239-
// Remove files
240-
REQUIRE(gcs_.remove_object(URI(file4)).ok());
241-
REQUIRE(gcs_.is_object(URI(file4), &is_object).ok());
242-
REQUIRE(!is_object);
243-
244-
// Remove directories
245-
REQUIRE(gcs_.remove_dir(URI(dir2)).ok());
246-
REQUIRE(gcs_.is_object(URI(file1), &is_object).ok());
247-
REQUIRE(!is_object);
248-
REQUIRE(gcs_.is_object(URI(file2), &is_object).ok());
249-
REQUIRE(!is_object);
250-
REQUIRE(gcs_.is_object(URI(file3), &is_object).ok());
251-
REQUIRE(!is_object);
252-
}
253-
254123
TEST_CASE_METHOD(
255124
GCSFx,
256125
"Test GCS filesystem I/O, multipart, serial",

‎test/src/unit-hdfs-filesystem.cc‎

Lines changed: 1 addition & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
*
66
* The MIT License
77
*
8-
* @copyright Copyright (c) 2017-2021 TileDB, Inc.
8+
* @copyright Copyright (c) 2017-2023 TileDB, Inc.
99
*
1010
* Permission is hereby granted, free of charge, to any person obtaining a copy
1111
* of this software and associated documentation files (the "Software"), to deal
@@ -118,42 +118,6 @@ TEST_CASE("Test HDFS filesystem", "[hdfs]") {
118118
}
119119
CHECK(allok);
120120

121-
std::vector<std::string> paths;
122-
st = hdfs.ls(URI("hdfs:///"), &paths);
123-
CHECK(st.ok());
124-
CHECK(paths.size() > 0);
125-
126-
// ls_with_sizes
127-
// Dir structure:
128-
// ...../subdir
129-
// ...../subdir/file
130-
// ...../subdir/subsubdir
131-
132-
std::string subdir = "hdfs://localhost:9000/tiledb_test/subdir";
133-
std::string file = subdir + "/file";
134-
std::string subsubdir = subdir + "/subsubdir";
135-
136-
CHECK(hdfs.create_dir(URI(subdir)).ok());
137-
CHECK(hdfs.create_dir(URI(subsubdir)).ok());
138-
CHECK(hdfs.touch(URI(file)).ok());
139-
140-
std::string s = "abcdef";
141-
CHECK(hdfs.write(URI(file), s.data(), s.size()).ok());
142-
143-
auto&& [status, rv] = hdfs.ls_with_sizes(URI(subdir));
144-
auto children = *rv;
145-
REQUIRE(status.ok());
146-
147-
REQUIRE(children.size() == 2);
148-
CHECK(children[0].path().native() == file);
149-
CHECK(children[1].path().native() == subsubdir.substr(0, subsubdir.size()));
150-
151-
CHECK(children[0].file_size() == s.size());
152-
// Directories don't get a size
153-
CHECK(children[1].file_size() == 0);
154-
// Cleanup
155-
CHECK(hdfs.remove_dir(URI(subdir)).ok());
156-
157121
uint64_t nbytes = 0;
158122
st = hdfs.file_size(URI("hdfs:///tiledb_test/tiledb_test_file"), &nbytes);
159123
CHECK(st.ok());

0 commit comments

Comments
 (0)