Add experimental C API for ls_recursive. (#4615)

This adds experimental C API for ls_recursive, which is currently only
supported over S3.

---
TYPE: FEATURE
DESC: Add experimental C API for ls_recursive.

---------

Co-authored-by: eric-hughes-tiledb <[email protected]>

Author: shaunrd0

CMakeLists.txt

@@ -352,6 +352,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/string/string_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/vfs_api_enum.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/vfs_api_external.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/vfs_api_experimental.h"
 )
 set(TILEDB_C_API_RELATIVE_HEADER_BASE "${CMAKE_CURRENT_SOURCE_DIR}")
 

test/CMakeLists.txt

@@ -203,6 +203,7 @@ set(TILEDB_UNIT_TEST_SOURCES
   src/unit-SubarrayPartitioner-sparse.cc
   src/unit-vfs.cc
   src/unit-win-filesystem.cc
+  "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc"
   )
 
 if (TILEDB_CPP_API)

tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc

@@ -0,0 +1,113 @@
+/**
+ * @file tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2023 TileDB, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * Tests the C API for tiledb_vfs_ls_recursive.
+ *
+ * TODO: This test is built and ran as part of tiledb_unit. Once we're able to
+ *      execute these tests in CI, we should build this test as a separate unit.
+ */
+
+#include <test/support/tdb_catch.h>
+#include "test/support/src/vfs_helpers.h"
+
+using namespace tiledb::test;
+
+TEST_CASE("C API: ls_recursive callback", "[vfs][ls-recursive]") {
+  using tiledb::sm::LsObjects;
+  S3Test s3_test({10, 50});
+  if (!s3_test.is_supported()) {
+    return;
+  }
+  auto expected = s3_test.expected_results();
+
+  vfs_config vfs_config;
+  tiledb_ctx_t* ctx;
+  tiledb_ctx_alloc(vfs_config.config, &ctx);
+  tiledb_vfs_t* vfs;
+  tiledb_vfs_alloc(ctx, vfs_config.config, &vfs);
+
+  LsObjects data;
+  tiledb_ls_callback_t cb = [](const char* path,
+                               size_t path_len,
+                               uint64_t object_size,
+                               void* data) -> int32_t {
+    auto* ls_data = static_cast<LsObjects*>(data);
+    ls_data->push_back({{path, path_len}, object_size});
+    return 1;
+  };
+
+  // This callback will return 0 exactly once. Traversal should stop immediately
+  // and not continue to the next object.
+  SECTION("callback stops traversal") {
+    cb = [](const char* path,
+            size_t path_len,
+            uint64_t object_size,
+            void* data) -> int32_t {
+      // There's no precheck here to push_back, so the vector size will match
+      // the number of times the callback was executed.
+      auto* ls_data = static_cast<LsObjects*>(data);
+      ls_data->push_back({{path, path_len}, object_size});
+      // Stop traversal after we collect 10 results.
+      return ls_data->size() != 10;
+    };
+    expected.resize(10);
+  }
+
+  CHECK(
+      tiledb_vfs_ls_recursive(ctx, vfs, s3_test.temp_dir_.c_str(), cb, &data) ==
+      TILEDB_OK);
+  CHECK(data.size() == expected.size());
+  CHECK(data == expected);
+}
+
+TEST_CASE("C API: ls_recursive throwing callback", "[vfs][ls-recursive]") {
+  using tiledb::sm::LsObjects;
+  S3Test s3_test({10, 50});
+  if (!s3_test.is_supported()) {
+    return;
+  }
+  auto expected = s3_test.expected_results();
+
+  vfs_config vfs_config;
+  tiledb_ctx_t* ctx;
+  tiledb_ctx_alloc(vfs_config.config, &ctx);
+  tiledb_vfs_t* vfs;
+  tiledb_vfs_alloc(ctx, vfs_config.config, &vfs);
+
+  LsObjects data;
+  tiledb_ls_callback_t cb =
+      [](const char*, size_t, uint64_t, void*) -> int32_t {
+    throw std::runtime_error("Throwing callback");
+  };
+
+  CHECK(
+      tiledb_vfs_ls_recursive(ctx, vfs, s3_test.temp_dir_.c_str(), cb, &data) ==
+      TILEDB_ERR);
+  CHECK(data.empty());
+}

tiledb/api/c_api/vfs/test/unit_capi_vfs.cc

@@ -675,3 +675,109 @@ TEST_CASE(
 
   CHECK(error != nullptr);
 }
+
+TEST_CASE("C API: tiledb_vfs_ls_recursive argument validation", "[capi][vfs]") {
+  /*
+   * No "success" sections here; too much overhead to set up.
+   */
+  ordinary_vfs x;
+  int32_t data;
+  auto cb = [](const char*, size_t, uint64_t, void*) { return 0; };
+  SECTION("null context") {
+    auto rc{tiledb_vfs_ls_recursive(nullptr, x.vfs, TEST_URI, cb, &data)};
+    CHECK(tiledb_status(rc) == TILEDB_INVALID_CONTEXT);
+  }
+  SECTION("null vfs") {
+    auto rc{tiledb_vfs_ls_recursive(x.ctx, nullptr, TEST_URI, cb, &data)};
+    CHECK(tiledb_status(rc) == TILEDB_ERR);
+  }
+  SECTION("null uri") {
+    auto rc{tiledb_vfs_ls_recursive(x.ctx, x.vfs, nullptr, cb, &data)};
+    CHECK(tiledb_status(rc) == TILEDB_ERR);
+  }
+  SECTION("null callback") {
+    auto rc{tiledb_vfs_ls_recursive(x.ctx, x.vfs, TEST_URI, nullptr, &data)};
+    CHECK(tiledb_status(rc) == TILEDB_ERR);
+  }
+  SECTION("null data ptr") {
+    auto rc{tiledb_vfs_ls_recursive(x.ctx, x.vfs, TEST_URI, cb, nullptr)};
+    CHECK(tiledb_status(rc) == TILEDB_ERR);
+  }
+}
+
+TEST_CASE(
+    "C API: VFS recursive ls unsupported backends",
+    "[capi][vfs][ls-recursive]") {
+  ordinary_vfs vfs;
+  int ls_data;
+  auto cb = [](const char*, size_t, uint64_t, void*) { return 0; };
+  // Recursive ls is currently only supported for S3.
+  tiledb::sm::URI uri{GENERATE(
+      "file:///path/",
+      "mem:///path/",
+      "azure://path/",
+      "gcs://path/",
+      "hdfs://path/")};
+  DYNAMIC_SECTION(
+      "Test recursive ls usupported backend over " << uri.backend_name()) {
+    if (!vfs.vfs->vfs()->supports_uri_scheme(uri)) {
+      return;
+    }
+    CHECK(
+        tiledb_vfs_ls_recursive(vfs.ctx, vfs.vfs, uri.c_str(), cb, &ls_data) ==
+        TILEDB_ERR);
+  }
+}
+
+TEST_CASE(
+    "C API: CallbackWrapper operator() validation",
+    "[ls-recursive][callback][wrapper]") {
+  tiledb::sm::LsObjects data;
+  auto cb = [](const char* path,
+               size_t path_len,
+               uint64_t object_size,
+               void* data) -> int32_t {
+    if (object_size > 100) {
+      // Throw if object size is greater than 100 bytes.
+      throw std::runtime_error("Throwing callback");
+    } else if (!std::string(path, path_len).ends_with(".txt")) {
+      // Reject non-txt files.
+      return 0;
+    }
+    auto* ls_data = static_cast<tiledb::sm::LsObjects*>(data);
+    ls_data->push_back({{path, path_len}, object_size});
+    return 1;
+  };
+  tiledb::sm::CallbackWrapper wrapper(cb, &data);
+
+  SECTION("Callback return 1 signals to continue traversal") {
+    CHECK(wrapper("file.txt", 10) == 1);
+    CHECK(data.size() == 1);
+  }
+  SECTION("Callback return 0 signals to stop traversal") {
+    CHECK_THROWS_AS(wrapper("some/dir/", 0) == 0, tiledb::sm::LsStopTraversal);
+  }
+  SECTION("Callback exception is propagated") {
+    CHECK_THROWS_WITH(wrapper("path", 101) == 0, "Throwing callback");
+  }
+}
+
+TEST_CASE(
+    "C API: CallbackWrapper construction validation",
+    "[ls-recursive][callback][wrapper]") {
+  using tiledb::sm::CallbackWrapper;
+  tiledb::sm::LsObjects data;
+  auto cb = [](const char*, size_t, uint64_t, void*) -> int32_t { return 1; };
+  SECTION("Null callback") {
+    CHECK_THROWS(CallbackWrapper(nullptr, &data));
+  }
+  SECTION("Null data") {
+    CHECK_THROWS(CallbackWrapper(cb, nullptr));
+  }
+  SECTION("Null callback and data") {
+    CHECK_THROWS(CallbackWrapper(nullptr, nullptr));
+  }
+  SECTION("Valid callback and data") {
+    CHECK_NOTHROW(CallbackWrapper(cb, &data));
+  }
+}

tiledb/api/c_api/vfs/vfs_api.cc

@@ -31,6 +31,7 @@
  */
 
 #include "tiledb/api/c_api_support/c_api_support.h"
+#include "vfs_api_experimental.h"
 #include "vfs_api_internal.h"
 
 namespace tiledb::api {
@@ -311,6 +312,23 @@ capi_return_t tiledb_vfs_touch(tiledb_vfs_t* vfs, const char* uri) {
   return TILEDB_OK;
 }
 
+capi_return_t tiledb_vfs_ls_recursive(
+    tiledb_vfs_t* vfs,
+    const char* path,
+    tiledb_ls_callback_t callback,
+    void* data) {
+  ensure_vfs_is_valid(vfs);
+  if (path == nullptr) {
+    throw CAPIStatusException("Invalid TileDB object: VFS passed a null path.");
+  } else if (callback == nullptr) {
+    throw CAPIStatusException(
+        "Invalid TileDB object: Callback function is null.");
+  }
+  ensure_output_pointer_is_valid(data);
+  vfs->ls_recursive(tiledb::sm::URI(path), callback, data);
+  return TILEDB_OK;
+}
+
 }  // namespace tiledb::api
 
 using tiledb::api::api_entry_context;
@@ -544,3 +562,14 @@ CAPI_INTERFACE(
     vfs_touch, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) {
   return api_entry_context<tiledb::api::tiledb_vfs_touch>(ctx, vfs, uri);
 }
+
+CAPI_INTERFACE(
+    vfs_ls_recursive,
+    tiledb_ctx_t* ctx,
+    tiledb_vfs_t* vfs,
+    const char* path,
+    tiledb_ls_callback_t callback,
+    void* data) {
+  return api_entry_context<tiledb::api::tiledb_vfs_ls_recursive>(
+      ctx, vfs, path, callback, data);
+}

tiledb/api/c_api/vfs/vfs_api_experimental.h

@@ -0,0 +1,102 @@
+/**
+ * @file tiledb/api/c_api/vfs/vfs_api_experimental.h
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2023 TileDB, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * This file declares the experimental VFS C API for TileDB.
+ */
+
+#ifndef TILEDB_VFS_API_EXPERIMENTAL_H
+#define TILEDB_VFS_API_EXPERIMENTAL_H
+
+#include "tiledb/api/c_api/api_external_common.h"
+#include "tiledb/api/c_api/context/context_api_external.h"
+#include "tiledb/api/c_api/vfs/vfs_api_external.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Typedef for ls_recursive callback function invoked on each object collected.
+ *
+ * @param path The path of a visited object for the relative filesystem.
+ * @param path_len The length of the path.
+ * @param object_size The size of the object at the current path.
+ * @param data Data passed to the callback used to store collected results.
+ */
+typedef int32_t (*tiledb_ls_callback_t)(
+    const char* path, size_t path_len, uint64_t object_size, void* data);
+
+/**
+ * Visits the children of `path` recursively, invoking the callback for each
+ * entry. The callback should return 1 to continue traversal, 0 to stop, or -1
+ * on error. The callback is responsible for writing gathered entries into the
+ * `data` buffer, for example using a pointer to a user-defined struct.
+ *
+ * Currently only S3 is supported, and the `path` must be a valid S3 URI.
+ *
+ * **Example:**
+ *
+ * @code{.c}
+ * int my_callback(
+ *     const char* path, size_t path_length, uint64_t file_size, void* data) {
+ *   MyCbStruct cb_data = static_cast<MyCbStruct*>(data);
+ *   // Perform custom callback behavior here.
+ *   return 1;  // Continue traversal to next entry.
+ * }
+ * MyCbStruct* cb_data = allocate_cb_struct();
+ *
+ * tiledb_vfs_ls_recursive(ctx, vfs, "s3://bucket/foo", my_callback, &cb_data);
+ * @endcode
+ *
+ * @param[in] ctx The TileDB context.
+ * @param[in] vfs The virtual filesystem object.
+ * @param[in] path The path in which the traversal will occur.
+ * @param[in] callback
+ * The callback function to be applied on every visited object.
+ *     The callback should return `0` if the iteration must stop, and `1`
+ *     if the iteration must continue. It takes as input the currently visited
+ *     path, the length of the currently visited path, the size of the file, and
+ *     user provided buffer for paths and object sizes in the form of a struct
+ *     pointer. The callback returns `-1` upon error. Note that `path` in the
+ *     callback will be an **absolute** path.
+ * @param[in] data Data pointer passed into the callback for storing results.
+ * @return `TILEDB_OK` for success and `TILEDB_ERR` for error.
+ */
+TILEDB_EXPORT capi_return_t tiledb_vfs_ls_recursive(
+    tiledb_ctx_t* ctx,
+    tiledb_vfs_t* vfs,
+    const char* path,
+    tiledb_ls_callback_t callback,
+    void* data) TILEDB_NOEXCEPT;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TILEDB_VFS_API_EXPERIMENTAL_H