Add experimental CPP API for ls_recursive. (#4625)

This adds the experimental CPP APIs for ls_recursive, which is currently
only supported over S3.

---
TYPE: FEATURE
DESC: Add experimental CPP API for ls_recursive.

Author: shaunrd0

test/src/unit-cppapi-vfs.cc

@@ -32,7 +32,9 @@
 
 #include <test/support/src/helpers.h>
 #include <test/support/tdb_catch.h>
+#include "test/support/src/vfs_helpers.h"
 #include "tiledb/sm/cpp_api/tiledb"
+#include "tiledb/sm/cpp_api/vfs_experimental.h"
 
 #ifdef _WIN32
 #include "tiledb/sm/filesystem/path_win.h"
@@ -500,3 +502,187 @@ TEST_CASE(
     }
   }
 }
+
+TEST_CASE("CPP API: VFS ls_recursive filter", "[cppapi][vfs][ls-recursive]") {
+  using namespace tiledb::test;
+  S3Test s3_test({10, 100, 0});
+  if (!s3_test.is_supported()) {
+    return;
+  }
+  auto expected_results = s3_test.expected_results();
+
+  vfs_config cfg;
+  tiledb::Context ctx(tiledb::Config(&cfg.config));
+  tiledb::VFS vfs(ctx);
+
+  tiledb::VFSExperimental::LsObjects ls_objects;
+  // Predicate filter to apply to ls_recursive.
+  tiledb::VFSExperimental::LsInclude include;
+  // Callback to populate ls_objects vector using a filter.
+  tiledb::VFSExperimental::LsCallback cb = [&](std::string_view path,
+                                               uint64_t size) {
+    if (include(path, size)) {
+      ls_objects.emplace_back(path, size);
+    }
+    return true;
+  };
+
+  SECTION("Default filter (include all)") {
+    include = [](std::string_view, uint64_t) { return true; };
+  }
+  SECTION("Custom filter (include none)") {
+    include = [](std::string_view, uint64_t) { return false; };
+  }
+
+  bool include_result = true;
+  SECTION("Custom filter (include half)") {
+    include = [&include_result](std::string_view, uint64_t) {
+      include_result = !include_result;
+      return include_result;
+    };
+  }
+
+  SECTION("Custom filter (search for test_file_50)") {
+    include = [](std::string_view object_name, uint64_t) {
+      return object_name.find("test_file_50") != std::string::npos;
+    };
+  }
+  SECTION("Custom filter (search for test_file_1*)") {
+    include = [](std::string_view object_name, uint64_t) {
+      return object_name.find("test_file_1") != std::string::npos;
+    };
+  }
+  SECTION("Custom filter (reject files over 50 bytes)") {
+    include = [](std::string_view, uint64_t size) { return size <= 50; };
+  }
+
+  // Test collecting results with LsInclude predicate.
+  auto results = tiledb::VFSExperimental::ls_recursive_filter(
+      ctx, vfs, s3_test.temp_dir_.to_string(), include);
+  std::erase_if(expected_results, [&include](const auto& object) {
+    return !include(object.first, object.second);
+  });
+  CHECK(results.size() == expected_results.size());
+  CHECK(expected_results == results);
+
+  // Test collecting results with LsCallback, writing data into ls_objects.
+  tiledb::VFSExperimental::ls_recursive(
+      ctx, vfs, s3_test.temp_dir_.to_string(), cb);
+  CHECK(ls_objects.size() == expected_results.size());
+  CHECK(expected_results == ls_objects);
+}
+
+TEST_CASE("CPP API: Callback stops traversal", "[cppapi][vfs][ls-recursive]") {
+  using namespace tiledb::test;
+  S3Test s3_test({10, 50, 15});
+  if (!s3_test.is_supported()) {
+    return;
+  }
+  auto expected_results = s3_test.expected_results();
+
+  vfs_config cfg;
+  tiledb::Context ctx(tiledb::Config(&cfg.config));
+  tiledb::VFS vfs(ctx);
+
+  tiledb::VFSExperimental::LsObjects ls_objects;
+  size_t cb_count = GENERATE(1, 10, 11, 50);
+  auto cb = [&](std::string_view path, uint64_t size) {
+    // Always emplace to check the callback is not invoked more than `cb_count`.
+    ls_objects.emplace_back(path, size);
+    // Signal to stop traversal when we have seen `cb_count` objects.
+    if (ls_objects.size() == cb_count) {
+      return false;
+    }
+    return true;
+  };
+  tiledb::VFSExperimental::ls_recursive(
+      ctx, vfs, s3_test.temp_dir_.to_string(), cb);
+  expected_results.resize(cb_count);
+  CHECK(ls_objects.size() == cb_count);
+  CHECK(ls_objects == expected_results);
+}
+
+TEST_CASE("CPP API: Throwing filter", "[cppapi][vfs][ls-recursive]") {
+  using namespace tiledb::test;
+  S3Test s3_test({0});
+  if (!s3_test.is_supported()) {
+    return;
+  }
+
+  vfs_config cfg;
+  tiledb::Context ctx(tiledb::Config(&cfg.config));
+  tiledb::VFS vfs(ctx);
+
+  tiledb::VFSExperimental::LsInclude filter = [](std::string_view,
+                                                 uint64_t) -> bool {
+    throw std::runtime_error("Throwing filter");
+  };
+  auto path = s3_test.temp_dir_.to_string();
+
+  // If the test directory is empty the filter should not throw.
+  SECTION("Throwing filter with 0 objects should not throw") {
+    CHECK_NOTHROW(
+        tiledb::VFSExperimental::ls_recursive_filter(ctx, vfs, path, filter));
+    CHECK_NOTHROW(
+        tiledb::VFSExperimental::ls_recursive(ctx, vfs, path, filter));
+  }
+  SECTION("Throwing filter with N objects should throw") {
+    vfs.touch(s3_test.temp_dir_.join_path("test_file").to_string());
+    CHECK_THROWS_AS(
+        tiledb::VFSExperimental::ls_recursive_filter(ctx, vfs, path, filter),
+        std::runtime_error);
+    CHECK_THROWS_WITH(
+        tiledb::VFSExperimental::ls_recursive_filter(ctx, vfs, path, filter),
+        Catch::Matchers::ContainsSubstring("Throwing filter"));
+    CHECK_THROWS_AS(
+        tiledb::VFSExperimental::ls_recursive(ctx, vfs, path, filter),
+        std::runtime_error);
+    CHECK_THROWS_WITH(
+        tiledb::VFSExperimental::ls_recursive(ctx, vfs, path, filter),
+        Catch::Matchers::ContainsSubstring("Throwing filter"));
+  }
+}
+
+TEST_CASE(
+    "CPP API: CallbackWrapperCPP construction validation",
+    "[ls-recursive][callback][wrapper]") {
+  using tiledb::sm::CallbackWrapperCPP;
+  tiledb::VFSExperimental::LsObjects data;
+  auto cb = [&](std::string_view, uint64_t) -> bool { return true; };
+  SECTION("Null callback") {
+    CHECK_THROWS(CallbackWrapperCPP(nullptr));
+  }
+  SECTION("Valid callback") {
+    CHECK_NOTHROW(CallbackWrapperCPP(cb));
+  }
+}
+
+TEST_CASE(
+    "CPP API: CallbackWrapperCPP operator() validation",
+    "[ls-recursive][callback][wrapper]") {
+  tiledb::VFSExperimental::LsObjects data;
+  auto cb = [&](std::string_view path, uint64_t object_size) -> bool {
+    if (object_size > 100) {
+      // Throw if object size is greater than 100 bytes.
+      throw std::runtime_error("Throwing callback");
+    } else if (!path.ends_with(".txt")) {
+      // Reject non-txt files.
+      return false;
+    }
+    data.emplace_back(path, object_size);
+    return true;
+  };
+  tiledb::sm::CallbackWrapperCPP wrapper(cb);
+
+  SECTION("Callback return true accepts object") {
+    CHECK(wrapper("file.txt", 10) == true);
+    CHECK(data.size() == 1);
+  }
+  SECTION("Callback return false rejects object") {
+    CHECK(wrapper("some/dir/", 0) == false);
+    CHECK(data.empty());
+  }
+  SECTION("Callback exception is propagated") {
+    CHECK_THROWS_WITH(wrapper("path", 101) == 0, "Throwing callback");
+  }
+}

tiledb/api/c_api/vfs/test/unit_capi_vfs.cc

@@ -730,7 +730,7 @@ TEST_CASE(
 }
 
 TEST_CASE(
-    "C API: CallbackWrapper operator() validation",
+    "C API: CallbackWrapperCAPI operator() validation",
     "[ls-recursive][callback][wrapper]") {
   tiledb::sm::LsObjects data;
   auto cb = [](const char* path,
@@ -748,7 +748,7 @@ TEST_CASE(
     ls_data->push_back({{path, path_len}, object_size});
     return 1;
   };
-  tiledb::sm::CallbackWrapper wrapper(cb, &data);
+  tiledb::sm::CallbackWrapperCAPI wrapper(cb, &data);
 
   SECTION("Callback return 1 signals to continue traversal") {
     CHECK(wrapper("file.txt", 10) == 1);
@@ -763,21 +763,21 @@ TEST_CASE(
 }
 
 TEST_CASE(
-    "C API: CallbackWrapper construction validation",
+    "C API: CallbackWrapperCAPI construction validation",
     "[ls-recursive][callback][wrapper]") {
-  using tiledb::sm::CallbackWrapper;
+  using tiledb::sm::CallbackWrapperCAPI;
   tiledb::sm::LsObjects data;
   auto cb = [](const char*, size_t, uint64_t, void*) -> int32_t { return 1; };
   SECTION("Null callback") {
-    CHECK_THROWS(CallbackWrapper(nullptr, &data));
+    CHECK_THROWS(CallbackWrapperCAPI(nullptr, &data));
   }
   SECTION("Null data") {
-    CHECK_THROWS(CallbackWrapper(cb, nullptr));
+    CHECK_THROWS(CallbackWrapperCAPI(cb, nullptr));
   }
   SECTION("Null callback and data") {
-    CHECK_THROWS(CallbackWrapper(nullptr, nullptr));
+    CHECK_THROWS(CallbackWrapperCAPI(nullptr, nullptr));
   }
   SECTION("Valid callback and data") {
-    CHECK_NOTHROW(CallbackWrapper(cb, &data));
+    CHECK_NOTHROW(CallbackWrapperCAPI(cb, &data));
   }
 }

tiledb/api/c_api/vfs/vfs_api_internal.h

@@ -151,7 +151,7 @@ struct tiledb_vfs_handle_t
       const tiledb::sm::URI& parent,
       tiledb_ls_callback_t cb,
       void* data) const {
-    tiledb::sm::CallbackWrapper wrapper(cb, data);
+    tiledb::sm::CallbackWrapperCAPI wrapper(cb, data);
     vfs_.ls_recursive(parent, wrapper);
   }
 };