Add support for Azure Data Lake Storage. (#5652)

teo-tsirpanis · web-flow · commit 371ae036dde9 · 2025-10-03T03:57:27.000+03:00
This PR updates the Azure VFS to support storage accounts with [hierarchical namespace](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-namespace) enabled, also known as Azure Data Lake Storage Gen2. Before the first Azure operation, we check whether the account support hierarchical namespace, and if it does, specialize some of the operations to use a `DatalakeServiceClient`. A dependency to `azure-storage-datalake-cpp` was also added. Automated testing is not currently implemented, due to lack of support by Azurite. Let me know if you want to test this on a real Azure Storage account. --- TYPE: FEATURE DESC: Added support for Azure Data Lake Storage.
diff --git a/cmake/inputs/Config.cmake.in b/cmake/inputs/Config.cmake.in
@@ -28,6 +28,7 @@ if(NOT @BUILD_SHARED_LIBS@) # NOT BUILD_SHARED_LIBS
   if(@TILEDB_AZURE@) # TILEDB_AZURE
     find_dependency(azure-identity-cpp)
     find_dependency(azure-storage-blobs-cpp)
+    find_dependency(azure-storage-files-datalake-cpp)
   endif()
   if(@TILEDB_GCS@) # TILEDB_GCS
     find_dependency(google_cloud_cpp_storage)
diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc
@@ -725,6 +725,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") {
   all_param_values["vfs.azure.storage_account_key"] = "";
   all_param_values["vfs.azure.storage_sas_token"] = "";
   all_param_values["vfs.azure.blob_endpoint"] = "";
+  all_param_values["vfs.azure.is_data_lake_endpoint"] = "";
   all_param_values["vfs.azure.block_list_block_size"] = "5242880";
   all_param_values["vfs.azure.max_parallel_ops"] =
       std::to_string(std::thread::hardware_concurrency());
@@ -797,6 +798,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") {
   vfs_param_values["azure.storage_account_key"] = "";
   vfs_param_values["azure.storage_sas_token"] = "";
   vfs_param_values["azure.blob_endpoint"] = "";
+  vfs_param_values["azure.is_data_lake_endpoint"] = "";
   vfs_param_values["azure.block_list_block_size"] = "5242880";
   vfs_param_values["azure.max_parallel_ops"] =
       std::to_string(std::thread::hardware_concurrency());
@@ -863,6 +865,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") {
   azure_param_values["storage_account_key"] = "";
   azure_param_values["storage_sas_token"] = "";
   azure_param_values["blob_endpoint"] = "";
+  azure_param_values["is_data_lake_endpoint"] = "";
   azure_param_values["block_list_block_size"] = "5242880";
   azure_param_values["max_parallel_ops"] =
       std::to_string(std::thread::hardware_concurrency());
diff --git a/test/src/unit-cppapi-config.cc b/test/src/unit-cppapi-config.cc
@@ -77,7 +77,7 @@ TEST_CASE("C++ API: Config iterator", "[cppapi][config]") {
     names.push_back(it->first);
   }
   // Check number of VFS params in default config object.
-  CHECK(names.size() == 67);
+  CHECK(names.size() == 68);
 }
 
 TEST_CASE("C++ API: Config Environment Variables", "[cppapi][config]") {
diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc
@@ -205,7 +205,7 @@ TEST_CASE("VFS: Test long local paths", "[vfs][long-paths]") {
 using AllBackends = std::tuple<LocalFsTest, GCSTest, GSTest, S3Test, AzureTest>;
 TEMPLATE_LIST_TEST_CASE(
     "VFS: URI semantics and file management", "[vfs][uri]", AllBackends) {
-  TestType fs({0});
+  TestType fs({});
   if (!fs.is_supported()) {
     return;
   }
@@ -218,19 +218,6 @@ TEMPLATE_LIST_TEST_CASE(
 
   URI path = fs.temp_dir_.add_trailing_slash();
 
-  // Set up
-  if (path.is_gcs() || path.is_s3() || path.is_azure()) {
-    if (vfs.is_bucket(path)) {
-      REQUIRE_NOTHROW(vfs.remove_bucket(path));
-    }
-    REQUIRE_NOTHROW(vfs.create_bucket(path));
-  } else {
-    if (vfs.is_dir(path)) {
-      REQUIRE_NOTHROW(vfs.remove_dir(path));
-    }
-    REQUIRE_NOTHROW(vfs.create_dir(path));
-  }
-
   /* Create the following file hierarchy:
    *
    * path/dir1/subdir/file1
@@ -480,21 +467,6 @@ TEMPLATE_LIST_TEST_CASE("VFS: File I/O", "[vfs][uri][file_io]", AllBackends) {
     CHECK_THROWS(vfs.file_size(non_existent));
   }
 
-  // Set up
-  if (path.is_gcs() || path.is_s3() || path.is_azure()) {
-    if (vfs.is_bucket(path)) {
-      REQUIRE_NOTHROW(vfs.remove_bucket(path));
-    }
-    REQUIRE_NOTHROW(vfs.create_bucket(path));
-  } else {
-    if (vfs.is_dir(path)) {
-      REQUIRE_NOTHROW(vfs.remove_dir(path));
-    }
-    REQUIRE_NOTHROW(vfs.create_dir(path));
-    // Bucket-specific operations are only valid for object store filesystems.
-    CHECK_THROWS(vfs.create_bucket(path));
-  }
-
   // Prepare buffers
   uint64_t buffer_size = multiplier * max_parallel_ops * chunk_size;
   auto write_buffer = new char[buffer_size];
@@ -801,6 +773,7 @@ TEST_CASE("VFS: Construct Azure Blob Storage endpoint URIs", "[azure][uri]") {
       config.set("vfs.azure.storage_account_name", "exampleaccount"));
   require_tiledb_ok(config.set("vfs.azure.blob_endpoint", custom_endpoint));
   require_tiledb_ok(config.set("vfs.azure.storage_sas_token", sas_token));
+  require_tiledb_ok(config.set("vfs.azure.is_data_lake_endpoint", "false"));
   if (sas_token.empty()) {
     // If the SAS token is empty, the VFS will try to connect to Microsoft Entra
     // ID to obtain credentials, which can take a long time because of retries.
diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt
@@ -524,10 +524,12 @@ if (TILEDB_AZURE)
 
   find_package(azure-identity-cpp CONFIG REQUIRED)
   find_package(azure-storage-blobs-cpp CONFIG REQUIRED)
+  find_package(azure-storage-files-datalake-cpp CONFIG REQUIRED)
   target_link_libraries(TILEDB_CORE_OBJECTS_ILIB
           INTERFACE
           Azure::azure-identity
           Azure::azure-storage-blobs
+          Azure::azure-storage-files-datalake
           )
 endif()
 
diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h
@@ -442,6 +442,12 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT;
  *    that at least one of these two options must be set (or both if shared
  *    key authentication is used). <br>
  *    **Default**: ""
+ * - `vfs.azure.is_data_lake_endpoint` <br>
+ *    Sets whether the Azure Storage account is known to have hierarchical
+ *    namespace enabled or disabled. This option can be used to reduce latency
+ *    when performing the first Azure request. If not set, the account's
+ *    capabilities will be automatically detected. <br>
+ *    **Default**: <unset>
  * - `vfs.azure.block_list_block_size` <br>
  *    The block size (in bytes) used in Azure blob block list writes.
  *    Any `uint64_t` value is acceptable. Note: `vfs.azure.block_list_block_size
diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc
@@ -188,6 +188,7 @@ const std::string Config::VFS_AZURE_STORAGE_ACCOUNT_NAME = "";
 const std::string Config::VFS_AZURE_STORAGE_ACCOUNT_KEY = "";
 const std::string Config::VFS_AZURE_STORAGE_SAS_TOKEN = "";
 const std::string Config::VFS_AZURE_BLOB_ENDPOINT = "";
+const std::string Config::VFS_AZURE_IS_DATA_LAKE_ENDPOINT = "";
 const std::string Config::VFS_AZURE_MAX_PARALLEL_OPS =
     Config::SM_IO_CONCURRENCY_LEVEL;
 const std::string Config::VFS_AZURE_BLOCK_LIST_BLOCK_SIZE = "5242880";
@@ -438,6 +439,9 @@ const std::map<std::string, std::string> default_config_values = {
     std::make_pair(
         "vfs.azure.storage_sas_token", Config::VFS_AZURE_STORAGE_SAS_TOKEN),
     std::make_pair("vfs.azure.blob_endpoint", Config::VFS_AZURE_BLOB_ENDPOINT),
+    std::make_pair(
+        "vfs.azure.is_data_lake_endpoint",
+        Config::VFS_AZURE_IS_DATA_LAKE_ENDPOINT),
     std::make_pair(
         "vfs.azure.max_parallel_ops", Config::VFS_AZURE_MAX_PARALLEL_OPS),
     std::make_pair(
@@ -878,6 +882,10 @@ Status Config::sanity_check(
       msg << "value " << param << " invalid canned acl for " << param;
       return Status_Error(msg.str());
     }
+  } else if (param == "vfs.azure.is_data_lake_endpoint") {
+    if (!value.empty()) {
+      RETURN_NOT_OK(utils::parse::convert(value, &v));
+    }
   }
 
   return Status::Ok();
diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h
@@ -488,6 +488,10 @@ class Config {
   /** Azure blob endpoint. */
   static const std::string VFS_AZURE_BLOB_ENDPOINT;
 
+  /** Whether the Azure storage account is known to support hierarchical
+   * namespace or not. */
+  static const std::string VFS_AZURE_IS_DATA_LAKE_ENDPOINT;
+
   /** Azure max parallel ops. */
   static const std::string VFS_AZURE_MAX_PARALLEL_OPS;
 
diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h
@@ -613,6 +613,12 @@ class Config {
    *    that at least one of these two options must be set (or both if shared
    *    key authentication is used). <br>
    *    **Default**: ""
+   * - `vfs.azure.is_data_lake_endpoint` <br>
+   *    Sets whether the Azure Storage account is known to have hierarchical
+   *    namespace enabled or disabled. This option can be used to reduce latency
+   *    when performing the first Azure request. If not specified, the account's
+   *    capabilities will be automatically detected. <br>
+   *    **Default**: <unset>
    * - `vfs.azure.block_list_block_size` <br>
    *    The block size (in bytes) used in Azure blob block list writes.
    *    Any `uint64_t` value is acceptable. Note:
diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc
diff --git a/tiledb/sm/filesystem/azure.h b/tiledb/sm/filesystem/azure.h
diff --git a/tiledb/sm/misc/parse_argument.h b/tiledb/sm/misc/parse_argument.h
diff --git a/vcpkg.json b/vcpkg.json

Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,7 @@ TEST_CASE("C++ API: Config iterator", "[cppapi][config]") {`
`77`	`77`	`names.push_back(it->first);`
`78`	`78`	`}`
`79`	`79`	`// Check number of VFS params in default config object.`
`80`		`- CHECK(names.size() == 67);`
	`80`	`+ CHECK(names.size() == 68);`
`81`	`81`	`}`
`82`	`82`
`83`	`83`	`TEST_CASE("C++ API: Config Environment Variables", "[cppapi][config]") {`