Merge pull request ClickHouse#79407 from ClickHouse/support-settings-for-delta-table-in-catalog

Allow to specify storage settings for DatabaseCatalog

Author: kssenii

ci/jobs/scripts/check_style/check-settings-style

@@ -37,7 +37,7 @@ ALL_DECLARATION_FILES="
     $ROOT_PATH/src/Storages/MySQL/MySQLSettings.cpp
     $ROOT_PATH/src/Storages/NATS/NATSSettings.cpp
     $ROOT_PATH/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp
-    $ROOT_PATH/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp
+    $ROOT_PATH/src/Storages/ObjectStorage/StorageObjectStorageSettings.h
     $ROOT_PATH/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.cpp
     $ROOT_PATH/src/Storages/RabbitMQ/RabbitMQSettings.cpp
     $ROOT_PATH/src/Storages/RocksDB/RocksDBSettings.cpp

src/Databases/DataLake/DatabaseDataLake.cpp

@@ -380,6 +380,8 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con
     const auto configuration = getConfiguration(storage_type);
 
     auto storage_settings = std::make_shared<StorageObjectStorageSettings>();
+    storage_settings->loadFromSettingsChanges(settings.allChanged());
+
     if (auto table_specific_properties = table_metadata.getDataLakeSpecificProperties();
         table_specific_properties.has_value())
     {

src/Databases/DataLake/DatabaseDataLakeSettings.cpp

@@ -4,6 +4,7 @@
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Databases/DataLake/DatabaseDataLakeSettings.h>
+#include <Storages/ObjectStorage/StorageObjectStorageSettings.h>
 #include <Common/Exception.h>
 
 namespace DB
@@ -28,7 +29,8 @@ namespace ErrorCodes
     DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \
 
 #define LIST_OF_DATABASE_ICEBERG_SETTINGS(M, ALIAS) \
-    DATABASE_ICEBERG_RELATED_SETTINGS(M, ALIAS)
+    DATABASE_ICEBERG_RELATED_SETTINGS(M, ALIAS) \
+    LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS(M, ALIAS) \
 
 DECLARE_SETTINGS_TRAITS(DatabaseDataLakeSettingsTraits, LIST_OF_DATABASE_ICEBERG_SETTINGS)
 IMPLEMENT_SETTINGS_TRAITS(DatabaseDataLakeSettingsTraits, LIST_OF_DATABASE_ICEBERG_SETTINGS)
@@ -88,4 +90,12 @@ void DatabaseDataLakeSettings::loadFromQuery(const ASTStorage & storage_def)
     }
 }
 
+SettingsChanges DatabaseDataLakeSettings::allChanged() const
+{
+    SettingsChanges changes;
+    for (const auto & setting : impl->allChanged())
+        changes.emplace_back(setting.getName(), setting.getValue());
+    return changes;
+}
+
 }

src/Databases/DataLake/DatabaseDataLakeSettings.h

@@ -2,6 +2,7 @@
 
 #include <Core/BaseSettingsFwdMacros.h>
 #include <Core/FormatFactorySettings.h>
+#include <Storages/ObjectStorage/StorageObjectStorageSettings.h>
 #include <Core/SettingsEnums.h>
 #include <Core/SettingsFields.h>
 
@@ -19,7 +20,11 @@ class SettingsChanges;
     M(CLASS_NAME, Bool) \
     M(CLASS_NAME, DatabaseDataLakeCatalogType) \
 
-DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(DatabaseDataLakeSettings, DECLARE_SETTING_TRAIT)
+#define LIST_OF_DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \
+    DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \
+    STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M)
+
+LIST_OF_DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(DatabaseDataLakeSettings, DECLARE_SETTING_TRAIT)
 
 struct DatabaseDataLakeSettings
 {
@@ -34,6 +39,8 @@ struct DatabaseDataLakeSettings
 
     void applyChanges(const SettingsChanges & changes);
 
+    SettingsChanges allChanged() const;
+
 private:
     std::unique_ptr<DatabaseDataLakeSettingsImpl> impl;
 };

src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp

@@ -11,37 +11,6 @@
 namespace DB
 {
 
-// clang-format off
-
-#define STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(DECLARE, ALIAS) \
-    DECLARE(Bool, allow_dynamic_metadata_for_data_lakes, false, R"(
-If enabled, indicates that metadata is taken from iceberg specification that is pulled from cloud before each query.
-)", 0) \
-    DECLARE(Bool, allow_experimental_delta_kernel_rs, false, R"(
-If enabled, the engine would use delta-kernel-rs for DeltaLake metadata parsing
-)", 0) \
-    DECLARE(Bool, delta_lake_read_schema_same_as_table_schema, false, R"(
-Whether delta-lake read schema is the same as table schema.
-)", 0) \
-    DECLARE(String, iceberg_metadata_file_path, "", R"(
-Explicit path to desired Iceberg metadata file, should be relative to path in object storage. Make sense for table function use case only.
-)", 0) \
-    DECLARE(String, iceberg_metadata_table_uuid, "", R"(
-Explicit table UUID to read metadata for. Ignored if iceberg_metadata_file_path is set.
-)", 0) \
-    DECLARE(Bool, iceberg_recent_metadata_file_by_last_updated_ms_field, false, R"(
-If enabled, the engine would use the metadata file with the most recent last_updated_ms json field. Does not make sense to use with iceberg_metadata_file_path.
-)", 0) \
-    DECLARE(Bool, iceberg_use_version_hint, false, R"(
-Get latest metadata path from version-hint.text file.
-)", 0) \
-
-// clang-format on
-
-#define LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS(M, ALIAS) \
-    STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(M, ALIAS) \
-    LIST_OF_ALL_FORMAT_SETTINGS(M, ALIAS)
-
 DECLARE_SETTINGS_TRAITS(StorageObjectStorageSettingsTraits, LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS)
 IMPLEMENT_SETTINGS_TRAITS(StorageObjectStorageSettingsTraits, LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS)
 
@@ -93,4 +62,14 @@ bool StorageObjectStorageSettings::hasBuiltin(std::string_view name)
 {
     return StorageObjectStorageSettingsImpl::hasBuiltin(name);
 }
+
+void StorageObjectStorageSettings::loadFromSettingsChanges(const SettingsChanges & changes)
+{
+    for (const auto & [name, value] : changes)
+    {
+        if (impl->has(name))
+            impl->set(name, value);
+    }
+}
+
 }

src/Storages/ObjectStorage/StorageObjectStorageSettings.h

@@ -44,6 +44,33 @@ class SettingsChanges;
     M(CLASS_NAME, UInt64Auto) \
     M(CLASS_NAME, URI)
 
+// clang-format off
+
+#define STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(DECLARE, ALIAS) \
+    DECLARE(Bool, allow_dynamic_metadata_for_data_lakes, false, R"(
+If enabled, indicates that metadata is taken from iceberg specification that is pulled from cloud before each query.
+)", 0) \
+    DECLARE(Bool, allow_experimental_delta_kernel_rs, false, R"(
+If enabled, the engine would use delta-kernel-rs for DeltaLake metadata parsing
+)", 0) \
+    DECLARE(Bool, delta_lake_read_schema_same_as_table_schema, false, R"(
+Whether delta-lake read schema is the same as table schema.
+)", 0) \
+    DECLARE(String, iceberg_metadata_file_path, "", R"(
+Explicit path to desired Iceberg metadata file, should be relative to path in object storage. Make sense for table function use case only.
+)", 0) \
+    DECLARE(String, iceberg_metadata_table_uuid, "", R"(
+Explicit table UUID to read metadata for. Ignored if iceberg_metadata_file_path is set.
+)", 0) \
+    DECLARE(Bool, iceberg_recent_metadata_file_by_last_updated_ms_field, false, R"(
+If enabled, the engine would use the metadata file with the most recent last_updated_ms json field. Does not make sense to use with iceberg_metadata_file_path.
+)", 0) \
+    DECLARE(Bool, iceberg_use_version_hint, false, R"(
+Get latest metadata path from version-hint.text file.
+)", 0) \
+
+// clang-format on
+
 STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(StorageObjectStorageSettings, DECLARE_SETTING_TRAIT)
 
 struct StorageObjectStorageSettings
@@ -57,6 +84,8 @@ struct StorageObjectStorageSettings
 
     void loadFromQuery(ASTSetQuery & settings_ast);
 
+    void loadFromSettingsChanges(const SettingsChanges & changes);
+
     Field get(const std::string & name);
 
     static bool hasBuiltin(std::string_view name);
@@ -67,4 +96,8 @@ struct StorageObjectStorageSettings
 
 using StorageObjectStorageSettingsPtr = std::shared_ptr<StorageObjectStorageSettings>;
 
+#define LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS(M, ALIAS) \
+    STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(M, ALIAS) \
+    LIST_OF_ALL_FORMAT_SETTINGS(M, ALIAS)
+
 }