Merge pull request ClickHouse#79541 from ClickHouse/make-delta-kernel-default

Enable delta-kernel by default

Author: kssenii

src/Core/Settings.cpp

@@ -6652,9 +6652,9 @@ Enable PRQL - an alternative to SQL.
     DECLARE(Bool, enable_adaptive_memory_spill_scheduler, false, R"(
 Trigger processor to spill data into external storage adpatively. grace join is supported at present.
 )", EXPERIMENTAL) \
-    DECLARE(Bool, allow_experimental_delta_kernel_rs, false, R"(
+    DECLARE(Bool, allow_experimental_delta_kernel_rs, true, R"(
 Allow experimental delta-kernel-rs implementation.
-)", EXPERIMENTAL) \
+)", BETA) \
     DECLARE(Bool, make_distributed_plan, false, R"(
 Make distributed query plan.
 )", EXPERIMENTAL) \

src/Core/SettingsChangesHistory.cpp

@@ -84,7 +84,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"default_reader_bucket_count", 8, 8, "New experimental setting."},
             {"optimize_exchanges", 0, 0, "New experimental setting."},
             {"force_exchange_kind", "", "", "New experimental setting."},
-            {"allow_experimental_delta_kernel_rs", false, false, "New setting"},
+            {"allow_experimental_delta_kernel_rs", true, true, "New setting"},
             {"allow_experimental_database_hms_catalog", false, false, "Allow experimental database engine DataLakeCatalog with catalog_type = 'hive'"},
         });
         addSettingsChanges(settings_changes_history, "25.4",

src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h

@@ -18,7 +18,6 @@ namespace DB
 {
 namespace StorageObjectStorageSetting
 {
-extern const StorageObjectStorageSettingsBool allow_experimental_delta_kernel_rs;
 extern const StorageObjectStorageSettingsBool delta_lake_read_schema_same_as_table_schema;
 }
 namespace Setting
@@ -65,14 +64,15 @@ class DeltaLakeMetadata final : public IDataLakeMetadata
     {
 #if USE_DELTA_KERNEL_RS
         auto configuration_ptr = configuration.lock();
+
         const auto & storage_settings_ref = configuration_ptr->getSettingsRef();
         const auto & query_settings_ref = local_context->getSettingsRef();
 
-        bool enable_delta_kernel = storage_settings_ref[StorageObjectStorageSetting::allow_experimental_delta_kernel_rs];
-        if (query_settings_ref[Setting::allow_experimental_delta_kernel_rs].changed)
-            enable_delta_kernel = query_settings_ref[Setting::allow_experimental_delta_kernel_rs].value;
+        const auto storage_type = configuration_ptr->getType();
+        const bool supports_delta_kernel = storage_type == ObjectStorageType::S3 || storage_type == ObjectStorageType::Local;
 
-        if (enable_delta_kernel)
+        bool enable_delta_kernel = query_settings_ref[Setting::allow_experimental_delta_kernel_rs];
+        if (supports_delta_kernel && enable_delta_kernel)
         {
             return std::make_unique<DeltaLakeMetadataDeltaKernel>(
                 object_storage,

src/Storages/ObjectStorage/StorageObjectStorageSettings.h

@@ -49,9 +49,6 @@ class SettingsChanges;
 #define STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(DECLARE, ALIAS) \
     DECLARE(Bool, allow_dynamic_metadata_for_data_lakes, false, R"(
 If enabled, indicates that metadata is taken from iceberg specification that is pulled from cloud before each query.
-)", 0) \
-    DECLARE(Bool, allow_experimental_delta_kernel_rs, false, R"(
-If enabled, the engine would use delta-kernel-rs for DeltaLake metadata parsing
 )", 0) \
     DECLARE(Bool, delta_lake_read_schema_same_as_table_schema, false, R"(
 Whether delta-lake read schema is the same as table schema.
@@ -69,6 +66,9 @@ If enabled, the engine would use the metadata file with the most recent last_upd
 Get latest metadata path from version-hint.text file.
 )", 0) \
 
+#define OBSOLETE_SETTINGS(M, ALIAS) \
+    MAKE_OBSOLETE(M, Bool, allow_experimental_delta_kernel_rs, true) \
+
 // clang-format on
 
 STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(StorageObjectStorageSettings, DECLARE_SETTING_TRAIT)
@@ -98,6 +98,7 @@ using StorageObjectStorageSettingsPtr = std::shared_ptr<StorageObjectStorageSett
 
 #define LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS(M, ALIAS) \
     STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(M, ALIAS) \
+    OBSOLETE_SETTINGS(M, ALIAS) \
     LIST_OF_ALL_FORMAT_SETTINGS(M, ALIAS)
 
 }

tests/integration/test_database_delta/test.py

@@ -216,7 +216,7 @@ def test_complex_table_schema(started_cluster, use_delta_kernel):
     )
     complex_data = (
         node1.query(
-            f"SELECT * FROM complex_schema.`{schema_name}.{table_name}`"
+            f"SELECT * FROM complex_schema.`{schema_name}.{table_name}`", settings={"allow_experimental_delta_kernel_rs": use_delta_kernel}
         )
         .strip()
         .split("\t")

tests/integration/test_mask_sensitive_info/test.py

@@ -260,10 +260,7 @@ def test_create_table():
         f"S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '{password}', format = 'CSV')",
         f"S3('http://minio1:9001/root/data/test9.csv.gz', 'NOSIGN', 'CSV', 'gzip')",
         f"S3('http://minio1:9001/root/data/test10.csv.gz', 'minio', '{password}')",
-        (
-            f"DeltaLake('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')",
-            "DNS_ERROR",
-        ),
+        f"DeltaLake('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')",
         f"S3Queue('http://minio1:9001/root/data/', 'CSV') settings mode = 'ordered'",
         f"S3Queue('http://minio1:9001/root/data/', 'CSV', 'gzip') settings mode = 'ordered'",
         f"S3Queue('http://minio1:9001/root/data/', 'minio', '{password}', 'CSV') settings mode = 'ordered'",