Merge pull request ClickHouse#79967 from Avogar/disable-substream-marks-by-default

Disable write_marks_for_substreams_in_compact_parts by default

Author: Avogar

src/Core/SettingsChangesHistory.cpp

@@ -777,7 +777,7 @@ const VersionToSettingsChangesMap & getMergeTreeSettingsChangesHistory()
             {"shared_merge_tree_merge_worker_regular_timeout_ms", 10000, 10000, "New setting"},
             {"apply_patches_on_merge", true, true, "New setting"},
             {"remove_unused_patch_parts", true, true, "New setting"},
-            {"write_marks_for_substreams_in_compact_parts", false, true, "New setting"},
+            {"write_marks_for_substreams_in_compact_parts", false, false, "New setting"},
             /// Release closed. Please use 25.6
         });
         addSettingsChanges(merge_tree_settings_changes_history, "25.4",

src/Storages/MergeTree/MergeTreeSettings.cpp

@@ -252,7 +252,7 @@ namespace ErrorCodes
     This mode allows to use significantly less memory for storing discriminators
     in parts when there is mostly one variant or a lot of NULL values.
     )", 0) \
-    DECLARE(Bool, write_marks_for_substreams_in_compact_parts, true, R"(
+    DECLARE(Bool, write_marks_for_substreams_in_compact_parts, false, R"(
     Enables writing marks per each substream instead of per each column in Compact parts.
     It allows to read individual subcolumns from the data part efficiently.
     )", 0) \

tests/integration/test_merge_tree_s3/test.py

@@ -88,6 +88,7 @@ def create_table(node, table_name, **additional_settings):
         "old_parts_lifetime": 0,
         "index_granularity": 512,
         "temporary_directories_lifetime": 1,
+        "write_marks_for_substreams_in_compact_parts": 1,
     }
     settings.update(additional_settings)
 

tests/integration/test_merge_tree_s3_failover/test.py

@@ -123,7 +123,7 @@ def test_write_failover(
         ) ENGINE=MergeTree()
         ORDER BY id
         PARTITION BY dt
-        SETTINGS storage_policy='s3', min_bytes_for_wide_part={}
+        SETTINGS storage_policy='s3', min_bytes_for_wide_part={}, write_marks_for_substreams_in_compact_parts=1
         """.format(
             min_bytes_for_wide_part
         )

tests/integration/test_replicated_merge_tree_s3/test.py

@@ -135,7 +135,7 @@ def drop_table(cluster):
 def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_part):
     create_table(
         cluster,
-        additional_settings={"min_rows_for_wide_part": min_rows_for_wide_part},
+        additional_settings={"min_rows_for_wide_part": min_rows_for_wide_part, "write_marks_for_substreams_in_compact_parts": 1},
     )
 
     insert(cluster, node_idxs=[1, 2, 3], verify=True)

tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py

@@ -116,7 +116,7 @@ def drop_table(cluster):
 def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_part):
     create_table(
         cluster,
-        additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part),
+        additional_settings="min_rows_for_wide_part={}, write_marks_for_substreams_in_compact_parts={}".format(min_rows_for_wide_part, 1),
     )
 
     all_values = ""

tests/integration/test_s3_zero_copy_ttl/test.py

@@ -36,7 +36,7 @@ def test_ttl_move_and_s3(started_cluster):
             ORDER BY id
             PARTITION BY id
             TTL date TO DISK 's3_disk'
-            SETTINGS storage_policy='s3_and_default', temporary_directories_lifetime=1
+            SETTINGS storage_policy='s3_and_default', temporary_directories_lifetime=1, write_marks_for_substreams_in_compact_parts=1
             """.format(
                 i
             )

tests/performance/compact_part_subcolumns.xml

@@ -3,8 +3,8 @@
         <allow_experimental_json_type>1</allow_experimental_json_type>
     </settings>
 
-    <create_query>CREATE TABLE t_json_1 (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=100000000000</create_query>
-    <create_query>CREATE TABLE t_json_2 (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=100000000000</create_query>
+    <create_query>CREATE TABLE t_json_1 (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=100000000000, write_marks_for_substreams_in_compact_parts=1</create_query>
+    <create_query>CREATE TABLE t_json_2 (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=100000000000, write_marks_for_substreams_in_compact_parts=1</create_query>
 
     <fill_query>INSERT INTO t_json_1 SELECT toJSONString(map(number % 10, repeat('a', number % 100))) FROM numbers(10000000)</fill_query>
     <fill_query>INSERT INTO t_json_2 SELECT toJSONString(arrayMap(x -> tuple('a' || x, x > 0 ? repeat('a', 100) : 'a'), range(1000))::Map(String, String)) FROM numbers(200000)</fill_query>

tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference

@@ -1,6 +1,6 @@
 Using storage policy: s3_cache
 DROP TABLE IF EXISTS test_02241
-CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization = 1
+CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization = 1, write_marks_for_substreams_in_compact_parts=1
 SYSTEM STOP MERGES test_02241
 SYSTEM DROP FILESYSTEM CACHE
 SELECT file_segment_range_begin, file_segment_range_end, size, state
@@ -110,7 +110,7 @@ SELECT count() FROM test_02241 WHERE value LIKE '%010%'
 18816
 Using storage policy: local_cache
 DROP TABLE IF EXISTS test_02241
-CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization = 1
+CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization = 1, write_marks_for_substreams_in_compact_parts=1
 SYSTEM STOP MERGES test_02241
 SYSTEM DROP FILESYSTEM CACHE
 SELECT file_segment_range_begin, file_segment_range_end, size, state
@@ -220,7 +220,7 @@ SELECT count() FROM test_02241 WHERE value LIKE '%010%'
 18816
 Using storage policy: azure_cache
 DROP TABLE IF EXISTS test_02241
-CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='azure_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization = 1
+CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='azure_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization = 1, write_marks_for_substreams_in_compact_parts=1
 SYSTEM STOP MERGES test_02241
 SYSTEM DROP FILESYSTEM CACHE
 SELECT file_segment_range_begin, file_segment_range_end, size, state

tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh

@@ -14,7 +14,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
     echo "Using storage policy: $STORAGE_POLICY"
 
     $CLICKHOUSE_CLIENT --echo --query "DROP TABLE IF EXISTS test_02241"
-    $CLICKHOUSE_CLIENT --echo --query "CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization = 1"
+    $CLICKHOUSE_CLIENT --echo --query "CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization = 1, write_marks_for_substreams_in_compact_parts=1"
     $CLICKHOUSE_CLIENT --echo --query "SYSTEM STOP MERGES test_02241"
 
     echo "SYSTEM DROP FILESYSTEM CACHE"