Merge pull request ClickHouse#79148 from ClickHouse/fs-cache-add-setting-to-allow-background-download

kssenii · web-flow · commit 8513caa679e1 · 2025-04-17T10:34:47.000Z
Add a setting to enable/disable dynamic fs cache resize
diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
@@ -75,6 +75,7 @@ namespace FileCacheSetting
     extern const FileCacheSettingsBool write_cache_per_user_id_directory;
     extern const FileCacheSettingsUInt64 cache_hits_threshold;
     extern const FileCacheSettingsBool enable_filesystem_query_cache_limit;
+    extern const FileCacheSettingsBool allow_dynamic_cache_resize;
 }
 
 namespace
@@ -116,6 +117,7 @@ FileCache::FileCache(const std::string & cache_name, const FileCacheSettings & s
     , load_metadata_threads(settings[FileCacheSetting::load_metadata_threads])
     , load_metadata_asynchronously(settings[FileCacheSetting::load_metadata_asynchronously])
     , write_cache_per_user_directory(settings[FileCacheSetting::write_cache_per_user_id_directory])
+    , allow_dynamic_cache_resize(settings[FileCacheSetting::allow_dynamic_cache_resize])
     , keep_current_size_to_max_ratio(1 - settings[FileCacheSetting::keep_free_space_size_ratio])
     , keep_current_elements_to_max_ratio(1 - settings[FileCacheSetting::keep_free_space_elements_ratio])
     , keep_up_free_space_remove_batch(settings[FileCacheSetting::keep_free_space_remove_batch])
@@ -1679,8 +1681,10 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
         actual_settings[FileCacheSetting::background_download_max_file_segment_size] = new_settings[FileCacheSetting::background_download_max_file_segment_size];
     }
 
-    if (new_settings[FileCacheSetting::max_size] != actual_settings[FileCacheSetting::max_size]
-        || new_settings[FileCacheSetting::max_elements] != actual_settings[FileCacheSetting::max_elements])
+    const bool cache_size_changed = new_settings[FileCacheSetting::max_size] != actual_settings[FileCacheSetting::max_size]
+        || new_settings[FileCacheSetting::max_elements] != actual_settings[FileCacheSetting::max_elements];
+
+    if (allow_dynamic_cache_resize && cache_size_changed)
     {
         EvictionCandidates eviction_candidates;
         bool modified_size_limit = false;
@@ -1845,6 +1849,13 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
         chassert(main_priority->getSizeLimit(lockCache()) == actual_settings[FileCacheSetting::max_size]);
         chassert(main_priority->getElementsLimit(lockCache()) == actual_settings[FileCacheSetting::max_elements]);
     }
+    else if (cache_size_changed)
+    {
+        LOG_WARNING(
+            log, "Filesystem cache size was modified, "
+            "but dynamic cache resize is disabled, therefore cache size will not be changed without server restart. "
+            "To enable dynamic cache resize, add `allow_dynamic_cache_resize` to cache configuration");
+    }
 
     if (new_settings[FileCacheSetting::max_file_segment_size] != actual_settings[FileCacheSetting::max_file_segment_size])
     {
diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h
@@ -210,6 +210,7 @@ class FileCache : private boost::noncopyable
     std::atomic<bool> stop_loading_metadata = false;
     ThreadFromGlobalPool load_metadata_main_thread;
     const bool write_cache_per_user_directory;
+    const bool allow_dynamic_cache_resize;
 
     BackgroundSchedulePoolTaskHolder keep_up_free_space_ratio_task;
     const double keep_current_size_to_max_ratio;
diff --git a/src/Interpreters/Cache/FileCacheSettings.cpp b/src/Interpreters/Cache/FileCacheSettings.cpp
@@ -45,7 +45,8 @@ namespace ErrorCodes
     DECLARE(UInt64, cache_hits_threshold, FILECACHE_DEFAULT_HITS_THRESHOLD, "Number of cache hits required to cache corresponding file segment", 0) \
     DECLARE(Bool, enable_bypass_cache_with_threshold, false, "Undocumented. Not recommended for use", 0) \
     DECLARE(UInt64, bypass_cache_threshold, FILECACHE_BYPASS_THRESHOLD, "Undocumented. Not recommended for use", 0) \
-    DECLARE(Bool, write_cache_per_user_id_directory, false, "Private setting", 0)
+    DECLARE(Bool, write_cache_per_user_id_directory, false, "Internal ClickHouse Cloud setting", 0) \
+    DECLARE(Bool, allow_dynamic_cache_resize, false, "Allow dynamic resize of filesystem cache", 0)
 
 DECLARE_SETTINGS_TRAITS(FileCacheSettingsTraits, LIST_OF_FILE_CACHE_SETTINGS)
 IMPLEMENT_SETTINGS_TRAITS(FileCacheSettingsTraits, LIST_OF_FILE_CACHE_SETTINGS)
diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml
@@ -24,6 +24,7 @@
                 <cache_on_write_operations>1</cache_on_write_operations>
                 <cache_policy>LRU</cache_policy>
                 <slru_size_ratio>0.3</slru_size_ratio>
+                <allow_dynamic_cache_resize>1</allow_dynamic_cache_resize>
                 <keep_free_space_size_ratio>0.15</keep_free_space_size_ratio>
                 <keep_free_space_elements_ratio>0.15</keep_free_space_elements_ratio>
                 <background_download_queue_size_limit>50</background_download_queue_size_limit>
diff --git a/tests/config/config.d/storage_conf_02944.xml b/tests/config/config.d/storage_conf_02944.xml
@@ -19,6 +19,7 @@
                 <boundary_alignment>10</boundary_alignment>
                 <cache_on_write_operations>0</cache_on_write_operations>
                 <load_metadata_asynchronously>0</load_metadata_asynchronously>
+                <allow_dynamic_cache_resize>1</allow_dynamic_cache_resize>
             </s3_cache_02944>
         </disks>
     </storage_configuration>
diff --git a/tests/integration/test_filesystem_cache/config.d/cache_dynamic_resize.xml b/tests/integration/test_filesystem_cache/config.d/cache_dynamic_resize.xml
@@ -12,8 +12,19 @@
                 <max_elements>100</max_elements>
                 <max_file_segment_size>10</max_file_segment_size>
                 <boundary_alignment>10</boundary_alignment>
+                <allow_dynamic_cache_resize>1</allow_dynamic_cache_resize>
                 <path>./cache_dynamic_reload/</path>
             </cache_dynamic_resize>
+            <cache_dynamic_resize_disabled>
+                <type>cache</type>
+                <disk>hdd_blob</disk>
+                <max_size>100000</max_size>
+                <max_elements>100</max_elements>
+                <max_file_segment_size>10</max_file_segment_size>
+                <boundary_alignment>10</boundary_alignment>
+                <allow_dynamic_cache_resize>0</allow_dynamic_cache_resize>
+                <path>./cache_dynamic_reload_disabled/</path>
+            </cache_dynamic_resize_disabled>
         </disks>
     </storage_configuration>
     <filesystem_cache_log>
diff --git a/tests/integration/test_filesystem_cache/test.py b/tests/integration/test_filesystem_cache/test.py
@@ -479,7 +479,9 @@ def to_int(value):
         )
         assert r_cache_count_2 == r_cache_count
 
-        assert node.query("select current_size from system.filesystem_cache_settings where cache_name = 'force_cache_on_merges'") == node.query("select sum(downloaded_size) from system.filesystem_cache")
+        assert node.query(
+            "select current_size from system.filesystem_cache_settings where cache_name = 'force_cache_on_merges'"
+        ) == node.query("select sum(downloaded_size) from system.filesystem_cache")
 
         node.query("SYSTEM DROP FILESYSTEM CACHE")
         node.query("OPTIMIZE TABLE test FINAL")
@@ -644,6 +646,7 @@ def test_keep_up_size_ratio(cluster):
         time.sleep(1)
     assert elements <= expected
 
+
 cache_dynamic_resize_config = """
 <clickhouse>
     <storage_configuration>
@@ -661,6 +664,16 @@ def test_keep_up_size_ratio(cluster):
                 <boundary_alignment>10</boundary_alignment>
                 <path>./cache_dynamic_reload/</path>
             </cache_dynamic_resize>
+            <cache_dynamic_resize_disabled>
+                <type>cache</type>
+                <disk>hdd_blob</disk>
+                <max_size>{}</max_size>
+                <max_elements>{}</max_elements>
+                <max_file_segment_size>10</max_file_segment_size>
+                <boundary_alignment>10</boundary_alignment>
+                <allow_dynamic_cache_resize>0</allow_dynamic_cache_resize>
+                <path>./cache_dynamic_reload_disabled/</path>
+            </cache_dynamic_resize_disabled>
         </disks>
     </storage_configuration>
     <filesystem_cache_log>
@@ -670,6 +683,7 @@ def test_keep_up_size_ratio(cluster):
 </clickhouse>
 """
 
+
 def test_dynamic_resize(cluster):
     node = cluster.instances["cache_dynamic_resize"]
     max_elements = 20
@@ -692,25 +706,32 @@ def test_dynamic_resize(cluster):
     )
 
     def get_downloaded_size():
-        return int(node.query(
-            f"SELECT sum(downloaded_size) FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
-        ))
+        return int(
+            node.query(
+                f"SELECT sum(downloaded_size) FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
+            )
+        )
 
     def get_queue_size():
-        return int(node.query(
-            f"SELECT current_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
-        ))
+        return int(
+            node.query(
+                f"SELECT current_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
+            )
+        )
 
     def get_downloaded_elements():
-        return int(node.query(
-            f"SELECT count() FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
-        ))
+        return int(
+            node.query(
+                f"SELECT count() FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
+            )
+        )
 
     def get_queue_elements():
-        return int(node.query(
-            f"SELECT current_elements_num FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
-        ))
-
+        return int(
+            node.query(
+                f"SELECT current_elements_num FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
+            )
+        )
 
     size = get_downloaded_size()
     assert size > 100
@@ -720,9 +741,11 @@ def get_queue_elements():
     assert elements > 10
     assert elements == get_queue_elements()
 
-    default_config = cache_dynamic_resize_config.format(100000, 100)
-    new_config = cache_dynamic_resize_config.format(100000, 10)
-    node.replace_config("/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config)
+    default_config = cache_dynamic_resize_config.format(100000, 100, 100000, 100)
+    new_config = cache_dynamic_resize_config.format(100000, 10, 100000, 100)
+    node.replace_config(
+        "/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config
+    )
 
     node.query("SYSTEM RELOAD CONFIG")
 
@@ -731,43 +754,59 @@ def get_queue_elements():
 
     node.query(f"SYSTEM ENABLE FAILPOINT file_cache_dynamic_resize_fail_to_evict")
 
-    new_config = cache_dynamic_resize_config.format(100000, 5)
-    node.replace_config("/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config)
+    new_config = cache_dynamic_resize_config.format(100000, 5, 100000, 100)
+    node.replace_config(
+        "/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config
+    )
 
     node.query("SYSTEM RELOAD CONFIG")
 
     assert 10 == get_queue_elements()
     assert 10 == get_downloaded_elements()
 
-    assert 100000 == int(node.query(
-        f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
-    ))
-    assert 10 == int(node.query(
-        f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
-    ))
+    assert 100000 == int(
+        node.query(
+            f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
+        )
+    )
+    assert 10 == int(
+        node.query(
+            f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
+        )
+    )
 
     node.query(f"SYSTEM DISABLE FAILPOINT file_cache_dynamic_resize_fail_to_evict")
     node.query("SYSTEM RELOAD CONFIG")
 
     assert 5 == get_downloaded_elements()
     assert 5 == get_queue_elements()
 
-    assert 100000 == int(node.query(
-        f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
-    ))
-    assert 5 == int(node.query(
-        f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
-    ))
+    assert 100000 == int(
+        node.query(
+            f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
+        )
+    )
+    assert 5 == int(
+        node.query(
+            f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
+        )
+    )
 
-    node.replace_config("/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", default_config)
+    node.replace_config(
+        "/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", default_config
+    )
     node.query("SYSTEM RELOAD CONFIG")
 
-    assert 100000 == int(node.query(
-        f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
-    ))
-    assert 100 == int(node.query(
-        f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
-    ))
+    assert 100000 == int(
+        node.query(
+            f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
+        )
+    )
+    assert 100 == int(
+        node.query(
+            f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
+        )
+    )
 
 
 def test_filesystem_cache_log(cluster):
@@ -799,7 +838,11 @@ def test_filesystem_cache_log(cluster):
     )
 
     node.query("SYSTEM FLUSH LOGS")
-    assert 0 == int(node.query(f"SELECT count() FROM system.filesystem_cache_log WHERE query_id = '{query_id}'"))
+    assert 0 == int(
+        node.query(
+            f"SELECT count() FROM system.filesystem_cache_log WHERE query_id = '{query_id}'"
+        )
+    )
 
     query_id = "system_filesystem_cache_log_2"
     node.query(
@@ -808,4 +851,69 @@ def test_filesystem_cache_log(cluster):
     )
 
     node.query("SYSTEM FLUSH LOGS")
-    assert 0 < int(node.query(f"SELECT count() FROM system.filesystem_cache_log WHERE query_id = '{query_id}'"))
+    assert 0 < int(
+        node.query(
+            f"SELECT count() FROM system.filesystem_cache_log WHERE query_id = '{query_id}'"
+        )
+    )
+
+
+def test_dynamic_resize_disabled(cluster):
+    node = cluster.instances["cache_dynamic_resize"]
+    max_elements = 20
+    cache_name = "cache_dynamic_resize_disabled"
+    node.query(
+        f"""
+DROP TABLE IF EXISTS test;
+SYSTEM DROP FILESYSTEM CACHE;
+CREATE TABLE test (a String)
+ENGINE = MergeTree() ORDER BY tuple()
+SETTINGS disk = '{cache_name}', min_bytes_for_wide_part = 10485760;
+    """
+    )
+
+    node.query(
+        """
+INSERT INTO test SELECT randomString(200);
+SELECT * FROM test;
+    """
+    )
+
+    def get_downloaded_size():
+        return int(
+            node.query(
+                f"SELECT sum(downloaded_size) FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
+            )
+        )
+
+    def get_downloaded_elements():
+        return int(
+            node.query(
+                f"SELECT count() FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
+            )
+        )
+
+    size = get_downloaded_size()
+    assert size > 100
+
+    elements = get_downloaded_elements()
+    assert elements > 10
+
+    default_config = cache_dynamic_resize_config.format(100000, 100, 100000, 100)
+    new_config = cache_dynamic_resize_config.format(100000, 100, 100000, 10)
+    node.replace_config(
+        "/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config
+    )
+
+    node.query("SYSTEM RELOAD CONFIG")
+
+    assert size == get_downloaded_size()
+    assert elements == get_downloaded_elements()
+
+    assert node.contains_in_log(
+        f"FileCache({cache_name}): Filesystem cache size was modified, but dynamic cache resize is disabled"
+    )
+    # Return config back to initial state.
+    node.replace_config(
+        "/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", default_config
+    )
diff --git a/tests/queries/0_stateless/02344_describe_cache.reference b/tests/queries/0_stateless/02344_describe_cache.reference
@@ -1,2 +1,2 @@
 1
-02344_describe_cache_test	/var/lib/clickhouse/filesystem_caches/02344_describe_cache_test	102400	10000000	33554432	4194304	0	LRU	0.6	5	5000	4194304	16	0	0	0	10	0	0	0	268435456	0	1	0	0
+02344_describe_cache_test	/var/lib/clickhouse/filesystem_caches/02344_describe_cache_test	102400	10000000	33554432	4194304	0	LRU	0.6	5	5000	4194304	16	0	0	0	10	0	0	0	268435456	0	0	1	0	0

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`1`
`2`		`-02344_describe_cache_test /var/lib/clickhouse/filesystem_caches/02344_describe_cache_test 102400 10000000 33554432 4194304 0 LRU 0.6 5 5000 4194304 16 0 0 0 10 0 0 0 268435456 0 1 0 0`
	`2`	`+02344_describe_cache_test /var/lib/clickhouse/filesystem_caches/02344_describe_cache_test 102400 10000000 33554432 4194304 0 LRU 0.6 5 5000 4194304 16 0 0 0 10 0 0 0 268435456 0 0 1 0 0`