Skip to content

Commit b72d270

Browse files
ksseniikssenii
committed
Add a setting to allow dynamic cache resize
1 parent c554cb9 commit b72d270

File tree

7 files changed

+164
-40
lines changed

7 files changed

+164
-40
lines changed

‎src/Interpreters/Cache/FileCache.cpp‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ namespace FileCacheSetting
7575
extern const FileCacheSettingsBool write_cache_per_user_id_directory;
7676
extern const FileCacheSettingsUInt64 cache_hits_threshold;
7777
extern const FileCacheSettingsBool enable_filesystem_query_cache_limit;
78+
extern const FileCacheSettingsBool allow_dynamic_cache_resize;
7879
}
7980

8081
namespace
@@ -116,6 +117,7 @@ FileCache::FileCache(const std::string & cache_name, const FileCacheSettings & s
116117
, load_metadata_threads(settings[FileCacheSetting::load_metadata_threads])
117118
, load_metadata_asynchronously(settings[FileCacheSetting::load_metadata_asynchronously])
118119
, write_cache_per_user_directory(settings[FileCacheSetting::write_cache_per_user_id_directory])
120+
, allow_dynamic_cache_resize(settings[FileCacheSetting::allow_dynamic_cache_resize])
119121
, keep_current_size_to_max_ratio(1 - settings[FileCacheSetting::keep_free_space_size_ratio])
120122
, keep_current_elements_to_max_ratio(1 - settings[FileCacheSetting::keep_free_space_elements_ratio])
121123
, keep_up_free_space_remove_batch(settings[FileCacheSetting::keep_free_space_remove_batch])
@@ -1679,8 +1681,10 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
16791681
actual_settings[FileCacheSetting::background_download_max_file_segment_size] = new_settings[FileCacheSetting::background_download_max_file_segment_size];
16801682
}
16811683

1682-
if (new_settings[FileCacheSetting::max_size] != actual_settings[FileCacheSetting::max_size]
1683-
|| new_settings[FileCacheSetting::max_elements] != actual_settings[FileCacheSetting::max_elements])
1684+
const bool cache_size_changed = new_settings[FileCacheSetting::max_size] != actual_settings[FileCacheSetting::max_size]
1685+
|| new_settings[FileCacheSetting::max_elements] != actual_settings[FileCacheSetting::max_elements];
1686+
1687+
if (allow_dynamic_cache_resize && cache_size_changed)
16841688
{
16851689
EvictionCandidates eviction_candidates;
16861690
bool modified_size_limit = false;
@@ -1845,6 +1849,13 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
18451849
chassert(main_priority->getSizeLimit(lockCache()) == actual_settings[FileCacheSetting::max_size]);
18461850
chassert(main_priority->getElementsLimit(lockCache()) == actual_settings[FileCacheSetting::max_elements]);
18471851
}
1852+
else if (cache_size_changed)
1853+
{
1854+
LOG_WARNING(
1855+
log, "Filesystem cache size was modified, "
1856+
"but dynamic cache resize is disabled, therefore cache size will not be changed without server restart. "
1857+
"To enable dynamic cache resize, add `allow_dynamic_cache_resize` to cache configuration");
1858+
}
18481859

18491860
if (new_settings[FileCacheSetting::max_file_segment_size] != actual_settings[FileCacheSetting::max_file_segment_size])
18501861
{

‎src/Interpreters/Cache/FileCache.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ class FileCache : private boost::noncopyable
210210
std::atomic<bool> stop_loading_metadata = false;
211211
ThreadFromGlobalPool load_metadata_main_thread;
212212
const bool write_cache_per_user_directory;
213+
const bool allow_dynamic_cache_resize;
213214

214215
BackgroundSchedulePoolTaskHolder keep_up_free_space_ratio_task;
215216
const double keep_current_size_to_max_ratio;

‎src/Interpreters/Cache/FileCacheSettings.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ namespace ErrorCodes
4545
DECLARE(UInt64, cache_hits_threshold, FILECACHE_DEFAULT_HITS_THRESHOLD, "Number of cache hits required to cache corresponding file segment", 0) \
4646
DECLARE(Bool, enable_bypass_cache_with_threshold, false, "Undocumented. Not recommended for use", 0) \
4747
DECLARE(UInt64, bypass_cache_threshold, FILECACHE_BYPASS_THRESHOLD, "Undocumented. Not recommended for use", 0) \
48-
DECLARE(Bool, write_cache_per_user_id_directory, false, "Private setting", 0)
48+
DECLARE(Bool, write_cache_per_user_id_directory, false, "Private setting", 0) \
49+
DECLARE(Bool, allow_dynamic_cache_resize, false, "Allow dynamic resize of filesystem cache", 0)
4950

5051
DECLARE_SETTINGS_TRAITS(FileCacheSettingsTraits, LIST_OF_FILE_CACHE_SETTINGS)
5152
IMPLEMENT_SETTINGS_TRAITS(FileCacheSettingsTraits, LIST_OF_FILE_CACHE_SETTINGS)

‎tests/config/config.d/storage_conf.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
<cache_on_write_operations>1</cache_on_write_operations>
2525
<cache_policy>LRU</cache_policy>
2626
<slru_size_ratio>0.3</slru_size_ratio>
27+
<allow_dynamic_cache_resize>1</allow_dynamic_cache_resize>
2728
<keep_free_space_size_ratio>0.15</keep_free_space_size_ratio>
2829
<keep_free_space_elements_ratio>0.15</keep_free_space_elements_ratio>
2930
<background_download_queue_size_limit>50</background_download_queue_size_limit>

‎tests/config/config.d/storage_conf_02944.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
<boundary_alignment>10</boundary_alignment>
2020
<cache_on_write_operations>0</cache_on_write_operations>
2121
<load_metadata_asynchronously>0</load_metadata_asynchronously>
22+
<allow_dynamic_cache_resize>1</allow_dynamic_cache_resize>
2223
</s3_cache_02944>
2324
</disks>
2425
</storage_configuration>

‎tests/integration/test_filesystem_cache/config.d/cache_dynamic_resize.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,19 @@
1212
<max_elements>100</max_elements>
1313
<max_file_segment_size>10</max_file_segment_size>
1414
<boundary_alignment>10</boundary_alignment>
15+
<allow_dynamic_cache_resize>1</allow_dynamic_cache_resize>
1516
<path>./cache_dynamic_reload/</path>
1617
</cache_dynamic_resize>
18+
<cache_dynamic_resize_disabled>
19+
<type>cache</type>
20+
<disk>hdd_blob</disk>
21+
<max_size>100000</max_size>
22+
<max_elements>100</max_elements>
23+
<max_file_segment_size>10</max_file_segment_size>
24+
<boundary_alignment>10</boundary_alignment>
25+
<allow_dynamic_cache_resize>0</allow_dynamic_cache_resize>
26+
<path>./cache_dynamic_reload/</path>
27+
</cache_dynamic_resize_disabled>
1728
</disks>
1829
</storage_configuration>
1930
<filesystem_cache_log>

‎tests/integration/test_filesystem_cache/test.py‎

Lines changed: 135 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -479,7 +479,9 @@ def to_int(value):
479479
)
480480
assert r_cache_count_2 == r_cache_count
481481

482-
assert node.query("select current_size from system.filesystem_cache_settings where cache_name = 'force_cache_on_merges'") == node.query("select sum(downloaded_size) from system.filesystem_cache")
482+
assert node.query(
483+
"select current_size from system.filesystem_cache_settings where cache_name = 'force_cache_on_merges'"
484+
) == node.query("select sum(downloaded_size) from system.filesystem_cache")
483485

484486
node.query("SYSTEM DROP FILESYSTEM CACHE")
485487
node.query("OPTIMIZE TABLE test FINAL")
@@ -644,6 +646,7 @@ def test_keep_up_size_ratio(cluster):
644646
time.sleep(1)
645647
assert elements <= expected
646648

649+
647650
cache_dynamic_resize_config = """
648651
<clickhouse>
649652
<storage_configuration>
@@ -670,6 +673,7 @@ def test_keep_up_size_ratio(cluster):
670673
</clickhouse>
671674
"""
672675

676+
673677
def test_dynamic_resize(cluster):
674678
node = cluster.instances["cache_dynamic_resize"]
675679
max_elements = 20
@@ -692,25 +696,32 @@ def test_dynamic_resize(cluster):
692696
)
693697

694698
def get_downloaded_size():
695-
return int(node.query(
696-
f"SELECT sum(downloaded_size) FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
697-
))
699+
return int(
700+
node.query(
701+
f"SELECT sum(downloaded_size) FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
702+
)
703+
)
698704

699705
def get_queue_size():
700-
return int(node.query(
701-
f"SELECT current_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
702-
))
706+
return int(
707+
node.query(
708+
f"SELECT current_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
709+
)
710+
)
703711

704712
def get_downloaded_elements():
705-
return int(node.query(
706-
f"SELECT count() FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
707-
))
713+
return int(
714+
node.query(
715+
f"SELECT count() FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
716+
)
717+
)
708718

709719
def get_queue_elements():
710-
return int(node.query(
711-
f"SELECT current_elements_num FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
712-
))
713-
720+
return int(
721+
node.query(
722+
f"SELECT current_elements_num FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
723+
)
724+
)
714725

715726
size = get_downloaded_size()
716727
assert size > 100
@@ -722,7 +733,9 @@ def get_queue_elements():
722733

723734
default_config = cache_dynamic_resize_config.format(100000, 100)
724735
new_config = cache_dynamic_resize_config.format(100000, 10)
725-
node.replace_config("/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config)
736+
node.replace_config(
737+
"/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config
738+
)
726739

727740
node.query("SYSTEM RELOAD CONFIG")
728741

@@ -732,42 +745,58 @@ def get_queue_elements():
732745
node.query(f"SYSTEM ENABLE FAILPOINT file_cache_dynamic_resize_fail_to_evict")
733746

734747
new_config = cache_dynamic_resize_config.format(100000, 5)
735-
node.replace_config("/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config)
748+
node.replace_config(
749+
"/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config
750+
)
736751

737752
node.query("SYSTEM RELOAD CONFIG")
738753

739754
assert 10 == get_queue_elements()
740755
assert 10 == get_downloaded_elements()
741756

742-
assert 100000 == int(node.query(
743-
f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
744-
))
745-
assert 10 == int(node.query(
746-
f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
747-
))
757+
assert 100000 == int(
758+
node.query(
759+
f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
760+
)
761+
)
762+
assert 10 == int(
763+
node.query(
764+
f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
765+
)
766+
)
748767

749768
node.query(f"SYSTEM DISABLE FAILPOINT file_cache_dynamic_resize_fail_to_evict")
750769
node.query("SYSTEM RELOAD CONFIG")
751770

752771
assert 5 == get_downloaded_elements()
753772
assert 5 == get_queue_elements()
754773

755-
assert 100000 == int(node.query(
756-
f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
757-
))
758-
assert 5 == int(node.query(
759-
f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
760-
))
774+
assert 100000 == int(
775+
node.query(
776+
f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
777+
)
778+
)
779+
assert 5 == int(
780+
node.query(
781+
f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
782+
)
783+
)
761784

762-
node.replace_config("/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", default_config)
785+
node.replace_config(
786+
"/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", default_config
787+
)
763788
node.query("SYSTEM RELOAD CONFIG")
764789

765-
assert 100000 == int(node.query(
766-
f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
767-
))
768-
assert 100 == int(node.query(
769-
f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
770-
))
790+
assert 100000 == int(
791+
node.query(
792+
f"SELECT max_size FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
793+
)
794+
)
795+
assert 100 == int(
796+
node.query(
797+
f"SELECT max_elements FROM system.filesystem_cache_settings WHERE cache_name = '{cache_name}'"
798+
)
799+
)
771800

772801

773802
def test_filesystem_cache_log(cluster):
@@ -799,7 +828,11 @@ def test_filesystem_cache_log(cluster):
799828
)
800829

801830
node.query("SYSTEM FLUSH LOGS")
802-
assert 0 == int(node.query(f"SELECT count() FROM system.filesystem_cache_log WHERE query_id = '{query_id}'"))
831+
assert 0 == int(
832+
node.query(
833+
f"SELECT count() FROM system.filesystem_cache_log WHERE query_id = '{query_id}'"
834+
)
835+
)
803836

804837
query_id = "system_filesystem_cache_log_2"
805838
node.query(
@@ -808,4 +841,69 @@ def test_filesystem_cache_log(cluster):
808841
)
809842

810843
node.query("SYSTEM FLUSH LOGS")
811-
assert 0 < int(node.query(f"SELECT count() FROM system.filesystem_cache_log WHERE query_id = '{query_id}'"))
844+
assert 0 < int(
845+
node.query(
846+
f"SELECT count() FROM system.filesystem_cache_log WHERE query_id = '{query_id}'"
847+
)
848+
)
849+
850+
851+
def test_dynamic_resize_disabled(cluster):
852+
node = cluster.instances["cache_dynamic_resize"]
853+
max_elements = 20
854+
cache_name = "cache_dynamic_resize_disabled"
855+
node.query(
856+
f"""
857+
DROP TABLE IF EXISTS test;
858+
SYSTEM DROP FILESYSTEM CACHE;
859+
CREATE TABLE test (a String)
860+
ENGINE = MergeTree() ORDER BY tuple()
861+
SETTINGS disk = '{cache_name}', min_bytes_for_wide_part = 10485760;
862+
"""
863+
)
864+
865+
node.query(
866+
"""
867+
INSERT INTO test SELECT randomString(200);
868+
SELECT * FROM test;
869+
"""
870+
)
871+
872+
def get_downloaded_size():
873+
return int(
874+
node.query(
875+
f"SELECT sum(downloaded_size) FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
876+
)
877+
)
878+
879+
def get_downloaded_elements():
880+
return int(
881+
node.query(
882+
f"SELECT count() FROM system.filesystem_cache WHERE cache_name = '{cache_name}'"
883+
)
884+
)
885+
886+
size = get_downloaded_size()
887+
assert size > 100
888+
889+
elements = get_downloaded_elements()
890+
assert elements > 10
891+
892+
default_config = cache_dynamic_resize_config.format(100000, 100)
893+
new_config = cache_dynamic_resize_config.format(100000, 10)
894+
node.replace_config(
895+
"/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", new_config
896+
)
897+
898+
node.query("SYSTEM RELOAD CONFIG")
899+
900+
assert size == get_downloaded_size()
901+
assert elements == get_downloaded_elements()
902+
903+
assert node.contains_in_log(
904+
f"FileCache({cache_name}): To enable dynamic cache resize, add `allow_dynamic_cache_resize` to cache configuration"
905+
)
906+
# Return config back to initial state.
907+
node.replace_config(
908+
"/etc/clickhouse-server/config.d/cache_dynamic_resize.xml", default_config
909+
)

0 commit comments

Comments
 (0)