Merge pull request ClickHouse#78671 from rschu1ze/qcc_plaintext_condition

`system.query_condition_cache`: Add field for plaintext condition

Author: rschu1ze

docs/en/operations/query-condition-cache.md

@@ -58,6 +58,8 @@ To clear the query condition cache, run [`SYSTEM DROP QUERY CONDITION CACHE`](..
 
 The content of the cache is displayed in system table [system.query_condition_cache](system-tables/query_condition_cache.md).
 To calculate the current size of the query condition cache in MB, run `SELECT formatReadableSize(sum(entry_size)) FROM system.query_condition_cache`.
+If you like to investigate individual filter conditions, you can check field `condition` in `system.query_condition_cache`.
+Note that the field is only populated if the query runs with enabled setting [query_condition_cache_store_conditions_as_plaintext](settings/settings#query_condition_cache_store_conditions_as_plaintext).
 
 The number of query condition cache hits and misses since database start are shown as events "QueryConditionCacheHits" and "QueryConditionCacheMisses" in system table [system.events](system-tables/events.md).
 Both counters are only updated for `SELECT` queries which run with setting `use_query_condition_cache = true`, other queries do not affect "QueryCacheMisses".

docs/en/operations/system-tables/query_condition_cache.md

@@ -17,7 +17,8 @@ Columns:
 
 - `table_uuid` ([String](../../sql-reference/data-types/string.md)) — The table UUID.
 - `part_name` ([String](../../sql-reference/data-types/string.md)) — The part name.
-- `key_hash` ([String](/sql-reference/data-types/string.md)) — The hash of the filter condition.
+- `condition` ([String](/sql-reference/data-types/string.md)) — The hashed filter condition. Only set if setting query_condition_cache_store_conditions_as_plaintext = true.
+- `condition_hash` ([String](/sql-reference/data-types/string.md)) — The hash of the filter condition.
 - `entry_size` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of the entry in bytes.
 - `matching_marks` ([String](../../sql-reference/data-types/string.md)) — Matching marks.
 
@@ -32,7 +33,8 @@ Row 1:
 ──────
 table_uuid:     28270a24-ea27-49f6-99cd-97b9bee976ac
 part_name:      all_1_1_0
-key_hash:       5456494897146899690 -- 5.46 quintillion
+condition:      or(equals(b, 10000_UInt16), equals(c, 10000_UInt16))
+condition_hash: 5456494897146899690 -- 5.46 quintillion
 entry_size:     40
 matching_marks: 111111110000000000000000000000000000000000000000000000000111111110000000000000000
 

src/Core/Settings.cpp

@@ -4831,6 +4831,16 @@ and reuse this information as an ephemeral index for subsequent queries.
 
 Possible values:
 
+- 0 - Disabled
+- 1 - Enabled
+)", 0) \
+    DECLARE(Bool, query_condition_cache_store_conditions_as_plaintext, false, R"(
+Stores the filter condition for the [query condition cache](/operations/query-condition-cache) in plaintext.
+If enabled, system.query_condition_cache shows the verbatim filter condition which makes it easier to debug issues with the cache.
+Disabled by default because plaintext filter conditions may expose sensitive information.
+
+Possible values:
+
 - 0 - Disabled
 - 1 - Enabled
 )", 0) \

src/Core/SettingsChangesHistory.cpp

@@ -83,6 +83,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"allow_special_bool_values_inside_variant", true, false, "Don't allow special bool values during Variant type parsing"},
             {"cast_string_to_variant_use_inference", true, true, "New setting to enable/disable types inference during CAST from String to Variant"},
             {"distributed_cache_read_request_max_tries", 20, 20, "New setting"},
+            {"query_condition_cache_store_conditions_as_plaintext", false, false, "New setting"},
             {"min_os_cpu_wait_time_ratio_to_throw", 0, 2, "New setting"},
             {"max_os_cpu_wait_time_ratio_to_throw", 0, 6, "New setting"},
             /// Release closed. Please use 25.5

src/Interpreters/Cache/QueryConditionCache.cpp

@@ -19,10 +19,10 @@ QueryConditionCache::QueryConditionCache(const String & cache_policy, size_t max
 }
 
 void QueryConditionCache::write(
-    const UUID & table_id, const String & part_name, size_t condition_hash,
+    const UUID & table_id, const String & part_name, size_t condition_hash, const String & condition,
     const MarkRanges & mark_ranges, size_t marks_count, bool has_final_mark)
 {
-    Key key = {table_id, part_name, condition_hash};
+    Key key = {table_id, part_name, condition_hash, condition};
 
     auto load_func = [&](){ return std::make_shared<Entry>(marks_count); };
     auto [entry, inserted] = cache.getOrSet(key, load_func);
@@ -40,19 +40,20 @@ void QueryConditionCache::write(
 
     LOG_DEBUG(
         logger,
-        "{} entry for table_id: {}, part_name: {}, condition_hash: {}, marks_count: {}, has_final_mark: {}, ranges: {}",
+        "{} entry for table_id: {}, part_name: {}, condition_hash: {}, condition: {}, marks_count: {}, has_final_mark: {}, ranges: {}",
         inserted ? "Inserted" : "Updated",
         table_id,
         part_name,
         condition_hash,
+        condition,
         marks_count,
         has_final_mark,
         toString(mark_ranges));
 }
 
 std::optional<QueryConditionCache::MatchingMarks> QueryConditionCache::read(const UUID & table_id, const String & part_name, size_t condition_hash)
 {
-    Key key = {table_id, part_name, condition_hash};
+    Key key = {table_id, part_name, condition_hash, ""};
 
     if (auto entry = cache.get(key))
     {
@@ -130,7 +131,7 @@ size_t QueryConditionCache::KeyHasher::operator()(const Key & key) const
 size_t QueryConditionCache::QueryConditionCacheEntryWeight::operator()(const Entry & entry) const
 {
     /// Estimate the memory size of `std::vector<bool>` (it uses bit-packing internally)
-    size_t dynamic_memory = (entry.matching_marks.capacity() + 7) / 8; /// round up to bytes.
-    return dynamic_memory + sizeof(decltype(entry.matching_marks));
+    size_t memory = (entry.matching_marks.capacity() + 7) / 8; /// round up to bytes.
+    return memory + sizeof(decltype(entry.matching_marks));
 }
 }

src/Interpreters/Cache/QueryConditionCache.h

@@ -35,6 +35,10 @@ class QueryConditionCache
         const String part_name;
         const size_t condition_hash;
 
+        /// -- Additional members, conceptually not part of the key. Only included for pretty-printing
+        ///    in system.query_condition_cache:
+        const String condition;
+
         bool operator==(const Key & other) const;
     };
 
@@ -74,7 +78,7 @@ class QueryConditionCache
 
     /// Add an entry to the cache. The passed marks represent ranges of the column with matches of the predicate.
     void write(
-        const UUID & table_id, const String & part_name, size_t condition_hash,
+        const UUID & table_id, const String & part_name, size_t condition_hash, const String & condition,
         const MarkRanges & mark_ranges, size_t marks_count, bool has_final_mark);
 
     /// Check the cache if it contains an entry for the given table + part id and predicate hash.

src/Processors/QueryPlan/FilterStep.cpp

@@ -173,7 +173,7 @@ void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ
     pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type)
     {
         bool on_totals = stream_type == QueryPipelineBuilder::StreamType::Totals;
-        return std::make_shared<FilterTransform>(header, expression, filter_column_name, remove_filter_column, on_totals, nullptr, condition_hash);
+        return std::make_shared<FilterTransform>(header, expression, filter_column_name, remove_filter_column, on_totals, nullptr, condition);
     });
 
     if (!blocksHaveEqualStructure(pipeline.getHeader(), *output_header))
@@ -248,9 +248,9 @@ void FilterStep::updateOutputHeader()
         return;
 }
 
-void FilterStep::setQueryConditionHash(size_t condition_hash_)
+void FilterStep::setConditionForQueryConditionCache(size_t condition_hash_, const String & condition_)
 {
-    condition_hash = condition_hash_;
+    condition = {condition_hash_, condition_};
 }
 
 bool FilterStep::canUseType(const DataTypePtr & filter_type)

src/Processors/QueryPlan/FilterStep.h

@@ -26,7 +26,7 @@ class FilterStep : public ITransformingStep
     const String & getFilterColumnName() const { return filter_column_name; }
     bool removesFilterColumn() const { return remove_filter_column; }
 
-    void setQueryConditionHash(size_t condition_hash_);
+    void setConditionForQueryConditionCache(size_t condition_hash_, const String & condition_);
 
     static bool canUseType(const DataTypePtr & type);
 
@@ -42,7 +42,7 @@ class FilterStep : public ITransformingStep
     String filter_column_name;
     bool remove_filter_column;
 
-    std::optional<size_t> condition_hash;
+    std::optional<std::pair<size_t, String>> condition; /// for query condition cache
 };
 
 }

src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp

@@ -35,6 +35,7 @@ namespace Setting
     extern const SettingsBool query_plan_try_use_vector_search;
     extern const SettingsBool query_plan_convert_join_to_in;
     extern const SettingsBool use_query_condition_cache;
+    extern const SettingsBool query_condition_cache_store_conditions_as_plaintext;
     extern const SettingsBoolAuto query_plan_join_swap_table;
     extern const SettingsMaxThreads max_threads;
     extern const SettingsSeconds lock_acquire_timeout;
@@ -92,6 +93,7 @@ QueryPlanOptimizationSettings::QueryPlanOptimizationSettings(
     aggregation_in_order = from[Setting::query_plan_enable_optimizations] && from[Setting::optimize_aggregation_in_order] && from[Setting::query_plan_aggregation_in_order];
     optimize_projection = from[Setting::optimize_use_projections];
     use_query_condition_cache = from[Setting::use_query_condition_cache] && from[Setting::allow_experimental_analyzer];
+    query_condition_cache_store_conditions_as_plaintext = from[Setting::query_condition_cache_store_conditions_as_plaintext];
 
     optimize_use_implicit_projections = optimize_projection && from[Setting::optimize_use_implicit_projections];
     force_use_projection = optimize_projection && from[Setting::force_optimize_projection];

src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h

@@ -67,7 +67,8 @@ struct QueryPlanOptimizationSettings
     bool optimize_sorting_by_input_stream_properties;
     bool aggregation_in_order;
     bool optimize_projection;
-    bool use_query_condition_cache = false;
+    bool use_query_condition_cache;
+    bool query_condition_cache_store_conditions_as_plaintext;
 
     /// --- Third-pass optimizations (Processors/QueryPlan/QueryPlan.cpp)
     bool build_sets = true; /// this one doesn't have a corresponding setting