Merge pull request ClickHouse#90296 from ClickHouse/backport/25.8/89004

Backport ClickHouse#89004 to 25.8: Fix performance of skip index analysis

Author: clickhouse-gh[bot]

src/Processors/QueryPlan/ReadFromMergeTree.cpp

@@ -1798,7 +1798,8 @@ static void buildIndexes(
     {
         std::vector<size_t> index_sizes;
         index_sizes.reserve(skip_indexes.useful_indices.size());
-        for (const auto& part : parts)
+
+        for (const auto & part : parts)
         {
             auto &index_order = skip_indexes.per_part_index_orders.emplace_back();
             index_order.resize(skip_indexes.useful_indices.size());
@@ -1808,7 +1809,7 @@ static void buildIndexes(
 
             for (const auto &idx : skip_indexes.useful_indices)
             {
-                const auto *extension = idx.index->getDeserializedFormat(part.data_part->getDataPartStorage(), idx.index->getFileName()).extension;
+                const auto *extension = idx.index->getDeserializedFormat(part.data_part->checksums, idx.index->getFileName()).extension;
                 auto sz = part.data_part->getFileSizeOrZero(idx.index->getFileName() + extension);
                 index_sizes.emplace_back(sz);
             }

src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp

@@ -1721,7 +1721,7 @@ std::pair<MarkRanges, RangesInDataPartReadHints> MergeTreeDataSelectExecutor::fi
     VectorSimilarityIndexCache * vector_similarity_index_cache,
     LoggerPtr log)
 {
-    if (!index_helper->getDeserializedFormat(part->getDataPartStorage(), index_helper->getFileName()))
+    if (!index_helper->getDeserializedFormat(part->checksums, index_helper->getFileName()))
     {
         LOG_DEBUG(log, "File for index {} does not exist ({}.*). Skipping it.", backQuote(index_helper->index.name),
             (fs::path(part->getDataPartStorage().getFullPath()) / index_helper->getFileName()).string());

src/Storages/MergeTree/MergeTreeIndexMinMax.cpp

@@ -216,11 +216,11 @@ MergeTreeIndexConditionPtr MergeTreeIndexMinMax::createIndexCondition(
     return std::make_shared<MergeTreeIndexConditionMinMax>(index, filter_dag, context);
 }
 
-MergeTreeIndexFormat MergeTreeIndexMinMax::getDeserializedFormat(const IDataPartStorage & data_part_storage, const std::string & relative_path_prefix) const
+MergeTreeIndexFormat MergeTreeIndexMinMax::getDeserializedFormat(const MergeTreeDataPartChecksums & checksums, const std::string & relative_path_prefix) const
 {
-    if (data_part_storage.existsFile(relative_path_prefix + ".idx2"))
+    if (checksums.files.contains(relative_path_prefix + ".idx2"))
         return {2, ".idx2"};
-    if (data_part_storage.existsFile(relative_path_prefix + ".idx"))
+    if (checksums.files.contains(relative_path_prefix + ".idx"))
         return {1, ".idx"};
     return {0 /* unknown */, ""};
 }

src/Storages/MergeTree/MergeTreeIndexMinMax.h

@@ -82,7 +82,7 @@ class MergeTreeIndexMinMax : public IMergeTreeIndex
         const ActionsDAG::Node * predicate, ContextPtr context) const override;
 
     const char* getSerializedFileExtension() const override { return ".idx2"; }
-    MergeTreeIndexFormat getDeserializedFormat(const IDataPartStorage & data_part_storage, const std::string & path_prefix) const override; /// NOLINT
+    MergeTreeIndexFormat getDeserializedFormat(const MergeTreeDataPartChecksums & checksums, const std::string & path_prefix) const override; /// NOLINT
 };
 
 }

src/Storages/MergeTree/MergeTreeIndexReader.cpp

@@ -72,7 +72,7 @@ void MergeTreeIndexReader::initStreamIfNeeded()
     if (stream)
         return;
 
-    auto index_format = index->getDeserializedFormat(part->getDataPartStorage(), index->getFileName());
+    auto index_format = index->getDeserializedFormat(part->checksums, index->getFileName());
 
     stream = makeIndexReader(
         index_format.extension,

src/Storages/MergeTree/MergeTreeIndices.cpp

@@ -1,3 +1,4 @@
+#include <Storages/MergeTree/MergeTreeDataPartChecksum.h>
 #include <Storages/MergeTree/MergeTreeIndices.h>
 
 #include <Columns/IColumn.h>
@@ -23,9 +24,9 @@ Names IMergeTreeIndex::getColumnsRequiredForIndexCalc() const
 }
 
 MergeTreeIndexFormat
-IMergeTreeIndex::getDeserializedFormat(const IDataPartStorage & data_part_storage, const std::string & relative_path_prefix) const
+IMergeTreeIndex::getDeserializedFormat(const MergeTreeDataPartChecksums & checksums, const std::string & relative_path_prefix) const
 {
-    if (data_part_storage.existsFile(relative_path_prefix + ".idx"))
+    if (checksums.files.contains(relative_path_prefix + ".idx"))
         return {1, ".idx"};
     return {0 /*unknown*/, ""};
 }

src/Storages/MergeTree/MergeTreeIndices.h

@@ -70,9 +70,9 @@ evalOrRpnIndexStates(RPNEvaluationIndexUsefulnessState lhs, RPNEvaluationIndexUs
 
 class ActionsDAG;
 class Block;
-class IDataPartStorage;
 struct MergeTreeWriterSettings;
 struct SelectQueryInfo;
+struct MergeTreeDataPartChecksums;
 
 class GinIndexStore;
 using GinIndexStorePtr = std::shared_ptr<GinIndexStore>;
@@ -318,7 +318,7 @@ struct IMergeTreeIndex
     ///
     /// Return pair<extension, version>.
     virtual MergeTreeIndexFormat
-    getDeserializedFormat(const IDataPartStorage & data_part_storage, const std::string & relative_path_prefix) const;
+    getDeserializedFormat(const MergeTreeDataPartChecksums & checksums, const std::string & relative_path_prefix) const;
 
     virtual MergeTreeIndexGranulePtr createIndexGranule() const = 0;
 

src/Storages/MergeTree/PatchParts/RangesInPatchParts.cpp

@@ -267,7 +267,7 @@ MaybeMinMaxStats getPatchMinMaxStats(const DataPartPtr & patch_part, const MarkR
 
     auto index_ptr = MergeTreeIndexFactory::instance().get(*it);
     /// Check that index exists in data part. It may be absent for parts created in earlier versions.
-    if (!index_ptr->getDeserializedFormat(patch_part->getDataPartStorage(), index_ptr->getFileName()))
+    if (!index_ptr->getDeserializedFormat(patch_part->checksums, index_ptr->getFileName()))
         return {};
 
     size_t total_marks_without_final = patch_part->index_granularity->getMarksCountWithoutFinal();