Merge pull request ClickHouse#79802 from rschu1ze/annoy-compat

Remove compatibility hack for Annoy and Usearch indexes

Author: rschu1ze

src/Processors/QueryPlan/ReadFromMergeTree.cpp

@@ -34,7 +34,6 @@
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <Storages/LazilyReadInfo.h>
 #include <Storages/MergeTree/MergeTreeDataSelectExecutor.h>
-#include <Storages/MergeTree/MergeTreeIndexLegacyVectorSimilarity.h>
 #include <Storages/MergeTree/MergeTreeIndexMinMax.h>
 #include <Storages/MergeTree/MergeTreeIndexVectorSimilarity.h>
 #include <Storages/MergeTree/MergeTreePrefetchedReadPool.h>
@@ -1774,9 +1773,6 @@ static void buildIndexes(
             if (const auto * vector_similarity_index = typeid_cast<const MergeTreeIndexVectorSimilarity *>(index_helper.get()))
                 condition = vector_similarity_index->createIndexCondition(filter_dag.predicate, context, vector_search_parameters);
 #endif
-            if (const auto * legacy_vector_similarity_index = typeid_cast<const MergeTreeIndexLegacyVectorSimilarity *>(index_helper.get()))
-                condition = legacy_vector_similarity_index->createIndexCondition(filter_dag.predicate, context);
-
             if (!condition)
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown vector search index {}", index_helper->index.name);
         }

src/Storages/MergeTree/MergeTreeIndexLegacyVectorSimilarity.cpp

@@ -143,16 +143,6 @@ MergeTreeIndexFactory::MergeTreeIndexFactory()
     registerCreator("vector_similarity", vectorSimilarityIndexCreator);
     registerValidator("vector_similarity", vectorSimilarityIndexValidator);
 #endif
-    /// ------
-    /// TODO: remove this block at the end of 2024.
-    /// Index types 'annoy' and 'usearch' are no longer supported as of June 2024. Their successor is index type 'vector_similarity'.
-    /// To support loading tables with old indexes during a transition period, register dummy indexes which allow load/attaching but
-    /// throw an exception when the user attempts to use them.
-    registerCreator("annoy", legacyVectorSimilarityIndexCreator);
-    registerValidator("annoy", legacyVectorSimilarityIndexValidator);
-    registerCreator("usearch", legacyVectorSimilarityIndexCreator);
-    registerValidator("usearch", legacyVectorSimilarityIndexValidator);
-    /// ------
 
     registerCreator("gin", ginIndexCreator);
     registerValidator("gin", ginIndexValidator);

src/Storages/MergeTree/MergeTreeIndexLegacyVectorSimilarity.h

@@ -401,9 +401,6 @@ MergeTreeIndexPtr vectorSimilarityIndexCreator(const IndexDescription & index);
 void vectorSimilarityIndexValidator(const IndexDescription & index, bool attach);
 #endif
 
-MergeTreeIndexPtr legacyVectorSimilarityIndexCreator(const IndexDescription & index);
-void legacyVectorSimilarityIndexValidator(const IndexDescription & index, bool attach);
-
 MergeTreeIndexPtr ginIndexCreator(const IndexDescription & index);
 void ginIndexValidator(const IndexDescription & index, bool attach);
 

src/Storages/MergeTree/MergeTreeIndices.cpp

@@ -1,10 +1,20 @@
-Create tables with vector similarity indexs on Float32 and BFloat16 columns
-Run all combinations of vector search queries: column type X reference vector type
+Create tables with vector similarity indexs on Float64, Float32 and BFloat16 columns
+Run all combinations of vector search queries: column type x reference vector type
 5
 5
 5
 5
-Verify that the index is used for all combinations of vector search queries: column type X reference vector type
+5
+5
+5
+5
+5
+Check that the index is used for all combinations of vector search queries: column type x reference vector type
+Description: vector_similarity GRANULARITY 100000000
+Description: vector_similarity GRANULARITY 100000000
+Description: vector_similarity GRANULARITY 100000000
+Description: vector_similarity GRANULARITY 100000000
+Description: vector_similarity GRANULARITY 100000000
 Description: vector_similarity GRANULARITY 100000000
 Description: vector_similarity GRANULARITY 100000000
 Description: vector_similarity GRANULARITY 100000000

src/Storages/MergeTree/MergeTreeIndices.h

@@ -5,53 +5,94 @@
 
 SET allow_experimental_vector_similarity_index = 1;
 SET enable_analyzer = 1;
-SET parallel_replicas_local_plan=1; -- this setting is randomized, set it explicitly to have local plan for parallel replicas
+SET parallel_replicas_local_plan = 1; -- this setting is randomized, set it explicitly to force local plan for parallel replicas
 
+DROP TABLE IF EXISTS tab_f64;
 DROP TABLE IF EXISTS tab_f32;
 DROP TABLE IF EXISTS tab_bf16;
 
-SELECT 'Create tables with vector similarity indexs on Float32 and BFloat16 columns';
+SELECT 'Create tables with vector similarity indexs on Float64, Float32 and BFloat16 columns';
+
+CREATE TABLE tab_f64(id Int32, vec Array(Float64), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 2)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 2;
+INSERT INTO tab_f64 VALUES (0, [1.0, 0.0]), (1, [1.1, 0.0]), (2, [1.2, 0.0]), (3, [1.3, 0.0]), (4, [1.4, 0.0]), (5, [0.0, 2.0]), (6, [0.0, 2.1]), (7, [0.0, 2.2]), (8, [0.0, 2.3]), (9, [0.0, 2.4]);
 
 CREATE TABLE tab_f32(id Int32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 2)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 2;
 INSERT INTO tab_f32 VALUES (0, [1.0, 0.0]), (1, [1.1, 0.0]), (2, [1.2, 0.0]), (3, [1.3, 0.0]), (4, [1.4, 0.0]), (5, [0.0, 2.0]), (6, [0.0, 2.1]), (7, [0.0, 2.2]), (8, [0.0, 2.3]), (9, [0.0, 2.4]);
 
 CREATE TABLE tab_bf16(id Int32, vec Array(BFloat16), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance', 2)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 2;
 INSERT INTO tab_bf16 VALUES (0, [1.0, 0.0]), (1, [1.1, 0.0]), (2, [1.2, 0.0]), (3, [1.3, 0.0]), (4, [1.4, 0.0]), (5, [0.0, 2.0]), (6, [0.0, 2.1]), (7, [0.0, 2.2]), (8, [0.0, 2.3]), (9, [0.0, 2.4]);
 
+DROP FUNCTION IF EXISTS constF64;
+CREATE FUNCTION constF64 AS () -> [toFloat64(0.0), toFloat64(2.0)];
+
 DROP FUNCTION IF EXISTS constF32;
 CREATE FUNCTION constF32 AS () -> [toFloat32(0.0), toFloat32(2.0)];
 
 DROP FUNCTION IF EXISTS constBF16;
 CREATE FUNCTION constBF16 AS () -> [toBFloat16(0.0), toBFloat16(2.0)];
 
-SELECT 'Run all combinations of vector search queries: column type X reference vector type';
+SELECT 'Run all combinations of vector search queries: column type x reference vector type';
 
 SELECT id
-FROM tab_f32
+FROM tab_f64
+ORDER BY L2Distance(vec, constF64())
+LIMIT 1;
+
+SELECT id
+FROM tab_f64
 ORDER BY L2Distance(vec, constF32())
 LIMIT 1;
 
 SELECT id
-FROM tab_bf16
+FROM tab_f64
 ORDER BY L2Distance(vec, constBF16())
 LIMIT 1;
 
 SELECT id
 FROM tab_f32
-ORDER BY L2Distance(vec, (SELECT vec FROM tab_f32 WHERE id = 5)) -- subquery evaluates to const scalar
+ORDER BY L2Distance(vec, constF64())
+LIMIT 1;
+
+SELECT id
+FROM tab_f32
+ORDER BY L2Distance(vec, constF32())
+LIMIT 1;
+
+SELECT id
+FROM tab_f32
+ORDER BY L2Distance(vec, constBF16())
+LIMIT 1;
+
+SELECT id
+FROM tab_bf16
+ORDER BY L2Distance(vec, constF64())
 LIMIT 1;
 
 SELECT id
 FROM tab_bf16
-ORDER BY L2Distance(vec, (SELECT vec FROM tab_bf16 WHERE id = 5)) -- subquery evaluates to const scalar
+ORDER BY L2Distance(vec, constF32())
+LIMIT 1;
+
+SELECT id
+FROM tab_bf16
+ORDER BY L2Distance(vec, constBF16())
 LIMIT 1;
 
-SELECT 'Verify that the index is used for all combinations of vector search queries: column type X reference vector type';
+SELECT 'Check that the index is used for all combinations of vector search queries: column type x reference vector type';
 
 SELECT trimLeft(explain) AS explain FROM (
 EXPLAIN indexes = 1
 SELECT id
-FROM tab_f32
+FROM tab_f64
+ORDER BY L2Distance(vec, constF64())
+LIMIT 1
+)
+WHERE explain LIKE '%vector_similarity%';
+
+SELECT trimLeft(explain) AS explain FROM (
+EXPLAIN indexes = 1
+SELECT id
+FROM tab_f64
 ORDER BY L2Distance(vec, constF32())
 LIMIT 1
 )
@@ -60,7 +101,7 @@ WHERE explain LIKE '%vector_similarity%';
 SELECT trimLeft(explain) AS explain FROM (
 EXPLAIN indexes = 1
 SELECT id
-FROM tab_bf16
+FROM tab_f64
 ORDER BY L2Distance(vec, constBF16())
 LIMIT 1
 )
@@ -70,7 +111,25 @@ SELECT trimLeft(explain) AS explain FROM (
 EXPLAIN indexes = 1
 SELECT id
 FROM tab_f32
-ORDER BY L2Distance(vec, (SELECT vec from tab_f32 WHERE id = 5))
+ORDER BY L2Distance(vec, constF64())
+LIMIT 1
+)
+WHERE explain LIKE '%vector_similarity%';
+
+SELECT trimLeft(explain) AS explain FROM (
+EXPLAIN indexes = 1
+SELECT id
+FROM tab_f32
+ORDER BY L2Distance(vec, constF32())
+LIMIT 1
+)
+WHERE explain LIKE '%vector_similarity%';
+
+SELECT trimLeft(explain) AS explain FROM (
+EXPLAIN indexes = 1
+SELECT id
+FROM tab_f32
+ORDER BY L2Distance(vec, constBF16())
 LIMIT 1
 )
 WHERE explain LIKE '%vector_similarity%';
@@ -79,13 +138,33 @@ SELECT trimLeft(explain) AS explain FROM (
 EXPLAIN indexes = 1
 SELECT id
 FROM tab_bf16
-ORDER BY L2Distance(vec, (SELECT vec from tab_bf16 WHERE id = 5))
+ORDER BY L2Distance(vec, constF64())
+LIMIT 1
+)
+WHERE explain LIKE '%vector_similarity%';
+
+SELECT trimLeft(explain) AS explain FROM (
+EXPLAIN indexes = 1
+SELECT id
+FROM tab_bf16
+ORDER BY L2Distance(vec, constF32())
+LIMIT 1
+)
+WHERE explain LIKE '%vector_similarity%';
+
+SELECT trimLeft(explain) AS explain FROM (
+EXPLAIN indexes = 1
+SELECT id
+FROM tab_bf16
+ORDER BY L2Distance(vec, constBF16())
 LIMIT 1
 )
 WHERE explain LIKE '%vector_similarity%';
 
+DROP FUNCTION constF64;
 DROP FUNCTION constF32;
 DROP FUNCTION constBF16;
 
+DROP TABLE tab_f64;
 DROP TABLE tab_f32;
 DROP TABLE tab_bf16;