fix: dropping an IVFFlat index fails to clean up all associated tables. (#22604)

- Drop all index tables of an ivfflat index.

Approved by: @heni02, @ouyuanning, @aunjgr

Author: aptend

pkg/pb/plan/plan.pb.go

@@ -2107,7 +2107,7 @@ func (s *Scope) DropIndex(c *Compile) error {
 	if err != nil {
 		return err
 	}
-	newCt, err := makeNewDropConstraint(oldCt, qry.GetIndexName())
+	newCt, dropIndexTableNames, err := makeNewDropConstraint(oldCt, qry.GetIndexName())
 	if err != nil {
 		return err
 	}
@@ -2117,19 +2117,18 @@ func (s *Scope) DropIndex(c *Compile) error {
 	}
 
 	//2. drop index table
-	if qry.IndexTableName != "" {
-		if _, err = d.Relation(c.proc.Ctx, qry.IndexTableName, nil); err != nil {
+	for _, indexTableName := range dropIndexTableNames {
+		if _, err = d.Relation(c.proc.Ctx, indexTableName, nil); err != nil {
 			return err
 		}
 
-		if err = maybeDeleteAutoIncrement(c.proc.Ctx, c.proc.GetService(), d, qry.IndexTableName, c.proc.GetTxnOperator()); err != nil {
+		if err = maybeDeleteAutoIncrement(c.proc.Ctx, c.proc.GetService(), d, indexTableName, c.proc.GetTxnOperator()); err != nil {
 			return err
 		}
 
-		if err = d.Delete(c.proc.Ctx, qry.IndexTableName); err != nil {
+		if err = d.Delete(c.proc.Ctx, indexTableName); err != nil {
 			return err
 		}
-
 	}
 
 	//3. delete index object from mo_catalog.mo_indexes
@@ -2141,7 +2140,8 @@ func (s *Scope) DropIndex(c *Compile) error {
 	return nil
 }
 
-func makeNewDropConstraint(oldCt *engine.ConstraintDef, dropName string) (*engine.ConstraintDef, error) {
+func makeNewDropConstraint(oldCt *engine.ConstraintDef, dropName string) (*engine.ConstraintDef, []string, error) {
+	dropIndexTableNames := []string{}
 	// must fount dropName because of being checked in plan
 	for i := 0; i < len(oldCt.Cts); i++ {
 		ct := oldCt.Cts[i]
@@ -2154,13 +2154,16 @@ func makeNewDropConstraint(oldCt *engine.ConstraintDef, dropName string) (*engin
 			oldCt.Cts[i] = def
 		case *engine.IndexDef:
 			pred := func(index *plan.IndexDef) bool {
+				if index.IndexName == dropName && len(index.IndexTableName) > 0 {
+					dropIndexTableNames = append(dropIndexTableNames, index.IndexTableName)
+				}
 				return index.IndexName == dropName
 			}
 			def.Indexes = plan2.RemoveIf[*plan.IndexDef](def.Indexes, pred)
 			oldCt.Cts[i] = def
 		}
 	}
-	return oldCt, nil
+	return oldCt, dropIndexTableNames, nil
 }
 
 func MakeNewCreateConstraint(oldCt *engine.ConstraintDef, c engine.Constraint) (*engine.ConstraintDef, error) {

pkg/sql/compile/ddl.go

@@ -3422,7 +3422,6 @@ func buildDropIndex(stmt *tree.DropIndex, ctx CompilerContext) (*Plan, error) {
 
 	for _, indexdef := range tableDef.Indexes {
 		if dropIndex.IndexName == indexdef.IndexName {
-			dropIndex.IndexTableName = indexdef.IndexTableName
 			found = true
 			break
 		}

pkg/sql/plan/build_ddl.go

@@ -721,10 +721,9 @@ func DeepCopyDataDefinition(old *plan.DataDefinition) *plan.DataDefinition {
 	case *plan.DataDefinition_DropIndex:
 		newDf.Definition = &plan.DataDefinition_DropIndex{
 			DropIndex: &plan.DropIndex{
-				Database:       df.DropIndex.Database,
-				Table:          df.DropIndex.Table,
-				IndexName:      df.DropIndex.IndexName,
-				IndexTableName: df.DropIndex.IndexTableName,
+				Database:  df.DropIndex.Database,
+				Table:     df.DropIndex.Table,
+				IndexName: df.DropIndex.IndexName,
 			},
 		}
 

pkg/sql/plan/deepcopy.go

@@ -1494,7 +1494,6 @@ message DropIndex {
 	string database = 1;
 	string table = 2;
 	string index_name = 3;
-	string index_table_name = 4;
 }
 
 message TruncateTable {
@@ -1651,4 +1650,4 @@ message CloneTable {
 	ObjectRef src_obj_def = 4;
 	string dst_database_name = 5;
 	string dst_table_name = 6;
-}
+}

proto/plan.proto

@@ -634,5 +634,29 @@ a    b    c    d
 9776    [10, 3, 8, 5, 48, 26, 5, 16, 17, 0, 0, 2, 132, 53, 1, 16, 112, 6, 0, 0, 7, 2, 1, 48, 48, 15, 18, 31, 3, 0, 0, 9, 6, 10, 19, 27, 50, 46, 17, 9, 18, 1, 4, 48, 132, 23, 3, 5, 132, 9, 4, 3, 11, 0, 2, 46, 84, 12, 10, 10, 1, 0, 12, 76, 26, 22, 16, 26, 35, 15, 3, 16, 15, 1, 51, 132, 125, 8, 1, 2, 132, 51, 67, 91, 8, 0, 0, 30, 126, 39, 32, 38, 4, 0, 1, 12, 24, 2, 2, 2, 4, 7, 2, 19, 93, 19, 70, 92, 2, 3, 1, 21, 36, 58, 132, 94, 0, 0, 0, 0, 21, 25, 57, 48, 1, 0, 0, 1]    3    [0, 0, 0]
 9777    [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]    4    [0, 0, 0]
 9778    [41, 0, 0, 7, 1, 1, 20, 67, 9, 0, 0, 0, 0, 31, 120, 61, 25, 0, 0, 0, 0, 10, 120, 90, 32, 0, 0, 1, 13, 11, 22, 50, 4, 0, 2, 93, 40, 15, 37, 18, 12, 2, 2, 19, 8, 44, 120, 25, 120, 5, 0, 0, 0, 2, 48, 97, 102, 14, 3, 3, 11, 9, 34, 41, 0, 0, 4, 120, 56, 3, 4, 5, 6, 15, 37, 116, 28, 0, 0, 3, 120, 120, 24, 6, 2, 0, 1, 28, 53, 90, 51, 11, 11, 2, 12, 14, 8, 6, 4, 30, 9, 1, 4, 22, 25, 79, 120, 66, 5, 0, 0, 6, 42, 120, 91, 43, 15, 2, 4, 39, 12, 9, 9, 12, 15, 5, 24, 36]    4    [0, 0, 0]
-SET experimental_ivf_index = 0;
 drop database vecdb2;
+create database vecdb3;
+use vecdb3;
+CREATE TABLE `rag_table` (
+`id` int NOT NULL AUTO_INCREMENT,
+`title` varchar(200) DEFAULT NULL,
+`embedding` vecf32(8) DEFAULT NULL,
+`content` text DEFAULT NULL,
+PRIMARY KEY (`id`),
+KEY `embedding_idx` USING ivfflat (`embedding`) lists = 20  op_type 'vector_l2_ops'
+);
+select count(*) from mo_catalog.mo_indexes as i join mo_catalog.mo_tables as t on i.table_id = t.rel_id and t.relname = "rag_table" and i.algo="ivfflat"; -- 3 ivf index tables
+count(*)
+3
+select count(*) from mo_catalog.mo_tables where reldatabase = "vecdb3"; -- rag_table + 3 ivf index tables
+count(*)
+4
+drop index embedding_idx on rag_table;
+select count(*) from mo_catalog.mo_indexes as i join mo_catalog.mo_tables as t on i.table_id = t.rel_id and t.relname = "rag_table" and i.algo="ivfflat"; -- 0 ivf index tables
+count(*)
+0
+select count(*) from mo_catalog.mo_tables where reldatabase = "vecdb3"; -- rag_table + 0 ivf index tables
+count(*)
+1
+SET experimental_ivf_index = 0;
+drop database vecdb3;

test/distributed/cases/array/array_index.result

@@ -694,5 +694,30 @@ alter table vector_index_08 add column d vecf32(3) not null after c;
 select * from vector_index_08;
 
 -- post
-SET experimental_ivf_index = 0;
 drop database vecdb2;
+
+
+create database vecdb3;
+use vecdb3;
+
+CREATE TABLE `rag_table` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `title` varchar(200) DEFAULT NULL,
+  `embedding` vecf32(8) DEFAULT NULL,
+  `content` text DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `embedding_idx` USING ivfflat (`embedding`) lists = 20  op_type 'vector_l2_ops'
+);
+
+select count(*) from mo_catalog.mo_indexes as i join mo_catalog.mo_tables as t on i.table_id = t.rel_id and t.relname = "rag_table" and i.algo="ivfflat"; -- 3 ivf index tables
+
+select count(*) from mo_catalog.mo_tables where reldatabase = "vecdb3"; -- rag_table + 3 ivf index tables
+
+drop index embedding_idx on rag_table;
+
+select count(*) from mo_catalog.mo_indexes as i join mo_catalog.mo_tables as t on i.table_id = t.rel_id and t.relname = "rag_table" and i.algo="ivfflat"; -- 0 ivf index tables
+
+select count(*) from mo_catalog.mo_tables where reldatabase = "vecdb3"; -- rag_table + 0 ivf index tables
+
+SET experimental_ivf_index = 0;
+drop database vecdb3;