Skip to content

Commit a5db99b

Browse files
sivukhinsivukhin
authored
Merge pull request #1659 from tursodatabase/vector-search-fix-idx-type
vector search: fix idx type bug
2 parents 8262d23 + b1dbaa1 commit a5db99b

File tree

6 files changed

+31
-47
lines changed

6 files changed

+31
-47
lines changed

‎libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19266,7 +19266,8 @@ struct Index {
1926619266
u16 nKeyCol; /* Number of columns forming the key */
1926719267
u16 nColumn; /* Number of columns stored in the index */
1926819268
u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */
19269-
unsigned idxType:3; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK, 4:VECTOR INDEX */
19269+
unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */
19270+
unsigned idxIsVector:1; /* 0:Normal 1:VECTOR INDEX */
1927019271
unsigned bUnordered:1; /* Use this index for == or IN queries only */
1927119272
unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */
1927219273
unsigned isResized:1; /* True if resizeIndexObject() has been called */
@@ -19298,7 +19299,6 @@ struct Index {
1929819299
#define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */
1929919300
#define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */
1930019301
#define SQLITE_IDXTYPE_IPK 3 /* INTEGER PRIMARY KEY index */
19301-
#define SQLITE_IDXTYPE_VECTOR 4 /* libSQL vector index */
1930219302

1930319303
/* Return true if index X is a PRIMARY KEY index */
1930419304
#define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY)
@@ -19307,10 +19307,7 @@ struct Index {
1930719307
#define IsUniqueIndex(X) ((X)->onError!=OE_None)
1930819308

1930919309
/* Return true if index X is a vector index */
19310-
#define IsVectorIndex(X) ((X)->idxType==SQLITE_IDXTYPE_VECTOR)
19311-
19312-
/* Return true if index X is an user defined index (APPDEF or VECTOR) */
19313-
#define IsAppDefIndex(X) ((X)->idxType==SQLITE_IDXTYPE_APPDEF||(X)->idxType==SQLITE_IDXTYPE_VECTOR)
19310+
#define IsVectorIndex(X) ((X)->idxIsVector==1)
1931419311

1931519312
/* The Index.aiColumn[] values are normally positive integer. But
1931619313
** there are some negative values that have special meaning:
@@ -123188,7 +123185,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
123188123185
for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){
123189123186
pNext = pIndex->pNext;
123190123187
assert( pIndex->pSchema==pTable->pSchema
123191-
|| (IsVirtual(pTable) && !IsAppDefIndex(pIndex)) );
123188+
|| (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) );
123192123189
if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){
123193123190
char *zName = pIndex->zName;
123194123191
TESTONLY ( Index *pOld = ) sqlite3HashInsert(
@@ -126700,13 +126697,12 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
126700126697
goto exit_create_index;
126701126698
}
126702126699
if( vectorIdxRc >= 1 ){
126703-
idxType = SQLITE_IDXTYPE_VECTOR;
126704126700
/*
126705126701
* SQLite can use B-Tree indices in some optimizations (like SELECT COUNT(*) can use any full B-Tree index instead of PK index)
126706126702
* But, SQLite pretty conservative about usage of unordered indices - that's what we need here
126707126703
*/
126708126704
pIndex->bUnordered = 1;
126709-
pIndex->idxType = idxType;
126705+
pIndex->idxIsVector = 1;
126710126706
}
126711126707
if( vectorIdxRc == 1 ){
126712126708
skipRefill = 1;
@@ -126754,7 +126750,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
126754126750
for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
126755126751
int k;
126756126752
assert( IsUniqueIndex(pIdx) );
126757-
assert( !IsAppDefIndex(pIdx) );
126753+
assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF );
126758126754
assert( IsUniqueIndex(pIndex) );
126759126755

126760126756
if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue;
@@ -127035,7 +127031,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists
127035127031
pParse->checkSchema = 1;
127036127032
goto exit_drop_index;
127037127033
}
127038-
if( !IsAppDefIndex(pIndex) ){
127034+
if( pIndex->idxType!=SQLITE_IDXTYPE_APPDEF ){
127039127035
sqlite3ErrorMsg(pParse, "index associated with UNIQUE "
127040127036
"or PRIMARY KEY constraint cannot be dropped", 0);
127041127037
goto exit_drop_index;
@@ -177910,9 +177906,6 @@ static YYACTIONTYPE yy_reduce(
177910177906
case 242: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm indextype ON nm LP sortlist RP where_opt */
177911177907
{
177912177908
u8 idxType = SQLITE_IDXTYPE_APPDEF;
177913-
if( yymsp[-6].minor.yy421.pUsing!=0 ){
177914-
idxType = SQLITE_IDXTYPE_VECTOR;
177915-
}
177916177909
sqlite3CreateIndex(pParse, &yymsp[-8].minor.yy0, &yymsp[-7].minor.yy0,
177917177910
sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy402, yymsp[-11].minor.yy502,
177918177911
&yymsp[-12].minor.yy0, yymsp[0].minor.yy590, SQLITE_SO_ASC, yymsp[-9].minor.yy502, idxType, yymsp[-6].minor.yy421.pUsing);

‎libsql-ffi/bundled/src/sqlite3.c‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19266,7 +19266,8 @@ struct Index {
1926619266
u16 nKeyCol; /* Number of columns forming the key */
1926719267
u16 nColumn; /* Number of columns stored in the index */
1926819268
u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */
19269-
unsigned idxType:3; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK, 4:VECTOR INDEX */
19269+
unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */
19270+
unsigned idxIsVector:1; /* 0:Normal 1:VECTOR INDEX */
1927019271
unsigned bUnordered:1; /* Use this index for == or IN queries only */
1927119272
unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */
1927219273
unsigned isResized:1; /* True if resizeIndexObject() has been called */
@@ -19298,7 +19299,6 @@ struct Index {
1929819299
#define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */
1929919300
#define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */
1930019301
#define SQLITE_IDXTYPE_IPK 3 /* INTEGER PRIMARY KEY index */
19301-
#define SQLITE_IDXTYPE_VECTOR 4 /* libSQL vector index */
1930219302

1930319303
/* Return true if index X is a PRIMARY KEY index */
1930419304
#define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY)
@@ -19307,10 +19307,7 @@ struct Index {
1930719307
#define IsUniqueIndex(X) ((X)->onError!=OE_None)
1930819308

1930919309
/* Return true if index X is a vector index */
19310-
#define IsVectorIndex(X) ((X)->idxType==SQLITE_IDXTYPE_VECTOR)
19311-
19312-
/* Return true if index X is an user defined index (APPDEF or VECTOR) */
19313-
#define IsAppDefIndex(X) ((X)->idxType==SQLITE_IDXTYPE_APPDEF||(X)->idxType==SQLITE_IDXTYPE_VECTOR)
19310+
#define IsVectorIndex(X) ((X)->idxIsVector==1)
1931419311

1931519312
/* The Index.aiColumn[] values are normally positive integer. But
1931619313
** there are some negative values that have special meaning:
@@ -123188,7 +123185,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
123188123185
for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){
123189123186
pNext = pIndex->pNext;
123190123187
assert( pIndex->pSchema==pTable->pSchema
123191-
|| (IsVirtual(pTable) && !IsAppDefIndex(pIndex)) );
123188+
|| (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) );
123192123189
if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){
123193123190
char *zName = pIndex->zName;
123194123191
TESTONLY ( Index *pOld = ) sqlite3HashInsert(
@@ -126700,13 +126697,12 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
126700126697
goto exit_create_index;
126701126698
}
126702126699
if( vectorIdxRc >= 1 ){
126703-
idxType = SQLITE_IDXTYPE_VECTOR;
126704126700
/*
126705126701
* SQLite can use B-Tree indices in some optimizations (like SELECT COUNT(*) can use any full B-Tree index instead of PK index)
126706126702
* But, SQLite pretty conservative about usage of unordered indices - that's what we need here
126707126703
*/
126708126704
pIndex->bUnordered = 1;
126709-
pIndex->idxType = idxType;
126705+
pIndex->idxIsVector = 1;
126710126706
}
126711126707
if( vectorIdxRc == 1 ){
126712126708
skipRefill = 1;
@@ -126754,7 +126750,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
126754126750
for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
126755126751
int k;
126756126752
assert( IsUniqueIndex(pIdx) );
126757-
assert( !IsAppDefIndex(pIdx) );
126753+
assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF );
126758126754
assert( IsUniqueIndex(pIndex) );
126759126755

126760126756
if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue;
@@ -127035,7 +127031,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists
127035127031
pParse->checkSchema = 1;
127036127032
goto exit_drop_index;
127037127033
}
127038-
if( !IsAppDefIndex(pIndex) ){
127034+
if( pIndex->idxType!=SQLITE_IDXTYPE_APPDEF ){
127039127035
sqlite3ErrorMsg(pParse, "index associated with UNIQUE "
127040127036
"or PRIMARY KEY constraint cannot be dropped", 0);
127041127037
goto exit_drop_index;
@@ -177910,9 +177906,6 @@ static YYACTIONTYPE yy_reduce(
177910177906
case 242: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm indextype ON nm LP sortlist RP where_opt */
177911177907
{
177912177908
u8 idxType = SQLITE_IDXTYPE_APPDEF;
177913-
if( yymsp[-6].minor.yy421.pUsing!=0 ){
177914-
idxType = SQLITE_IDXTYPE_VECTOR;
177915-
}
177916177909
sqlite3CreateIndex(pParse, &yymsp[-8].minor.yy0, &yymsp[-7].minor.yy0,
177917177910
sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy402, yymsp[-11].minor.yy502,
177918177911
&yymsp[-12].minor.yy0, yymsp[0].minor.yy590, SQLITE_SO_ASC, yymsp[-9].minor.yy502, idxType, yymsp[-6].minor.yy421.pUsing);

‎libsql-sqlite3/src/build.c‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -833,7 +833,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
833833
for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){
834834
pNext = pIndex->pNext;
835835
assert( pIndex->pSchema==pTable->pSchema
836-
|| (IsVirtual(pTable) && !IsAppDefIndex(pIndex)) );
836+
|| (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) );
837837
if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){
838838
char *zName = pIndex->zName;
839839
TESTONLY ( Index *pOld = ) sqlite3HashInsert(
@@ -4345,13 +4345,12 @@ void sqlite3CreateIndex(
43454345
goto exit_create_index;
43464346
}
43474347
if( vectorIdxRc >= 1 ){
4348-
idxType = SQLITE_IDXTYPE_VECTOR;
43494348
/*
43504349
* SQLite can use B-Tree indices in some optimizations (like SELECT COUNT(*) can use any full B-Tree index instead of PK index)
43514350
* But, SQLite pretty conservative about usage of unordered indices - that's what we need here
43524351
*/
43534352
pIndex->bUnordered = 1;
4354-
pIndex->idxType = idxType;
4353+
pIndex->idxIsVector = 1;
43554354
}
43564355
if( vectorIdxRc == 1 ){
43574356
skipRefill = 1;
@@ -4399,7 +4398,7 @@ void sqlite3CreateIndex(
43994398
for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
44004399
int k;
44014400
assert( IsUniqueIndex(pIdx) );
4402-
assert( !IsAppDefIndex(pIdx) );
4401+
assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF );
44034402
assert( IsUniqueIndex(pIndex) );
44044403

44054404
if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue;
@@ -4680,7 +4679,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){
46804679
pParse->checkSchema = 1;
46814680
goto exit_drop_index;
46824681
}
4683-
if( !IsAppDefIndex(pIndex) ){
4682+
if( pIndex->idxType!=SQLITE_IDXTYPE_APPDEF ){
46844683
sqlite3ErrorMsg(pParse, "index associated with UNIQUE "
46854684
"or PRIMARY KEY constraint cannot be dropped", 0);
46864685
goto exit_drop_index;

‎libsql-sqlite3/src/parse.y‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1451,9 +1451,6 @@ paren_exprlist(A) ::= LP exprlist(X) RP. {A = X;}
14511451
cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D) indextype(T)
14521452
ON nm(Y) LP sortlist(Z) RP where_opt(W). {
14531453
u8 idxType = SQLITE_IDXTYPE_APPDEF;
1454-
if( T.pUsing!=0 ){
1455-
idxType = SQLITE_IDXTYPE_VECTOR;
1456-
}
14571454
sqlite3CreateIndex(pParse, &X, &D,
14581455
sqlite3SrcListAppend(pParse,0,&Y,0), Z, U,
14591456
&S, W, SQLITE_SO_ASC, NE, idxType, T.pUsing);

‎libsql-sqlite3/src/sqliteInt.h‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2799,7 +2799,8 @@ struct Index {
27992799
u16 nKeyCol; /* Number of columns forming the key */
28002800
u16 nColumn; /* Number of columns stored in the index */
28012801
u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */
2802-
unsigned idxType:3; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK, 4:VECTOR INDEX */
2802+
unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */
2803+
unsigned idxIsVector:1; /* 0:Normal 1:VECTOR INDEX */
28032804
unsigned bUnordered:1; /* Use this index for == or IN queries only */
28042805
unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */
28052806
unsigned isResized:1; /* True if resizeIndexObject() has been called */
@@ -2831,7 +2832,6 @@ struct Index {
28312832
#define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */
28322833
#define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */
28332834
#define SQLITE_IDXTYPE_IPK 3 /* INTEGER PRIMARY KEY index */
2834-
#define SQLITE_IDXTYPE_VECTOR 4 /* libSQL vector index */
28352835

28362836
/* Return true if index X is a PRIMARY KEY index */
28372837
#define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY)
@@ -2840,10 +2840,7 @@ struct Index {
28402840
#define IsUniqueIndex(X) ((X)->onError!=OE_None)
28412841

28422842
/* Return true if index X is a vector index */
2843-
#define IsVectorIndex(X) ((X)->idxType==SQLITE_IDXTYPE_VECTOR)
2844-
2845-
/* Return true if index X is an user defined index (APPDEF or VECTOR) */
2846-
#define IsAppDefIndex(X) ((X)->idxType==SQLITE_IDXTYPE_APPDEF||(X)->idxType==SQLITE_IDXTYPE_VECTOR)
2843+
#define IsVectorIndex(X) ((X)->idxIsVector==1)
28472844

28482845
/* The Index.aiColumn[] values are normally positive integer. But
28492846
** there are some negative values that have special meaning:

‎libsql-sqlite3/test/libsql_vector_index.test‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,17 @@ set testprefix vector
3030

3131
sqlite3_db_config_lookaside db 0 0 0
3232

33-
do_execsql_test vector-integrity {
34-
CREATE TABLE t_integrity( v FLOAT32(3) );
35-
CREATE INDEX t_integrity_idx ON t_integrity( libsql_vector_idx(v) );
36-
INSERT INTO t_integrity VALUES (vector('[1,2,3]'));
33+
do_execsql_test vector-pragmas {
34+
CREATE TABLE t_pragmas( v FLOAT32(3) );
35+
CREATE INDEX t_pragmas_idx ON t_pragmas( libsql_vector_idx(v) );
36+
INSERT INTO t_pragmas VALUES (vector('[1,2,3]'));
3737
PRAGMA integrity_check;
38-
} {{row 1 missing from index t_integrity_idx} {wrong # of entries in index t_integrity_idx}}
38+
PRAGMA index_list='t_pragmas';
39+
} {
40+
{row 1 missing from index t_pragmas_idx}
41+
{wrong # of entries in index t_pragmas_idx}
42+
0 t_pragmas_idx 0 c 0
43+
}
3944

4045
do_execsql_test vector-typename {
4146
CREATE TABLE t_type_spaces( v FLOAT32 ( 3 ) );

0 commit comments

Comments
 (0)