Skip to content

Commit e7de104

Browse files
sivukhinsivukhin
authored
Merge pull request #1645 from tursodatabase/fix-fts5-crash
Fix fts5 crash
2 parents 9120ce6 + 0d41105 commit e7de104

File tree

5 files changed

+133
-85
lines changed

5 files changed

+133
-85
lines changed

‎libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c‎

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
** README.md
2929
** configure
3030
** configure.ac
31+
** ext/fts5/fts5_tokenize.c
3132
** ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java
3233
** ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java
3334
** ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java
@@ -259750,40 +259751,46 @@ static int fts5TriCreate(
259750259751
Fts5Tokenizer **ppOut
259751259752
){
259752259753
int rc = SQLITE_OK;
259753-
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
259754-
UNUSED_PARAM(pUnused);
259755-
if( pNew==0 ){
259756-
rc = SQLITE_NOMEM;
259754+
TrigramTokenizer *pNew = 0;
259755+
259756+
if( nArg%2 ){
259757+
rc = SQLITE_ERROR;
259757259758
}else{
259758-
int i;
259759-
pNew->bFold = 1;
259760-
pNew->iFoldParam = 0;
259761-
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
259762-
const char *zArg = azArg[i+1];
259763-
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
259764-
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
259765-
rc = SQLITE_ERROR;
259759+
pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
259760+
UNUSED_PARAM(pUnused);
259761+
if( pNew==0 ){
259762+
rc = SQLITE_NOMEM;
259763+
}else{
259764+
int i;
259765+
pNew->bFold = 1;
259766+
pNew->iFoldParam = 0;
259767+
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
259768+
const char *zArg = azArg[i+1];
259769+
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
259770+
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
259771+
rc = SQLITE_ERROR;
259772+
}else{
259773+
pNew->bFold = (zArg[0]=='0');
259774+
}
259775+
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
259776+
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
259777+
rc = SQLITE_ERROR;
259778+
}else{
259779+
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
259780+
}
259766259781
}else{
259767-
pNew->bFold = (zArg[0]=='0');
259768-
}
259769-
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
259770-
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
259771259782
rc = SQLITE_ERROR;
259772-
}else{
259773-
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
259774259783
}
259775-
}else{
259776-
rc = SQLITE_ERROR;
259777259784
}
259778-
}
259779259785

259780-
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
259781-
rc = SQLITE_ERROR;
259782-
}
259786+
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
259787+
rc = SQLITE_ERROR;
259788+
}
259783259789

259784-
if( rc!=SQLITE_OK ){
259785-
fts5TriDelete((Fts5Tokenizer*)pNew);
259786-
pNew = 0;
259790+
if( rc!=SQLITE_OK ){
259791+
fts5TriDelete((Fts5Tokenizer*)pNew);
259792+
pNew = 0;
259793+
}
259787259794
}
259788259795
}
259789259796
*ppOut = (Fts5Tokenizer*)pNew;

‎libsql-ffi/bundled/bindings/bindgen.rs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -940,7 +940,7 @@ extern "C" {
940940
extern "C" {
941941
pub fn sqlite3_vmprintf(
942942
arg1: *const ::std::os::raw::c_char,
943-
arg2: va_list,
943+
arg2: *mut __va_list_tag,
944944
) -> *mut ::std::os::raw::c_char;
945945
}
946946
extern "C" {
@@ -956,7 +956,7 @@ extern "C" {
956956
arg1: ::std::os::raw::c_int,
957957
arg2: *mut ::std::os::raw::c_char,
958958
arg3: *const ::std::os::raw::c_char,
959-
arg4: va_list,
959+
arg4: *mut __va_list_tag,
960960
) -> *mut ::std::os::raw::c_char;
961961
}
962962
extern "C" {
@@ -2503,7 +2503,7 @@ extern "C" {
25032503
pub fn sqlite3_str_vappendf(
25042504
arg1: *mut sqlite3_str,
25052505
zFormat: *const ::std::os::raw::c_char,
2506-
arg2: va_list,
2506+
arg2: *mut __va_list_tag,
25072507
);
25082508
}
25092509
extern "C" {
@@ -3524,4 +3524,12 @@ extern "C" {
35243524
extern "C" {
35253525
pub static sqlite3_wal_manager: libsql_wal_manager;
35263526
}
3527-
pub type __builtin_va_list = *mut ::std::os::raw::c_char;
3527+
pub type __builtin_va_list = [__va_list_tag; 1usize];
3528+
#[repr(C)]
3529+
#[derive(Debug, Copy, Clone)]
3530+
pub struct __va_list_tag {
3531+
pub gp_offset: ::std::os::raw::c_uint,
3532+
pub fp_offset: ::std::os::raw::c_uint,
3533+
pub overflow_arg_area: *mut ::std::os::raw::c_void,
3534+
pub reg_save_area: *mut ::std::os::raw::c_void,
3535+
}

‎libsql-ffi/bundled/src/sqlite3.c‎

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
** README.md
2929
** configure
3030
** configure.ac
31+
** ext/fts5/fts5_tokenize.c
3132
** ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java
3233
** ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java
3334
** ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java
@@ -259750,40 +259751,46 @@ static int fts5TriCreate(
259750259751
Fts5Tokenizer **ppOut
259751259752
){
259752259753
int rc = SQLITE_OK;
259753-
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
259754-
UNUSED_PARAM(pUnused);
259755-
if( pNew==0 ){
259756-
rc = SQLITE_NOMEM;
259754+
TrigramTokenizer *pNew = 0;
259755+
259756+
if( nArg%2 ){
259757+
rc = SQLITE_ERROR;
259757259758
}else{
259758-
int i;
259759-
pNew->bFold = 1;
259760-
pNew->iFoldParam = 0;
259761-
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
259762-
const char *zArg = azArg[i+1];
259763-
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
259764-
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
259765-
rc = SQLITE_ERROR;
259759+
pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
259760+
UNUSED_PARAM(pUnused);
259761+
if( pNew==0 ){
259762+
rc = SQLITE_NOMEM;
259763+
}else{
259764+
int i;
259765+
pNew->bFold = 1;
259766+
pNew->iFoldParam = 0;
259767+
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
259768+
const char *zArg = azArg[i+1];
259769+
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
259770+
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
259771+
rc = SQLITE_ERROR;
259772+
}else{
259773+
pNew->bFold = (zArg[0]=='0');
259774+
}
259775+
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
259776+
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
259777+
rc = SQLITE_ERROR;
259778+
}else{
259779+
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
259780+
}
259766259781
}else{
259767-
pNew->bFold = (zArg[0]=='0');
259768-
}
259769-
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
259770-
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
259771259782
rc = SQLITE_ERROR;
259772-
}else{
259773-
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
259774259783
}
259775-
}else{
259776-
rc = SQLITE_ERROR;
259777259784
}
259778-
}
259779259785

259780-
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
259781-
rc = SQLITE_ERROR;
259782-
}
259786+
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
259787+
rc = SQLITE_ERROR;
259788+
}
259783259789

259784-
if( rc!=SQLITE_OK ){
259785-
fts5TriDelete((Fts5Tokenizer*)pNew);
259786-
pNew = 0;
259790+
if( rc!=SQLITE_OK ){
259791+
fts5TriDelete((Fts5Tokenizer*)pNew);
259792+
pNew = 0;
259793+
}
259787259794
}
259788259795
}
259789259796
*ppOut = (Fts5Tokenizer*)pNew;

‎libsql-sqlite3/ext/fts5/fts5_tokenize.c‎

Lines changed: 33 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1290,40 +1290,46 @@ static int fts5TriCreate(
12901290
Fts5Tokenizer **ppOut
12911291
){
12921292
int rc = SQLITE_OK;
1293-
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
1294-
UNUSED_PARAM(pUnused);
1295-
if( pNew==0 ){
1296-
rc = SQLITE_NOMEM;
1293+
TrigramTokenizer *pNew = 0;
1294+
1295+
if( nArg%2 ){
1296+
rc = SQLITE_ERROR;
12971297
}else{
1298-
int i;
1299-
pNew->bFold = 1;
1300-
pNew->iFoldParam = 0;
1301-
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
1302-
const char *zArg = azArg[i+1];
1303-
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
1304-
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
1305-
rc = SQLITE_ERROR;
1298+
pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
1299+
UNUSED_PARAM(pUnused);
1300+
if( pNew==0 ){
1301+
rc = SQLITE_NOMEM;
1302+
}else{
1303+
int i;
1304+
pNew->bFold = 1;
1305+
pNew->iFoldParam = 0;
1306+
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
1307+
const char *zArg = azArg[i+1];
1308+
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
1309+
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
1310+
rc = SQLITE_ERROR;
1311+
}else{
1312+
pNew->bFold = (zArg[0]=='0');
1313+
}
1314+
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
1315+
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
1316+
rc = SQLITE_ERROR;
1317+
}else{
1318+
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
1319+
}
13061320
}else{
1307-
pNew->bFold = (zArg[0]=='0');
1308-
}
1309-
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
1310-
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
13111321
rc = SQLITE_ERROR;
1312-
}else{
1313-
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
13141322
}
1315-
}else{
1316-
rc = SQLITE_ERROR;
13171323
}
1318-
}
13191324

1320-
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
1321-
rc = SQLITE_ERROR;
1322-
}
1325+
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
1326+
rc = SQLITE_ERROR;
1327+
}
13231328

1324-
if( rc!=SQLITE_OK ){
1325-
fts5TriDelete((Fts5Tokenizer*)pNew);
1326-
pNew = 0;
1329+
if( rc!=SQLITE_OK ){
1330+
fts5TriDelete((Fts5Tokenizer*)pNew);
1331+
pNew = 0;
1332+
}
13271333
}
13281334
}
13291335
*ppOut = (Fts5Tokenizer*)pNew;

‎libsql/tests/integration_tests.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,26 @@ async fn debug_print_row() {
596596
);
597597
}
598598

599+
#[tokio::test]
600+
async fn fts5_invalid_tokenizer() {
601+
let db = Database::open(":memory:").unwrap();
602+
let conn = db.connect().unwrap();
603+
assert!(conn
604+
.execute(
605+
"CREATE VIRTUAL TABLE t USING fts5(s, tokenize='trigram case_sensitive ')",
606+
(),
607+
)
608+
.await
609+
.is_err());
610+
assert!(conn
611+
.execute(
612+
"CREATE VIRTUAL TABLE t USING fts5(s, tokenize='trigram remove_diacritics ')",
613+
(),
614+
)
615+
.await
616+
.is_err());
617+
}
618+
599619
#[cfg(feature = "serde")]
600620
#[tokio::test]
601621
async fn deserialize_row() {

0 commit comments

Comments
 (0)