Add FTS4 unicode61 tokenizer support

Author: looi

sqlite3.go

@@ -8,7 +8,7 @@ package sqlite3
 /*
 #cgo CFLAGS: -std=gnu99
 #cgo CFLAGS: -DSQLITE_ENABLE_RTREE -DSQLITE_THREADSAFE
-#cgo CFLAGS: -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS3_PARENTHESIS
+#cgo CFLAGS: -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS4_UNICODE61
 #include <sqlite3-binding.h>
 #include <stdlib.h>
 #include <string.h>

sqlite3_fts3_test.go

@@ -81,3 +81,47 @@ func TestFTS3(t *testing.T) {
 		t.Fatal("Result should be only one")
 	}
 }
+
+func TestFTS4(t *testing.T) {
+	tempFilename := TempFilename()
+	db, err := sql.Open("sqlite3", tempFilename)
+	if err != nil {
+		t.Fatal("Failed to open database:", err)
+	}
+	defer os.Remove(tempFilename)
+	defer db.Close()
+
+	_, err = db.Exec("DROP TABLE foo")
+	_, err = db.Exec("CREATE VIRTUAL TABLE foo USING fts4(tokenize=unicode61, id INTEGER PRIMARY KEY, value TEXT)")
+	if err != nil {
+		t.Fatal("Failed to create table:", err)
+	}
+
+	_, err = db.Exec("INSERT INTO foo(id, value) VALUES(?, ?)", 1, `février`)
+	if err != nil {
+		t.Fatal("Failed to insert value:", err)
+	}
+
+	rows, err := db.Query("SELECT value FROM foo WHERE value MATCH 'fevrier'")
+	if err != nil {
+		t.Fatal("Unable to query foo table:", err)
+	}
+	defer rows.Close()
+
+	var value string
+	if !rows.Next() {
+		t.Fatal("Result should be only one")
+	}
+
+	if err := rows.Scan(&value); err != nil {
+		t.Fatal("Unable to scan results:", err)
+	}
+
+	if value != `février` {
+		t.Fatal("Value should be `février`, but:", value)
+	}
+
+	if rows.Next() {
+		t.Fatal("Result should be only one")
+	}
+}