From 912701ea3f661ccafe64b8c4ea2fc84cb4c157d1 Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 7 Oct 2025 17:53:43 +0300 Subject: [PATCH 1/3] Add Index tests --- .../ut/indexes/kqp_indexes_fulltext_ut.cpp | 128 ++++++++++++++++++ ydb/core/kqp/ut/indexes/ya.make | 3 +- 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp new file mode 100644 index 000000000000..84385036e1bd --- /dev/null +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp @@ -0,0 +1,128 @@ +#include + + +namespace NKikimr::NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +Y_UNIT_TEST_SUITE(KqpFulltextIndexes) { + +TKikimrRunner Kikimr() { + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableFulltextIndex(true); + auto settings = TKikimrSettings().SetFeatureFlags(featureFlags); + return TKikimrRunner(settings); +} + +void CreateTexts(NQuery::TQueryClient& db) { + TString query = R"sql( + CREATE TABLE `/Root/Texts` ( + Key Uint64, + Text String, + Data String, + PRIMARY KEY (Key) + ); + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +} + +void FillTexts(NQuery::TQueryClient& db) { + TString query = R"sql( + INSERT INTO `/Root/Texts` (Key, Text, Data) VALUES + (100, "Cats chase small animals.", "cats data"), + (200, "Dogs chase small cats.", "dogs data"), + (300, "Cats love cats.", "cats cats data"), + (400, "Fox love dogs.", "fox data") + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +} + +void AddIndex(NQuery::TQueryClient& db) { + TString query = R"sql( + ALTER TABLE `/Root/Texts` ADD INDEX fulltext_idx + GLOBAL USING fulltext + ON (Text) + WITH (layout=flat, tokenizer=standard, use_filter_lowercase=true) + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +} + +void AddIndexCovered(NQuery::TQueryClient& db) { + TString query = R"sql( + ALTER TABLE `/Root/Texts` ADD INDEX fulltext_idx + GLOBAL USING fulltext + ON (Text) COVER (Data) + WITH (layout=flat, tokenizer=standard, use_filter_lowercase=true) + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +} + +TResultSet ReadIndex(NQuery::TQueryClient& db) { + TString query = R"sql( + SELECT * FROM `/Root/Texts/fulltext_idx/indexImplTable`; + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + return result.GetResultSet(0); +} + +Y_UNIT_TEST(AddIndex) { + auto kikimr = Kikimr(); + auto db = kikimr.GetQueryClient(); + + CreateTexts(db); + FillTexts(db); + AddIndex(db); + + auto index = ReadIndex(db); + CompareYson(R"([ + [[100u];"animals"]; + [[100u];"cats"]; + [[200u];"cats"]; + [[300u];"cats"]; + [[100u];"chase"]; + [[200u];"chase"]; + [[200u];"dogs"]; + [[400u];"dogs"]; + [[400u];"fox"]; + [[300u];"love"]; + [[400u];"love"]; + [[100u];"small"]; + [[200u];"small"] + ])", NYdb::FormatResultSetYson(index)); +} + +Y_UNIT_TEST(AddIndexCovered) { + auto kikimr = Kikimr(); + auto db = kikimr.GetQueryClient(); + + CreateTexts(db); + FillTexts(db); + AddIndexCovered(db); + + auto index = ReadIndex(db); + CompareYson(R"([ + [["cats data"];[100u];"animals"]; + [["cats data"];[100u];"cats"]; + [["dogs data"];[200u];"cats"]; + [["cats cats data"];[300u];"cats"]; + [["cats data"];[100u];"chase"]; + [["dogs data"];[200u];"chase"]; + [["dogs data"];[200u];"dogs"]; + [["fox data"];[400u];"dogs"]; + [["fox data"];[400u];"fox"]; + [["cats cats data"];[300u];"love"]; + [["fox data"];[400u];"love"]; + [["cats data"];[100u];"small"]; + [["dogs data"];[200u];"small"] + ])", NYdb::FormatResultSetYson(index)); +} + +} + +} diff --git a/ydb/core/kqp/ut/indexes/ya.make b/ydb/core/kqp/ut/indexes/ya.make index 5451c7ac2fa0..0b5e3d1cc618 100644 --- a/ydb/core/kqp/ut/indexes/ya.make +++ b/ydb/core/kqp/ut/indexes/ya.make @@ -11,9 +11,10 @@ ELSE() ENDIF() SRCS( - kqp_indexes_ut.cpp + kqp_indexes_fulltext_ut.cpp kqp_indexes_multishard_ut.cpp kqp_indexes_prefixed_vector_ut.cpp + kqp_indexes_ut.cpp kqp_indexes_vector_ut.cpp ) From 56ab6a7da345fa509465bb9d351aa2d6ed981929 Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 7 Oct 2025 18:28:09 +0300 Subject: [PATCH 2/3] more test todos --- .../ut/indexes/kqp_indexes_fulltext_ut.cpp | 209 +++++++++++++++++- 1 file changed, 202 insertions(+), 7 deletions(-) diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp index 84385036e1bd..c346ff8a2639 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp @@ -28,13 +28,22 @@ void CreateTexts(NQuery::TQueryClient& db) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } -void FillTexts(NQuery::TQueryClient& db) { +void UpsertTexts(NQuery::TQueryClient& db) { TString query = R"sql( - INSERT INTO `/Root/Texts` (Key, Text, Data) VALUES + UPSERT INTO `/Root/Texts` (Key, Text, Data) VALUES (100, "Cats chase small animals.", "cats data"), (200, "Dogs chase small cats.", "dogs data"), (300, "Cats love cats.", "cats cats data"), - (400, "Fox love dogs.", "fox data") + (400, "Foxes love dogs.", "fox data") + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +} + +void UpsertRow(NQuery::TQueryClient& db) { + TString query = R"sql( + UPSERT INTO `/Root/Texts` (Key, Text, Data) VALUES + (250, "Dogs are big animals.", "new dogs data") )sql"; auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -76,7 +85,7 @@ Y_UNIT_TEST(AddIndex) { auto db = kikimr.GetQueryClient(); CreateTexts(db); - FillTexts(db); + UpsertTexts(db); AddIndex(db); auto index = ReadIndex(db); @@ -89,7 +98,7 @@ Y_UNIT_TEST(AddIndex) { [[200u];"chase"]; [[200u];"dogs"]; [[400u];"dogs"]; - [[400u];"fox"]; + [[400u];"foxes"]; [[300u];"love"]; [[400u];"love"]; [[100u];"small"]; @@ -102,7 +111,7 @@ Y_UNIT_TEST(AddIndexCovered) { auto db = kikimr.GetQueryClient(); CreateTexts(db); - FillTexts(db); + UpsertTexts(db); AddIndexCovered(db); auto index = ReadIndex(db); @@ -115,13 +124,199 @@ Y_UNIT_TEST(AddIndexCovered) { [["dogs data"];[200u];"chase"]; [["dogs data"];[200u];"dogs"]; [["fox data"];[400u];"dogs"]; - [["fox data"];[400u];"fox"]; + [["fox data"];[400u];"foxes"]; [["cats cats data"];[300u];"love"]; [["fox data"];[400u];"love"]; [["cats data"];[100u];"small"]; [["dogs data"];[200u];"small"] ])", NYdb::FormatResultSetYson(index)); } + +Y_UNIT_TEST(UpsertRow) { + auto kikimr = Kikimr(); + auto db = kikimr.GetQueryClient(); + + CreateTexts(db); + UpsertTexts(db); + AddIndex(db); + return; // TODO: upserts are not implemented + UpsertRow(db); + + auto index = ReadIndex(db); + CompareYson(R"([ + + ])", NYdb::FormatResultSetYson(index)); +} + +Y_UNIT_TEST(UpsertRowCovered) { + auto kikimr = Kikimr(); + auto db = kikimr.GetQueryClient(); + + CreateTexts(db); + UpsertTexts(db); + AddIndexCovered(db); + return; // TODO: upserts are not implemented + UpsertRow(db); + + auto index = ReadIndex(db); + CompareYson(R"([ + + ])", NYdb::FormatResultSetYson(index)); +} + +Y_UNIT_TEST(InsertRow) { + // TODO: inserts are not implemented +} + +Y_UNIT_TEST(InsertRowCovered) { + // TODO: inserts are not implemented +} + +Y_UNIT_TEST(DeleteRow) { + // TODO: deletes are not implemented + + // TODO: test delete by key and filter +} + +Y_UNIT_TEST(DeleteRowCovered) { + // TODO: deletes are not implemented + + // TODO: test delete by key and filter +} + +Y_UNIT_TEST(UpdateRow) { + // TODO: deletes are not implemented + + // TODO: test update of key, text, data +} + +Y_UNIT_TEST(UpdateRowCovered) { + // TODO: deletes are not implemented + + // TODO: test update of key, text, data +} + +Y_UNIT_TEST(CreateTable) { + auto kikimr = Kikimr(); + auto db = kikimr.GetQueryClient(); + + { // CreateTexts + TString query = R"sql( + CREATE TABLE `/Root/Texts` ( + Key Uint64, + Text String, + Data String, + PRIMARY KEY (Key), + INDEX fulltext_idx + GLOBAL USING fulltext + ON (Text) + WITH (layout=flat, tokenizer=standard, use_filter_lowercase=true) + ); + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + return; // TODO: upserts are not implemented + UpsertTexts(db); + + auto index = ReadIndex(db); + CompareYson(R"([ + [[100u];"animals"]; + [[100u];"cats"]; + [[200u];"cats"]; + [[300u];"cats"]; + [[100u];"chase"]; + [[200u];"chase"]; + [[200u];"dogs"]; + [[400u];"dogs"]; + [[400u];"foxes"]; + [[300u];"love"]; + [[400u];"love"]; + [[100u];"small"]; + [[200u];"small"] + ])", NYdb::FormatResultSetYson(index)); +} + +Y_UNIT_TEST(CreateTableCovered) { + auto kikimr = Kikimr(); + auto db = kikimr.GetQueryClient(); + + { // CreateTexts + TString query = R"sql( + CREATE TABLE `/Root/Texts` ( + Key Uint64, + Text String, + Data String, + PRIMARY KEY (Key), + INDEX fulltext_idx + GLOBAL USING fulltext + ON (Text) COVER (Data) + WITH (layout=flat, tokenizer=standard, use_filter_lowercase=true) + ); + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + return; // TODO: upserts are not implemented + UpsertTexts(db); + + auto index = ReadIndex(db); + CompareYson(R"([ + [["cats data"];[100u];"animals"]; + [["cats data"];[100u];"cats"]; + [["dogs data"];[200u];"cats"]; + [["cats cats data"];[300u];"cats"]; + [["cats data"];[100u];"chase"]; + [["dogs data"];[200u];"chase"]; + [["dogs data"];[200u];"dogs"]; + [["fox data"];[400u];"dogs"]; + [["fox data"];[400u];"foxes"]; + [["cats cats data"];[300u];"love"]; + [["fox data"];[400u];"love"]; + [["cats data"];[100u];"small"]; + [["dogs data"];[200u];"small"] + ])", NYdb::FormatResultSetYson(index)); +} + +Y_UNIT_TEST(NoBulkUpsert) { + // NKikimrConfig::TFeatureFlags featureFlags; + // featureFlags.SetEnableVectorIndex(true); + // auto setting = NKikimrKqp::TKqpSetting(); + // auto serverSettings = TKikimrSettings() + // .SetFeatureFlags(featureFlags) + // .SetKqpSettings({setting}); + + // TKikimrRunner kikimr(serverSettings); + // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + + // auto db = kikimr.GetTableClient(); + // auto session = DoCreateTableAndVectorIndex(db, true); + + // const TString originalPostingTable = ReadTablePartToYson(session, "/Root/TestTable/index1/indexImplPostingTable"); + + // // BulkUpsert to the table with index should fail + // { + // NYdb::TValueBuilder rows; + // rows.BeginList(); + // rows.AddListItem() + // .BeginStruct() + // .AddMember("pk").Int64(11) + // .AddMember("emb").String("\x77\x77\x03") + // .AddMember("data").String("43") + // .EndStruct(); + // rows.EndList(); + // auto result = db.BulkUpsert("/Root/TestTable", rows.Build()).GetValueSync(); + // auto issues = result.GetIssues().ToString(); + // UNIT_ASSERT_C(result.GetStatus() == EStatus::SCHEME_ERROR, result.GetStatus()); + // UNIT_ASSERT_C(issues.contains("Only async-indexed tables are supported by BulkUpsert"), issues); + // } + + // const TString postingTable1_bulk = ReadTablePartToYson(session, "/Root/TestTable/index1/indexImplPostingTable"); + // UNIT_ASSERT_STRINGS_EQUAL(originalPostingTable, postingTable1_bulk); +} + +Y_UNIT_TEST(NoIndexImplTableUpdates) { +} } From 4709dde8c136699f582e76cc2713e56665ea1bc5 Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 8 Oct 2025 15:19:06 +0300 Subject: [PATCH 3/3] fill tests --- .../ut/indexes/kqp_indexes_fulltext_ut.cpp | 73 ++++++++++--------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp index c346ff8a2639..a6958b7352d2 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_fulltext_ut.cpp @@ -279,43 +279,48 @@ Y_UNIT_TEST(CreateTableCovered) { } Y_UNIT_TEST(NoBulkUpsert) { - // NKikimrConfig::TFeatureFlags featureFlags; - // featureFlags.SetEnableVectorIndex(true); - // auto setting = NKikimrKqp::TKqpSetting(); - // auto serverSettings = TKikimrSettings() - // .SetFeatureFlags(featureFlags) - // .SetKqpSettings({setting}); - - // TKikimrRunner kikimr(serverSettings); - // kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); - - // auto db = kikimr.GetTableClient(); - // auto session = DoCreateTableAndVectorIndex(db, true); - - // const TString originalPostingTable = ReadTablePartToYson(session, "/Root/TestTable/index1/indexImplPostingTable"); - - // // BulkUpsert to the table with index should fail - // { - // NYdb::TValueBuilder rows; - // rows.BeginList(); - // rows.AddListItem() - // .BeginStruct() - // .AddMember("pk").Int64(11) - // .AddMember("emb").String("\x77\x77\x03") - // .AddMember("data").String("43") - // .EndStruct(); - // rows.EndList(); - // auto result = db.BulkUpsert("/Root/TestTable", rows.Build()).GetValueSync(); - // auto issues = result.GetIssues().ToString(); - // UNIT_ASSERT_C(result.GetStatus() == EStatus::SCHEME_ERROR, result.GetStatus()); - // UNIT_ASSERT_C(issues.contains("Only async-indexed tables are supported by BulkUpsert"), issues); - // } - - // const TString postingTable1_bulk = ReadTablePartToYson(session, "/Root/TestTable/index1/indexImplPostingTable"); - // UNIT_ASSERT_STRINGS_EQUAL(originalPostingTable, postingTable1_bulk); + auto kikimr = Kikimr(); + auto db = kikimr.GetQueryClient(); + + CreateTexts(db); + AddIndex(db); + + { // BulkUpsert + NYdb::TValueBuilder rows; + rows.BeginList(); + rows.AddListItem() + .BeginStruct() + .AddMember("Key").Uint64(900) + .EndStruct(); + rows.EndList(); + auto result = kikimr.GetTableClient().BulkUpsert("/Root/Texts", rows.Build()).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Only async-indexed tables are supported by BulkUpsert"); + } + + auto index = ReadIndex(db); + CompareYson(R"([])", NYdb::FormatResultSetYson(index)); } Y_UNIT_TEST(NoIndexImplTableUpdates) { + auto kikimr = Kikimr(); + auto db = kikimr.GetQueryClient(); + + CreateTexts(db); + AddIndex(db); + + { // UpsertRow + TString query = R"sql( + UPSERT INTO `/Root/Texts/fulltext_idx/indexImplTable` (__ydb_token, Key) VALUES + ("dogs", 901) + )sql"; + auto result = db.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Writing to index implementation tables is not allowed"); + } + + auto index = ReadIndex(db); + CompareYson(R"([])", NYdb::FormatResultSetYson(index)); } }