Skip to content

Commit f017809

Browse files
KKouldKKould
authored
fix: attach table does not carry the indexes of the original table (#18465)
* fix: attach table does not carry the indexes of the original table * chore: fix test * chore: fix test * chore: fix test * chore: fix test * chore: fix test * chore: fix test * chore: add more index type on attach table test * chore: fix test * chore: fix test
1 parent ca842e5 commit f017809

File tree

5 files changed

+88
-0
lines changed

5 files changed

+88
-0
lines changed

โ€Žsrc/query/ee/src/attach_table/handler.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ impl AttachTableHandler for RealAttachTableHandler {
105105
field_comments,
106106
drop_on: None,
107107
statistics: stat,
108+
indexes: snapshot_hint.indexes,
108109
..Default::default()
109110
};
110111
let req = CreateTableReq {

โ€Žsrc/query/storages/fuse/src/fuse_table.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,7 @@ impl FuseTable {
728728
table_info.meta.comment = comments.table_comment;
729729
// TODO assert about field comments
730730
table_info.meta.field_comments = comments.field_comments;
731+
table_info.meta.indexes = hint.indexes;
731732
}
732733
}
733734

โ€Žsrc/query/storages/fuse/src/operations/snapshot_hint.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
use std::collections::BTreeMap;
1516
use std::io::Read;
1617
use std::io::Write;
1718
use std::time::Instant;
1819

1920
use bytes::Buf;
2021
use databend_common_catalog::table_context::TableContext;
2122
use databend_common_exception::Result;
23+
use databend_common_meta_app::schema::TableIndex;
2224
use databend_common_meta_app::schema::TableMeta;
2325
use log::info;
2426
use log::warn;
@@ -46,6 +48,8 @@ impl<'a> SnapshotHintWriter<'a> {
4648
pub struct SnapshotHint {
4749
pub snapshot_full_path: String,
4850
pub entity_comments: EntityComments,
51+
#[serde(default)]
52+
pub indexes: BTreeMap<String, TableIndex>,
4953
}
5054

5155
impl SnapshotHint {
@@ -122,6 +126,7 @@ async fn try_read_legacy_hint(
122126
table_comment: "".to_string(),
123127
field_comments: vec![],
124128
},
129+
indexes: Default::default(),
125130
}))
126131
}
127132
Err(e) if e.kind() == opendal::ErrorKind::NotFound => Ok(None),
@@ -167,6 +172,7 @@ impl SnapshotHintWriter<'_> {
167172
let hint = SnapshotHint {
168173
snapshot_full_path: last_snapshot_path,
169174
entity_comments: EntityComments::from(table_meta),
175+
indexes: table_meta.indexes.clone(),
170176
};
171177

172178
let mut bytes = vec![];

โ€Žtests/suites/5_ee/04_attach_read_only/04_0008_attach_table_index.result

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
init table
2+
4
3+
check index
4+
idx_inverted INVERTED default att_phrases
5+
idx_text NGRAM default att_phrases
6+
idx_vector VECTOR default att_phrases
7+
>>>> REFRESH NGRAM INDEX idx_text ON phrases;
8+
idx_inverted INVERTED default att_phrases
9+
idx_text NGRAM default att_phrases
10+
idx_vector VECTOR default att_phrases
11+
>>>> DROP NGRAM INDEX idx_text ON phrases;
12+
idx_inverted INVERTED default att_phrases
13+
idx_text NGRAM default att_phrases
14+
idx_vector VECTOR default att_phrases
15+
>>>> REFRESH INVERTED INDEX idx_inverted ON phrases;
16+
idx_inverted INVERTED default att_phrases
17+
idx_text NGRAM default att_phrases
18+
idx_vector VECTOR default att_phrases
19+
>>>> DROP INVERTED INDEX idx_inverted ON phrases;
20+
idx_inverted INVERTED default att_phrases
21+
idx_text NGRAM default att_phrases
22+
idx_vector VECTOR default att_phrases
23+
>>>> DROP VECTOR INDEX idx_vector ON phrases;
24+
idx_inverted INVERTED default att_phrases
25+
idx_text NGRAM default att_phrases
26+
idx_vector VECTOR default att_phrases

โ€Žtests/suites/5_ee/04_attach_read_only/04_0008_attach_table_index.sh

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env bash
2+
3+
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
4+
. "$CURDIR"/../../../shell_env.sh
5+
6+
echo "drop table if exists phrases" | $BENDSQL_CLIENT_CONNECT
7+
echo "create table phrases (id INT, text STRING, text_0 STRING, embedding Vector(8), NGRAM INDEX idx_text (text), VECTOR INDEX idx_vector (embedding) m=10 ef_construct=40 distance='cosine') 's3://testbucket/admin/data/' connection=(access_key_id ='minioadmin' secret_access_key ='minioadmin' endpoint_url='${STORAGE_S3_ENDPOINT_URL}')" | $BENDSQL_CLIENT_CONNECT
8+
echo "CREATE INVERTED INDEX idx_inverted ON phrases(text_0);" | $BENDSQL_CLIENT_CONNECT
9+
10+
echo "init table"
11+
echo "INSERT INTO phrases VALUES(1, 'apple banana cherry', 'apple banana cherry', [0.61418788, 0.34545306, 0.14638622, 0.53249639, 0.09139293, 0.84940919, 0.105433, 0.4156201]),(2, 'banana date fig', 'banana date fig', [0.21828953, 0.87852734, 0.64221122, 0.24536394, 0.81689593, 0.86341877, 0.7218334, 0.45028494]),(3, 'cherry elderberry fig', 'cherry elderberry fig', [0.43279006, 0.45523681, 0.76060274, 0.66284758, 0.19131476, 0.13564463, 0.88712212, 0.93279565]),(4, 'date grape kiwi', 'date grape kiwi', [0.79671359, 0.86079789, 0.94477631, 0.5116732, 0.29733205, 0.33645561, 0.41380333, 0.75909903])" | $BENDSQL_CLIENT_CONNECT
12+
13+
storage_prefix=$(mysql -uroot -h127.0.0.1 -P3307 -e "set global hide_options_in_show_create_table=0;show create table phrases" | grep -i snapshot_location | awk -F'SNAPSHOT_LOCATION='"'"'|_ss' '{print $2}')
14+
15+
# attach table
16+
echo "drop table if exists att_phrases" | $BENDSQL_CLIENT_CONNECT
17+
echo "attach table att_phrases 's3://testbucket/admin/data/$storage_prefix' connection=(access_key_id ='minioadmin' secret_access_key ='minioadmin' endpoint_url='${STORAGE_S3_ENDPOINT_URL}');" | $BENDSQL_CLIENT_CONNECT
18+
19+
# check index
20+
echo "check index"
21+
echo "select name, type, database, table from system.indexes where database = 'default' and table = 'att_phrases';" | $BENDSQL_CLIENT_CONNECT
22+
23+
# refresh ngram index
24+
stmt "REFRESH NGRAM INDEX idx_text ON phrases;"
25+
echo "drop table if exists att_phrases" | $BENDSQL_CLIENT_CONNECT
26+
echo "attach table att_phrases 's3://testbucket/admin/data/$storage_prefix' connection=(access_key_id ='minioadmin' secret_access_key ='minioadmin' endpoint_url='${STORAGE_S3_ENDPOINT_URL}');" | $BENDSQL_CLIENT_CONNECT
27+
echo "select name, type, database, table from system.indexes where database = 'default' and table = 'att_phrases';" | $BENDSQL_CLIENT_CONNECT
28+
29+
# drop ngram index
30+
stmt "DROP NGRAM INDEX idx_text ON phrases;"
31+
echo "drop table if exists att_phrases" | $BENDSQL_CLIENT_CONNECT
32+
echo "attach table att_phrases 's3://testbucket/admin/data/$storage_prefix' connection=(access_key_id ='minioadmin' secret_access_key ='minioadmin' endpoint_url='${STORAGE_S3_ENDPOINT_URL}');" | $BENDSQL_CLIENT_CONNECT
33+
echo "select name, type, database, table from system.indexes where database = 'default' and table = 'att_phrases';" | $BENDSQL_CLIENT_CONNECT
34+
35+
# refresh inverted index
36+
stmt "REFRESH INVERTED INDEX idx_inverted ON phrases;"
37+
echo "drop table if exists att_phrases" | $BENDSQL_CLIENT_CONNECT
38+
echo "attach table att_phrases 's3://testbucket/admin/data/$storage_prefix' connection=(access_key_id ='minioadmin' secret_access_key ='minioadmin' endpoint_url='${STORAGE_S3_ENDPOINT_URL}');" | $BENDSQL_CLIENT_CONNECT
39+
echo "select name, type, database, table from system.indexes where database = 'default' and table = 'att_phrases';" | $BENDSQL_CLIENT_CONNECT
40+
41+
# drop inverted index
42+
stmt "DROP INVERTED INDEX idx_inverted ON phrases;"
43+
echo "drop table if exists att_phrases" | $BENDSQL_CLIENT_CONNECT
44+
echo "attach table att_phrases 's3://testbucket/admin/data/$storage_prefix' connection=(access_key_id ='minioadmin' secret_access_key ='minioadmin' endpoint_url='${STORAGE_S3_ENDPOINT_URL}');" | $BENDSQL_CLIENT_CONNECT
45+
echo "select name, type, database, table from system.indexes where database = 'default' and table = 'att_phrases';" | $BENDSQL_CLIENT_CONNECT
46+
47+
# drop vector index
48+
stmt "DROP VECTOR INDEX idx_vector ON phrases;"
49+
echo "drop table if exists att_phrases" | $BENDSQL_CLIENT_CONNECT
50+
echo "attach table att_phrases 's3://testbucket/admin/data/$storage_prefix' connection=(access_key_id ='minioadmin' secret_access_key ='minioadmin' endpoint_url='${STORAGE_S3_ENDPOINT_URL}');" | $BENDSQL_CLIENT_CONNECT
51+
echo "select name, type, database, table from system.indexes where database = 'default' and table = 'att_phrases';" | $BENDSQL_CLIENT_CONNECT
52+
53+
echo "drop table if exists phrases" | $BENDSQL_CLIENT_CONNECT
54+
echo "drop table if exists att_phrases" | $BENDSQL_CLIENT_CONNECT

0 commit comments

Comments
ย (0)