feat(query): Support create/drop Vector(HNSW) index (#18077)

* feat(query): Support create/drop vector index

* fix

* fix tests

* fix tests

Author: b41sh

Cargo.lock

@@ -84,7 +84,7 @@ members = [
     "src/query/ee_features/storage_encryption",
     "src/query/ee_features/stream_handler",
     "src/query/ee_features/storage_quota",
-    "src/query/ee_features/inverted_index",
+    "src/query/ee_features/table_index",
     "src/query/ee_features/virtual_column",
     "src/query/ee_features/resources_management",
     "src/query/service",
@@ -193,14 +193,13 @@ databend-enterprise-attach-table = { path = "src/query/ee_features/attach_table"
 databend-enterprise-data-mask-feature = { path = "src/query/ee_features/data_mask" }
 databend-enterprise-fail-safe = { path = "src/query/ee_features/fail_safe" }
 databend-enterprise-hilbert-clustering = { path = "src/query/ee_features/hilbert_clustering" }
-databend-enterprise-inverted-index = { path = "src/query/ee_features/inverted_index" }
 databend-enterprise-meta = { path = "src/meta/ee" }
-databend-enterprise-ngram-index = { path = "src/query/ee_features/ngram_index" }
 databend-enterprise-query = { path = "src/query/ee" }
 databend-enterprise-resources-management = { path = "src/query/ee_features/resources_management" }
 databend-enterprise-storage-encryption = { path = "src/query/ee_features/storage_encryption" }
 databend-enterprise-storage-quota = { path = "src/query/ee_features/storage_quota" }
 databend-enterprise-stream-handler = { path = "src/query/ee_features/stream_handler" }
+databend-enterprise-table-index = { path = "src/query/ee_features/table_index" }
 databend-enterprise-vacuum-handler = { path = "src/query/ee_features/vacuum_handler" }
 databend-enterprise-virtual-column = { path = "src/query/ee_features/virtual_column" }
 databend-functions-scalar-arithmetic = { path = "src/query/functions/src/scalars/arithmetic" }

Cargo.toml

@@ -81,6 +81,8 @@ pub enum Feature {
     WorkloadGroup,
     #[serde(alias = "system_history", alias = "SYSTEM_HISTORY")]
     SystemHistory,
+    #[serde(alias = "vector_index", alias = "VECTOR_INDEX")]
+    VectorIndex,
     #[serde(other)]
     Unknown,
 }
@@ -131,6 +133,7 @@ impl fmt::Display for Feature {
             Feature::NgramIndex => write!(f, "ngram_index"),
             Feature::WorkloadGroup => write!(f, "workload_group"),
             Feature::SystemHistory => write!(f, "system_history"),
+            Feature::VectorIndex => write!(f, "vector_index"),
             Feature::Unknown => write!(f, "unknown"),
         }
     }
@@ -179,7 +182,8 @@ impl Feature {
             | (Feature::AttacheTable, Feature::AttacheTable)
             | (Feature::StorageEncryption, Feature::StorageEncryption)
             | (Feature::HilbertClustering, Feature::HilbertClustering)
-            | (Feature::NgramIndex, Feature::NgramIndex) => Ok(true),
+            | (Feature::NgramIndex, Feature::NgramIndex)
+            | (Feature::VectorIndex, Feature::VectorIndex) => Ok(true),
             (_, _) => Ok(false),
         }
     }
@@ -363,6 +367,11 @@ mod tests {
             serde_json::from_str::<Feature>("\"system_history\"").unwrap()
         );
 
+        assert_eq!(
+            Feature::VectorIndex,
+            serde_json::from_str::<Feature>("\"VectorIndex\"").unwrap()
+        );
+
         assert_eq!(
             Feature::Unknown,
             serde_json::from_str::<Feature>("\"ssss\"").unwrap()

src/common/license/src/license.rs

@@ -153,6 +153,7 @@ use databend_common_meta_app::schema::TableIdList;
 use databend_common_meta_app::schema::TableIdToName;
 use databend_common_meta_app::schema::TableIdent;
 use databend_common_meta_app::schema::TableIndex;
+use databend_common_meta_app::schema::TableIndexType;
 use databend_common_meta_app::schema::TableInfo;
 use databend_common_meta_app::schema::TableMeta;
 use databend_common_meta_app::schema::TableNameIdent;
@@ -1021,12 +1022,12 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
                             IndexColumnIdNotFound::new(*column_id, &index.name),
                         )));
                     }
-                    if index_column_ids.contains(column_id) {
+                    if index_column_ids.contains(&(*column_id, index.index_type.clone())) {
                         return Err(KVAppError::AppError(AppError::DuplicatedIndexColumnId(
                             DuplicatedIndexColumnId::new(*column_id, &index.name),
                         )));
                     }
-                    index_column_ids.insert(column_id);
+                    index_column_ids.insert((*column_id, index.index_type.clone()));
                 }
             }
         }
@@ -2450,7 +2451,7 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
 
             // column_id can not be duplicated
             for (name, index) in indexes.iter() {
-                if *name == req.name {
+                if *name == req.name || index.index_type != req.index_type {
                     continue;
                 }
                 for column_id in &req.column_ids {
@@ -2474,6 +2475,7 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
                         &req.tenant,
                         req.table_id,
                         &req.name,
+                        &req.index_type,
                         &old_index.version,
                     )?;
                     mark_delete_op = Some(TxnOp::put(m_key, m_value));
@@ -2545,15 +2547,34 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
             let indexes = &mut table_meta.indexes;
             if !indexes.contains_key(&req.name) && !req.if_exists {
                 return Err(KVAppError::AppError(AppError::UnknownIndex(
-                    UnknownError::<IndexName>::new(req.name.clone(), "drop table index"),
+                    UnknownError::<IndexName>::new(
+                        req.name.clone(),
+                        format!("drop {} index", req.index_type),
+                    ),
                 )));
             }
             let Some(index) = indexes.remove(&req.name) else {
                 return Ok(());
             };
+            if index.index_type != req.index_type {
+                return Err(KVAppError::AppError(AppError::UnknownIndex(
+                    UnknownError::<IndexName>::new(
+                        req.name.clone(),
+                        format!(
+                            "drop {} index, but the index is {}",
+                            req.index_type, index.index_type
+                        ),
+                    ),
+                )));
+            }
 
-            let (m_key, m_value) =
-                mark_table_index_as_deleted(&req.tenant, req.table_id, &req.name, &index.version)?;
+            let (m_key, m_value) = mark_table_index_as_deleted(
+                &req.tenant,
+                req.table_id,
+                &req.name,
+                &req.index_type,
+                &index.version,
+            )?;
 
             let txn_req = TxnRequest::new(
                 vec![
@@ -4248,15 +4269,21 @@ pub fn mark_table_index_as_deleted(
     tenant: &Tenant,
     table_id: u64,
     index_name: &str,
+    index_type: &TableIndexType,
     index_version: &str,
 ) -> Result<(String, Vec<u8>), MetaError> {
     let marked_deleted_table_index_id_ident = MarkedDeletedTableIndexIdIdent::new_generic(
         tenant,
         MarkedDeletedTableIndexId::new(table_id, index_name.to_owned(), index_version.to_owned()),
     );
+    let deleted_index_type = match index_type {
+        TableIndexType::Inverted => MarkedDeletedIndexType::INVERTED,
+        TableIndexType::Ngram => MarkedDeletedIndexType::NGRAM,
+        TableIndexType::Vector => MarkedDeletedIndexType::VECTOR,
+    };
     let marked_deleted_table_index_meta = MarkedDeletedIndexMeta {
         dropped_on: Utc::now(),
-        index_type: MarkedDeletedIndexType::INVERTED,
+        index_type: deleted_index_type,
     };
 
     Ok((