refactor(query): auto generate virtual columns for variant column (#17707)

* refactor(query): auto generate virtual columns for variant column

* fix

* support update

* fix unit tests

* fix explain

* fix tests

* support cast type

* fix

* fix tests

* add virutal column builder tests

* fix virtual block count

* fix tests

* fix virtual column conflict

* fix

* create virtual column for json array

* fix

* virtual column builder add sample

Author: b41sh

Cargo.lock

@@ -351,7 +351,7 @@ jaq-interpret = "1.5.0"
 jaq-parse = "1.0.3"
 jaq-std = "1.6.0"
 jiff = { version = "0.2.1", features = ["serde", "tzdb-bundle-always"] }
-jsonb = "0.5.0"
+jsonb = "0.5.1"
 jwt-simple = { version = "0.12.10", default-features = false, features = ["pure-rust"] }
 lenient_semver = "0.4.2"
 levenshtein_automata = "0.2.1"

Cargo.toml

@@ -87,7 +87,7 @@ fn test_decode_v122_table_meta() -> anyhow::Result<()> {
         202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99,
         202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 226, 1, 1, 1, 234, 1, 6, 10, 1, 97, 18, 1, 98,
         130, 2, 55, 10, 31, 10, 7, 102, 105, 101, 108, 100, 95, 48, 18, 2, 10, 0, 18, 2, 50, 0, 18,
-        4, 58, 2, 10, 0, 24, 19, 32, 129, 188, 193, 150, 11, 18, 6, 10, 1, 97, 18, 1, 98, 24, 130,
+        4, 58, 2, 10, 0, 24, 19, 32, 128, 188, 193, 150, 11, 18, 6, 10, 1, 97, 18, 1, 98, 24, 129,
         188, 193, 150, 11, 32, 10, 160, 6, 122, 168, 6, 24, 160, 6, 122, 168, 6, 24,
     ];
 
@@ -185,7 +185,7 @@ fn s(ss: impl ToString) -> String {
 fn test_decode_v122_virtual_data_schema() -> anyhow::Result<()> {
     let virtual_data_schema = vec![
         10, 31, 10, 7, 102, 105, 101, 108, 100, 95, 48, 18, 2, 10, 0, 18, 2, 50, 0, 18, 4, 58, 2,
-        10, 0, 24, 19, 32, 129, 188, 193, 150, 11, 18, 6, 10, 1, 97, 18, 1, 98, 24, 130, 188, 193,
+        10, 0, 24, 19, 32, 128, 188, 193, 150, 11, 18, 6, 10, 1, 97, 18, 1, 98, 24, 129, 188, 193,
         150, 11, 32, 10, 160, 6, 122, 168, 6, 24,
     ];
 

src/meta/proto-conv/tests/it/v122_virtual_schema.rs

@@ -53,8 +53,7 @@ pub struct VirtualColumnField {
     pub source_column_id: u32,
     /// The source column name.
     pub source_name: String,
-    /// The virtual column id, it is not a real column,
-    /// but is used to identify a virtual column when reading data.
+    /// The virtual column id
     pub column_id: u32,
     /// The virtual column name.
     pub name: String,

src/query/catalog/src/plan/pushdown.rs

@@ -19,7 +19,6 @@ databend-common-catalog = { workspace = true }
 databend-common-config = { workspace = true }
 databend-common-exception = { workspace = true }
 databend-common-expression = { workspace = true }
-databend-common-functions = { workspace = true }
 databend-common-io = { workspace = true }
 databend-common-license = { workspace = true }
 databend-common-management = { workspace = true }
@@ -29,7 +28,6 @@ databend-common-meta-store = { workspace = true }
 databend-common-meta-types = { workspace = true }
 databend-common-metrics = { workspace = true }
 databend-common-pipeline-core = { workspace = true }
-databend-common-pipeline-sinks = { workspace = true }
 databend-common-pipeline-sources = { workspace = true }
 databend-common-pipeline-transforms = { workspace = true }
 databend-common-sql = { workspace = true }

src/query/ee/Cargo.toml

@@ -85,7 +85,6 @@ impl HilbertClusteringHandler for RealHilbertClusteringHandler {
                 fuse_table.schema_with_stream(),
                 fuse_table.get_operator(),
                 &push_downs,
-                fuse_table.get_storage_format(),
                 chunk.to_vec(),
             )
             .await?;