Skip to content

Commit 9044f0e

Browse files
forsaken628forsaken628
authored
chore(query): refactoring BlockEntry (#18131)
* pub * block * add_entry * x * refine * refine * update * x * x * x * x * refine * to_column * fix * fix
1 parent 385ae72 commit 9044f0e

File tree

157 files changed

+1422
-1875
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

157 files changed

+1422
-1875
lines changed

‎src/query/catalog/src/plan/internal_column.rs‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ use databend_common_expression::ColumnId;
3232
use databend_common_expression::FromData;
3333
use databend_common_expression::Scalar;
3434
use databend_common_expression::TableDataType;
35-
use databend_common_expression::Value;
3635
use databend_common_expression::BASE_BLOCK_IDS_COLUMN_ID;
3736
use databend_common_expression::BASE_ROW_ID_COLUMN_ID;
3837
use databend_common_expression::BLOCK_NAME_COLUMN_ID;
@@ -227,14 +226,17 @@ impl InternalColumn {
227226

228227
UInt64Type::from_data(row_ids).into()
229228
}
230-
InternalColumnType::BlockName => {
231-
BlockEntry::from_arg_scalar::<StringType>(meta.block_location.clone())
232-
}
233-
InternalColumnType::SegmentName => {
234-
BlockEntry::from_arg_scalar::<StringType>(meta.segment_location.clone())
235-
}
236-
InternalColumnType::SnapshotName => BlockEntry::from_arg_scalar::<StringType>(
229+
InternalColumnType::BlockName => BlockEntry::new_const_column_arg::<StringType>(
230+
meta.block_location.clone(),
231+
num_rows,
232+
),
233+
InternalColumnType::SegmentName => BlockEntry::new_const_column_arg::<StringType>(
234+
meta.segment_location.clone(),
235+
num_rows,
236+
),
237+
InternalColumnType::SnapshotName => BlockEntry::new_const_column_arg::<StringType>(
237238
meta.snapshot_location.clone().unwrap_or_default(),
239+
num_rows,
238240
),
239241
InternalColumnType::BaseRowId => {
240242
let uuid =
@@ -260,9 +262,10 @@ impl InternalColumn {
260262
}
261263
InternalColumnType::BaseBlockIds => {
262264
assert!(meta.base_block_ids.is_some());
263-
BlockEntry::new(
265+
BlockEntry::new_const_column(
264266
DataType::Array(Box::new(DataType::Decimal(DecimalSize::default_128()))),
265-
Value::Scalar(meta.base_block_ids.clone().unwrap()),
267+
meta.base_block_ids.clone().unwrap(),
268+
num_rows,
266269
)
267270
}
268271
InternalColumnType::SearchMatched => {

‎src/query/catalog/src/plan/stream_column.rs‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -124,12 +124,7 @@ impl StreamColumnMeta {
124124

125125
pub fn build_origin_block_row_num(num_rows: usize) -> BlockEntry {
126126
let row_ids = (0..num_rows as u64).collect();
127-
let column = Value::Column(UInt64Type::from_data(row_ids));
128-
129-
BlockEntry::new(
130-
DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt64))),
131-
column.wrap_nullable(None),
132-
)
127+
UInt64Type::from_data(row_ids).wrap_nullable(None).into()
133128
}
134129

135130
#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)]
@@ -198,10 +193,14 @@ impl StreamColumn {
198193
pub fn generate_column_values(&self, meta: &StreamColumnMeta, num_rows: usize) -> BlockEntry {
199194
match &self.column_type {
200195
StreamColumnType::OriginVersion | StreamColumnType::RowVersion => unreachable!(),
201-
StreamColumnType::OriginBlockId => BlockEntry::new(
202-
DataType::Nullable(Box::new(DataType::Decimal(DecimalSize::default_128()))),
203-
meta.build_origin_block_id(),
204-
),
196+
StreamColumnType::OriginBlockId => {
197+
BlockEntry::new(meta.build_origin_block_id(), || {
198+
(
199+
DataType::Nullable(Box::new(DataType::Decimal(DecimalSize::default_128()))),
200+
num_rows,
201+
)
202+
})
203+
}
205204
StreamColumnType::OriginRowNum => build_origin_block_row_num(num_rows),
206205
}
207206
}

‎src/query/ee/tests/it/storages/fuse/operations/virtual_columns_builder.rs‎

Lines changed: 73 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,11 @@ use std::str::FromStr;
1616

1717
use databend_common_base::base::tokio;
1818
use databend_common_exception::Result;
19-
use databend_common_expression::types::DataType;
2019
use databend_common_expression::types::Int32Type;
21-
use databend_common_expression::types::NumberDataType;
2220
use databend_common_expression::types::VariantType;
23-
use databend_common_expression::BlockEntry;
2421
use databend_common_expression::ColumnId;
2522
use databend_common_expression::DataBlock;
2623
use databend_common_expression::FromData;
27-
use databend_common_expression::Value;
2824
use databend_common_expression::VariantDataType;
2925
use databend_common_storages_fuse::io::VirtualColumnBuilder;
3026
use databend_common_storages_fuse::FuseTable;
@@ -62,36 +58,27 @@ async fn test_virtual_column_builder() -> Result<()> {
6258

6359
let block = DataBlock::new(
6460
vec![
65-
BlockEntry::new(
66-
DataType::Number(NumberDataType::Int32),
67-
Value::Column(Int32Type::from_data(vec![1, 2, 3])),
68-
),
69-
BlockEntry::new(
70-
DataType::Nullable(Box::new(DataType::Variant)),
71-
Value::Column(VariantType::from_opt_data(vec![
72-
Some(
73-
OwnedJsonb::from_str(
74-
r#"{"a": 1, "b": {"c": "x"}, "e": [{"f": 100}, 200]}"#,
75-
)
76-
.unwrap()
77-
.to_vec(),
78-
),
79-
Some(
80-
OwnedJsonb::from_str(
81-
r#"{"a": 2, "b": {"c": "y", "d": true}, "e": [{"f": 300}, 400]}"#,
82-
)
61+
(Int32Type::from_data(vec![1, 2, 3])).into(),
62+
(VariantType::from_opt_data(vec![
63+
Some(
64+
OwnedJsonb::from_str(r#"{"a": 1, "b": {"c": "x"}, "e": [{"f": 100}, 200]}"#)
8365
.unwrap()
8466
.to_vec(),
85-
),
86-
Some(
87-
OwnedJsonb::from_str(
88-
r#"{"a": 3, "b": {"d": false}, "e": [{"f": 500}, 600]}"#,
89-
)
67+
),
68+
Some(
69+
OwnedJsonb::from_str(
70+
r#"{"a": 2, "b": {"c": "y", "d": true}, "e": [{"f": 300}, 400]}"#,
71+
)
72+
.unwrap()
73+
.to_vec(),
74+
),
75+
Some(
76+
OwnedJsonb::from_str(r#"{"a": 3, "b": {"d": false}, "e": [{"f": 500}, 600]}"#)
9077
.unwrap()
9178
.to_vec(),
92-
), // 'c' is missing here
93-
])),
94-
),
79+
), // 'c' is missing here
80+
]))
81+
.into(),
9582
],
9683
3,
9784
);
@@ -161,13 +148,8 @@ async fn test_virtual_column_builder() -> Result<()> {
161148

162149
let block = DataBlock::new(
163150
vec![
164-
BlockEntry::new(
165-
DataType::Number(NumberDataType::Int32),
166-
Value::Column(Int32Type::from_data(vec![1, 2, 3, 4, 5, 6, 7, 8])),
167-
),
168-
BlockEntry::new(
169-
DataType::Nullable(Box::new(DataType::Variant)),
170-
Value::Column(VariantType::from_opt_data(vec![
151+
Int32Type::from_data(vec![1, 2, 3, 4, 5, 6, 7, 8]).into(),
152+
VariantType::from_opt_data(vec![
171153
Some(
172154
OwnedJsonb::from_str(r#"{"id":1, "create": "3/06", "text": "a", "user": {"id": 1}}"#)
173155
.unwrap()
@@ -208,8 +190,7 @@ async fn test_virtual_column_builder() -> Result<()> {
208190
.unwrap()
209191
.to_vec(),
210192
),
211-
])),
212-
),
193+
]).into(),
213194
],
214195
8,
215196
);
@@ -266,32 +247,60 @@ async fn test_virtual_column_builder() -> Result<()> {
266247
.unwrap();
267248
assert_eq!(meta_geo_lat.data_type, VariantDataType::Float64);
268249

269-
let block = DataBlock::new(
270-
vec![
271-
BlockEntry::new(
272-
DataType::Number(NumberDataType::Int32),
273-
Value::Column(Int32Type::from_data(vec![1, 2, 3, 4, 5, 6, 7, 8])),
250+
let entries = vec![
251+
Int32Type::from_data(vec![1, 2, 3, 4, 5, 6, 7, 8]).into(),
252+
VariantType::from_opt_data(vec![
253+
// Consistent types
254+
Some(
255+
OwnedJsonb::from_str(
256+
r#"{"int_col": 1, "str_col": "a", "bool_col": true, "float_col": 1.1}"#,
257+
)
258+
.unwrap()
259+
.to_vec(),
274260
),
275-
BlockEntry::new(
276-
DataType::Nullable(Box::new(DataType::Variant)),
277-
Value::Column(VariantType::from_opt_data(vec![
278-
// Consistent types
279-
Some(OwnedJsonb::from_str(r#"{"int_col": 1, "str_col": "a", "bool_col": true, "float_col": 1.1}"#).unwrap().to_vec()),
280-
Some(OwnedJsonb::from_str(r#"{"int_col": 2, "str_col": "b", "bool_col": false, "float_col": 2.2}"#).unwrap().to_vec()),
281-
// Mixed types -> Variant/Jsonb
282-
Some(OwnedJsonb::from_str(r#"{"mixed_col": 10}"#).unwrap().to_vec()),
283-
Some(OwnedJsonb::from_str(r#"{"mixed_col": "hello"}"#).unwrap().to_vec()),
284-
// Int/UInt coercion
285-
Some(OwnedJsonb::from_str(r#"{"num_coerce1": 5, "num_coerce2": 10000000000}"#).unwrap().to_vec()), // UInt64 > i64::MAX
286-
Some(OwnedJsonb::from_str(r#"{"num_coerce1": -5, "num_coerce2": 50}"#).unwrap().to_vec()), // Int64
287-
// All Nulls (will be discarded later, but type is inferred first if exists)
288-
Some(OwnedJsonb::from_str(r#"{"all_null_col": null}"#).unwrap().to_vec()),
289-
Some(OwnedJsonb::from_str(r#"{}"#).unwrap().to_vec()),
290-
])),
291-
),
292-
],
293-
8, // Number of rows
294-
);
261+
Some(
262+
OwnedJsonb::from_str(
263+
r#"{"int_col": 2, "str_col": "b", "bool_col": false, "float_col": 2.2}"#,
264+
)
265+
.unwrap()
266+
.to_vec(),
267+
),
268+
// Mixed types -> Variant/Jsonb
269+
Some(
270+
OwnedJsonb::from_str(r#"{"mixed_col": 10}"#)
271+
.unwrap()
272+
.to_vec(),
273+
),
274+
Some(
275+
OwnedJsonb::from_str(r#"{"mixed_col": "hello"}"#)
276+
.unwrap()
277+
.to_vec(),
278+
),
279+
// Int/UInt coercion
280+
Some(
281+
OwnedJsonb::from_str(r#"{"num_coerce1": 5, "num_coerce2": 10000000000}"#)
282+
.unwrap()
283+
.to_vec(),
284+
), // UInt64 > i64::MAX
285+
Some(
286+
OwnedJsonb::from_str(r#"{"num_coerce1": -5, "num_coerce2": 50}"#)
287+
.unwrap()
288+
.to_vec(),
289+
), // Int64
290+
// All Nulls (will be discarded later, but type is inferred first if exists)
291+
Some(
292+
OwnedJsonb::from_str(r#"{"all_null_col": null}"#)
293+
.unwrap()
294+
.to_vec(),
295+
),
296+
Some(OwnedJsonb::from_str(r#"{}"#).unwrap().to_vec()),
297+
])
298+
.into(),
299+
];
300+
301+
let block = DataBlock::new(
302+
entries, 8, // Number of rows
303+
);
295304

296305
let result = builder.add_block(&block, &write_settings, &location)?;
297306

‎src/query/expression/src/aggregate/group_hash.rs‎

Lines changed: 5 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -565,7 +565,6 @@ mod tests {
565565
use crate::BlockEntry;
566566
use crate::DataBlock;
567567
use crate::FromData;
568-
use crate::Scalar;
569568
use crate::Value;
570569

571570
fn merge_hash_slice(ls: &[u64]) -> u64 {
@@ -576,24 +575,10 @@ mod tests {
576575
fn test_value_spread() -> Result<()> {
577576
let data = DataBlock::new(
578577
vec![
579-
BlockEntry::new(
580-
Int32Type::data_type(),
581-
Value::Column(Int32Type::from_data(vec![3, 1, 2, 2, 4, 3, 7, 0, 3])),
582-
),
583-
BlockEntry::new(
584-
StringType::data_type(),
585-
Value::Scalar(Scalar::String("a".to_string())),
586-
),
587-
BlockEntry::new(
588-
Int32Type::data_type(),
589-
Value::Column(Int32Type::from_data(vec![3, 1, 3, 2, 2, 3, 4, 3, 3])),
590-
),
591-
BlockEntry::new(
592-
StringType::data_type(),
593-
Value::Column(StringType::from_data(vec![
594-
"a", "b", "c", "d", "e", "f", "g", "h", "i",
595-
])),
596-
),
578+
Int32Type::from_data(vec![3, 1, 2, 2, 4, 3, 7, 0, 3]).into(),
579+
BlockEntry::new_const_column_arg::<StringType>("a".to_string(), 9),
580+
Int32Type::from_data(vec![3, 1, 3, 2, 2, 3, 4, 3, 3]).into(),
581+
StringType::from_data(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i"]).into(),
597582
],
598583
9,
599584
);
@@ -603,7 +588,7 @@ mod tests {
603588
let mut target = vec![0; data.num_rows()];
604589
for (i, entry) in data.columns().iter().enumerate() {
605590
let indices = [0, 3, 8];
606-
group_hash_value_spread(&indices, entry.value.to_owned(), i == 0, &mut target)?;
591+
group_hash_value_spread(&indices, entry.value(), i == 0, &mut target)?;
607592
}
608593

609594
assert_eq!(

0 commit comments

Comments
 (0)