Skip to content

Commit 749610a

Browse files
sundy-lisundy-li
authored
fix(query): backport to 680 stick the created_by infos in parquet writer (#17223)
* fix(query): stick the created_by infos in parquet writer (#17220) * fix(query): stick the created_by infos in parquet writer * fix(query): stick the created_by infos in parquet writer * fix(query): stick the created_by infos in parquet writer * fix(query): stick the created_by infos in parquet writer * fix(query): stick the created_by infos in parquet writer * fix(query): fix register function working with nullable scalar (#17217) * fix(query): fix register function working with nullable scalar * fix(query): fix register function working with nullable scalar * fix(query): increase pool * Update 19_0005_fuzz_cte.sh * Update mysql_source.rs * fix(query): fix register function working with nullable scalar * fix(query): rollback tests
1 parent 5cd121f commit 749610a

File tree

20 files changed

+185
-51
lines changed

20 files changed

+185
-51
lines changed

‎src/query/expression/src/register_vectorize.rs‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -283,8 +283,7 @@ pub fn passthrough_nullable_1_arg<I1: ArgType, O: ArgType>(
283283

284284
match out {
285285
Value::Column(out) => Value::Column(NullableColumn::new(out, args_validity)),
286-
Value::Scalar(out) if args_validity.get_bit(0) => Value::Scalar(Some(out)),
287-
_ => Value::Scalar(None),
286+
Value::Scalar(out) => Value::Scalar(Some(out)),
288287
}
289288
}
290289
_ => Value::Scalar(None),
@@ -308,15 +307,15 @@ pub fn passthrough_nullable_2_arg<I1: ArgType, I2: ArgType, O: ArgType>(
308307
if let Some(validity) = ctx.validity.as_ref() {
309308
args_validity = &args_validity & validity;
310309
}
310+
311311
ctx.validity = Some(args_validity.clone());
312312
match (arg1.value(), arg2.value()) {
313313
(Some(arg1), Some(arg2)) => {
314314
let out = func(arg1, arg2, ctx);
315315

316316
match out {
317317
Value::Column(out) => Value::Column(NullableColumn::new(out, args_validity)),
318-
Value::Scalar(out) if args_validity.get_bit(0) => Value::Scalar(Some(out)),
319-
_ => Value::Scalar(None),
318+
Value::Scalar(out) => Value::Scalar(Some(out)),
320319
}
321320
}
322321
_ => Value::Scalar(None),
@@ -352,8 +351,7 @@ pub fn passthrough_nullable_3_arg<I1: ArgType, I2: ArgType, I3: ArgType, O: ArgT
352351

353352
match out {
354353
Value::Column(out) => Value::Column(NullableColumn::new(out, args_validity)),
355-
Value::Scalar(out) if args_validity.get_bit(0) => Value::Scalar(Some(out)),
356-
_ => Value::Scalar(None),
354+
Value::Scalar(out) => Value::Scalar(Some(out)),
357355
}
358356
}
359357
_ => Value::Scalar(None),
@@ -397,8 +395,7 @@ pub fn passthrough_nullable_4_arg<
397395

398396
match out {
399397
Value::Column(out) => Value::Column(NullableColumn::new(out, args_validity)),
400-
Value::Scalar(out) if args_validity.get_bit(0) => Value::Scalar(Some(out)),
401-
_ => Value::Scalar(None),
398+
Value::Scalar(out) => Value::Scalar(Some(out)),
402399
}
403400
}
404401
_ => Value::Scalar(None),
@@ -427,8 +424,7 @@ pub fn combine_nullable_1_arg<I1: ArgType, O: ArgType>(
427424
out.column,
428425
&args_validity & &out.validity,
429426
)),
430-
Value::Scalar(out) if args_validity.get_bit(0) => Value::Scalar(out),
431-
_ => Value::Scalar(None),
427+
Value::Scalar(out) => Value::Scalar(out),
432428
}
433429
}
434430
_ => Value::Scalar(None),
@@ -465,8 +461,7 @@ pub fn combine_nullable_2_arg<I1: ArgType, I2: ArgType, O: ArgType>(
465461
out.column,
466462
&args_validity & &out.validity,
467463
)),
468-
Value::Scalar(out) if args_validity.get_bit(0) => Value::Scalar(out),
469-
_ => Value::Scalar(None),
464+
Value::Scalar(out) => Value::Scalar(out),
470465
}
471466
}
472467
_ => Value::Scalar(None),
@@ -505,8 +500,7 @@ pub fn combine_nullable_3_arg<I1: ArgType, I2: ArgType, I3: ArgType, O: ArgType>
505500
out.column,
506501
&args_validity & &out.validity,
507502
)),
508-
Value::Scalar(out) if args_validity.get_bit(0) => Value::Scalar(out),
509-
_ => Value::Scalar(None),
503+
Value::Scalar(out) => Value::Scalar(out),
510504
}
511505
}
512506
_ => Value::Scalar(None),
@@ -552,8 +546,7 @@ pub fn combine_nullable_4_arg<I1: ArgType, I2: ArgType, I3: ArgType, I4: ArgType
552546
out.column,
553547
&args_validity & &out.validity,
554548
)),
555-
Value::Scalar(out) if args_validity.get_bit(0) => Value::Scalar(out),
556-
_ => Value::Scalar(None),
549+
Value::Scalar(out) => Value::Scalar(out),
557550
}
558551
}
559552
_ => Value::Scalar(None),

‎src/query/functions/tests/it/scalars/mod.rs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,3 +271,43 @@ fn list_all_builtin_functions() {
271271
fn check_ambiguity() {
272272
BUILTIN_FUNCTIONS.check_ambiguity()
273273
}
274+
275+
#[test]
276+
fn test_if_function() -> Result<()> {
277+
use databend_common_expression::types::*;
278+
use databend_common_expression::FromData;
279+
use databend_common_expression::Scalar;
280+
let raw_expr = parser::parse_raw_expr("if(eq(n,1), sum_sid + 1,100)", &[
281+
("n", UInt8Type::data_type()),
282+
("sum_sid", Int32Type::data_type().wrap_nullable()),
283+
]);
284+
let expr = type_check::check(&raw_expr, &BUILTIN_FUNCTIONS)?;
285+
let block = DataBlock::new(
286+
vec![
287+
BlockEntry {
288+
data_type: UInt8Type::data_type(),
289+
value: Value::Column(UInt8Type::from_data(vec![2_u8, 1])),
290+
},
291+
BlockEntry {
292+
data_type: Int32Type::data_type().wrap_nullable(),
293+
value: Value::Scalar(Scalar::Number(NumberScalar::Int32(2400_i32))),
294+
},
295+
],
296+
2,
297+
);
298+
let func_ctx = FunctionContext::default();
299+
let evaluator = Evaluator::new(&block, &func_ctx, &BUILTIN_FUNCTIONS);
300+
let result = evaluator.run(&expr).unwrap();
301+
let result = result
302+
.as_column()
303+
.unwrap()
304+
.clone()
305+
.as_nullable()
306+
.unwrap()
307+
.clone();
308+
309+
let bm = Bitmap::from_iter([true, true]);
310+
assert_eq!(result.validity, bm);
311+
assert_eq!(result.column, Int64Type::from_data(vec![100, 2401]));
312+
Ok(())
313+
}

‎src/query/storages/stage/src/append/parquet_file/writer_processor.rs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,19 +71,35 @@ pub struct ParquetFileWriter {
7171
const MAX_BUFFER_SIZE: usize = 64 * 1024 * 1024;
7272
// this is number of rows, not size
7373
const MAX_ROW_GROUP_SIZE: usize = 1024 * 1024;
74+
const CREATE_BY_LEN: usize = 24; // "Databend 1.2.333-nightly".len();
7475

7576
fn create_writer(
7677
arrow_schema: Arc<Schema>,
7778
targe_file_size: Option<usize>,
7879
) -> Result<ArrowWriter<Vec<u8>>> {
80+
// example: 1.2.333-nightly
81+
// tags may contain other items like `1.2.680-p2`, we will fill it with `1.2.680-p2.....`
82+
let mut create_by = format!(
83+
"Databend {}.{}.{}-{:.<7}",
84+
DATABEND_SEMVER.major,
85+
DATABEND_SEMVER.minor,
86+
DATABEND_SEMVER.patch,
87+
DATABEND_SEMVER.pre.as_str()
88+
);
89+
90+
if create_by.len() != CREATE_BY_LEN {
91+
create_by = format!("{:.<24}", create_by);
92+
create_by.truncate(24);
93+
}
94+
7995
let props = WriterProperties::builder()
8096
.set_compression(TableCompression::Zstd.into())
8197
.set_max_row_group_size(MAX_ROW_GROUP_SIZE)
8298
.set_encoding(Encoding::PLAIN)
8399
.set_dictionary_enabled(false)
84100
.set_statistics_enabled(EnabledStatistics::Chunk)
85101
.set_bloom_filter_enabled(false)
86-
.set_created_by(format!("Databend {}", *DATABEND_SEMVER))
102+
.set_created_by(create_by)
87103
.build();
88104
let buf_size = match targe_file_size {
89105
Some(n) if n < MAX_BUFFER_SIZE => n,

‎tests/sqllogictests/suites/query/cte/basic_r_cte.test‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,5 +227,64 @@ select cte1.a from cte1;
227227
8
228228
9
229229

230+
231+
statement ok
232+
create table train(
233+
train_id varchar(8) not null ,
234+
departure_station varchar(32) not null,
235+
arrival_station varchar(32) not null,
236+
seat_count int not null
237+
);
238+
239+
statement ok
240+
create table passenger(
241+
passenger_id varchar(16) not null,
242+
departure_station varchar(32) not null,
243+
arrival_station varchar(32) not null
244+
);
245+
246+
statement ok
247+
create table city(city varchar(32));
248+
249+
statement ok
250+
insert into city
251+
with t as (select 1 n union select 2 union select 3 union select 4 union select 5)
252+
,t1 as(select row_number()over() rn from t ,t t2,t t3)
253+
select concat('城市',rn::varchar) city from t1 where rn<=5;
254+
255+
statement ok
256+
insert into train
257+
select concat('G',row_number()over()::varchar),c1.city,c2.city, n from city c1, city c2, (select 600 n union select 800 union select 1200 union select 1600) a ;
258+
259+
statement ok
260+
insert into passenger
261+
select concat('P',substr((100000000+row_number()over())::varchar,2)),c1.city,c2.city from city c1, city c2 ,city c3, city c4, city c5,
262+
city c6, (select 1 n union select 2 union select 3 union select 4) c7,(select 1 n union select 2) c8;
263+
264+
265+
query III
266+
with
267+
t0 as (
268+
select
269+
train_id,
270+
seat_count,
271+
sum(seat_count) over (
272+
partition by departure_station, arrival_station order by train_id
273+
) ::int sum_sid
274+
from
275+
train
276+
)
277+
select
278+
sum(case when n=1 then sum_sid+1 else 0 end::int),
279+
sum(sum_sid),
280+
sum(seat_count)
281+
from
282+
t0,(select 1 n union all select 2);
283+
----
284+
261700 523200 210000
285+
286+
statement ok
287+
use default;
288+
230289
statement ok
231-
drop table t1;
290+
drop database db;

‎tests/sqllogictests/suites/stage/formats/parquet/options/null_if.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ remove @data/unload/parquet/null_if/
2626
query
2727
copy into @data/unload/parquet/null_if from string
2828
----
29-
3 56 379
29+
3 56 387
3030

3131
statement ok
3232
drop file format if exists parquet_null_if

‎tests/sqllogictests/suites/stage/formats/parquet/options/parquet_missing_uuid.test‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ remove @data/parquet/unload/uuid
1010
query
1111
copy into @data/parquet/unload/uuid/ from (select 1 as a) file_format = (type = parquet)
1212
----
13-
1 1 366
13+
1 1 374
1414

1515
query error column id doesn't exist
1616
copy into t_uuid from @data/parquet/unload/uuid file_format = (type = parquet) RETURN_FAILED_ONLY=TRUE
@@ -22,7 +22,7 @@ select * from t_uuid
2222
query
2323
copy into @data/parquet/unload/uuid/ from (select 1 as a) file_format = (type = parquet)
2424
----
25-
1 1 366
25+
1 1 374
2626

2727
statement ok
2828
truncate table t_uuid

‎tests/suites/0_stateless/05_hints/05_0001_set_var.result‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,5 @@ America/Toronto
2323
1
2424
2022-02-02 03:00:00
2525
2022-02-02 03:00:00
26-
1 13 419
26+
1 13 427
2727
Asia/Shanghai

‎tests/suites/0_stateless/18_rbac/18_0002_ownership_cover.result‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
1
33
200
44
=== test stage ===
5-
1 8 392
5+
1 8 400
66
0
77
=== test udf ===
88
2

‎tests/suites/0_stateless/18_rbac/18_0007_privilege_access.result‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ Error: APIError: QueryFailed: [1063]Permission denied: privilege READ is require
101101
Error: APIError: QueryFailed: [1063]Permission denied: No privilege on database root_db for user b.
102102
Error: APIError: QueryFailed: [1063]Permission denied: No privilege on table root_table for user b.
103103
Error: APIError: QueryFailed: [1063]Permission denied: No privilege on table root_table for user b.
104-
1 1 366
104+
1 1 374
105105
Error: APIError: QueryFailed: [1063]Permission denied: privilege [Select] is required on 'default'.'default'.'t1' for user 'b'@'%' with roles [public]
106106
Error: APIError: QueryFailed: [1063]Permission denied: privilege [Read] is required on STAGE s3 for user 'b'@'%' with roles [public]. Note: Please ensure that your current role have the appropriate permissions to create a new Warehouse|Database|Table|UDF|Stage.
107107
Error: APIError: QueryFailed: [1063]Permission denied: privilege [Select] is required on 'default'.'default'.'t' for user 'b'@'%' with roles [public]

‎tests/suites/0_stateless/19_fuzz/19_0005_fuzz_cte.result‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
OK

0 commit comments

Comments
 (0)