Skip to content

Commit 91b5011

Browse files
KKouldKKould
authored
feat: support the to_binary with format (#18525)
* feat: support the `to_binary` with format * chore: codefmt * chore: codefmt * chore: codefmt * chore: codefmt * chore: codefmt
1 parent 3c3255b commit 91b5011

File tree

7 files changed

+124
-5
lines changed

7 files changed

+124
-5
lines changed

‎src/query/functions/src/scalars/binary.rs‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,12 @@ use databend_common_expression::FunctionRegistry;
3636
use databend_common_expression::Value;
3737

3838
pub fn register(registry: &mut FunctionRegistry) {
39-
registry.register_aliases("to_hex", &["hex"]);
40-
registry.register_aliases("from_hex", &["unhex"]);
39+
registry.register_aliases("to_hex", &["hex", "hex_encode"]);
40+
registry.register_aliases("from_hex", &["unhex", "hex_decode_binary"]);
41+
registry.register_aliases("try_from_hex", &["try_hex_decode_binary"]);
42+
registry.register_aliases("to_base64", &["base64_encode"]);
43+
registry.register_aliases("from_base64", &["base64_decode_binary"]);
44+
registry.register_aliases("try_from_base64", &["try_base64_decode_binary"]);
4145

4246
registry.register_passthrough_nullable_1_arg::<BinaryType, NumberType<u64>, _, _>(
4347
"length",
@@ -138,6 +142,32 @@ pub fn register(registry: &mut FunctionRegistry) {
138142
},
139143
);
140144

145+
registry.register_passthrough_nullable_2_arg::<StringType, StringType, BinaryType, _, _>(
146+
"to_binary",
147+
|_, _, _| FunctionDomain::Full,
148+
|val, format, ctx| {
149+
let Some(format) = format.as_scalar() else {
150+
ctx.set_error(
151+
0,
152+
"`format` parameter must be a scalar constant, not a column or expression",
153+
);
154+
return Value::Scalar(Vec::new());
155+
};
156+
match format.to_ascii_lowercase().as_str() {
157+
"hex" => eval_unhex(val, ctx),
158+
"base64" => eval_from_base64(val, ctx),
159+
"utf-8" => match val {
160+
Value::Scalar(val) => Value::Scalar(val.as_bytes().to_vec()),
161+
Value::Column(col) => Value::Column(col.into()),
162+
},
163+
_ => {
164+
ctx.set_error(0, "The format option only supports hex, base64, and utf-8");
165+
Value::Scalar(Vec::new())
166+
}
167+
}
168+
},
169+
);
170+
141171
registry.register_combine_nullable_1_arg::<StringType, BinaryType, _, _>(
142172
"try_to_binary",
143173
|_, _| FunctionDomain::Full,
@@ -150,6 +180,28 @@ pub fn register(registry: &mut FunctionRegistry) {
150180
},
151181
);
152182

183+
registry.register_combine_nullable_2_arg::<StringType, StringType, BinaryType, _, _>(
184+
"try_to_binary",
185+
|_, _, _| FunctionDomain::Full,
186+
|val, format, ctx| {
187+
let Some(format) = format.as_scalar() else {
188+
return Value::Scalar(None);
189+
};
190+
match format.to_ascii_lowercase().as_str() {
191+
"hex" => error_to_null(eval_unhex)(val, ctx),
192+
"base64" => error_to_null(eval_from_base64)(val, ctx),
193+
"utf-8" => match val {
194+
Value::Scalar(val) => Value::Scalar(Some(val.as_bytes().to_vec())),
195+
Value::Column(col) => {
196+
let validity = Bitmap::new_constant(true, col.len());
197+
Value::Column(NullableColumn::new_unchecked(col.into(), validity))
198+
}
199+
},
200+
_ => Value::Scalar(None),
201+
}
202+
},
203+
);
204+
153205
registry.register_passthrough_nullable_1_arg::<BinaryType, StringType, _, _>(
154206
"to_hex",
155207
|_, _| FunctionDomain::Full,

‎src/query/functions/tests/it/scalars/testdata/function_list.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ array_get -> get
66
array_length -> length
77
array_size -> length
88
array_slice -> slice
9+
base64_decode_binary -> from_base64
10+
base64_encode -> to_base64
911
between_dows -> between_days
1012
between_doys -> between_days
1113
between_epochs -> between_seconds
@@ -28,6 +30,8 @@ diff_doys -> diff_days
2830
diff_epochs -> diff_seconds
2931
diff_isodows -> diff_days
3032
hex -> to_hex
33+
hex_decode_binary -> from_hex
34+
hex_encode -> to_hex
3135
intdiv -> div
3236
ipv4_num_to_string -> inet_ntoa
3337
ipv4_string_to_num -> inet_aton
@@ -95,6 +99,8 @@ to_start_of_iso_week -> to_monday
9599
to_text -> to_string
96100
to_varchar -> to_string
97101
trunc -> truncate
102+
try_base64_decode_binary -> try_from_base64
103+
try_hex_decode_binary -> try_from_hex
98104
try_ipv4_num_to_string -> try_inet_ntoa
99105
try_ipv4_string_to_num -> try_inet_aton
100106
try_json_object -> try_object_construct
@@ -3944,6 +3950,8 @@ Functions overloads:
39443950
7 to_binary(Geography NULL) :: Binary NULL
39453951
8 to_binary(String) :: Binary
39463952
9 to_binary(String NULL) :: Binary NULL
3953+
10 to_binary(String, String) :: Binary
3954+
11 to_binary(String NULL, String NULL) :: Binary NULL
39473955
0 to_bitmap(String) :: Bitmap
39483956
1 to_bitmap(String NULL) :: Bitmap NULL
39493957
2 to_bitmap(UInt64) :: Bitmap
@@ -4616,6 +4624,8 @@ Functions overloads:
46164624
7 try_to_binary(Geography NULL) :: Binary NULL
46174625
8 try_to_binary(String) :: Binary NULL
46184626
9 try_to_binary(String NULL) :: Binary NULL
4627+
10 try_to_binary(String, String) :: Binary NULL
4628+
11 try_to_binary(String NULL, String NULL) :: Binary NULL
46194629
0 try_to_boolean(Variant) :: Boolean NULL
46204630
1 try_to_boolean(Variant NULL) :: Boolean NULL
46214631
2 try_to_boolean(String) :: Boolean NULL

‎src/query/sql/src/planner/semantic/type_check.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3644,6 +3644,10 @@ impl<'a> TypeChecker<'a> {
36443644
Ascii::new("stream_has_data"),
36453645
Ascii::new("getvariable"),
36463646
Ascii::new("equal_null"),
3647+
Ascii::new("hex_decode_string"),
3648+
Ascii::new("base64_decode_string"),
3649+
Ascii::new("try_hex_decode_string"),
3650+
Ascii::new("try_base64_decode_string"),
36473651
];
36483652
FUNCTIONS
36493653
}
@@ -4302,6 +4306,36 @@ impl<'a> TypeChecker<'a> {
43024306
Some(self.resolve_map_access(span, expr, paths))
43034307
}
43044308
}
4309+
(func_name, &[expr])
4310+
if matches!(
4311+
func_name,
4312+
"hex_decode_string"
4313+
| "try_hex_decode_string"
4314+
| "base64_decode_string"
4315+
| "try_base64_decode_string"
4316+
) =>
4317+
{
4318+
Some(self.resolve(&Expr::Cast {
4319+
span,
4320+
expr: Box::new(Expr::FunctionCall {
4321+
span,
4322+
func: ASTFunctionCall {
4323+
distinct: false,
4324+
name: Identifier::from_name(
4325+
span,
4326+
func_name.replace("_string", "_binary"),
4327+
),
4328+
args: vec![expr.clone()],
4329+
params: vec![],
4330+
order_by: vec![],
4331+
window: None,
4332+
lambda: None,
4333+
},
4334+
}),
4335+
target_type: TypeName::String,
4336+
pg_style: false,
4337+
}))
4338+
}
43054339
_ => None,
43064340
}
43074341
}

‎tests/sqllogictests/suites/base/03_common/03_0041_insert_into_binary.test‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ statement ok
1111
CREATE TABLE IF NOT EXISTS t1(id Int, v binary) Engine = Fuse
1212

1313
statement ok
14-
INSERT INTO t1 (id, v) VALUES(1, to_binary('aaa')),(2, from_hex('616161')),(3, from_base64('YWFh'))
14+
INSERT INTO t1 (id, v) VALUES(1, to_binary('aaa')),(2, from_hex('616161')),(3, from_base64('YWFh')),(4, to_binary('aaa', 'utf-8')),(5, to_binary('616161', 'hex')),(6, to_binary('YWFh', 'base64'))
1515

1616
statement ok
17-
INSERT INTO t1 (id, v) VALUES(4, 'aaa')
17+
INSERT INTO t1 (id, v) VALUES(7, 'aaa')
1818

1919
query IT
2020
SELECT id, v FROM t1 order by id
@@ -23,6 +23,9 @@ SELECT id, v FROM t1 order by id
2323
2 616161
2424
3 616161
2525
4 616161
26+
5 616161
27+
6 616161
28+
7 616161
2629

2730
statement ok
2831
ALTER TABLE t1 MODIFY COLUMN v string
@@ -34,6 +37,9 @@ SELECT id, v FROM t1 order by id
3437
2 aaa
3538
3 aaa
3639
4 aaa
40+
5 aaa
41+
6 aaa
42+
7 aaa
3743

3844
statement ok
3945
ALTER TABLE t1 MODIFY COLUMN v binary
@@ -45,6 +51,9 @@ SELECT id, v FROM t1 order by id
4551
2 616161
4652
3 616161
4753
4 616161
54+
5 616161
55+
6 616161
56+
7 616161
4857

4958
statement ok
5059
create table t2(a int, b binary NOT NULL DEFAULT 'abc', c double default 'inf', e float default 'nan' );

‎tests/sqllogictests/suites/query/functions/02_0019_function_strings_hex.test‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,12 @@ select hex(null)
1818
----
1919
NULL
2020

21+
query T
22+
SELECT FROM_HEX(TO_HEX('abc'))::STRING
23+
----
24+
abc
25+
26+
query T
27+
SELECT HEX_DECODE_STRING(TO_HEX('abc'))
28+
----
29+
abc

‎tests/sqllogictests/suites/query/functions/02_0024_function_strings_base_64.test‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ SELECT FROM_BASE64(TO_BASE64('abc'))::STRING
88
----
99
abc
1010

11+
query T
12+
SELECT BASE64_DECODE_STRING(TO_BASE64('abc'))
13+
----
14+
abc
15+
1116
query T
1217
SELECT TO_BASE64(NULL)
1318
----

‎tests/suites/0_stateless/20+_others/20_0013_pretty_error.result‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ Error: APIError: QueryFailed: [1008]error:
1616
--> SQL:1:8
1717
|
1818
1 | select base64(1)
19-
| ^^^^^^^^^ no function matches the given name: 'base64', do you mean 'to_base64'?
19+
| ^^^^^^^^^ no function matches the given name: 'base64', do you mean 'base64_encode', 'base64_decode_binary', 'base64_decode_string', 'to_base64'?
2020

2121

2222
Error: APIError: QueryFailed: [1065]error:

0 commit comments

Comments
 (0)