Skip to content

Commit 61f108a

Browse files
BohuTANGBohuTANG
authored
Merge pull request #10529 from b41sh/fix-insert-nested
fix(query): fix insert nested data types with expr
2 parents 6a14011 + 9aaaa2d commit 61f108a

File tree

10 files changed

+77
-25
lines changed

10 files changed

+77
-25
lines changed

‎src/common/io/src/cursor_ext/cursor_read_number_ext.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,19 @@ where B: AsRef<[u8]>
103103
fn read_int_text<T: FromLexical>(&mut self) -> Result<T> {
104104
let buf = self.remaining_slice();
105105
let (n_in, n_out) = collect_number(buf);
106+
if n_in == 0 {
107+
return Err(ErrorCode::BadBytes("number is not exist"));
108+
}
106109
let n = read_num_text_exact(&buf[..n_out])?;
107110
self.consume(n_in);
108111
Ok(n)
109112
}
110113

111114
fn read_float_text<T: FromLexical>(&mut self) -> Result<T> {
112115
let (n_in, n_out) = collect_number(self.remaining_slice());
116+
if n_in == 0 {
117+
return Err(ErrorCode::BadBytes("number is not exist"));
118+
}
113119
let n = read_num_text_exact(&self.remaining_slice()[..n_out])?;
114120
self.consume(n_in);
115121
Ok(n)

‎src/query/formats/src/field_decoder/fast_values.rs‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,12 @@ impl FastFieldDecoderValues {
9999
}
100100
}
101101

102+
fn pop_inner_values(&self, column: &mut ColumnBuilder, size: usize) {
103+
for _ in 0..size {
104+
let _ = column.pop();
105+
}
106+
}
107+
102108
pub fn read_field<R: AsRef<[u8]>>(
103109
&self,
104110
column: &mut ColumnBuilder,
@@ -285,10 +291,16 @@ impl FastFieldDecoderValues {
285291
break;
286292
}
287293
if idx != 0 {
288-
reader.must_ignore_byte(b',')?;
294+
if let Err(err) = reader.must_ignore_byte(b',') {
295+
self.pop_inner_values(&mut column.builder, idx);
296+
return Err(err.into());
297+
}
289298
}
290299
let _ = reader.ignore_white_spaces();
291-
self.read_field(&mut column.builder, reader, positions)?;
300+
if let Err(err) = self.read_field(&mut column.builder, reader, positions) {
301+
self.pop_inner_values(&mut column.builder, idx);
302+
return Err(err);
303+
}
292304
}
293305
column.commit_row();
294306
Ok(())
@@ -311,24 +323,41 @@ impl FastFieldDecoderValues {
311323
break;
312324
}
313325
if idx != 0 {
314-
reader.must_ignore_byte(b',')?;
326+
if let Err(err) = reader.must_ignore_byte(b',') {
327+
self.pop_inner_values(&mut map_builder[KEY], idx);
328+
self.pop_inner_values(&mut map_builder[VALUE], idx);
329+
return Err(err.into());
330+
}
315331
}
316332
let _ = reader.ignore_white_spaces();
317-
self.read_field(&mut map_builder[KEY], reader, positions)?;
333+
if let Err(err) = self.read_field(&mut map_builder[KEY], reader, positions) {
334+
self.pop_inner_values(&mut map_builder[KEY], idx);
335+
self.pop_inner_values(&mut map_builder[VALUE], idx);
336+
return Err(err);
337+
}
318338
// check duplicate map keys
319339
let key = map_builder[KEY].pop().unwrap();
320340
if set.contains(&key) {
321-
column.commit_row();
341+
self.pop_inner_values(&mut map_builder[KEY], idx);
342+
self.pop_inner_values(&mut map_builder[VALUE], idx);
322343
return Err(ErrorCode::BadBytes(
323344
"map keys have to be unique".to_string(),
324345
));
325346
}
326347
set.insert(key.clone());
327348
map_builder[KEY].push(key.as_ref());
328349
let _ = reader.ignore_white_spaces();
329-
reader.must_ignore_byte(b':')?;
350+
if let Err(err) = reader.must_ignore_byte(b':') {
351+
self.pop_inner_values(&mut map_builder[KEY], idx + 1);
352+
self.pop_inner_values(&mut map_builder[VALUE], idx);
353+
return Err(err.into());
354+
}
330355
let _ = reader.ignore_white_spaces();
331-
self.read_field(&mut map_builder[VALUE], reader, positions)?;
356+
if let Err(err) = self.read_field(&mut map_builder[VALUE], reader, positions) {
357+
self.pop_inner_values(&mut map_builder[KEY], idx + 1);
358+
self.pop_inner_values(&mut map_builder[VALUE], idx);
359+
return Err(err);
360+
}
332361
}
333362
column.commit_row();
334363
Ok(())
@@ -341,15 +370,30 @@ impl FastFieldDecoderValues {
341370
positions: &mut VecDeque<usize>,
342371
) -> Result<()> {
343372
reader.must_ignore_byte(b'(')?;
344-
for (idx, field) in fields.iter_mut().enumerate() {
373+
for idx in 0..fields.len() {
345374
let _ = reader.ignore_white_spaces();
346375
if idx != 0 {
347-
reader.must_ignore_byte(b',')?;
376+
if let Err(err) = reader.must_ignore_byte(b',') {
377+
for field in fields.iter_mut().take(idx) {
378+
self.pop_inner_values(field, 1);
379+
}
380+
return Err(err.into());
381+
}
348382
}
349383
let _ = reader.ignore_white_spaces();
350-
self.read_field(field, reader, positions)?;
384+
if let Err(err) = self.read_field(&mut fields[idx], reader, positions) {
385+
for field in fields.iter_mut().take(idx) {
386+
self.pop_inner_values(field, 1);
387+
}
388+
return Err(err);
389+
}
390+
}
391+
if let Err(err) = reader.must_ignore_byte(b')') {
392+
for field in fields.iter_mut() {
393+
self.pop_inner_values(field, 1);
394+
}
395+
return Err(err.into());
351396
}
352-
reader.must_ignore_byte(b')')?;
353397
Ok(())
354398
}
355399

‎src/query/formats/src/field_decoder/tsv.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,6 @@ impl FieldDecoderRowBased for FieldDecoderTSV {
143143
// check duplicate map keys
144144
let key = map_builder[KEY].pop().unwrap();
145145
if set.contains(&key) {
146-
column.commit_row();
147146
return Err(ErrorCode::BadBytes(
148147
"map keys have to be unique".to_string(),
149148
));

‎src/query/formats/src/field_decoder/values.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,6 @@ impl FieldDecoderRowBased for FieldDecoderValues {
173173
// check duplicate map keys
174174
let key = map_builder[KEY].pop().unwrap();
175175
if set.contains(&key) {
176-
column.commit_row();
177176
return Err(ErrorCode::BadBytes(
178177
"map keys have to be unique".to_string(),
179178
));

‎src/query/service/src/interpreters/interpreter_insert.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,8 @@ impl ValueSource {
673673
for col in columns.iter_mut().take(pop_count) {
674674
col.pop();
675675
}
676+
// rollback to start position of the row
677+
reader.rollback(start_pos_of_row + 1);
676678
skip_to_next_row(reader, 1)?;
677679
let end_pos_of_row = reader.position();
678680

‎tests/sqllogictests/suites/base/03_common/03_0023_insert_into_array‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -324,21 +324,21 @@ statement ok
324324
CREATE TABLE t15(id Int, arr Array(String)) Engine = Fuse
325325

326326
statement ok
327-
INSERT INTO t15 (id, arr) VALUES(1, ['aa', 'bb']), (2, ['cc', 'dd']), (3, ['ee', 'ff'])
327+
INSERT INTO t15 (id, arr) VALUES(1, ['aa', 'bb']), (2, ['cc', 'dd']), (3, [12, 34])
328328

329329
query IT
330330
select * from t15
331331
----
332332
1 ['aa','bb']
333333
2 ['cc','dd']
334-
3 ['ee','ff']
334+
3 ['12','34']
335335

336336
query TT
337337
select arr[1], arr[2] from t15
338338
----
339339
aa bb
340340
cc dd
341-
ee ff
341+
12 34
342342

343343
statement ok
344344
drop table if exists t16
@@ -347,19 +347,21 @@ statement ok
347347
CREATE TABLE t16(id Int, arr Array(Int64)) Engine = Fuse
348348

349349
statement ok
350-
INSERT INTO t16 (id, arr) VALUES(1, [1,2,3,4]), (2, [5,6,7,8])
350+
INSERT INTO t16 (id, arr) VALUES(1, [1,2,3,4]), (2, [5,6,7,8]), (3, ['9','10','11'])
351351

352352
query IT
353353
select * from t16
354354
----
355355
1 [1,2,3,4]
356356
2 [5,6,7,8]
357+
3 [9,10,11]
357358

358359
query II
359360
select arr[1], arr[2] from t16
360361
----
361362
1 2
362363
5 6
364+
9 10
363365

364366
query II
365367
select arr[1], arr[2] from t16 where arr[2] = 6 order by arr[3] desc

‎tests/sqllogictests/suites/base/03_common/03_0026_insert_into_tuple‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ statement ok
4747
CREATE TABLE IF NOT EXISTS t3(id Int, t Tuple(a Tuple(m Int64, n Int64), b Tuple(x Int64, y Int64))) Engine = Fuse
4848

4949
statement ok
50-
INSERT INTO t3 (id, t) VALUES(1, ((10, 11), (20, 21))), (2, ((30, 31), (40, 41)))
50+
INSERT INTO t3 (id, t) VALUES(1, ((10, 11), (20, 21))), (2, (('30', '31'), (40, 41)))
5151

5252
query IT
5353
select * from t3

‎tests/sqllogictests/suites/base/03_common/03_0037_insert_into_map‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,26 +11,26 @@ statement ok
1111
CREATE TABLE IF NOT EXISTS t1(id Int, m Map(Int64, String)) Engine = Fuse
1212

1313
statement ok
14-
INSERT INTO t1 (id, m) VALUES(1, {100:'abc',200:'def'}),(2, {300:'mn'}),(3, {});
14+
INSERT INTO t1 (id, m) VALUES(1, {100:'abc',200:'def'}),(2, {'300':123}),(3, {});
1515

1616
query IT
1717
select * from t1
1818
----
1919
1 {100:'abc',200:'def'}
20-
2 {300:'mn'}
20+
2 {300:'123'}
2121
3 {}
2222

2323
query TTTT
2424
select m[100], m[200], m[300], m[400] from t1
2525
----
2626
abc def NULL NULL
27-
NULL NULL mn NULL
27+
NULL NULL 123 NULL
2828
NULL NULL NULL NULL
2929

3030
query IT
31-
select * from t1 where m[300] = 'mn'
31+
select * from t1 where m[300] = '123'
3232
----
33-
2 {300:'mn'}
33+
2 {300:'123'}
3434

3535
statement error 1001
3636
INSERT INTO t1 (id, m) VALUES(1, {100:'k1',100:'k2'})

‎tests/suites/1_stateful/00_copy/00_0002_copy_from_fs_on_error.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ echo "truncate table wrong_csv" | $MYSQL_CLIENT_CONNECT
5151

5252
WRONG_CSV="COPY INTO wrong_csv FROM 'fs://${DATADIR}/wrong_sample.csv' FILE_FORMAT = (type = CSV field_delimiter = ',' record_delimiter = '\n' skip_header = 0) ON_ERROR=abort_2"
5353

54-
echo "$WRONG_CSV" | $MYSQL_CLIENT_CONNECT 2>&1 | grep -c "bad field end"
54+
echo "$WRONG_CSV" | $MYSQL_CLIENT_CONNECT 2>&1 | grep -c "fail to decode column"
5555
echo "select count(1) from wrong_csv" | $MYSQL_CLIENT_CONNECT
5656
echo "truncate table wrong_csv" | $MYSQL_CLIENT_CONNECT
5757

‎tests/suites/1_stateful/05_formats/05_02_csv/05_02_02_csv_spaces.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ EOF
4040
curl -H "insert_sql:insert into test_csv file_format = (type = CSV)" -F "upload=@/tmp/whitespace.csv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
4141
echo "select * from test_csv" | $MYSQL_CLIENT_CONNECT
4242

43-
curl -s -H "insert_sql:insert into test_csv_number file_format = (type = CSV)" -F "upload=@/tmp/whitespace_number1.csv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "bad field"
43+
curl -s -H "insert_sql:insert into test_csv_number file_format = (type = CSV)" -F "upload=@/tmp/whitespace_number1.csv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "fail to decode column"
4444
curl -s -H "insert_sql:insert into test_csv_number file_format = (type = CSV)" -F "upload=@/tmp/whitespace_number2.csv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "bad field"
4545

4646
echo "drop table if exists test_csv" | $MYSQL_CLIENT_CONNECT

0 commit comments

Comments
 (0)