Skip to content

Commit fa2db67

Browse files
elianddbelianddb
authored
Merge pull request #3270 from dolthub/elian/9969
dolthub/dolt#9969: Fix `ENCLOSED BY` ignoring terminator inside of field when using `LOAD DATA INFILE`
2 parents fff320b + c6c7daf commit fa2db67

15 files changed

+236
-69
lines changed

‎.gitattributes‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,22 @@
1-
enginetest/testdata/test1.txt binary
2-
enginetest/testdata/test2.csv binary
3-
enginetest/testdata/test3.csv binary
4-
enginetest/testdata/test3backwards.csv binary
5-
enginetest/testdata/test4.txt binary
6-
enginetest/testdata/test5.txt binary
7-
enginetest/testdata/test6.csv binary
8-
enginetest/testdata/test7.txt binary
9-
enginetest/testdata/test8.txt binary
10-
enginetest/testdata/test9.txt binary
11-
enginetest/testdata/test10.txt binary
12-
enginetest/testdata/simple_json.txt binary
1+
enginetest/testdata/test1.txt binary
2+
enginetest/testdata/test2.csv binary
3+
enginetest/testdata/test3.csv binary
4+
enginetest/testdata/test3backwards.csv binary
5+
enginetest/testdata/test4.txt binary
6+
enginetest/testdata/test5.txt binary
7+
enginetest/testdata/test6.csv binary
8+
enginetest/testdata/test7.txt binary
9+
enginetest/testdata/test8.txt binary
10+
enginetest/testdata/test9.txt binary
11+
enginetest/testdata/test10.txt binary
12+
enginetest/testdata/simple_json.txt binary
13+
enginetest/testdata/loaddata_null_in_field.dat binary
14+
enginetest/testdata/loaddata_lborder_null.dat binary
15+
enginetest/testdata/loaddata_enc_esc_eq.dat binary
16+
enginetest/testdata/loaddata_eof.dat binary
17+
enginetest/testdata/loaddata_term_in_field.dat binary
18+
enginetest/testdata/loaddata_mixed_escapes.dat binary
19+
enginetest/testdata/loaddata_enclosed.dat binary
20+
enginetest/testdata/loaddata_single_quotes.dat binary
21+
enginetest/testdata/loaddata_nulls.dat binary
22+
enginetest/testdata/loaddata_escape.dat binary

‎enginetest/queries/load_queries.go‎

Lines changed: 98 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,103 @@ import (
2525

2626
var LoadDataScripts = []ScriptTest{
2727
{
28-
Name: "LOAD DATA applies column defaults when \\N provided",
28+
// https://github.com/dolthub/dolt/issues/9969
29+
Name: "LOAD DATA with ENCLOSED BY and ESCAPED BY parsing",
30+
SetUpScript: []string{
31+
"create table t1(pk int primary key, c1 longtext)",
32+
"LOAD DATA INFILE './testdata/loaddata_term_in_field.dat' INTO TABLE t1 FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"'",
33+
"create table t2(pk int primary key, c1 longtext)",
34+
"LOAD DATA INFILE './testdata/loaddata_escape.dat' INTO TABLE t2 FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\\\\'",
35+
"create table t3(a varchar(20), b varchar(20))",
36+
"LOAD DATA INFILE './testdata/loaddata_enclosed.dat' INTO TABLE t3 FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"'",
37+
"create table t4(a varchar(20), b varchar(20))",
38+
"LOAD DATA INFILE './testdata/loaddata_mixed_escapes.dat' INTO TABLE t4 FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\\\\'",
39+
"create table t5(a text, b text)",
40+
"LOAD DATA INFILE './testdata/loaddata_single_quotes.dat' INTO TABLE t5 FIELDS TERMINATED BY ',' ENCLOSED BY ''''",
41+
"create table t6(pk int, a varchar(20), b varchar(20))",
42+
"LOAD DATA INFILE './testdata/loaddata_nulls.dat' INTO TABLE t6 FIELDS TERMINATED BY ','",
43+
"create table t7(i int, v text)",
44+
"LOAD DATA INFILE './testdata/loaddata_eof.dat' INTO TABLE t7 FIELDS TERMINATED BY ',' ENCLOSED BY '$' ESCAPED BY '$'",
45+
"create table t8(i int, v text)",
46+
"LOAD DATA INFILE './testdata/loaddata_enc_esc_eq.dat' INTO TABLE t8 FIELDS TERMINATED BY ',' ENCLOSED BY '$' ESCAPED BY '$'",
47+
"create table t9(i int, v text)",
48+
"LOAD DATA INFILE './testdata/loaddata_lborder_null.dat' INTO TABLE t9 FIELDS TERMINATED BY ',' ENCLOSED BY '' ESCAPED BY ''",
49+
"create table t10(i int, v text)",
50+
"LOAD DATA INFILE './testdata/loaddata_null_in_field.dat' INTO TABLE t10 FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY ''",
51+
},
52+
Assertions: []ScriptTestAssertion{
53+
{
54+
Query: "select * from t1",
55+
Expected: []sql.Row{{1, "foo,bar"}},
56+
},
57+
{
58+
Query: "select * from t2",
59+
Expected: []sql.Row{{1, "foo,bar"}},
60+
},
61+
{
62+
Query: "select * from t3 ORDER BY a",
63+
Expected: []sql.Row{
64+
{"a\"b", "cd\"ef"},
65+
{"field1", "field2"},
66+
{"foo,bar", "baz,qux"},
67+
},
68+
},
69+
{
70+
Query: "select * from t4",
71+
Expected: []sql.Row{
72+
{nil, "\x1A"},
73+
{"a,b", "c,d"},
74+
{"hello\nworld", "foo\tbar"},
75+
},
76+
},
77+
{
78+
Query: "select * from t5",
79+
Expected: []sql.Row{
80+
{"Field A", "Field B"},
81+
{"Field 1", "Field 2"},
82+
{"Field 3", "Field 4"},
83+
{"Field 5", "Field 6"},
84+
},
85+
},
86+
{
87+
Query: "select * from t6 ORDER BY pk",
88+
Expected: []sql.Row{
89+
{1, "hello", "world"},
90+
{2, nil, "test"},
91+
{3, "", "empty"},
92+
{4, nil, nil},
93+
},
94+
},
95+
{
96+
Query: "select * from t7",
97+
Expected: []sql.Row{
98+
{1, "foo $0 $b $n $t $Z $N bar"},
99+
{2, "$foo $ bar$"},
100+
},
101+
},
102+
{
103+
Query: "select * from t8",
104+
Expected: []sql.Row{
105+
{1, "foo $0 $b $n $t $Z $N bar"},
106+
{2, "foo $ bar"},
107+
},
108+
},
109+
{
110+
Query: "select * from t9",
111+
Expected: []sql.Row{
112+
{1, "\x00foo bar"},
113+
},
114+
},
115+
{
116+
Query: "select * from t10",
117+
Expected: []sql.Row{
118+
{1, "foo \x00 bar"},
119+
},
120+
},
121+
},
122+
},
123+
{
124+
Name: "LOAD DATA does not apply column defaults when \\N provided",
29125
SetUpScript: []string{
30126
"create table t (pk int primary key, c1 int default 1, c2 int)",
31127
// Explicitly use Windows-style line endings to be robust on Windows CI
@@ -34,7 +130,7 @@ var LoadDataScripts = []ScriptTest{
34130
Assertions: []ScriptTestAssertion{
35131
{
36132
Query: "select * from t",
37-
Expected: []sql.Row{{1, 1, 1}},
133+
Expected: []sql.Row{{1, nil, 1}},
38134
},
39135
},
40136
},

‎enginetest/testdata/loaddata_enc_esc_eq.dat‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
$1$,$foo $0 $b $n $t $Z $N bar$
2+
$2$,$foo $$ bar$

‎enginetest/testdata/loaddata_enclosed.dat‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"field1","field2"
2+
"a""b","cd""ef"
3+
"foo,bar","baz,qux"

‎enginetest/testdata/loaddata_eof.dat‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
$1$,$foo $0 $b $n $t $Z $N bar$
2+
$2$,$foo $$ bar$

‎enginetest/testdata/loaddata_escape.dat‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"1","foo\,bar"

‎enginetest/testdata/loaddata_lborder_null.dat‎

10 Bytes
Binary file not shown.

‎enginetest/testdata/loaddata_mixed_escapes.dat‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"hello\nworld","foo\tbar"
2+
"a\,b","c\,d"
3+
"\N","\Z"

‎enginetest/testdata/loaddata_null_in_field.dat‎

15 Bytes
Binary file not shown.

‎enginetest/testdata/loaddata_nulls.dat‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
1,hello,world
2+
2,\N,test
3+
3,,empty
4+
4,\N,\N

0 commit comments

Comments
 (0)