Skip to content

Commit d6e6042

Browse files
xitongsysxitongsys
committed
add err in StrToParquetType
1 parent 4053bcc commit d6e6042

File tree

7 files changed

+143
-71
lines changed

7 files changed

+143
-71
lines changed

‎common/common.go‎

Lines changed: 78 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,7 +1026,8 @@ func TransposeTable(table [][]interface{}) [][]interface{} {
10261026
// If a single record is not valid by the arrow definitions we assign it
10271027
// default value which we chose.
10281028
func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
1029-
el *parquet.SchemaElement) []interface{} {
1029+
el *parquet.SchemaElement) ([]interface{}, error) {
1030+
var err error
10301031
recs := make([]interface{}, len)
10311032
switch field.Type.(type) {
10321033
case *arrow.Int8Type:
@@ -1038,11 +1039,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
10381039
} else {
10391040
rec = 0
10401041
}
1041-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1042+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
10421043
el.Type,
10431044
el.ConvertedType,
10441045
int(el.GetTypeLength()),
10451046
int(el.GetScale()))
1047+
if err != nil {
1048+
return nil, err
1049+
}
10461050
}
10471051
case *arrow.Int16Type:
10481052
arr := col.(*array.Int16)
@@ -1053,11 +1057,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
10531057
} else {
10541058
rec = 0
10551059
}
1056-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1060+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
10571061
el.Type,
10581062
el.ConvertedType,
10591063
int(el.GetTypeLength()),
10601064
int(el.GetScale()))
1065+
if err != nil {
1066+
return nil, err
1067+
}
10611068
}
10621069
case *arrow.Int32Type:
10631070
arr := col.(*array.Int32)
@@ -1068,11 +1075,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
10681075
} else {
10691076
rec = 0
10701077
}
1071-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1078+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
10721079
el.Type,
10731080
el.ConvertedType,
10741081
int(el.GetTypeLength()),
10751082
int(el.GetScale()))
1083+
if err != nil {
1084+
return nil, err
1085+
}
10761086
}
10771087
case *arrow.Int64Type:
10781088
arr := col.(*array.Int64)
@@ -1083,11 +1093,15 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
10831093
} else {
10841094
rec = 0
10851095
}
1086-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1096+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
10871097
el.Type,
10881098
el.ConvertedType,
10891099
int(el.GetTypeLength()),
10901100
int(el.GetScale()))
1101+
1102+
if err != nil {
1103+
return nil, err
1104+
}
10911105
}
10921106
case *arrow.Uint8Type:
10931107
arr := col.(*array.Uint8)
@@ -1098,11 +1112,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
10981112
} else {
10991113
rec = 0
11001114
}
1101-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1115+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
11021116
el.Type,
11031117
el.ConvertedType,
11041118
int(el.GetTypeLength()),
11051119
int(el.GetScale()))
1120+
if err != nil {
1121+
return nil, err
1122+
}
11061123
}
11071124
case *arrow.Uint16Type:
11081125
arr := col.(*array.Uint16)
@@ -1113,11 +1130,15 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
11131130
} else {
11141131
rec = 0
11151132
}
1116-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1133+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
11171134
el.Type,
11181135
el.ConvertedType,
11191136
int(el.GetTypeLength()),
11201137
int(el.GetScale()))
1138+
1139+
if err != nil {
1140+
return nil, err
1141+
}
11211142
}
11221143
case *arrow.Uint32Type:
11231144
arr := col.(*array.Uint32)
@@ -1128,11 +1149,15 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
11281149
} else {
11291150
rec = 0
11301151
}
1131-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1152+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
11321153
el.Type,
11331154
el.ConvertedType,
11341155
int(el.GetTypeLength()),
11351156
int(el.GetScale()))
1157+
1158+
if err != nil {
1159+
return nil, err
1160+
}
11361161
}
11371162
case *arrow.Uint64Type:
11381163
arr := col.(*array.Uint64)
@@ -1143,11 +1168,15 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
11431168
} else {
11441169
rec = 0
11451170
}
1146-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1171+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
11471172
el.Type,
11481173
el.ConvertedType,
11491174
int(el.GetTypeLength()),
11501175
int(el.GetScale()))
1176+
1177+
if err != nil {
1178+
return nil, err
1179+
}
11511180
}
11521181
case *arrow.Float32Type:
11531182
arr := col.(*array.Float32)
@@ -1158,11 +1187,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
11581187
} else {
11591188
rec = 0
11601189
}
1161-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1190+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
11621191
el.Type,
11631192
el.ConvertedType,
11641193
int(el.GetTypeLength()),
11651194
int(el.GetScale()))
1195+
if err != nil {
1196+
return nil, err
1197+
}
11661198
}
11671199
case *arrow.Float64Type:
11681200
arr := col.(*array.Float64)
@@ -1173,11 +1205,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
11731205
} else {
11741206
rec = 0
11751207
}
1176-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1208+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
11771209
el.Type,
11781210
el.ConvertedType,
11791211
int(el.GetTypeLength()),
11801212
int(el.GetScale()))
1213+
if err != nil {
1214+
return nil, err
1215+
}
11811216
}
11821217
case *arrow.Date32Type:
11831218
arr := col.(*array.Date32)
@@ -1188,11 +1223,15 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
11881223
} else {
11891224
rec = 0
11901225
}
1191-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1226+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
11921227
el.Type,
11931228
el.ConvertedType,
11941229
int(el.GetTypeLength()),
11951230
int(el.GetScale()))
1231+
1232+
if err != nil {
1233+
return nil, err
1234+
}
11961235
}
11971236
case *arrow.Date64Type:
11981237
arr := col.(*array.Date64)
@@ -1203,11 +1242,15 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
12031242
} else {
12041243
rec = 0
12051244
}
1206-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1245+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
12071246
el.Type,
12081247
el.ConvertedType,
12091248
int(el.GetTypeLength()),
12101249
int(el.GetScale()))
1250+
1251+
if err != nil {
1252+
return nil, err
1253+
}
12111254
}
12121255
case *arrow.BinaryType:
12131256
arr := col.(*array.Binary)
@@ -1218,11 +1261,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
12181261
} else {
12191262
rec = []byte("")
12201263
}
1221-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1264+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
12221265
el.Type,
12231266
el.ConvertedType,
12241267
int(el.GetTypeLength()),
12251268
int(el.GetScale()))
1269+
if err != nil {
1270+
return nil, err
1271+
}
12261272
}
12271273
case *arrow.StringType:
12281274
arr := col.(*array.String)
@@ -1233,11 +1279,15 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
12331279
} else {
12341280
rec = ""
12351281
}
1236-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1282+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
12371283
el.Type,
12381284
el.ConvertedType,
12391285
int(el.GetTypeLength()),
12401286
int(el.GetScale()))
1287+
1288+
if err != nil {
1289+
return nil, err
1290+
}
12411291
}
12421292
case *arrow.BooleanType:
12431293
arr := col.(*array.Boolean)
@@ -1248,11 +1298,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
12481298
} else {
12491299
rec = false
12501300
}
1251-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1301+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
12521302
el.Type,
12531303
el.ConvertedType,
12541304
int(el.GetTypeLength()),
12551305
int(el.GetScale()))
1306+
if err != nil {
1307+
return nil, err
1308+
}
12561309
}
12571310
case *arrow.Time32Type:
12581311
arr := col.(*array.Time32)
@@ -1263,11 +1316,14 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
12631316
} else {
12641317
rec = 0
12651318
}
1266-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1319+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
12671320
el.Type,
12681321
el.ConvertedType,
12691322
int(el.GetTypeLength()),
12701323
int(el.GetScale()))
1324+
if err != nil {
1325+
return nil, err
1326+
}
12711327
}
12721328
case *arrow.TimestampType:
12731329
arr := col.(*array.Timestamp)
@@ -1278,12 +1334,15 @@ func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
12781334
} else {
12791335
rec = 0
12801336
}
1281-
recs[i] = types.StrToParquetType(fmt.Sprintf("%v", rec),
1337+
recs[i], err = types.StrToParquetType(fmt.Sprintf("%v", rec),
12821338
el.Type,
12831339
el.ConvertedType,
12841340
int(el.GetTypeLength()),
12851341
int(el.GetScale()))
1342+
if err != nil {
1343+
return nil, err
1344+
}
12861345
}
12871346
}
1288-
return recs
1347+
return recs, nil
12891348
}

‎marshal/json.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,10 @@ func MarshalJSON(ss []interface{}, schemaHandler *schema.SchemaHandler) (tb *map
247247
} else {
248248
table := res[node.PathMap.Path]
249249
pT, cT := schema.Type, schema.ConvertedType
250-
val := types.JSONTypeToParquetType(node.Val, pT, cT, int(schema.GetTypeLength()), int(schema.GetScale()))
250+
val, err := types.JSONTypeToParquetType(node.Val, pT, cT, int(schema.GetTypeLength()), int(schema.GetScale()))
251+
if err != nil {
252+
return nil, err
253+
}
251254

252255
table.Values = append(table.Values, val)
253256
table.DefinitionLevels = append(table.DefinitionLevels, node.DL)

‎types/converter_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,20 @@ func TestINT96(t *testing.T) {
1919
}
2020

2121
func TestDECIMAL(t *testing.T) {
22-
a1 := StrToParquetType("1.23", parquet.TypePtr(parquet.Type_INT32), parquet.ConvertedTypePtr(parquet.ConvertedType_DECIMAL), 9, 2).(int32)
23-
sa1 := DECIMAL_INT_ToString(int64(a1), 9, 2)
22+
a1, _ := StrToParquetType("1.23", parquet.TypePtr(parquet.Type_INT32), parquet.ConvertedTypePtr(parquet.ConvertedType_DECIMAL), 9, 2)
23+
sa1 := DECIMAL_INT_ToString(int64(a1.(int32)), 9, 2)
2424
if sa1 != "1.23" {
2525
t.Error("DECIMAL_INT_ToString error: ", a1, sa1)
2626
}
2727

28-
a2 := StrToParquetType("1.230", parquet.TypePtr(parquet.Type_INT64), parquet.ConvertedTypePtr(parquet.ConvertedType_DECIMAL), 9, 3).(int64)
29-
sa2 := DECIMAL_INT_ToString(int64(a2), 9, 3)
28+
a2, _ := StrToParquetType("1.230", parquet.TypePtr(parquet.Type_INT64), parquet.ConvertedTypePtr(parquet.ConvertedType_DECIMAL), 9, 3)
29+
sa2 := DECIMAL_INT_ToString(int64(a2.(int64)), 9, 3)
3030
if sa2 != "1.230" {
3131
t.Error("DECIMAL_INT_ToString error: ", a2, sa2)
3232
}
3333

34-
a3 := StrToParquetType("11.230", parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_DECIMAL), 9, 3).(string)
35-
sa3 := DECIMAL_BYTE_ARRAY_ToString([]byte(a3), 9, 3)
34+
a3, _ := StrToParquetType("11.230", parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_DECIMAL), 9, 3)
35+
sa3 := DECIMAL_BYTE_ARRAY_ToString([]byte(a3.(string)), 9, 3)
3636
if sa3 != "11.230" {
3737
t.Error("DECIMAL_BYTE_ARRAY_ToString error: ", a3, sa3)
3838
}

0 commit comments

Comments
 (0)