Skip to content

Commit 7bee10a

Browse files
Yifeng-SigmaYifeng-Sigma
authored
[SIG-45543] return microsecond rather than nanosec in arrowToRecord() for timestamp tz/ntz/ltz to avoid future date overflow (#180)
1 parent 669e036 commit 7bee10a

File tree

2 files changed

+18
-18
lines changed

2 files changed

+18
-18
lines changed

‎converter.go‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1048,7 +1048,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
10481048
}
10491049
defer newCol.Release()
10501050
case timestampNtzType:
1051-
tb := array.NewTimestampBuilder(pool, &arrow.TimestampType{Unit: arrow.Nanosecond})
1051+
tb := array.NewTimestampBuilder(pool, &arrow.TimestampType{Unit: arrow.Microsecond})
10521052
if col.DataType().ID() == arrow.STRUCT {
10531053
structData := col.(*array.Struct)
10541054
epochArray, ok := structData.Field(0).(*array.Int64)
@@ -1066,7 +1066,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
10661066
for i := 0; i < int(numRows); i++ {
10671067
if !col.IsNull(i) {
10681068
val := time.Unix(epoch[i], int64(fraction[i]))
1069-
tb.Append(arrow.Timestamp(val.UnixNano()))
1069+
tb.Append(arrow.Timestamp(val.UnixMicro()))
10701070
} else {
10711071
tb.AppendNull()
10721072
}
@@ -1075,7 +1075,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
10751075
for i, t := range col.(*array.Int64).Int64Values() {
10761076
if !col.IsNull(i) {
10771077
val := time.Unix(0, t*int64(math.Pow10(9-int(srcColumnMeta.Scale)))).UTC()
1078-
tb.Append(arrow.Timestamp(val.UnixNano()))
1078+
tb.Append(arrow.Timestamp(val.UnixMicro()))
10791079
} else {
10801080
tb.AppendNull()
10811081
}
@@ -1089,7 +1089,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
10891089
for i, t := range timestampValues {
10901090
if !col.IsNull(i) {
10911091
val := time.Unix(0, int64(t)*int64(math.Pow10(9-int(srcColumnMeta.Scale)))).UTC()
1092-
tb.Append(arrow.Timestamp(val.UnixNano()))
1092+
tb.Append(arrow.Timestamp(val.UnixMicro()))
10931093
} else {
10941094
tb.AppendNull()
10951095
}
@@ -1099,7 +1099,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
10991099
defer newCol.Release()
11001100
tb.Release()
11011101
case timestampLtzType:
1102-
tb := array.NewTimestampBuilder(pool, &arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone: loc.String()})
1102+
tb := array.NewTimestampBuilder(pool, &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: loc.String()})
11031103
if col.DataType().ID() == arrow.STRUCT {
11041104
structData := col.(*array.Struct)
11051105

@@ -1108,7 +1108,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
11081108
for i := 0; i < int(numRows); i++ {
11091109
if !col.IsNull(i) {
11101110
val := time.Unix(epoch[i], int64(fraction[i]))
1111-
tb.Append(arrow.Timestamp(val.UnixNano()))
1111+
tb.Append(arrow.Timestamp(val.UnixMicro()))
11121112
} else {
11131113
tb.AppendNull()
11141114
}
@@ -1119,7 +1119,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
11191119
q := t / int64(math.Pow10(int(srcColumnMeta.Scale)))
11201120
r := t % int64(math.Pow10(int(srcColumnMeta.Scale)))
11211121
val := time.Unix(q, r)
1122-
tb.Append(arrow.Timestamp(val.UnixNano()))
1122+
tb.Append(arrow.Timestamp(val.UnixMicro()))
11231123
} else {
11241124
tb.AppendNull()
11251125
}
@@ -1135,7 +1135,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
11351135
q := int64(t) / int64(math.Pow10(int(srcColumnMeta.Scale)))
11361136
r := int64(t) % int64(math.Pow10(int(srcColumnMeta.Scale)))
11371137
val := time.Unix(q, r)
1138-
tb.Append(arrow.Timestamp(val.UnixNano()))
1138+
tb.Append(arrow.Timestamp(val.UnixMicro()))
11391139
} else {
11401140
tb.AppendNull()
11411141
}
@@ -1145,7 +1145,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
11451145
defer newCol.Release()
11461146
tb.Release()
11471147
case timestampTzType:
1148-
tb := array.NewTimestampBuilder(pool, &arrow.TimestampType{Unit: arrow.Nanosecond})
1148+
tb := array.NewTimestampBuilder(pool, &arrow.TimestampType{Unit: arrow.Microsecond})
11491149
structData, ok := col.(*array.Struct)
11501150
if !ok {
11511151
return nil, fmt.Errorf("expect type *array.Struct but get %s", col.DataType())
@@ -1166,7 +1166,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
11661166
loc := Location(int(timezone[i]) - 1440)
11671167
tt := time.Unix(epoch[i], 0)
11681168
val := tt.In(loc)
1169-
tb.Append(arrow.Timestamp(val.UnixNano()))
1169+
tb.Append(arrow.Timestamp(val.UnixMicro()))
11701170
} else {
11711171
tb.AppendNull()
11721172
}
@@ -1195,7 +1195,7 @@ func arrowToRecord(record arrow.Record, pool memory.Allocator, rowType []execRes
11951195
loc := Location(int(timezone[i]) - 1440)
11961196
tt := time.Unix(epoch[i], int64(fraction[i]))
11971197
val := tt.In(loc)
1198-
tb.Append(arrow.Timestamp(val.UnixNano()))
1198+
tb.Append(arrow.Timestamp(val.UnixMicro()))
11991199
} else {
12001200
tb.AppendNull()
12011201
}
@@ -1222,9 +1222,9 @@ func recordToSchema(sc *arrow.Schema, rowType []execResponseRowType, loc *time.L
12221222
case timeType:
12231223
t = &arrow.Time64Type{Unit: arrow.Nanosecond}
12241224
case timestampNtzType, timestampTzType:
1225-
t = &arrow.TimestampType{Unit: arrow.Nanosecond}
1225+
t = &arrow.TimestampType{Unit: arrow.Microsecond}
12261226
case timestampLtzType:
1227-
t = &arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone: loc.String()}
1227+
t = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: loc.String()}
12281228
default:
12291229
converted = false
12301230
}

‎converter_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -952,7 +952,7 @@ func TestArrowToRecord(t *testing.T) {
952952
compare: func(src interface{}, convertedRec arrow.Record) int {
953953
srcvs := src.([]time.Time)
954954
for i, t := range convertedRec.Column(0).(*array.Timestamp).TimestampValues() {
955-
if srcvs[i].UnixNano() != int64(t) {
955+
if srcvs[i].UnixMicro() != int64(t) {
956956
return i
957957
}
958958
}
@@ -975,7 +975,7 @@ func TestArrowToRecord(t *testing.T) {
975975
compare: func(src interface{}, convertedRec arrow.Record) int {
976976
srcvs := src.([]time.Time)
977977
for i, t := range convertedRec.Column(0).(*array.Timestamp).TimestampValues() {
978-
if srcvs[i].UnixNano() != int64(t) {
978+
if srcvs[i].UnixMicro() != int64(t) {
979979
return i
980980
}
981981
}
@@ -997,7 +997,7 @@ func TestArrowToRecord(t *testing.T) {
997997
compare: func(src interface{}, convertedRec arrow.Record) int {
998998
srcvs := src.([]time.Time)
999999
for i, t := range convertedRec.Column(0).(*array.Timestamp).TimestampValues() {
1000-
if srcvs[i].UnixNano() != int64(t) {
1000+
if srcvs[i].UnixMicro() != int64(t) {
10011001
return i
10021002
}
10031003
}
@@ -1020,7 +1020,7 @@ func TestArrowToRecord(t *testing.T) {
10201020
compare: func(src interface{}, convertedRec arrow.Record) int {
10211021
srcvs := src.([]time.Time)
10221022
for i, t := range convertedRec.Column(0).(*array.Timestamp).TimestampValues() {
1023-
if srcvs[i].UnixNano() != int64(t) {
1023+
if srcvs[i].UnixMicro() != int64(t) {
10241024
return i
10251025
}
10261026
}
@@ -1045,7 +1045,7 @@ func TestArrowToRecord(t *testing.T) {
10451045
compare: func(src interface{}, convertedRec arrow.Record) int {
10461046
srcvs := src.([]time.Time)
10471047
for i, t := range convertedRec.Column(0).(*array.Timestamp).TimestampValues() {
1048-
if srcvs[i].Unix() != time.Unix(0, int64(t)).Unix() {
1048+
if srcvs[i].Unix() != time.Unix(0, int64(t) * 1000).Unix() {
10491049
return i
10501050
}
10511051
}

0 commit comments

Comments
 (0)