Skip to content

Commit f751a45

Browse files
ErigaraRoman Shanin
andauthored
chore(parquet/record/field): dont truncate timestamps on display (#8266)
# Which issue does this PR close? - Closes #8265. # Rationale for this change It more convenient to debug tings when timestamps are properly shown. # What changes are included in this PR? Fix in methods and tests to reflect changes. # Are these changes tested? Changes are covered by adjusted tests. # Are there any user-facing changes? Yes, when user would display `Row` or individual `Field` timestamps would be shown with proper precision. Co-authored-by: Roman Shanin <[email protected]>
1 parent 77df2ee commit f751a45

File tree

1 file changed

+57
-33
lines changed

1 file changed

+57
-33
lines changed

parquet/src/record/api.rs

Lines changed: 57 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -928,29 +928,22 @@ fn convert_date_to_string(value: i32) -> String {
928928
format!("{}", dt.format("%Y-%m-%d"))
929929
}
930930

931-
/// Helper method to convert Parquet timestamp into a string.
932-
/// Input `value` is a number of seconds since the epoch in UTC.
933-
/// Datetime is displayed in local timezone.
934-
#[inline]
935-
fn convert_timestamp_secs_to_string(value: i64) -> String {
936-
let dt = Utc.timestamp_opt(value, 0).unwrap();
937-
format!("{}", dt.format("%Y-%m-%d %H:%M:%S %:z"))
938-
}
939-
940931
/// Helper method to convert Parquet timestamp into a string.
941932
/// Input `value` is a number of milliseconds since the epoch in UTC.
942-
/// Datetime is displayed in local timezone.
933+
/// Datetime is displayed in UTC timezone.
943934
#[inline]
944935
fn convert_timestamp_millis_to_string(value: i64) -> String {
945-
convert_timestamp_secs_to_string(value / 1000)
936+
let dt = Utc.timestamp_millis_opt(value).unwrap();
937+
format!("{}", dt.format("%Y-%m-%d %H:%M:%S%.3f %:z"))
946938
}
947939

948940
/// Helper method to convert Parquet timestamp into a string.
949941
/// Input `value` is a number of microseconds since the epoch in UTC.
950-
/// Datetime is displayed in local timezone.
942+
/// Datetime is displayed in UTC timezone.
951943
#[inline]
952944
fn convert_timestamp_micros_to_string(value: i64) -> String {
953-
convert_timestamp_secs_to_string(value / 1000000)
945+
let dt = Utc.timestamp_micros(value).unwrap();
946+
format!("{}", dt.format("%Y-%m-%d %H:%M:%S%.6f %:z"))
954947
}
955948

956949
/// Helper method to convert Parquet time (milliseconds since midnight) into a string.
@@ -1278,44 +1271,75 @@ mod tests {
12781271

12791272
#[test]
12801273
fn test_convert_timestamp_millis_to_string() {
1281-
fn check_datetime_conversion(y: u32, m: u32, d: u32, h: u32, mi: u32, s: u32) {
1274+
fn check_datetime_conversion(
1275+
(y, m, d, h, mi, s, milli): (u32, u32, u32, u32, u32, u32, u32),
1276+
exp: &str,
1277+
) {
12821278
let datetime = chrono::NaiveDate::from_ymd_opt(y as i32, m, d)
12831279
.unwrap()
1284-
.and_hms_opt(h, mi, s)
1280+
.and_hms_milli_opt(h, mi, s, milli)
12851281
.unwrap();
12861282
let dt = Utc.from_utc_datetime(&datetime);
12871283
let res = convert_timestamp_millis_to_string(dt.timestamp_millis());
1288-
let exp = format!("{}", dt.format("%Y-%m-%d %H:%M:%S %:z"));
12891284
assert_eq!(res, exp);
12901285
}
12911286

1292-
check_datetime_conversion(1969, 9, 10, 1, 2, 3);
1293-
check_datetime_conversion(2010, 1, 2, 13, 12, 54);
1294-
check_datetime_conversion(2011, 1, 3, 8, 23, 1);
1295-
check_datetime_conversion(2012, 4, 5, 11, 6, 32);
1296-
check_datetime_conversion(2013, 5, 12, 16, 38, 0);
1297-
check_datetime_conversion(2014, 11, 28, 21, 15, 12);
1287+
check_datetime_conversion((1969, 9, 10, 1, 2, 3, 4), "1969-09-10 01:02:03.004 +00:00");
1288+
check_datetime_conversion(
1289+
(2010, 1, 2, 13, 12, 54, 42),
1290+
"2010-01-02 13:12:54.042 +00:00",
1291+
);
1292+
check_datetime_conversion((2011, 1, 3, 8, 23, 1, 27), "2011-01-03 08:23:01.027 +00:00");
1293+
check_datetime_conversion((2012, 4, 5, 11, 6, 32, 0), "2012-04-05 11:06:32.000 +00:00");
1294+
check_datetime_conversion(
1295+
(2013, 5, 12, 16, 38, 0, 15),
1296+
"2013-05-12 16:38:00.015 +00:00",
1297+
);
1298+
check_datetime_conversion(
1299+
(2014, 11, 28, 21, 15, 12, 59),
1300+
"2014-11-28 21:15:12.059 +00:00",
1301+
);
12981302
}
12991303

13001304
#[test]
13011305
fn test_convert_timestamp_micros_to_string() {
1302-
fn check_datetime_conversion(y: u32, m: u32, d: u32, h: u32, mi: u32, s: u32) {
1306+
fn check_datetime_conversion(
1307+
(y, m, d, h, mi, s, micro): (u32, u32, u32, u32, u32, u32, u32),
1308+
exp: &str,
1309+
) {
13031310
let datetime = chrono::NaiveDate::from_ymd_opt(y as i32, m, d)
13041311
.unwrap()
1305-
.and_hms_opt(h, mi, s)
1312+
.and_hms_micro_opt(h, mi, s, micro)
13061313
.unwrap();
13071314
let dt = Utc.from_utc_datetime(&datetime);
13081315
let res = convert_timestamp_micros_to_string(dt.timestamp_micros());
1309-
let exp = format!("{}", dt.format("%Y-%m-%d %H:%M:%S %:z"));
13101316
assert_eq!(res, exp);
13111317
}
13121318

1313-
check_datetime_conversion(1969, 9, 10, 1, 2, 3);
1314-
check_datetime_conversion(2010, 1, 2, 13, 12, 54);
1315-
check_datetime_conversion(2011, 1, 3, 8, 23, 1);
1316-
check_datetime_conversion(2012, 4, 5, 11, 6, 32);
1317-
check_datetime_conversion(2013, 5, 12, 16, 38, 0);
1318-
check_datetime_conversion(2014, 11, 28, 21, 15, 12);
1319+
check_datetime_conversion(
1320+
(1969, 9, 10, 1, 2, 3, 4),
1321+
"1969-09-10 01:02:03.000004 +00:00",
1322+
);
1323+
check_datetime_conversion(
1324+
(2010, 1, 2, 13, 12, 54, 42),
1325+
"2010-01-02 13:12:54.000042 +00:00",
1326+
);
1327+
check_datetime_conversion(
1328+
(2011, 1, 3, 8, 23, 1, 27),
1329+
"2011-01-03 08:23:01.000027 +00:00",
1330+
);
1331+
check_datetime_conversion(
1332+
(2012, 4, 5, 11, 6, 32, 0),
1333+
"2012-04-05 11:06:32.000000 +00:00",
1334+
);
1335+
check_datetime_conversion(
1336+
(2013, 5, 12, 16, 38, 0, 15),
1337+
"2013-05-12 16:38:00.000015 +00:00",
1338+
);
1339+
check_datetime_conversion(
1340+
(2014, 11, 28, 21, 15, 12, 59),
1341+
"2014-11-28 21:15:12.000059 +00:00",
1342+
);
13191343
}
13201344

13211345
#[test]
@@ -2000,11 +2024,11 @@ mod tests {
20002024
);
20012025
assert_eq!(
20022026
Field::TimestampMillis(12345678).to_json_value(),
2003-
Value::String("1970-01-01 03:25:45 +00:00".to_string())
2027+
Value::String("1970-01-01 03:25:45.678 +00:00".to_string())
20042028
);
20052029
assert_eq!(
20062030
Field::TimestampMicros(12345678901).to_json_value(),
2007-
Value::String(convert_timestamp_micros_to_string(12345678901))
2031+
Value::String("1970-01-01 03:25:45.678901 +00:00".to_string())
20082032
);
20092033
assert_eq!(
20102034
Field::TimeMillis(47445123).to_json_value(),

0 commit comments

Comments
 (0)