Skip to content

Commit 5a51a51

Browse files
lqiu96lqiu96
authored
chore: Use custom timestamp validator for ISO8601 timestamps with more than nanosecond precision (#4017)
* chore: Use custom timestamp validator for ISO8601 timestamps with more than nanosecond precision * chore: Rename helper method to validateTimestamp * chore: Address GCA comments * chore: Address last GCA comment * chore: Update to use assert helper method
1 parent 57ffe1d commit 5a51a51

File tree

2 files changed

+136
-67
lines changed

2 files changed

+136
-67
lines changed

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java‎

Lines changed: 63 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.google.api.services.bigquery.model.RangeValue;
2727
import com.google.auto.value.AutoValue;
2828
import com.google.cloud.Timestamp;
29+
import com.google.common.annotations.VisibleForTesting;
2930
import com.google.common.base.Function;
3031
import com.google.common.collect.ImmutableList;
3132
import com.google.common.collect.ImmutableMap;
@@ -44,6 +45,8 @@
4445
import java.util.HashMap;
4546
import java.util.List;
4647
import java.util.Map;
48+
import java.util.regex.Matcher;
49+
import java.util.regex.Pattern;
4750
import javax.annotation.Nullable;
4851
import org.threeten.extra.PeriodDuration;
4952

@@ -76,7 +79,7 @@
7679
@AutoValue
7780
public abstract class QueryParameterValue implements Serializable {
7881

79-
private static final DateTimeFormatter timestampFormatter =
82+
static final DateTimeFormatter TIMESTAMP_FORMATTER =
8083
new DateTimeFormatterBuilder()
8184
.parseLenient()
8285
.append(DateTimeFormatter.ISO_LOCAL_DATE)
@@ -94,15 +97,21 @@ public abstract class QueryParameterValue implements Serializable {
9497
.optionalEnd()
9598
.toFormatter()
9699
.withZone(ZoneOffset.UTC);
97-
private static final DateTimeFormatter timestampValidator =
100+
private static final DateTimeFormatter TIMESTAMP_VALIDATOR =
98101
new DateTimeFormatterBuilder()
99102
.parseLenient()
100-
.append(timestampFormatter)
103+
.append(TIMESTAMP_FORMATTER)
101104
.optionalStart()
102105
.appendOffsetId()
103106
.optionalEnd()
104107
.toFormatter()
105108
.withZone(ZoneOffset.UTC);
109+
// Regex to identify >9 digits in the fraction part (e.g. `.123456789123`)
110+
// Matches the dot, followed by 10+ digits (fractional part), followed by non-digits (like `+00`)
111+
// or end of string
112+
private static final Pattern ISO8601_TIMESTAMP_HIGH_PRECISION_PATTERN =
113+
Pattern.compile("\\.(\\d{10,})(?:\\D|$)");
114+
106115
private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
107116
private static final DateTimeFormatter timeFormatter =
108117
DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS");
@@ -303,6 +312,9 @@ public static QueryParameterValue bytes(byte[] value) {
303312
/**
304313
* Creates a {@code QueryParameterValue} object with a type of TIMESTAMP.
305314
*
315+
* <p>This method only supports microsecond precision for timestamp. To use higher precision,
316+
* prefer {@link #timestamp(String)} with an ISO8601 String
317+
*
306318
* @param value Microseconds since epoch, e.g. 1733945416000000 corresponds to 2024-12-11
307319
* 19:30:16.929Z
308320
*/
@@ -311,8 +323,14 @@ public static QueryParameterValue timestamp(Long value) {
311323
}
312324

313325
/**
314-
* Creates a {@code QueryParameterValue} object with a type of TIMESTAMP. Must be in the format
315-
* "yyyy-MM-dd HH:mm:ss.SSSSSSZZ", e.g. "2014-08-19 12:41:35.220000+00:00".
326+
* Creates a {@code QueryParameterValue} object with a type of TIMESTAMP.
327+
*
328+
* <p>This method supports up to picosecond precision (12 digits) for timestamp. Input should
329+
* conform to ISO8601 format.
330+
*
331+
* <p>Must be in the format "yyyy-MM-dd HH:mm:ss.SSSSSS{SSSSSSS}ZZ", e.g. "2014-08-19
332+
* 12:41:35.123456+00:00" for microsecond precision and "2014-08-19 12:41:35.123456789123+00:00"
333+
* for picosecond precision
316334
*/
317335
public static QueryParameterValue timestamp(String value) {
318336
return of(value, StandardSQLTypeName.TIMESTAMP);
@@ -481,12 +499,15 @@ private static <T> String valueToStringOrNull(T value, StandardSQLTypeName type)
481499
throw new IllegalArgumentException("Cannot convert RANGE to String value");
482500
case TIMESTAMP:
483501
if (value instanceof Long) {
502+
// Timestamp passed as a Long only support Microsecond precision
484503
Timestamp timestamp = Timestamp.ofTimeMicroseconds((Long) value);
485-
return timestampFormatter.format(
504+
return TIMESTAMP_FORMATTER.format(
486505
Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()));
487506
} else if (value instanceof String) {
488-
// verify that the String is in the right format
489-
checkFormat(value, timestampValidator);
507+
// Timestamp passed as a String can support up picosecond precision, however,
508+
// DateTimeFormatter only supports nanosecond precision. Higher than nanosecond
509+
// requires a custom validator.
510+
validateTimestamp((String) value);
490511
return (String) value;
491512
}
492513
break;
@@ -521,9 +542,42 @@ private static <T> String valueToStringOrNull(T value, StandardSQLTypeName type)
521542
"Type " + type + " incompatible with " + value.getClass().getCanonicalName());
522543
}
523544

545+
/**
546+
* Internal helper method to check that the timestamp follows the expected String input of ISO8601
547+
* string. Allows the fractional portion of the timestamp to support up to 12 digits of precision
548+
* (up to picosecond).
549+
*
550+
* @throws IllegalArgumentException if timestamp is invalid or exceeds picosecond precision
551+
*/
552+
@VisibleForTesting
553+
static void validateTimestamp(String timestamp) {
554+
// Check if the string has greater than nanosecond precision (>9 digits in fractional second)
555+
Matcher matcher = ISO8601_TIMESTAMP_HIGH_PRECISION_PATTERN.matcher(timestamp);
556+
if (matcher.find()) {
557+
// Group 1 is the fractional second part of the ISO8601 string
558+
String fraction = matcher.group(1);
559+
// Pos 10-12 of the fractional second are guaranteed to be digits. The regex only
560+
// matches the fraction section as long as they are digits.
561+
if (fraction.length() > 12) {
562+
throw new IllegalArgumentException(
563+
"Fractional second portion of ISO8601 only supports up to picosecond (12 digits) in BigQuery");
564+
}
565+
566+
// Replace the entire fractional second portion with just the nanosecond portion.
567+
// The new timestamp will be validated against the JDK's DateTimeFormatter
568+
String truncatedFraction = fraction.substring(0, 9);
569+
timestamp =
570+
new StringBuilder(timestamp)
571+
.replace(matcher.start(1), matcher.end(1), truncatedFraction)
572+
.toString();
573+
}
574+
575+
// It is valid as long as DateTimeFormatter doesn't throw an exception
576+
checkFormat(timestamp, TIMESTAMP_VALIDATOR);
577+
}
578+
524579
private static void checkFormat(Object value, DateTimeFormatter formatter) {
525580
try {
526-
527581
formatter.parse((String) value);
528582
} catch (DateTimeParseException e) {
529583
throw new IllegalArgumentException(e.getMessage(), e);

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java‎

Lines changed: 73 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,9 @@
1616

1717
package com.google.cloud.bigquery;
1818

19+
import static com.google.cloud.bigquery.QueryParameterValue.TIMESTAMP_FORMATTER;
1920
import static com.google.common.truth.Truth.assertThat;
20-
import static java.time.temporal.ChronoField.HOUR_OF_DAY;
21-
import static java.time.temporal.ChronoField.MINUTE_OF_HOUR;
22-
import static java.time.temporal.ChronoField.NANO_OF_SECOND;
23-
import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
21+
import static org.junit.Assert.assertThrows;
2422

2523
import com.google.api.services.bigquery.model.QueryParameterType;
2624
import com.google.common.collect.ImmutableMap;
@@ -29,9 +27,6 @@
2927
import java.text.ParseException;
3028
import java.time.Instant;
3129
import java.time.Period;
32-
import java.time.ZoneOffset;
33-
import java.time.format.DateTimeFormatter;
34-
import java.time.format.DateTimeFormatterBuilder;
3530
import java.util.ArrayList;
3631
import java.util.Date;
3732
import java.util.HashMap;
@@ -43,25 +38,6 @@
4338

4439
public class QueryParameterValueTest {
4540

46-
private static final DateTimeFormatter TIMESTAMPFORMATTER =
47-
new DateTimeFormatterBuilder()
48-
.parseLenient()
49-
.append(DateTimeFormatter.ISO_LOCAL_DATE)
50-
.appendLiteral(' ')
51-
.appendValue(HOUR_OF_DAY, 2)
52-
.appendLiteral(':')
53-
.appendValue(MINUTE_OF_HOUR, 2)
54-
.optionalStart()
55-
.appendLiteral(':')
56-
.appendValue(SECOND_OF_MINUTE, 2)
57-
.optionalStart()
58-
.appendFraction(NANO_OF_SECOND, 6, 9, true)
59-
.optionalStart()
60-
.appendOffset("+HHMM", "+00:00")
61-
.optionalEnd()
62-
.toFormatter()
63-
.withZone(ZoneOffset.UTC);
64-
6541
private static final QueryParameterValue QUERY_PARAMETER_VALUE =
6642
QueryParameterValue.newBuilder()
6743
.setType(StandardSQLTypeName.STRING)
@@ -326,11 +302,9 @@ public void testStringArray() {
326302

327303
@Test
328304
public void testTimestampFromLong() {
329-
QueryParameterValue value = QueryParameterValue.timestamp(1408452095220000L);
330-
assertThat(value.getValue()).isEqualTo("2014-08-19 12:41:35.220000+00:00");
331-
assertThat(value.getType()).isEqualTo(StandardSQLTypeName.TIMESTAMP);
332-
assertThat(value.getArrayType()).isNull();
333-
assertThat(value.getArrayValues()).isNull();
305+
// Expects output to be ISO8601 string with microsecond precision
306+
assertTimestampValue(
307+
QueryParameterValue.timestamp(1408452095220000L), "2014-08-19 12:41:35.220000+00:00");
334308
}
335309

336310
@Test
@@ -340,43 +314,77 @@ public void testTimestampWithFormatter() {
340314
long secs = Math.floorDiv(timestampInMicroseconds, microseconds);
341315
int nano = (int) Math.floorMod(timestampInMicroseconds, microseconds) * 1000;
342316
Instant instant = Instant.ofEpochSecond(secs, nano);
343-
String expected = TIMESTAMPFORMATTER.format(instant);
344-
assertThat(expected)
345-
.isEqualTo(QueryParameterValue.timestamp(timestampInMicroseconds).getValue());
317+
String expected = TIMESTAMP_FORMATTER.format(instant);
318+
assertTimestampValue(QueryParameterValue.timestamp(timestampInMicroseconds), expected);
346319
}
347320

348321
@Test
349-
public void testTimestamp() {
350-
QueryParameterValue value = QueryParameterValue.timestamp("2014-08-19 12:41:35.220000+00:00");
351-
assertThat(value.getValue()).isEqualTo("2014-08-19 12:41:35.220000+00:00");
352-
assertThat(value.getType()).isEqualTo(StandardSQLTypeName.TIMESTAMP);
353-
assertThat(value.getArrayType()).isNull();
354-
assertThat(value.getArrayValues()).isNull();
322+
public void testTimestampFromString() {
323+
assertTimestampValue(
324+
QueryParameterValue.timestamp("2014-08-19 12:41:35.220000+00:00"),
325+
"2014-08-19 12:41:35.220000+00:00");
326+
assertTimestampValue(
327+
QueryParameterValue.timestamp("2025-08-19 12:34:56.123456789+00:00"),
328+
"2025-08-19 12:34:56.123456789+00:00");
329+
330+
// The following test cases test more than nanosecond precision
331+
// 10 digits of precision (1 digit more than nanosecond)
332+
assertTimestampValue(
333+
QueryParameterValue.timestamp("2025-12-08 12:34:56.1234567890+00:00"),
334+
"2025-12-08 12:34:56.1234567890+00:00");
335+
// 12 digits (picosecond precision)
336+
assertTimestampValue(
337+
QueryParameterValue.timestamp("2025-12-08 12:34:56.123456789123+00:00"),
338+
"2025-12-08 12:34:56.123456789123+00:00");
339+
340+
// More than picosecond precision
341+
assertThrows(
342+
IllegalArgumentException.class,
343+
() -> QueryParameterValue.timestamp("2025-12-08 12:34:56.1234567891234+00:00"));
344+
assertThrows(
345+
IllegalArgumentException.class,
346+
() ->
347+
QueryParameterValue.timestamp("2025-12-08 12:34:56.123456789123456789123456789+00:00"));
355348
}
356349

357350
@Test
358351
public void testTimestampWithDateTimeFormatterBuilder() {
359-
QueryParameterValue value = QueryParameterValue.timestamp("2019-02-14 12:34:45.938993Z");
360-
assertThat(value.getValue()).isEqualTo("2019-02-14 12:34:45.938993Z");
361-
assertThat(value.getType()).isEqualTo(StandardSQLTypeName.TIMESTAMP);
362-
assertThat(value.getArrayType()).isNull();
363-
assertThat(value.getArrayValues()).isNull();
364-
QueryParameterValue value1 = QueryParameterValue.timestamp("2019-02-14 12:34:45.938993+0000");
365-
assertThat(value1.getValue()).isEqualTo("2019-02-14 12:34:45.938993+0000");
366-
assertThat(value1.getType()).isEqualTo(StandardSQLTypeName.TIMESTAMP);
367-
assertThat(value1.getArrayType()).isNull();
368-
assertThat(value1.getArrayValues()).isNull();
369-
QueryParameterValue value2 = QueryParameterValue.timestamp("2019-02-14 12:34:45.102+00:00");
370-
assertThat(value2.getValue()).isEqualTo("2019-02-14 12:34:45.102+00:00");
371-
assertThat(value2.getType()).isEqualTo(StandardSQLTypeName.TIMESTAMP);
372-
assertThat(value2.getArrayType()).isNull();
373-
assertThat(value2.getArrayValues()).isNull();
352+
assertTimestampValue(
353+
QueryParameterValue.timestamp("2019-02-14 12:34:45.938993Z"),
354+
"2019-02-14 12:34:45.938993Z");
355+
assertTimestampValue(
356+
QueryParameterValue.timestamp("2019-02-14 12:34:45.938993+0000"),
357+
"2019-02-14 12:34:45.938993+0000");
358+
assertTimestampValue(
359+
QueryParameterValue.timestamp("2019-02-14 12:34:45.102+00:00"),
360+
"2019-02-14 12:34:45.102+00:00");
374361
}
375362

376-
@Test(expected = IllegalArgumentException.class)
377-
public void testInvalidTimestamp() {
363+
@Test
364+
public void testInvalidTimestampStringValues() {
365+
assertThrows(IllegalArgumentException.class, () -> QueryParameterValue.timestamp("abc"));
366+
378367
// missing the time
379-
QueryParameterValue.timestamp("2014-08-19");
368+
assertThrows(IllegalArgumentException.class, () -> QueryParameterValue.timestamp("2014-08-19"));
369+
370+
// missing the hour
371+
assertThrows(
372+
IllegalArgumentException.class, () -> QueryParameterValue.timestamp("2014-08-19 12"));
373+
374+
// can't have the 'T' separator
375+
assertThrows(
376+
IllegalArgumentException.class, () -> QueryParameterValue.timestamp("2014-08-19T12"));
377+
assertThrows(
378+
IllegalArgumentException.class,
379+
() -> QueryParameterValue.timestamp("2014-08-19T12:34:00.123456"));
380+
381+
// Fractional part has picosecond length, but fractional part is not a valid number
382+
assertThrows(
383+
IllegalArgumentException.class,
384+
() -> QueryParameterValue.timestamp("2014-08-19 12:34:00.123456789abc+00:00"));
385+
assertThrows(
386+
IllegalArgumentException.class,
387+
() -> QueryParameterValue.timestamp("2014-08-19 12:34:00.123456abc789+00:00"));
380388
}
381389

382390
@Test
@@ -683,4 +691,11 @@ private static void testRangeDataEquals(String start, String end, FieldElementTy
683691
assertThat(queryParameterValue.getStructValues()).isNull();
684692
assertThat(queryParameterValue.getValue()).isNull();
685693
}
694+
695+
private void assertTimestampValue(QueryParameterValue value, String expectedStringValue) {
696+
assertThat(value.getValue()).isEqualTo(expectedStringValue);
697+
assertThat(value.getType()).isEqualTo(StandardSQLTypeName.TIMESTAMP);
698+
assertThat(value.getArrayType()).isNull();
699+
assertThat(value.getArrayValues()).isNull();
700+
}
686701
}

0 commit comments

Comments
 (0)