Skip to content

Commit 7198340

Browse files
lqiu96lqiu96
authored
test: Add integration tests for picosecond support (#4030)
* test: Add integration tests for picosecond support * chore: Add a micro -> pico exact timestamp test case * chore: Add additional test cases * chore: Fix test cases with 'Z' * chore: Test if first query has an impact * chore: Remove flaky test for now * chore: Remove testing comment
1 parent adbe2e0 commit 7198340

File tree

3 files changed

+336
-8
lines changed

3 files changed

+336
-8
lines changed

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -328,9 +328,9 @@ public static QueryParameterValue timestamp(Long value) {
328328
* <p>This method supports up to picosecond precision (12 digits) for timestamp. Input should
329329
* conform to ISO8601 format.
330330
*
331-
* <p>Must be in the format "yyyy-MM-dd HH:mm:ss.SSSSSS{SSSSSSS}ZZ", e.g. "2014-08-19
332-
* 12:41:35.123456+00:00" for microsecond precision and "2014-08-19 12:41:35.123456789123+00:00"
333-
* for picosecond precision
331+
* <p>Should be in the format "yyyy-MM-dd HH:mm:ss.SSSSSS{SSSSSSS}Z", e.g. "2014-08-19
332+
* 12:41:35.123456Z" for microsecond precision and "2014-08-19 12:41:35.123456789123Z" for
333+
* picosecond precision
334334
*/
335335
public static QueryParameterValue timestamp(String value) {
336336
return of(value, StandardSQLTypeName.TIMESTAMP);

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/testing/RemoteBigQueryHelper.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -120,18 +120,27 @@ public static RemoteBigQueryHelper create(String projectId, InputStream keyStrea
120120
* credentials.
121121
*/
122122
public static RemoteBigQueryHelper create() {
123+
return create(BigQueryOptions.newBuilder());
124+
}
125+
126+
/**
127+
* Creates a {@code RemoteBigQueryHelper} object using default project id and authentication
128+
* credentials.
129+
*
130+
* @param bigqueryOptionsBuilder Custom BigqueryOptions.Builder with some pre-defined settings
131+
*/
132+
public static RemoteBigQueryHelper create(BigQueryOptions.Builder bigqueryOptionsBuilder) {
123133
HttpTransportOptions transportOptions = BigQueryOptions.getDefaultHttpTransportOptions();
124134
transportOptions =
125135
transportOptions.toBuilder()
126136
.setConnectTimeout(connectTimeout)
127137
.setReadTimeout(connectTimeout)
128138
.build();
129-
BigQueryOptions bigqueryOptions =
130-
BigQueryOptions.newBuilder()
139+
BigQueryOptions.Builder builder =
140+
bigqueryOptionsBuilder
131141
.setRetrySettings(retrySettings())
132-
.setTransportOptions(transportOptions)
133-
.build();
134-
return new RemoteBigQueryHelper(bigqueryOptions);
142+
.setTransportOptions(transportOptions);
143+
return new RemoteBigQueryHelper(builder.build());
135144
}
136145

137146
private static RetrySettings retrySettings() {

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITHighPrecisionTimestamp.java‎

Lines changed: 319 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,319 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery.it;
17+
18+
import static org.junit.Assert.assertEquals;
19+
import static org.junit.Assert.assertFalse;
20+
import static org.junit.Assert.assertNotNull;
21+
import static org.junit.Assert.assertThrows;
22+
import static org.junit.Assert.assertTrue;
23+
24+
import com.google.cloud.bigquery.BigQuery;
25+
import com.google.cloud.bigquery.BigQueryException;
26+
import com.google.cloud.bigquery.BigQueryOptions;
27+
import com.google.cloud.bigquery.DataFormatOptions;
28+
import com.google.cloud.bigquery.DatasetId;
29+
import com.google.cloud.bigquery.DatasetInfo;
30+
import com.google.cloud.bigquery.Field;
31+
import com.google.cloud.bigquery.InsertAllRequest;
32+
import com.google.cloud.bigquery.InsertAllResponse;
33+
import com.google.cloud.bigquery.QueryJobConfiguration;
34+
import com.google.cloud.bigquery.QueryParameterValue;
35+
import com.google.cloud.bigquery.Schema;
36+
import com.google.cloud.bigquery.StandardSQLTypeName;
37+
import com.google.cloud.bigquery.StandardTableDefinition;
38+
import com.google.cloud.bigquery.Table;
39+
import com.google.cloud.bigquery.TableId;
40+
import com.google.cloud.bigquery.TableInfo;
41+
import com.google.cloud.bigquery.TableResult;
42+
import com.google.cloud.bigquery.testing.RemoteBigQueryHelper;
43+
import com.google.protobuf.Timestamp;
44+
import java.util.Collections;
45+
import java.util.List;
46+
import java.util.Map;
47+
import java.util.UUID;
48+
import java.util.stream.Collectors;
49+
import java.util.stream.StreamSupport;
50+
import org.junit.AfterClass;
51+
import org.junit.BeforeClass;
52+
import org.junit.Test;
53+
54+
public class ITHighPrecisionTimestamp {
55+
56+
public static final String TEST_HIGH_PRECISION_TIMESTAMP_TABLE_NAME =
57+
"test_high_precision_timestamp";
58+
private static BigQuery bigquery;
59+
private static final String DATASET = RemoteBigQueryHelper.generateDatasetName();
60+
private static TableId defaultTableId;
61+
public static final long TIMESTAMP_PICOSECOND_PRECISION = 12L;
62+
private static final Field TIMESTAMP_HIGH_PRECISION_FIELD_SCHEMA =
63+
Field.newBuilder("timestampHighPrecisionField", StandardSQLTypeName.TIMESTAMP)
64+
.setTimestampPrecision(TIMESTAMP_PICOSECOND_PRECISION)
65+
.build();
66+
private static final Schema TABLE_SCHEMA = Schema.of(TIMESTAMP_HIGH_PRECISION_FIELD_SCHEMA);
67+
68+
private static final String TIMESTAMP1 = "2025-01-01T12:34:56.123456789123Z";
69+
private static final String TIMESTAMP2 = "1970-01-01T12:34:56.123456789123Z";
70+
private static final String TIMESTAMP3 = "2000-01-01T12:34:56.123456789123Z";
71+
72+
@BeforeClass
73+
public static void beforeClass() {
74+
BigQueryOptions.Builder builder =
75+
BigQueryOptions.newBuilder()
76+
.setDataFormatOptions(
77+
DataFormatOptions.newBuilder()
78+
.timestampFormatOptions(DataFormatOptions.TimestampFormatOptions.ISO8601_STRING)
79+
.build());
80+
RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(builder);
81+
bigquery = bigqueryHelper.getOptions().getService();
82+
83+
// Create a new dataset
84+
DatasetInfo info = DatasetInfo.newBuilder(DATASET).build();
85+
bigquery.create(info);
86+
87+
StandardTableDefinition tableDefinition =
88+
StandardTableDefinition.newBuilder().setSchema(TABLE_SCHEMA).build();
89+
defaultTableId = TableId.of(DATASET, TEST_HIGH_PRECISION_TIMESTAMP_TABLE_NAME);
90+
91+
// Create a new table that can be re-used by the test cases
92+
Table createdTable = bigquery.create(TableInfo.of(defaultTableId, tableDefinition));
93+
assertNotNull(createdTable);
94+
95+
// Populate with some starter data
96+
Map<String, Object> timestamp1 =
97+
Collections.singletonMap("timestampHighPrecisionField", TIMESTAMP1);
98+
Map<String, Object> timestamp2 =
99+
Collections.singletonMap("timestampHighPrecisionField", TIMESTAMP2);
100+
Map<String, Object> timestamp3 =
101+
Collections.singletonMap("timestampHighPrecisionField", TIMESTAMP3);
102+
InsertAllRequest request =
103+
InsertAllRequest.newBuilder(defaultTableId)
104+
.addRow(timestamp1)
105+
.addRow(timestamp2)
106+
.addRow(timestamp3)
107+
.build();
108+
InsertAllResponse response = bigquery.insertAll(request);
109+
assertFalse(response.hasErrors());
110+
assertEquals(0, response.getInsertErrors().size());
111+
}
112+
113+
@AfterClass
114+
public static void afterClass() {
115+
if (bigquery != null) {
116+
bigquery.delete(defaultTableId);
117+
RemoteBigQueryHelper.forceDelete(bigquery, DATASET);
118+
}
119+
}
120+
121+
private static String generateTempTableName() {
122+
return String.format(
123+
"insert_temp_%s%s",
124+
UUID.randomUUID().toString().substring(0, 6), TEST_HIGH_PRECISION_TIMESTAMP_TABLE_NAME);
125+
}
126+
127+
@Test
128+
public void query_highPrecisionTimestamp() throws InterruptedException {
129+
String sql =
130+
String.format("SELECT timestampHighPrecisionField FROM %s;", defaultTableId.getTable());
131+
QueryJobConfiguration queryJobConfiguration =
132+
QueryJobConfiguration.newBuilder(sql)
133+
.setDefaultDataset(DatasetId.of(DATASET))
134+
.setUseLegacySql(false)
135+
.build();
136+
TableResult result = bigquery.query(queryJobConfiguration);
137+
assertNotNull(result.getJobId());
138+
String[] expected = new String[] {TIMESTAMP1, TIMESTAMP2, TIMESTAMP3};
139+
List<String> timestamps =
140+
StreamSupport.stream(result.getValues().spliterator(), false)
141+
.map(x -> (String) x.get(0).getValue())
142+
.collect(Collectors.toList());
143+
assertEquals(expected.length, timestamps.size());
144+
for (int i = 0; i < timestamps.size(); i++) {
145+
assertEquals(expected[i], timestamps.get(i));
146+
}
147+
}
148+
149+
@Test
150+
public void insert_highPrecisionTimestamp_ISOValidFormat() {
151+
StandardTableDefinition tableDefinition =
152+
StandardTableDefinition.newBuilder().setSchema(TABLE_SCHEMA).build();
153+
String tempTableName = generateTempTableName();
154+
TableId tableId = TableId.of(DATASET, tempTableName);
155+
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
156+
assertNotNull(createdTable);
157+
158+
Map<String, Object> timestampISO =
159+
Collections.singletonMap("timestampHighPrecisionField", "2025-01-01T12:34:56.123456Z");
160+
InsertAllRequest request = InsertAllRequest.newBuilder(tableId).addRow(timestampISO).build();
161+
InsertAllResponse response = bigquery.insertAll(request);
162+
assertFalse(response.hasErrors());
163+
assertEquals(0, response.getInsertErrors().size());
164+
165+
bigquery.delete(tableId);
166+
}
167+
168+
@Test
169+
public void insert_highPrecisionTimestamp_invalidFormats() {
170+
StandardTableDefinition tableDefinition =
171+
StandardTableDefinition.newBuilder().setSchema(TABLE_SCHEMA).build();
172+
String tempTable = generateTempTableName();
173+
TableId tableId = TableId.of(DATASET, tempTable);
174+
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
175+
assertNotNull(createdTable);
176+
177+
Map<String, Object> timestampInMicros =
178+
Collections.singletonMap("timestampHighPrecisionField", 123456);
179+
Map<String, Object> timestampInMicrosString =
180+
Collections.singletonMap("timestampHighPrecisionField", "123456");
181+
Map<String, Object> timestampNegative =
182+
Collections.singletonMap("timestampHighPrecisionField", -123456);
183+
Map<String, Object> timestampFloat =
184+
Collections.singletonMap("timestampHighPrecisionField", 1000.0);
185+
Map<String, Object> timestampProtobuf =
186+
Collections.singletonMap(
187+
"timestampHighPrecisionField",
188+
Timestamp.newBuilder().setSeconds(123456789).setNanos(123456789).build());
189+
Map<String, Object> timestampProtobufNegative =
190+
Collections.singletonMap(
191+
"timestampHighPrecisionField",
192+
Timestamp.newBuilder().setSeconds(-123456789).setNanos(-123456789).build());
193+
InsertAllRequest request =
194+
InsertAllRequest.newBuilder(tableId)
195+
.addRow(timestampInMicros)
196+
.addRow(timestampInMicrosString)
197+
.addRow(timestampNegative)
198+
.addRow(timestampFloat)
199+
.addRow(timestampProtobuf)
200+
.addRow(timestampProtobufNegative)
201+
.build();
202+
InsertAllResponse response = bigquery.insertAll(request);
203+
assertTrue(response.hasErrors());
204+
assertEquals(request.getRows().size(), response.getInsertErrors().size());
205+
206+
bigquery.delete(tableId);
207+
}
208+
209+
@Test
210+
public void queryNamedParameter_highPrecisionTimestamp() throws InterruptedException {
211+
String query =
212+
String.format(
213+
"SELECT * FROM %s.%s WHERE timestampHighPrecisionField >= CAST(@timestampParam AS TIMESTAMP(12))",
214+
DATASET, defaultTableId.getTable());
215+
216+
QueryJobConfiguration queryConfig =
217+
QueryJobConfiguration.newBuilder(query)
218+
.setDefaultDataset(DATASET)
219+
.setUseLegacySql(false)
220+
.addNamedParameter(
221+
"timestampParam",
222+
QueryParameterValue.timestamp("2000-01-01 12:34:56.123456789123Z"))
223+
.build();
224+
225+
TableResult result = bigquery.query(queryConfig);
226+
assertNotNull(result);
227+
String[] expected = new String[] {TIMESTAMP1, TIMESTAMP3};
228+
List<String> timestamps =
229+
StreamSupport.stream(result.getValues().spliterator(), false)
230+
.map(x -> (String) x.get(0).getValue())
231+
.collect(Collectors.toList());
232+
assertEquals(expected.length, timestamps.size());
233+
for (int i = 0; i < timestamps.size(); i++) {
234+
assertEquals(expected[i], timestamps.get(i));
235+
}
236+
}
237+
238+
@Test
239+
public void queryNamedParameter_highPrecisionTimestamp_microsLong() throws InterruptedException {
240+
String query =
241+
String.format(
242+
"SELECT * FROM %s.%s WHERE timestampHighPrecisionField >= CAST(@timestampParam AS TIMESTAMP(12))",
243+
DATASET, defaultTableId.getTable());
244+
245+
QueryJobConfiguration queryConfig =
246+
QueryJobConfiguration.newBuilder(query)
247+
.setDefaultDataset(DATASET)
248+
.setUseLegacySql(false)
249+
.addNamedParameter(
250+
"timestampParam",
251+
QueryParameterValue.timestamp(
252+
946730096123456L)) // micros for 2000-01-01 12:34:56.123456Z
253+
.build();
254+
255+
TableResult result = bigquery.query(queryConfig);
256+
assertNotNull(result);
257+
// Exact timestamp for TIMESTAMP3 is `2000-01-01T12:34:56.123456789123Z` and for the micros
258+
// is `2000-01-01T12:34:56.123456Z`. The micros value gets cast to 12 digits of precision, so
259+
// it becomes `2000-01-01T12:34:56.123456000000Z`. We do expect it as part of the query.
260+
String[] expected = new String[] {TIMESTAMP1, TIMESTAMP3};
261+
List<String> timestamps =
262+
StreamSupport.stream(result.getValues().spliterator(), false)
263+
.map(x -> (String) x.get(0).getValue())
264+
.collect(Collectors.toList());
265+
assertEquals(expected.length, timestamps.size());
266+
for (int i = 0; i < timestamps.size(); i++) {
267+
assertEquals(expected[i], timestamps.get(i));
268+
}
269+
}
270+
271+
@Test
272+
public void queryNamedParameter_highPrecisionTimestamp_microsISOString()
273+
throws InterruptedException {
274+
String query =
275+
String.format(
276+
"SELECT * FROM %s.%s WHERE timestampHighPrecisionField >= CAST(@timestampParam AS TIMESTAMP(12))",
277+
DATASET, defaultTableId.getTable());
278+
279+
QueryJobConfiguration queryConfig =
280+
QueryJobConfiguration.newBuilder(query)
281+
.setDefaultDataset(DATASET)
282+
.setUseLegacySql(false)
283+
.addNamedParameter(
284+
"timestampParam", QueryParameterValue.timestamp("2000-01-01 12:34:56.123456Z"))
285+
.build();
286+
287+
TableResult result = bigquery.query(queryConfig);
288+
assertNotNull(result);
289+
List<String> timestamps =
290+
StreamSupport.stream(result.getValues().spliterator(), false)
291+
.map(x -> (String) x.get(0).getValue())
292+
.collect(Collectors.toList());
293+
String[] expected = new String[] {TIMESTAMP1, TIMESTAMP3};
294+
assertEquals(expected.length, timestamps.size());
295+
for (int i = 0; i < timestamps.size(); i++) {
296+
assertEquals(expected[i], timestamps.get(i));
297+
}
298+
}
299+
300+
@Test
301+
public void queryNamedParameter_highPrecisionTimestamp_noExplicitCastInQuery_fails() {
302+
String query =
303+
String.format(
304+
"SELECT * FROM %s.%s WHERE timestampHighPrecisionField >= @timestampParam",
305+
DATASET, defaultTableId.getTable());
306+
307+
QueryJobConfiguration queryConfig =
308+
QueryJobConfiguration.newBuilder(query)
309+
.setDefaultDataset(DATASET)
310+
.setUseLegacySql(false)
311+
.addNamedParameter(
312+
"timestampParam", QueryParameterValue.timestamp("2000-01-01 12:34:56.123456789123"))
313+
.build();
314+
315+
BigQueryException exception =
316+
assertThrows(BigQueryException.class, () -> bigquery.query(queryConfig));
317+
assertEquals("Invalid argument type passed to a function", exception.getMessage());
318+
}
319+
}

0 commit comments

Comments
 (0)