Skip to content

Commit c31b1ad

Browse files
chalmerlowechalmerlowe
authored
Merge branch 'main' into feat-374142081-add-source-column-match
2 parents 5686794 + 69a2c2b commit c31b1ad

File tree

6 files changed

+199
-1
lines changed

6 files changed

+199
-1
lines changed

‎google/cloud/bigquery/external_config.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -896,6 +896,21 @@ def date_format(self) -> Optional[str]:
896896
def date_format(self, value: Optional[str]):
897897
self._properties["dateFormat"] = value
898898

899+
@property
900+
def datetime_format(self) -> Optional[str]:
901+
"""Optional[str]: Format used to parse DATETIME values. Supports C-style
902+
and SQL-style values.
903+
904+
See:
905+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.datetime_format
906+
"""
907+
result = self._properties.get("datetimeFormat")
908+
return typing.cast(str, result)
909+
910+
@datetime_format.setter
911+
def datetime_format(self, value: Optional[str]):
912+
self._properties["datetimeFormat"] = value
913+
899914
@property
900915
def time_zone(self) -> Optional[str]:
901916
"""Optional[str]: Time zone used when parsing timestamp values that do not
@@ -913,6 +928,34 @@ def time_zone(self) -> Optional[str]:
913928
def time_zone(self, value: Optional[str]):
914929
self._properties["timeZone"] = value
915930

931+
@property
932+
def time_format(self) -> Optional[str]:
933+
"""Optional[str]: Format used to parse TIME values. Supports C-style and SQL-style values.
934+
935+
See:
936+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.time_format
937+
"""
938+
result = self._properties.get("timeFormat")
939+
return typing.cast(str, result)
940+
941+
@time_format.setter
942+
def time_format(self, value: Optional[str]):
943+
self._properties["timeFormat"] = value
944+
945+
@property
946+
def timestamp_format(self) -> Optional[str]:
947+
"""Optional[str]: Format used to parse TIMESTAMP values. Supports C-style and SQL-style values.
948+
949+
See:
950+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.timestamp_format
951+
"""
952+
result = self._properties.get("timestampFormat")
953+
return typing.cast(str, result)
954+
955+
@timestamp_format.setter
956+
def timestamp_format(self, value: Optional[str]):
957+
self._properties["timestampFormat"] = value
958+
916959
@property
917960
def connection_id(self):
918961
"""Optional[str]: [Experimental] ID of a BigQuery Connection API

‎google/cloud/bigquery/job/load.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,6 +595,19 @@ def date_format(self) -> Optional[str]:
595595
def date_format(self, value: Optional[str]):
596596
self._set_sub_prop("dateFormat", value)
597597

598+
@property
599+
def datetime_format(self) -> Optional[str]:
600+
"""Optional[str]: Date format used for parsing DATETIME values.
601+
602+
See:
603+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.datetime_format
604+
"""
605+
return self._get_sub_prop("datetimeFormat")
606+
607+
@datetime_format.setter
608+
def datetime_format(self, value: Optional[str]):
609+
self._set_sub_prop("datetimeFormat", value)
610+
598611
@property
599612
def time_zone(self) -> Optional[str]:
600613
"""Optional[str]: Default time zone that will apply when parsing timestamp
@@ -609,6 +622,32 @@ def time_zone(self) -> Optional[str]:
609622
def time_zone(self, value: Optional[str]):
610623
self._set_sub_prop("timeZone", value)
611624

625+
@property
626+
def time_format(self) -> Optional[str]:
627+
"""Optional[str]: Date format used for parsing TIME values.
628+
629+
See:
630+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.time_format
631+
"""
632+
return self._get_sub_prop("timeFormat")
633+
634+
@time_format.setter
635+
def time_format(self, value: Optional[str]):
636+
self._set_sub_prop("timeFormat", value)
637+
638+
@property
639+
def timestamp_format(self) -> Optional[str]:
640+
"""Optional[str]: Date format used for parsing TIMESTAMP values.
641+
642+
See:
643+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.timestamp_format
644+
"""
645+
return self._get_sub_prop("timestampFormat")
646+
647+
@timestamp_format.setter
648+
def timestamp_format(self, value: Optional[str]):
649+
self._set_sub_prop("timestampFormat", value)
650+
612651
@property
613652
def time_partitioning(self):
614653
"""Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based
@@ -964,13 +1003,34 @@ def date_format(self):
9641003
"""
9651004
return self.configuration.date_format
9661005

1006+
@property
1007+
def datetime_format(self):
1008+
"""See
1009+
:attr:`google.cloud.bigquery.job.LoadJobConfig.datetime_format`.
1010+
"""
1011+
return self.configuration.datetime_format
1012+
9671013
@property
9681014
def time_zone(self):
9691015
"""See
9701016
:attr:`google.cloud.bigquery.job.LoadJobConfig.time_zone`.
9711017
"""
9721018
return self.configuration.time_zone
9731019

1020+
@property
1021+
def time_format(self):
1022+
"""See
1023+
:attr:`google.cloud.bigquery.job.LoadJobConfig.time_format`.
1024+
"""
1025+
return self.configuration.time_format
1026+
1027+
@property
1028+
def timestamp_format(self):
1029+
"""See
1030+
:attr:`google.cloud.bigquery.job.LoadJobConfig.timestamp_format`.
1031+
"""
1032+
return self.configuration.timestamp_format
1033+
9741034
@property
9751035
def schema_update_options(self):
9761036
"""See

‎samples/geography/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
attrs==25.3.0
2-
certifi==2025.7.9
2+
certifi==2025.7.14
33
cffi==1.17.1
44
charset-normalizer==3.4.2
55
click===8.1.8; python_version == '3.9'

‎tests/unit/job/test_load.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,22 @@ def _setUpConstants(self):
4040
self.REFERENCE_FILE_SCHEMA_URI = "gs://path/to/reference"
4141
self.SOURCE_COLUMN_MATCH = "NAME"
4242
self.DATE_FORMAT = "%Y-%m-%d"
43+
self.DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
4344
self.TIME_ZONE = "UTC"
45+
self.TIME_FORMAT = "%H:%M:%S"
46+
self.TIMESTAMP_FORMAT = "YYYY-MM-DD HH:MM:SS.SSSSSSZ"
4447

4548
def _make_resource(self, started=False, ended=False):
4649
resource = super(TestLoadJob, self)._make_resource(started, ended)
4750
config = resource["configuration"]["load"]
4851
config["sourceUris"] = [self.SOURCE1]
4952
config["sourceColumnMatch"] = self.SOURCE_COLUMN_MATCH
5053
config["dateFormat"] = self.DATE_FORMAT
54+
config["datetimeFormat"] = self.DATETIME_FORMAT
5155
config["timeZone"] = self.TIME_ZONE
56+
config["timeFormat"] = self.TIME_FORMAT
57+
config["timestampFormat"] = self.TIMESTAMP_FORMAT
58+
5259
config["destinationTable"] = {
5360
"projectId": self.PROJECT,
5461
"datasetId": self.DS_ID,
@@ -162,10 +169,22 @@ def _verifyResourceProperties(self, job, resource):
162169
self.assertEqual(job.date_format, config["dateFormat"])
163170
else:
164171
self.assertIsNone(job.date_format)
172+
if "datetimeFormat" in config:
173+
self.assertEqual(job.datetime_format, config["datetimeFormat"])
174+
else:
175+
self.assertIsNone(job.datetime_format)
165176
if "timeZone" in config:
166177
self.assertEqual(job.time_zone, config["timeZone"])
167178
else:
168179
self.assertIsNone(job.time_zone)
180+
if "timeFormat" in config:
181+
self.assertEqual(job.time_format, config["timeFormat"])
182+
else:
183+
self.assertIsNone(job.time_format)
184+
if "timestampFormat" in config:
185+
self.assertEqual(job.timestamp_format, config["timestampFormat"])
186+
else:
187+
self.assertIsNone(job.timestamp_format)
169188

170189
if "sourceColumnMatch" in config:
171190
# job.source_column_match will be an Enum, config[...] is a string
@@ -219,7 +238,10 @@ def test_ctor(self):
219238
self.assertIsNone(job.reference_file_schema_uri)
220239
self.assertIsNone(job.source_column_match)
221240
self.assertIsNone(job.date_format)
241+
self.assertIsNone(job.datetime_format)
222242
self.assertIsNone(job.time_zone)
243+
self.assertIsNone(job.time_format)
244+
self.assertIsNone(job.timestamp_format)
223245

224246
def test_ctor_w_config(self):
225247
from google.cloud.bigquery.schema import SchemaField
@@ -617,8 +639,12 @@ def test_begin_w_alternate_client(self):
617639
"schemaUpdateOptions": [SchemaUpdateOption.ALLOW_FIELD_ADDITION],
618640
"sourceColumnMatch": self.SOURCE_COLUMN_MATCH,
619641
"dateFormat": self.DATE_FORMAT,
642+
"datetimeFormat": self.DATETIME_FORMAT,
620643
"timeZone": self.TIME_ZONE,
644+
"timeFormat": self.TIME_FORMAT,
645+
"timestampFormat": self.TIMESTAMP_FORMAT,
621646
}
647+
622648
RESOURCE["configuration"]["load"] = LOAD_CONFIGURATION
623649
conn1 = make_connection()
624650
client1 = _make_client(project=self.PROJECT, connection=conn1)
@@ -648,7 +674,10 @@ def test_begin_w_alternate_client(self):
648674
config.reference_file_schema_uri = "gs://path/to/reference"
649675
config.source_column_match = SourceColumnMatch(self.SOURCE_COLUMN_MATCH)
650676
config.date_format = self.DATE_FORMAT
677+
config.datetime_format = self.DATETIME_FORMAT
651678
config.time_zone = self.TIME_ZONE
679+
config.time_format = self.TIME_FORMAT
680+
config.timestamp_format = self.TIMESTAMP_FORMAT
652681

653682
with mock.patch(
654683
"google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes"

‎tests/unit/job/test_load_config.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -873,6 +873,22 @@ def test_date_format_setter(self):
873873
config.date_format = date_format
874874
self.assertEqual(config._properties["load"]["dateFormat"], date_format)
875875

876+
def test_datetime_format_missing(self):
877+
config = self._get_target_class()()
878+
self.assertIsNone(config.datetime_format)
879+
880+
def test_datetime_format_hit(self):
881+
datetime_format = "%Y-%m-%dT%H:%M:%S"
882+
config = self._get_target_class()()
883+
config._properties["load"]["datetimeFormat"] = datetime_format
884+
self.assertEqual(config.datetime_format, datetime_format)
885+
886+
def test_datetime_format_setter(self):
887+
datetime_format = "YYYY/MM/DD HH24:MI:SS"
888+
config = self._get_target_class()()
889+
config.datetime_format = datetime_format
890+
self.assertEqual(config._properties["load"]["datetimeFormat"], datetime_format)
891+
876892
def test_time_zone_missing(self):
877893
config = self._get_target_class()()
878894
self.assertIsNone(config.time_zone)
@@ -889,6 +905,40 @@ def test_time_zone_setter(self):
889905
config.time_zone = time_zone
890906
self.assertEqual(config._properties["load"]["timeZone"], time_zone)
891907

908+
def test_time_format_missing(self):
909+
config = self._get_target_class()()
910+
self.assertIsNone(config.time_format)
911+
912+
def test_time_format_hit(self):
913+
time_format = "%H:%M:%S"
914+
config = self._get_target_class()()
915+
config._properties["load"]["timeFormat"] = time_format
916+
self.assertEqual(config.time_format, time_format)
917+
918+
def test_time_format_setter(self):
919+
time_format = "HH24:MI:SS"
920+
config = self._get_target_class()()
921+
config.time_format = time_format
922+
self.assertEqual(config._properties["load"]["timeFormat"], time_format)
923+
924+
def test_timestamp_format_missing(self):
925+
config = self._get_target_class()()
926+
self.assertIsNone(config.timestamp_format)
927+
928+
def test_timestamp_format_hit(self):
929+
timestamp_format = "%Y-%m-%dT%H:%M:%S.%fZ"
930+
config = self._get_target_class()()
931+
config._properties["load"]["timestampFormat"] = timestamp_format
932+
self.assertEqual(config.timestamp_format, timestamp_format)
933+
934+
def test_timestamp_format_setter(self):
935+
timestamp_format = "YYYY/MM/DD HH24:MI:SS.FF6 TZR"
936+
config = self._get_target_class()()
937+
config.timestamp_format = timestamp_format
938+
self.assertEqual(
939+
config._properties["load"]["timestampFormat"], timestamp_format
940+
)
941+
892942
def test_parquet_options_missing(self):
893943
config = self._get_target_class()()
894944
self.assertIsNone(config.parquet_options)

‎tests/unit/test_external_config.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ class TestExternalConfig(unittest.TestCase):
2828
SOURCE_URIS = ["gs://foo", "gs://bar"]
2929
SOURCE_COLUMN_MATCH = SourceColumnMatch.NAME
3030
DATE_FORMAT = "MM/DD/YYYY"
31+
DATETIME_FORMAT = "MM/DD/YYYY HH24:MI:SS"
3132
TIME_ZONE = "America/Los_Angeles"
33+
TIME_FORMAT = "HH24:MI:SS"
34+
TIMESTAMP_FORMAT = "MM/DD/YYYY HH24:MI:SS.FF6 TZR"
3235

3336
BASE_RESOURCE = {
3437
"sourceFormat": "",
@@ -38,7 +41,10 @@ class TestExternalConfig(unittest.TestCase):
3841
"ignoreUnknownValues": False,
3942
"compression": "compression",
4043
"dateFormat": DATE_FORMAT,
44+
"datetimeFormat": DATETIME_FORMAT,
4145
"timeZone": TIME_ZONE,
46+
"timeFormat": TIME_FORMAT,
47+
"timestampFormat": TIMESTAMP_FORMAT,
4248
}
4349

4450
def test_from_api_repr_base(self):
@@ -86,7 +92,11 @@ def test_to_api_repr_base(self):
8692
ec.schema = [schema.SchemaField("full_name", "STRING", mode="REQUIRED")]
8793

8894
ec.date_format = self.DATE_FORMAT
95+
ec.datetime_format = self.DATETIME_FORMAT
8996
ec.time_zone = self.TIME_ZONE
97+
ec.time_format = self.TIME_FORMAT
98+
ec.timestamp_format = self.TIMESTAMP_FORMAT
99+
90100
exp_schema = {
91101
"fields": [{"name": "full_name", "type": "STRING", "mode": "REQUIRED"}]
92102
}
@@ -101,7 +111,10 @@ def test_to_api_repr_base(self):
101111
"connectionId": "path/to/connection",
102112
"schema": exp_schema,
103113
"dateFormat": self.DATE_FORMAT,
114+
"datetimeFormat": self.DATETIME_FORMAT,
104115
"timeZone": self.TIME_ZONE,
116+
"timeFormat": self.TIME_FORMAT,
117+
"timestampFormat": self.TIMESTAMP_FORMAT,
105118
}
106119
self.assertEqual(got_resource, exp_resource)
107120

@@ -152,7 +165,10 @@ def _verify_base(self, ec):
152165
self.assertEqual(ec.max_bad_records, 17)
153166
self.assertEqual(ec.source_uris, self.SOURCE_URIS)
154167
self.assertEqual(ec.date_format, self.DATE_FORMAT)
168+
self.assertEqual(ec.datetime_format, self.DATETIME_FORMAT)
155169
self.assertEqual(ec.time_zone, self.TIME_ZONE)
170+
self.assertEqual(ec.time_format, self.TIME_FORMAT)
171+
self.assertEqual(ec.timestamp_format, self.TIMESTAMP_FORMAT)
156172

157173
def test_to_api_repr_source_format(self):
158174
ec = external_config.ExternalConfig("CSV")

0 commit comments

Comments
 (0)