Skip to content

Commit 5281f90

Browse files
chalmerlowechalmerlowe
authored
Merge branch 'main' into renovate/all
2 parents 3459267 + 54d3dc6 commit 5281f90

File tree

5 files changed

+198
-0
lines changed

5 files changed

+198
-0
lines changed

‎google/cloud/bigquery/external_config.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -862,6 +862,21 @@ def date_format(self) -> Optional[str]:
862862
def date_format(self, value: Optional[str]):
863863
self._properties["dateFormat"] = value
864864

865+
@property
866+
def datetime_format(self) -> Optional[str]:
867+
"""Optional[str]: Format used to parse DATETIME values. Supports C-style
868+
and SQL-style values.
869+
870+
See:
871+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.datetime_format
872+
"""
873+
result = self._properties.get("datetimeFormat")
874+
return typing.cast(str, result)
875+
876+
@datetime_format.setter
877+
def datetime_format(self, value: Optional[str]):
878+
self._properties["datetimeFormat"] = value
879+
865880
@property
866881
def time_zone(self) -> Optional[str]:
867882
"""Optional[str]: Time zone used when parsing timestamp values that do not
@@ -879,6 +894,34 @@ def time_zone(self) -> Optional[str]:
879894
def time_zone(self, value: Optional[str]):
880895
self._properties["timeZone"] = value
881896

897+
@property
898+
def time_format(self) -> Optional[str]:
899+
"""Optional[str]: Format used to parse TIME values. Supports C-style and SQL-style values.
900+
901+
See:
902+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.time_format
903+
"""
904+
result = self._properties.get("timeFormat")
905+
return typing.cast(str, result)
906+
907+
@time_format.setter
908+
def time_format(self, value: Optional[str]):
909+
self._properties["timeFormat"] = value
910+
911+
@property
912+
def timestamp_format(self) -> Optional[str]:
913+
"""Optional[str]: Format used to parse TIMESTAMP values. Supports C-style and SQL-style values.
914+
915+
See:
916+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.timestamp_format
917+
"""
918+
result = self._properties.get("timestampFormat")
919+
return typing.cast(str, result)
920+
921+
@timestamp_format.setter
922+
def timestamp_format(self, value: Optional[str]):
923+
self._properties["timestampFormat"] = value
924+
882925
@property
883926
def connection_id(self):
884927
"""Optional[str]: [Experimental] ID of a BigQuery Connection API

‎google/cloud/bigquery/job/load.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,19 @@ def date_format(self) -> Optional[str]:
561561
def date_format(self, value: Optional[str]):
562562
self._set_sub_prop("dateFormat", value)
563563

564+
@property
565+
def datetime_format(self) -> Optional[str]:
566+
"""Optional[str]: Date format used for parsing DATETIME values.
567+
568+
See:
569+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.datetime_format
570+
"""
571+
return self._get_sub_prop("datetimeFormat")
572+
573+
@datetime_format.setter
574+
def datetime_format(self, value: Optional[str]):
575+
self._set_sub_prop("datetimeFormat", value)
576+
564577
@property
565578
def time_zone(self) -> Optional[str]:
566579
"""Optional[str]: Default time zone that will apply when parsing timestamp
@@ -575,6 +588,32 @@ def time_zone(self) -> Optional[str]:
575588
def time_zone(self, value: Optional[str]):
576589
self._set_sub_prop("timeZone", value)
577590

591+
@property
592+
def time_format(self) -> Optional[str]:
593+
"""Optional[str]: Date format used for parsing TIME values.
594+
595+
See:
596+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.time_format
597+
"""
598+
return self._get_sub_prop("timeFormat")
599+
600+
@time_format.setter
601+
def time_format(self, value: Optional[str]):
602+
self._set_sub_prop("timeFormat", value)
603+
604+
@property
605+
def timestamp_format(self) -> Optional[str]:
606+
"""Optional[str]: Date format used for parsing TIMESTAMP values.
607+
608+
See:
609+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.timestamp_format
610+
"""
611+
return self._get_sub_prop("timestampFormat")
612+
613+
@timestamp_format.setter
614+
def timestamp_format(self, value: Optional[str]):
615+
self._set_sub_prop("timestampFormat", value)
616+
578617
@property
579618
def time_partitioning(self):
580619
"""Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based
@@ -923,13 +962,34 @@ def date_format(self):
923962
"""
924963
return self.configuration.date_format
925964

965+
@property
966+
def datetime_format(self):
967+
"""See
968+
:attr:`google.cloud.bigquery.job.LoadJobConfig.datetime_format`.
969+
"""
970+
return self.configuration.datetime_format
971+
926972
@property
927973
def time_zone(self):
928974
"""See
929975
:attr:`google.cloud.bigquery.job.LoadJobConfig.time_zone`.
930976
"""
931977
return self.configuration.time_zone
932978

979+
@property
980+
def time_format(self):
981+
"""See
982+
:attr:`google.cloud.bigquery.job.LoadJobConfig.time_format`.
983+
"""
984+
return self.configuration.time_format
985+
986+
@property
987+
def timestamp_format(self):
988+
"""See
989+
:attr:`google.cloud.bigquery.job.LoadJobConfig.timestamp_format`.
990+
"""
991+
return self.configuration.timestamp_format
992+
933993
@property
934994
def schema_update_options(self):
935995
"""See

‎tests/unit/job/test_load.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,21 @@ def _setUpConstants(self):
3838
self.OUTPUT_ROWS = 345
3939
self.REFERENCE_FILE_SCHEMA_URI = "gs://path/to/reference"
4040
self.DATE_FORMAT = "%Y-%m-%d"
41+
self.DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
4142
self.TIME_ZONE = "UTC"
43+
self.TIME_FORMAT = "%H:%M:%S"
44+
self.TIMESTAMP_FORMAT = "YYYY-MM-DD HH:MM:SS.SSSSSSZ"
4245

4346
def _make_resource(self, started=False, ended=False):
4447
resource = super(TestLoadJob, self)._make_resource(started, ended)
4548
config = resource["configuration"]["load"]
4649
config["sourceUris"] = [self.SOURCE1]
4750
config["dateFormat"] = self.DATE_FORMAT
51+
config["datetimeFormat"] = self.DATETIME_FORMAT
4852
config["timeZone"] = self.TIME_ZONE
53+
config["timeFormat"] = self.TIME_FORMAT
54+
config["timestampFormat"] = self.TIMESTAMP_FORMAT
55+
4956
config["destinationTable"] = {
5057
"projectId": self.PROJECT,
5158
"datasetId": self.DS_ID,
@@ -159,10 +166,22 @@ def _verifyResourceProperties(self, job, resource):
159166
self.assertEqual(job.date_format, config["dateFormat"])
160167
else:
161168
self.assertIsNone(job.date_format)
169+
if "datetimeFormat" in config:
170+
self.assertEqual(job.datetime_format, config["datetimeFormat"])
171+
else:
172+
self.assertIsNone(job.datetime_format)
162173
if "timeZone" in config:
163174
self.assertEqual(job.time_zone, config["timeZone"])
164175
else:
165176
self.assertIsNone(job.time_zone)
177+
if "timeFormat" in config:
178+
self.assertEqual(job.time_format, config["timeFormat"])
179+
else:
180+
self.assertIsNone(job.time_format)
181+
if "timestampFormat" in config:
182+
self.assertEqual(job.timestamp_format, config["timestampFormat"])
183+
else:
184+
self.assertIsNone(job.timestamp_format)
166185

167186
def test_ctor(self):
168187
client = _make_client(project=self.PROJECT)
@@ -206,7 +225,10 @@ def test_ctor(self):
206225
self.assertIsNone(job.schema_update_options)
207226
self.assertIsNone(job.reference_file_schema_uri)
208227
self.assertIsNone(job.date_format)
228+
self.assertIsNone(job.datetime_format)
209229
self.assertIsNone(job.time_zone)
230+
self.assertIsNone(job.time_format)
231+
self.assertIsNone(job.timestamp_format)
210232

211233
def test_ctor_w_config(self):
212234
from google.cloud.bigquery.schema import SchemaField
@@ -603,8 +625,12 @@ def test_begin_w_alternate_client(self):
603625
},
604626
"schemaUpdateOptions": [SchemaUpdateOption.ALLOW_FIELD_ADDITION],
605627
"dateFormat": self.DATE_FORMAT,
628+
"datetimeFormat": self.DATETIME_FORMAT,
606629
"timeZone": self.TIME_ZONE,
630+
"timeFormat": self.TIME_FORMAT,
631+
"timestampFormat": self.TIMESTAMP_FORMAT,
607632
}
633+
608634
RESOURCE["configuration"]["load"] = LOAD_CONFIGURATION
609635
conn1 = make_connection()
610636
client1 = _make_client(project=self.PROJECT, connection=conn1)
@@ -633,7 +659,10 @@ def test_begin_w_alternate_client(self):
633659
config.schema_update_options = [SchemaUpdateOption.ALLOW_FIELD_ADDITION]
634660
config.reference_file_schema_uri = "gs://path/to/reference"
635661
config.date_format = self.DATE_FORMAT
662+
config.datetime_format = self.DATETIME_FORMAT
636663
config.time_zone = self.TIME_ZONE
664+
config.time_format = self.TIME_FORMAT
665+
config.timestamp_format = self.TIMESTAMP_FORMAT
637666

638667
with mock.patch(
639668
"google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes"

‎tests/unit/job/test_load_config.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -844,6 +844,22 @@ def test_date_format_setter(self):
844844
config.date_format = date_format
845845
self.assertEqual(config._properties["load"]["dateFormat"], date_format)
846846

847+
def test_datetime_format_missing(self):
848+
config = self._get_target_class()()
849+
self.assertIsNone(config.datetime_format)
850+
851+
def test_datetime_format_hit(self):
852+
datetime_format = "%Y-%m-%dT%H:%M:%S"
853+
config = self._get_target_class()()
854+
config._properties["load"]["datetimeFormat"] = datetime_format
855+
self.assertEqual(config.datetime_format, datetime_format)
856+
857+
def test_datetime_format_setter(self):
858+
datetime_format = "YYYY/MM/DD HH24:MI:SS"
859+
config = self._get_target_class()()
860+
config.datetime_format = datetime_format
861+
self.assertEqual(config._properties["load"]["datetimeFormat"], datetime_format)
862+
847863
def test_time_zone_missing(self):
848864
config = self._get_target_class()()
849865
self.assertIsNone(config.time_zone)
@@ -860,6 +876,40 @@ def test_time_zone_setter(self):
860876
config.time_zone = time_zone
861877
self.assertEqual(config._properties["load"]["timeZone"], time_zone)
862878

879+
def test_time_format_missing(self):
880+
config = self._get_target_class()()
881+
self.assertIsNone(config.time_format)
882+
883+
def test_time_format_hit(self):
884+
time_format = "%H:%M:%S"
885+
config = self._get_target_class()()
886+
config._properties["load"]["timeFormat"] = time_format
887+
self.assertEqual(config.time_format, time_format)
888+
889+
def test_time_format_setter(self):
890+
time_format = "HH24:MI:SS"
891+
config = self._get_target_class()()
892+
config.time_format = time_format
893+
self.assertEqual(config._properties["load"]["timeFormat"], time_format)
894+
895+
def test_timestamp_format_missing(self):
896+
config = self._get_target_class()()
897+
self.assertIsNone(config.timestamp_format)
898+
899+
def test_timestamp_format_hit(self):
900+
timestamp_format = "%Y-%m-%dT%H:%M:%S.%fZ"
901+
config = self._get_target_class()()
902+
config._properties["load"]["timestampFormat"] = timestamp_format
903+
self.assertEqual(config.timestamp_format, timestamp_format)
904+
905+
def test_timestamp_format_setter(self):
906+
timestamp_format = "YYYY/MM/DD HH24:MI:SS.FF6 TZR"
907+
config = self._get_target_class()()
908+
config.timestamp_format = timestamp_format
909+
self.assertEqual(
910+
config._properties["load"]["timestampFormat"], timestamp_format
911+
)
912+
863913
def test_parquet_options_missing(self):
864914
config = self._get_target_class()()
865915
self.assertIsNone(config.parquet_options)

‎tests/unit/test_external_config.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@
2626
class TestExternalConfig(unittest.TestCase):
2727
SOURCE_URIS = ["gs://foo", "gs://bar"]
2828
DATE_FORMAT = "MM/DD/YYYY"
29+
DATETIME_FORMAT = "MM/DD/YYYY HH24:MI:SS"
2930
TIME_ZONE = "America/Los_Angeles"
31+
TIME_FORMAT = "HH24:MI:SS"
32+
TIMESTAMP_FORMAT = "MM/DD/YYYY HH24:MI:SS.FF6 TZR"
3033

3134
BASE_RESOURCE = {
3235
"sourceFormat": "",
@@ -36,7 +39,10 @@ class TestExternalConfig(unittest.TestCase):
3639
"ignoreUnknownValues": False,
3740
"compression": "compression",
3841
"dateFormat": DATE_FORMAT,
42+
"datetimeFormat": DATETIME_FORMAT,
3943
"timeZone": TIME_ZONE,
44+
"timeFormat": TIME_FORMAT,
45+
"timestampFormat": TIMESTAMP_FORMAT,
4046
}
4147

4248
def test_from_api_repr_base(self):
@@ -84,7 +90,11 @@ def test_to_api_repr_base(self):
8490
ec.schema = [schema.SchemaField("full_name", "STRING", mode="REQUIRED")]
8591

8692
ec.date_format = self.DATE_FORMAT
93+
ec.datetime_format = self.DATETIME_FORMAT
8794
ec.time_zone = self.TIME_ZONE
95+
ec.time_format = self.TIME_FORMAT
96+
ec.timestamp_format = self.TIMESTAMP_FORMAT
97+
8898
exp_schema = {
8999
"fields": [{"name": "full_name", "type": "STRING", "mode": "REQUIRED"}]
90100
}
@@ -99,7 +109,10 @@ def test_to_api_repr_base(self):
99109
"connectionId": "path/to/connection",
100110
"schema": exp_schema,
101111
"dateFormat": self.DATE_FORMAT,
112+
"datetimeFormat": self.DATETIME_FORMAT,
102113
"timeZone": self.TIME_ZONE,
114+
"timeFormat": self.TIME_FORMAT,
115+
"timestampFormat": self.TIMESTAMP_FORMAT,
103116
}
104117
self.assertEqual(got_resource, exp_resource)
105118

@@ -136,7 +149,10 @@ def _verify_base(self, ec):
136149
self.assertEqual(ec.max_bad_records, 17)
137150
self.assertEqual(ec.source_uris, self.SOURCE_URIS)
138151
self.assertEqual(ec.date_format, self.DATE_FORMAT)
152+
self.assertEqual(ec.datetime_format, self.DATETIME_FORMAT)
139153
self.assertEqual(ec.time_zone, self.TIME_ZONE)
154+
self.assertEqual(ec.time_format, self.TIME_FORMAT)
155+
self.assertEqual(ec.timestamp_format, self.TIMESTAMP_FORMAT)
140156

141157
def test_to_api_repr_source_format(self):
142158
ec = external_config.ExternalConfig("CSV")

0 commit comments

Comments
 (0)