feat: Add new options to LoadJobConfig and ExternalConfig

This commit introduces new configuration options for BigQuery load jobs and external table definitions, aligning with recent updates to the underlying protos.

New options added:

- `time_zone`: Specifies the default timezone for parsing timestamps.
  (Applies to LoadJobConfig, ExternalConfig; CSV & JSON)
- `date_format`: Specifies the format for parsing DATE values.
  (Applies to LoadJobConfig, ExternalConfig; CSV & JSON)
- `datetime_format`: Specifies the format for parsing DATETIME values.
  (Applies to LoadJobConfig, ExternalConfig; CSV & JSON)
- `time_format`: Specifies the format for parsing TIME values.
  (Applies to LoadJobConfig, ExternalConfig; CSV & JSON)
- `timestamp_format`: Specifies the format for parsing TIMESTAMP values.
  (Applies to LoadJobConfig, ExternalConfig; CSV & JSON)
- `null_markers`: A list of strings to be interpreted as NULL.
  (Applies to LoadJobConfig, ExternalConfig (via CSVOptions); CSV only)
- `source_column_name_match_option`: Controls how source columns are matched to the schema.
  (Applies to LoadJobConfig, ExternalConfig (via CSVOptions); CSV only)

Changes include:
- Added corresponding properties (getters/setters) to `LoadJobConfig`, `LoadJob`, `ExternalConfig`, and `CSVOptions`.
- Updated docstrings and type hints for all new attributes.
- Updated unit tests to cover the new options, ensuring they are correctly handled during object initialization, serialization to API representation, and deserialization from API responses.

Author: google-labs-jules[bot]

google/cloud/bigquery/external_config.py

@@ -23,7 +23,7 @@
 import base64
 import copy
 import typing
-from typing import Any, Dict, FrozenSet, Iterable, Optional, Union
+from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Union
 
 from google.cloud.bigquery._helpers import _to_bytes
 from google.cloud.bigquery._helpers import _bytes_to_json
@@ -474,6 +474,36 @@ def skip_leading_rows(self):
     def skip_leading_rows(self, value):
         self._properties["skipLeadingRows"] = str(value)
 
+    @property
+    def null_markers(self) -> Optional[List[str]]:
+        """Optional[List[str]]: A list of strings represented as SQL NULL value.
+
+        See
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.null_marker
+        (Note: API doc refers to null_marker singular, but proto is null_markers plural and a list)
+        """
+        return self._properties.get("nullMarkers")
+
+    @null_markers.setter
+    def null_markers(self, value: Optional[List[str]]):
+        self._properties["nullMarkers"] = value
+
+    @property
+    def source_column_name_match_option(self) -> Optional[str]:
+        """Optional[str]: Controls the strategy used to match loaded columns to the schema.
+        Acceptable values are: "POSITION", "NAME".
+
+        See
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.source_column_match
+        (Note: This field is documented under ExternalDataConfiguration in the REST API docs but seems
+         more appropriate here for CSVOptions, matching the proto structure for external tables)
+        """
+        return self._properties.get("sourceColumnMatch")
+
+    @source_column_name_match_option.setter
+    def source_column_name_match_option(self, value: Optional[str]):
+        self._properties["sourceColumnMatch"] = value
+
     def to_api_repr(self) -> dict:
         """Build an API representation of this object.
 
@@ -848,6 +878,82 @@ def schema(self, value):
             prop = {"fields": [field.to_api_repr() for field in value]}
         self._properties["schema"] = prop
 
+    @property
+    def time_zone(self) -> Optional[str]:
+        """Optional[str]: Default time zone that will apply when parsing
+        timestamp values that have no specific time zone.
+
+        (Valid for CSV and NEWLINE_DELIMITED_JSON)
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.time_zone
+        """
+        return self._properties.get("timeZone")
+
+    @time_zone.setter
+    def time_zone(self, value: Optional[str]):
+        self._properties["timeZone"] = value
+
+    @property
+    def date_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing DATE values.
+
+        (Valid for CSV and NEWLINE_DELIMITED_JSON)
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.date_format
+        """
+        return self._properties.get("dateFormat")
+
+    @date_format.setter
+    def date_format(self, value: Optional[str]):
+        self._properties["dateFormat"] = value
+
+    @property
+    def datetime_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing DATETIME values.
+
+        (Valid for CSV and NEWLINE_DELIMITED_JSON)
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.datetime_format
+        """
+        return self._properties.get("datetimeFormat")
+
+    @datetime_format.setter
+    def datetime_format(self, value: Optional[str]):
+        self._properties["datetimeFormat"] = value
+
+    @property
+    def time_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing TIME values.
+
+        (Valid for CSV and NEWLINE_DELIMITED_JSON)
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.time_format
+        """
+        return self._properties.get("timeFormat")
+
+    @time_format.setter
+    def time_format(self, value: Optional[str]):
+        self._properties["timeFormat"] = value
+
+    @property
+    def timestamp_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing TIMESTAMP values.
+
+        (Valid for CSV and NEWLINE_DELIMITED_JSON)
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.timestamp_format
+        """
+        return self._properties.get("timestampFormat")
+
+    @timestamp_format.setter
+    def timestamp_format(self, value: Optional[str]):
+        self._properties["timestampFormat"] = value
+
     @property
     def connection_id(self):
         """Optional[str]: [Experimental] ID of a BigQuery Connection API

google/cloud/bigquery/job/load.py

@@ -548,6 +548,104 @@ def source_format(self):
     def source_format(self, value):
         self._set_sub_prop("sourceFormat", value)
 
+    @property
+    def time_zone(self):
+        """Optional[str]: Default time zone that will apply when parsing
+        timestamp values that have no specific time zone.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.time_zone
+        """
+        return self._get_sub_prop("timeZone")
+
+    @time_zone.setter
+    def time_zone(self, value: Optional[str]):
+        self._set_sub_prop("timeZone", value)
+
+    @property
+    def date_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing DATE values.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.date_format
+        """
+        return self._get_sub_prop("dateFormat")
+
+    @date_format.setter
+    def date_format(self, value: Optional[str]):
+        self._set_sub_prop("dateFormat", value)
+
+    @property
+    def datetime_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing DATETIME values.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.datetime_format
+        """
+        return self._get_sub_prop("datetimeFormat")
+
+    @datetime_format.setter
+    def datetime_format(self, value: Optional[str]):
+        self._set_sub_prop("datetimeFormat", value)
+
+    @property
+    def time_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing TIME values.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.time_format
+        """
+        return self._get_sub_prop("timeFormat")
+
+    @time_format.setter
+    def time_format(self, value: Optional[str]):
+        self._set_sub_prop("timeFormat", value)
+
+    @property
+    def timestamp_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing TIMESTAMP values.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.timestamp_format
+        """
+        return self._get_sub_prop("timestampFormat")
+
+    @timestamp_format.setter
+    def timestamp_format(self, value: Optional[str]):
+        self._set_sub_prop("timestampFormat", value)
+
+    @property
+    def null_markers(self) -> Optional[List[str]]:
+        """Optional[List[str]]: A list of strings represented as SQL NULL value in a CSV file.
+
+        (CSV only).
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.null_markers
+        """
+        return self._get_sub_prop("nullMarkers")
+
+    @null_markers.setter
+    def null_markers(self, value: Optional[List[str]]):
+        self._set_sub_prop("nullMarkers", value)
+
+    @property
+    def source_column_name_match_option(self) -> Optional[str]:
+        """Optional[str]: Controls the strategy used to match loaded columns to the schema.
+
+        (CSV only).
+        Acceptable values are based on the SourceColumnMatch enum in the proto.
+        Example values: "MATCH_BY_NAME", "MATCH_BY_POSITION".
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.source_column_match
+        """
+        return self._get_sub_prop("sourceColumnMatch")
+
+    @source_column_name_match_option.setter
+    def source_column_name_match_option(self, value: Optional[str]):
+        self._set_sub_prop("sourceColumnMatch", value)
+
     @property
     def time_partitioning(self):
         """Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based
@@ -889,6 +987,55 @@ def clustering_fields(self):
         """
         return self.configuration.clustering_fields
 
+    @property
+    def time_zone(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.time_zone`.
+        """
+        return self.configuration.time_zone
+
+    @property
+    def date_format(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.date_format`.
+        """
+        return self.configuration.date_format
+
+    @property
+    def datetime_format(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.datetime_format`.
+        """
+        return self.configuration.datetime_format
+
+    @property
+    def time_format(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.time_format`.
+        """
+        return self.configuration.time_format
+
+    @property
+    def timestamp_format(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.timestamp_format`.
+        """
+        return self.configuration.timestamp_format
+
+    @property
+    def null_markers(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.null_markers`.
+        """
+        return self.configuration.null_markers
+
+    @property
+    def source_column_name_match_option(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.source_column_name_match_option`.
+        """
+        return self.configuration.source_column_name_match_option
+
     @property
     def schema_update_options(self):
         """See

tests/unit/job/test_load.py

@@ -37,11 +37,25 @@ def _setUpConstants(self):
         self.OUTPUT_BYTES = 23456
         self.OUTPUT_ROWS = 345
         self.REFERENCE_FILE_SCHEMA_URI = "gs://path/to/reference"
+        self.TIME_ZONE = "UTC"
+        self.DATE_FORMAT = "%Y-%m-%d"
+        self.DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
+        self.TIME_FORMAT = "%H:%M:%S"
+        self.TIMESTAMP_FORMAT = "YYYY-MM-DD HH:MM:SS.SSSSSSZ"
+        self.NULL_MARKERS = ["N/A", "\\N"]
+        self.SOURCE_COLUMN_NAME_MATCH_OPTION = "MATCH_BY_NAME"
 
     def _make_resource(self, started=False, ended=False):
         resource = super(TestLoadJob, self)._make_resource(started, ended)
         config = resource["configuration"]["load"]
         config["sourceUris"] = [self.SOURCE1]
+        config["timeZone"] = self.TIME_ZONE
+        config["dateFormat"] = self.DATE_FORMAT
+        config["datetimeFormat"] = self.DATETIME_FORMAT
+        config["timeFormat"] = self.TIME_FORMAT
+        config["timestampFormat"] = self.TIMESTAMP_FORMAT
+        config["nullMarkers"] = self.NULL_MARKERS
+        config["sourceColumnMatch"] = self.SOURCE_COLUMN_NAME_MATCH_OPTION
         config["destinationTable"] = {
             "projectId": self.PROJECT,
             "datasetId": self.DS_ID,
@@ -153,6 +167,37 @@ def _verifyResourceProperties(self, job, resource):
         else:
             self.assertIsNone(job.destination_encryption_configuration)
 
+        if "timeZone" in config:
+            self.assertEqual(job.time_zone, config["timeZone"])
+        else:
+            self.assertIsNone(job.time_zone)
+        if "dateFormat" in config:
+            self.assertEqual(job.date_format, config["dateFormat"])
+        else:
+            self.assertIsNone(job.date_format)
+        if "datetimeFormat" in config:
+            self.assertEqual(job.datetime_format, config["datetimeFormat"])
+        else:
+            self.assertIsNone(job.datetime_format)
+        if "timeFormat" in config:
+            self.assertEqual(job.time_format, config["timeFormat"])
+        else:
+            self.assertIsNone(job.time_format)
+        if "timestampFormat" in config:
+            self.assertEqual(job.timestamp_format, config["timestampFormat"])
+        else:
+            self.assertIsNone(job.timestamp_format)
+        if "nullMarkers" in config:
+            self.assertEqual(job.null_markers, config["nullMarkers"])
+        else:
+            self.assertIsNone(job.null_markers)
+        if "sourceColumnMatch" in config:
+            self.assertEqual(
+                job.source_column_name_match_option, config["sourceColumnMatch"]
+            )
+        else:
+            self.assertIsNone(job.source_column_name_match_option)
+
     def test_ctor(self):
         client = _make_client(project=self.PROJECT)
         job = self._make_one(self.JOB_ID, [self.SOURCE1], self.TABLE_REF, client)
@@ -194,6 +239,13 @@ def test_ctor(self):
         self.assertIsNone(job.clustering_fields)
         self.assertIsNone(job.schema_update_options)
         self.assertIsNone(job.reference_file_schema_uri)
+        self.assertIsNone(job.time_zone)
+        self.assertIsNone(job.date_format)
+        self.assertIsNone(job.datetime_format)
+        self.assertIsNone(job.time_format)
+        self.assertIsNone(job.timestamp_format)
+        self.assertIsNone(job.null_markers)
+        self.assertIsNone(job.source_column_name_match_option)
 
     def test_ctor_w_config(self):
         from google.cloud.bigquery.schema import SchemaField
@@ -571,6 +623,13 @@ def test_begin_w_alternate_client(self):
                 ]
             },
             "schemaUpdateOptions": [SchemaUpdateOption.ALLOW_FIELD_ADDITION],
+            "timeZone": self.TIME_ZONE,
+            "dateFormat": self.DATE_FORMAT,
+            "datetimeFormat": self.DATETIME_FORMAT,
+            "timeFormat": self.TIME_FORMAT,
+            "timestampFormat": self.TIMESTAMP_FORMAT,
+            "nullMarkers": self.NULL_MARKERS,
+            "sourceColumnMatch": self.SOURCE_COLUMN_NAME_MATCH_OPTION,
         }
         RESOURCE["configuration"]["load"] = LOAD_CONFIGURATION
         conn1 = make_connection()
@@ -599,6 +658,13 @@ def test_begin_w_alternate_client(self):
         config.write_disposition = WriteDisposition.WRITE_TRUNCATE
         config.schema_update_options = [SchemaUpdateOption.ALLOW_FIELD_ADDITION]
         config.reference_file_schema_uri = "gs://path/to/reference"
+        config.time_zone = self.TIME_ZONE
+        config.date_format = self.DATE_FORMAT
+        config.datetime_format = self.DATETIME_FORMAT
+        config.time_format = self.TIME_FORMAT
+        config.timestamp_format = self.TIMESTAMP_FORMAT
+        config.null_markers = self.NULL_MARKERS
+        config.source_column_name_match_option = self.SOURCE_COLUMN_NAME_MATCH_OPTION
         with mock.patch(
             "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes"
         ) as final_attributes: