Merge branch 'main' into feat-374142081-add-date-format

Author: chalmerlowe

google/cloud/bigquery/external_config.py

@@ -862,6 +862,22 @@ def date_format(self) -> Optional[str]:
     def date_format(self, value: Optional[str]):
         self._properties["dateFormat"] = value
 
+    def time_zone(self) -> Optional[str]:
+        """Optional[str]: Time zone used when parsing timestamp values that do not
+        have specific time zone information (e.g. 2024-04-20 12:34:56). The expected
+        format is an IANA timezone string (e.g. America/Los_Angeles).
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.time_zone
+        """
+
+        result = self._properties.get("timeZone")
+        return typing.cast(str, result)
+
+    @time_zone.setter
+    def time_zone(self, value: Optional[str]):
+        self._properties["timeZone"] = value
+
     @property
     def connection_id(self):
         """Optional[str]: [Experimental] ID of a BigQuery Connection API

google/cloud/bigquery/job/load.py

@@ -561,6 +561,19 @@ def date_format(self) -> Optional[str]:
     def date_format(self, value: Optional[str]):
         self._set_sub_prop("dateFormat", value)
 
+    def time_zone(self) -> Optional[str]:
+        """Optional[str]: Default time zone that will apply when parsing timestamp
+        values that have no specific time zone.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.time_zone
+        """
+        return self._get_sub_prop("timeZone")
+
+    @time_zone.setter
+    def time_zone(self, value: Optional[str]):
+        self._set_sub_prop("timeZone", value)
+
     @property
     def time_partitioning(self):
         """Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based
@@ -909,6 +922,12 @@ def date_format(self):
         """
         return self.configuration.date_format
 
+    def time_zone(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.time_zone`.
+        """
+        return self.configuration.time_zone
+
     @property
     def schema_update_options(self):
         """See

tests/unit/job/test_load.py

@@ -37,15 +37,15 @@ def _setUpConstants(self):
         self.OUTPUT_BYTES = 23456
         self.OUTPUT_ROWS = 345
         self.REFERENCE_FILE_SCHEMA_URI = "gs://path/to/reference"
-
         self.DATE_FORMAT = "%Y-%m-%d"
+        self.TIME_ZONE = "UTC"
 
     def _make_resource(self, started=False, ended=False):
         resource = super(TestLoadJob, self)._make_resource(started, ended)
         config = resource["configuration"]["load"]
         config["sourceUris"] = [self.SOURCE1]
-
         config["dateFormat"] = self.DATE_FORMAT
+        config["timeZone"] = self.TIME_ZONE
         config["destinationTable"] = {
             "projectId": self.PROJECT,
             "datasetId": self.DS_ID,
@@ -156,6 +156,10 @@ def _verifyResourceProperties(self, job, resource):
             )
         else:
             self.assertIsNone(job.destination_encryption_configuration)
+        if "timeZone" in config:
+            self.assertEqual(job.time_zone, config["timeZone"])
+        else:
+            self.assertIsNone(job.time_zone)
 
         if "dateFormat" in config:
             self.assertEqual(job.date_format, config["dateFormat"])
@@ -203,8 +207,8 @@ def test_ctor(self):
         self.assertIsNone(job.clustering_fields)
         self.assertIsNone(job.schema_update_options)
         self.assertIsNone(job.reference_file_schema_uri)
-
         self.assertIsNone(job.date_format)
+        self.assertIsNone(job.time_zone)
 
     def test_ctor_w_config(self):
         from google.cloud.bigquery.schema import SchemaField
@@ -442,6 +446,24 @@ def test_from_api_repr_w_properties(self):
         self.assertIs(job._client, client)
         self._verifyResourceProperties(job, RESOURCE)
 
+    def test_to_api_repr(self):
+        self._setUpConstants()
+        client = _make_client(project=self.PROJECT)
+        RESOURCE = self._make_resource(ended=False)
+
+        klass = self._get_target_class()
+        job = klass.from_api_repr(RESOURCE, client)
+        api_repr = job.to_api_repr()
+
+        # as per the documentation in load.py -> LoadJob.to_api_repr(),
+        # the return value from to_api_repr should not include statistics
+        expected = {
+            "jobReference": RESOURCE["jobReference"],
+            "configuration": RESOURCE["configuration"],
+        }
+
+        self.assertEqual(api_repr, expected)
+
     def test_begin_w_already_running(self):
         conn = make_connection()
         client = _make_client(project=self.PROJECT, connection=conn)
@@ -583,6 +605,7 @@ def test_begin_w_alternate_client(self):
             },
             "schemaUpdateOptions": [SchemaUpdateOption.ALLOW_FIELD_ADDITION],
             "dateFormat": self.DATE_FORMAT,
+            "timeZone": self.TIME_ZONE,
         }
         RESOURCE["configuration"]["load"] = LOAD_CONFIGURATION
         conn1 = make_connection()
@@ -611,8 +634,8 @@ def test_begin_w_alternate_client(self):
         config.write_disposition = WriteDisposition.WRITE_TRUNCATE
         config.schema_update_options = [SchemaUpdateOption.ALLOW_FIELD_ADDITION]
         config.reference_file_schema_uri = "gs://path/to/reference"
-
         config.date_format = self.DATE_FORMAT
+        config.time_zone = self.TIME_ZONE
 
         with mock.patch(
             "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes"

tests/unit/job/test_load_config.py

@@ -844,6 +844,22 @@ def test_date_format_setter(self):
         config.date_format = date_format
         self.assertEqual(config._properties["load"]["dateFormat"], date_format)
 
+    def test_time_zone_missing(self):
+        config = self._get_target_class()()
+        self.assertIsNone(config.time_zone)
+
+    def test_time_zone_hit(self):
+        time_zone = "UTC"
+        config = self._get_target_class()()
+        config._properties["load"]["timeZone"] = time_zone
+        self.assertEqual(config.time_zone, time_zone)
+
+    def test_time_zone_setter(self):
+        time_zone = "America/New_York"
+        config = self._get_target_class()()
+        config.time_zone = time_zone
+        self.assertEqual(config._properties["load"]["timeZone"], time_zone)
+
     def test_parquet_options_missing(self):
         config = self._get_target_class()()
         self.assertIsNone(config.parquet_options)
@@ -917,3 +933,114 @@ def test_column_name_character_map_none(self):
             config._properties["load"]["columnNameCharacterMap"],
             ColumnNameCharacterMap.COLUMN_NAME_CHARACTER_MAP_UNSPECIFIED,
         )
+
+    RESOURCE = {
+        "load": {
+            "allowJaggedRows": True,
+            "createDisposition": "CREATE_NEVER",
+            "encoding": "UTF-8",
+            "fieldDelimiter": ",",
+            "ignoreUnknownValues": True,
+            "maxBadRecords": 10,
+            "nullMarker": "\\N",
+            "quote": '"',
+            "schema": {
+                "fields": [
+                    {"name": "name", "type": "STRING", "mode": "NULLABLE"},
+                    {"name": "age", "type": "INTEGER", "mode": "NULLABLE"},
+                ]
+            },
+            "skipLeadingRows": "1",
+            "sourceFormat": "CSV",
+            "timePartitioning": {
+                "type": "DAY",
+                "field": "transaction_date",
+            },
+            "useAvroLogicalTypes": True,
+            "writeDisposition": "WRITE_TRUNCATE",
+            "timeZone": "America/New_York",
+            "parquetOptions": {"enableListInference": True},
+            "columnNameCharacterMap": "V2",
+            "someNewField": "some-value",
+        }
+    }
+
+    def test_from_api_repr(self):
+        from google.cloud.bigquery.job import (
+            CreateDisposition,
+            LoadJobConfig,
+            SourceFormat,
+            WriteDisposition,
+        )
+        from google.cloud.bigquery.schema import SchemaField
+        from google.cloud.bigquery.table import TimePartitioning, TimePartitioningType
+
+        from google.cloud.bigquery.job.load import ColumnNameCharacterMap
+
+        config = LoadJobConfig.from_api_repr(self.RESOURCE)
+
+        self.assertTrue(config.allow_jagged_rows)
+        self.assertEqual(config.create_disposition, CreateDisposition.CREATE_NEVER)
+        self.assertEqual(config.encoding, "UTF-8")
+        self.assertEqual(config.field_delimiter, ",")
+        self.assertTrue(config.ignore_unknown_values)
+        self.assertEqual(config.max_bad_records, 10)
+        self.assertEqual(config.null_marker, "\\N")
+        self.assertEqual(config.quote_character, '"')
+        self.assertEqual(
+            config.schema,
+            [SchemaField("name", "STRING"), SchemaField("age", "INTEGER")],
+        )
+        self.assertEqual(config.skip_leading_rows, 1)
+        self.assertEqual(config.source_format, SourceFormat.CSV)
+        self.assertEqual(
+            config.time_partitioning,
+            TimePartitioning(type_=TimePartitioningType.DAY, field="transaction_date"),
+        )
+        self.assertTrue(config.use_avro_logical_types)
+        self.assertEqual(config.write_disposition, WriteDisposition.WRITE_TRUNCATE)
+        self.assertEqual(config.time_zone, "America/New_York")
+        self.assertTrue(config.parquet_options.enable_list_inference)
+        self.assertEqual(config.column_name_character_map, ColumnNameCharacterMap.V2)
+        self.assertEqual(config._properties["load"]["someNewField"], "some-value")
+
+    def test_to_api_repr(self):
+        from google.cloud.bigquery.job import (
+            CreateDisposition,
+            LoadJobConfig,
+            SourceFormat,
+            WriteDisposition,
+        )
+        from google.cloud.bigquery.schema import SchemaField
+        from google.cloud.bigquery.table import TimePartitioning, TimePartitioningType
+        from google.cloud.bigquery.format_options import ParquetOptions
+        from google.cloud.bigquery.job.load import ColumnNameCharacterMap
+
+        config = LoadJobConfig()
+        config.allow_jagged_rows = True
+        config.create_disposition = CreateDisposition.CREATE_NEVER
+        config.encoding = "UTF-8"
+        config.field_delimiter = ","
+        config.ignore_unknown_values = True
+        config.max_bad_records = 10
+        config.null_marker = r"\N"
+        config.quote_character = '"'
+        config.schema = [SchemaField("name", "STRING"), SchemaField("age", "INTEGER")]
+        config.skip_leading_rows = 1
+        config.source_format = SourceFormat.CSV
+        config.time_partitioning = TimePartitioning(
+            type_=TimePartitioningType.DAY, field="transaction_date"
+        )
+        config.use_avro_logical_types = True
+        config.write_disposition = WriteDisposition.WRITE_TRUNCATE
+        config.time_zone = "America/New_York"
+        parquet_options = ParquetOptions()
+        parquet_options.enable_list_inference = True
+        config.parquet_options = parquet_options
+        config.column_name_character_map = ColumnNameCharacterMap.V2
+        config._properties["load"]["someNewField"] = "some-value"
+
+        api_repr = config.to_api_repr()
+
+        expected = self.RESOURCE
+        self.assertEqual(api_repr, expected)

tests/unit/test_external_config.py

@@ -25,8 +25,8 @@
 
 class TestExternalConfig(unittest.TestCase):
     SOURCE_URIS = ["gs://foo", "gs://bar"]
-
     DATE_FORMAT = "MM/DD/YYYY"
+    TIME_ZONE = "America/Los_Angeles"
 
     BASE_RESOURCE = {
         "sourceFormat": "",
@@ -36,6 +36,7 @@ class TestExternalConfig(unittest.TestCase):
         "ignoreUnknownValues": False,
         "compression": "compression",
         "dateFormat": DATE_FORMAT,
+        "timeZone": TIME_ZONE,
     }
 
     def test_from_api_repr_base(self):
@@ -83,6 +84,7 @@ def test_to_api_repr_base(self):
         ec.schema = [schema.SchemaField("full_name", "STRING", mode="REQUIRED")]
 
         ec.date_format = self.DATE_FORMAT
+        ec.time_zone = self.TIME_ZONE
         exp_schema = {
             "fields": [{"name": "full_name", "type": "STRING", "mode": "REQUIRED"}]
         }
@@ -97,6 +99,7 @@ def test_to_api_repr_base(self):
             "connectionId": "path/to/connection",
             "schema": exp_schema,
             "dateFormat": self.DATE_FORMAT,
+            "timeZone": self.TIME_ZONE,
         }
         self.assertEqual(got_resource, exp_resource)
 
@@ -132,8 +135,8 @@ def _verify_base(self, ec):
         self.assertEqual(ec.ignore_unknown_values, False)
         self.assertEqual(ec.max_bad_records, 17)
         self.assertEqual(ec.source_uris, self.SOURCE_URIS)
-
         self.assertEqual(ec.date_format, self.DATE_FORMAT)
+        self.assertEqual(ec.time_zone, self.TIME_ZONE)
 
     def test_to_api_repr_source_format(self):
         ec = external_config.ExternalConfig("CSV")