Merge branch 'main' into pangea-v1alpha

Author: chalmerlowe

.github/.OwlBot.lock.yaml

@@ -13,5 +13,5 @@
 # limitations under the License.
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
-  digest: sha256:2ed982f884312e4883e01b5ab8af8b6935f0216a5a2d82928d273081fc3be562
-# created: 2024-11-12T12:09:45.821174897Z
+  digest: sha256:8e3e7e18255c22d1489258d0374c901c01f9c4fd77a12088670cd73d580aa737
+# created: 2024-12-17T00:59:58.625514486Z

.kokoro/docker/docs/requirements.txt

@@ -2,11 +2,11 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --allow-unsafe --generate-hashes requirements.in
+#    pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in
 #
-argcomplete==3.5.1 \
-    --hash=sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363 \
-    --hash=sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4
+argcomplete==3.5.2 \
+    --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \
+    --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb
     # via nox
 colorlog==6.9.0 \
     --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \
@@ -23,7 +23,7 @@ filelock==3.16.1 \
 nox==2024.10.9 \
     --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \
     --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95
-    # via -r requirements.in
+    # via -r synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in
 packaging==24.2 \
     --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \
     --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f
@@ -32,11 +32,41 @@ platformdirs==4.3.6 \
     --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
     --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
     # via virtualenv
-tomli==2.0.2 \
-    --hash=sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38 \
-    --hash=sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed
+tomli==2.2.1 \
+    --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \
+    --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \
+    --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \
+    --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \
+    --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \
+    --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \
+    --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \
+    --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \
+    --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \
+    --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \
+    --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \
+    --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \
+    --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \
+    --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \
+    --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \
+    --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \
+    --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \
+    --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \
+    --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \
+    --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \
+    --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \
+    --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \
+    --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \
+    --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \
+    --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \
+    --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \
+    --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \
+    --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \
+    --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \
+    --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \
+    --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \
+    --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7
     # via nox
-virtualenv==20.27.1 \
-    --hash=sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba \
-    --hash=sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4
+virtualenv==20.28.0 \
+    --hash=sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0 \
+    --hash=sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa
     # via nox

.kokoro/requirements.txt

@@ -254,9 +254,9 @@ jeepney==0.8.0 \
     # via
     #   keyring
     #   secretstorage
-jinja2==3.1.4 \
-    --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \
-    --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d
+jinja2==3.1.5 \
+    --hash=sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb \
+    --hash=sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb
     # via gcp-releasetool
 keyring==25.4.1 \
     --hash=sha256:5426f817cf7f6f007ba5ec722b1bcad95a75b27d780343772ad76b17cb47b0bf \

google/cloud/bigquery/schema.py

@@ -19,8 +19,9 @@
 import collections
 import copy
 import enum
-from typing import Any, Dict, Iterable, Optional, Union, cast
+from typing import Any, cast, Dict, Iterable, Optional, Union
 
+from google.cloud.bigquery import _helpers
 from google.cloud.bigquery import standard_sql
 from google.cloud.bigquery._helpers import (
     _isinstance_or_raise,
@@ -240,32 +241,9 @@ def __init__(
             self._properties["rangeElementType"] = {"type": range_element_type}
         if isinstance(range_element_type, FieldElementType):
             self._properties["rangeElementType"] = range_element_type.to_api_repr()
-        if isinstance(rounding_mode, RoundingMode):
-            self._properties["roundingMode"] = rounding_mode.name
-        if isinstance(rounding_mode, str):
-            self._properties["roundingMode"] = rounding_mode
-        if isinstance(foreign_type_definition, str):
-            self._properties["foreignTypeDefinition"] = foreign_type_definition
-
-        # The order of operations is important:
-        # If field_type is FOREIGN, then foreign_type_definition must be set.
-        if field_type != "FOREIGN":
-            self._properties["type"] = field_type
-        else:
-            if self._properties.get("foreignTypeDefinition") is None:
-                raise ValueError(
-                    "If the 'field_type' is 'FOREIGN', then 'foreign_type_definition' is required."
-                )
-            self._properties["type"] = field_type
-
-        self._fields = tuple(fields)
+        if fields:  # Don't set the property if it's not set.
+            self._properties["fields"] = [field.to_api_repr() for field in fields]
 
-    @staticmethod
-    def __get_int(api_repr, name):
-        v = api_repr.get(name, _DEFAULT_VALUE)
-        if v is not _DEFAULT_VALUE:
-            v = int(v)
-        return v
 
     @classmethod
     def from_api_repr(cls, api_repr: dict) -> "SchemaField":
@@ -279,48 +257,19 @@ def from_api_repr(cls, api_repr: dict) -> "SchemaField":
         Returns:
             google.cloud.bigquery.schema.SchemaField: The ``SchemaField`` object.
         """
-        field_type = api_repr["type"].upper()
+        placeholder = cls("this_will_be_replaced", "PLACEHOLDER")
 
-        # Handle optional properties with default values
-        mode = api_repr.get("mode", "NULLABLE")
-        description = api_repr.get("description", _DEFAULT_VALUE)
-        fields = api_repr.get("fields", ())
-        policy_tags = api_repr.get("policyTags", _DEFAULT_VALUE)
+        # Note: we don't make a copy of api_repr because this can cause
+        # unnecessary slowdowns, especially on deeply nested STRUCT / RECORD
+        # fields. See https://github.com/googleapis/python-bigquery/issues/6
+        placeholder._properties = api_repr
 
-        default_value_expression = api_repr.get("defaultValueExpression", None)
-
-        if policy_tags is not None and policy_tags is not _DEFAULT_VALUE:
-            policy_tags = PolicyTagList.from_api_repr(policy_tags)
-
-        if api_repr.get("rangeElementType"):
-            range_element_type = cast(dict, api_repr.get("rangeElementType"))
-            element_type = range_element_type.get("type")
-        else:
-            element_type = None
-
-        rounding_mode = api_repr.get("roundingMode")
-        foreign_type_definition = api_repr.get("foreignTypeDefinition")
-
-        return cls(
-            field_type=field_type,
-            fields=[cls.from_api_repr(f) for f in fields],
-            mode=mode.upper(),
-            default_value_expression=default_value_expression,
-            description=description,
-            name=api_repr["name"],
-            policy_tags=policy_tags,
-            precision=cls.__get_int(api_repr, "precision"),
-            scale=cls.__get_int(api_repr, "scale"),
-            max_length=cls.__get_int(api_repr, "maxLength"),
-            range_element_type=element_type,
-            rounding_mode=rounding_mode,
-            foreign_type_definition=foreign_type_definition,
-        )
+        return placeholder
 
     @property
     def name(self):
         """str: The name of the field."""
-        return self._properties["name"]
+        return self._properties.get("name", "")
 
     @property
     def field_type(self):
@@ -329,7 +278,10 @@ def field_type(self):
         See:
         https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableFieldSchema.FIELDS.type
         """
-        return self._properties["type"]
+        type_ = self._properties.get("type")
+        if type_ is None:  # Shouldn't happen, but some unit tests do this.
+            return None
+        return cast(str, type_).upper()
 
     @property
     def mode(self):
@@ -338,7 +290,7 @@ def mode(self):
         See:
         https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableFieldSchema.FIELDS.mode
         """
-        return self._properties.get("mode")
+        return cast(str, self._properties.get("mode", "NULLABLE")).upper()
 
     @property
     def is_nullable(self):
@@ -358,17 +310,17 @@ def description(self):
     @property
     def precision(self):
         """Optional[int]: Precision (number of digits) for the NUMERIC field."""
-        return self._properties.get("precision")
+        return _helpers._int_or_none(self._properties.get("precision"))
 
     @property
     def scale(self):
         """Optional[int]: Scale (digits after decimal) for the NUMERIC field."""
-        return self._properties.get("scale")
+        return _helpers._int_or_none(self._properties.get("scale"))
 
     @property
     def max_length(self):
         """Optional[int]: Maximum length for the STRING or BYTES field."""
-        return self._properties.get("maxLength")
+        return _helpers._int_or_none(self._properties.get("maxLength"))
 
     @property
     def range_element_type(self):
@@ -404,7 +356,7 @@ def fields(self):
 
         Must be empty unset if ``field_type`` is not 'RECORD'.
         """
-        return self._fields
+        return tuple(_to_schema_fields(self._properties.get("fields", [])))
 
     @property
     def policy_tags(self):
@@ -420,15 +372,10 @@ def to_api_repr(self) -> dict:
         Returns:
             Dict: A dictionary representing the SchemaField in a serialized form.
         """
-        answer = self._properties.copy()
-
-        # If this is a RECORD type, then sub-fields are also included,
-        # add this to the serialized representation.
-        if self.field_type.upper() in _STRUCT_TYPES:
-            answer["fields"] = [f.to_api_repr() for f in self.fields]
-
-        # Done; return the serialized dictionary.
-        return answer
+        # Note: we don't make a copy of _properties because this can cause
+        # unnecessary slowdowns, especially on deeply nested STRUCT / RECORD
+        # fields. See https://github.com/googleapis/python-bigquery/issues/6
+        return self._properties
 
     def _key(self):
         """A tuple key that uniquely describes this field.
@@ -464,7 +411,7 @@ def _key(self):
             self.mode.upper(),  # pytype: disable=attribute-error
             self.default_value_expression,
             self.description,
-            self._fields,
+            self.fields,
             policy_tags,
         )
 

tests/unit/job/test_load_config.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import warnings
 
 import pytest
@@ -571,16 +572,34 @@ def test_schema_setter_valid_mappings_list(self):
             config._properties["load"]["schema"], {"fields": [full_name_repr, age_repr]}
         )
 
-    def test_schema_setter_invalid_mappings_list(self):
+    def test_schema_setter_allows_unknown_properties(self):
         config = self._get_target_class()()
 
         schema = [
-            {"name": "full_name", "type": "STRING", "mode": "REQUIRED"},
-            {"name": "age", "typeoo": "INTEGER", "mode": "REQUIRED"},
+            {
+                "name": "full_name",
+                "type": "STRING",
+                "mode": "REQUIRED",
+                "someNewProperty": "test-value",
+            },
+            {
+                "name": "age",
+                # Note: This type should be included, too. Avoid client-side
+                # validation, as it could prevent backwards-compatible
+                # evolution of the server-side behavior.
+                "typo": "INTEGER",
+                "mode": "REQUIRED",
+                "anotherNewProperty": "another-test",
+            },
         ]
 
-        with self.assertRaises(Exception):
-            config.schema = schema
+        # Make sure the setter doesn't mutate schema.
+        expected_schema = copy.deepcopy(schema)
+
+        config.schema = schema
+
+        # _properties should include all fields, including unknown ones.
+        assert config._properties["load"]["schema"]["fields"] == expected_schema
 
     def test_schema_setter_unsetting_schema(self):
         from google.cloud.bigquery.schema import SchemaField

tests/unit/test_schema.py

@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+import copy
+import unittest
+from unittest import mock
+
+import pytest
+
 from google.cloud import bigquery
 from google.cloud.bigquery.enums import RoundingMode
 from google.cloud.bigquery.standard_sql import StandardSqlStructType
@@ -22,11 +29,6 @@
     SerDeInfo,
 )
 
-import unittest
-from unittest import mock
-
-import pytest
-
 
 class TestSchemaField(unittest.TestCase):
     @staticmethod
@@ -877,13 +879,32 @@ def test_schema_fields_sequence(self):
         result = self._call_fut(schema)
         self.assertEqual(result, schema)
 
-    def test_invalid_mapping_representation(self):
+    def test_unknown_properties(self):
         schema = [
-            {"name": "full_name", "type": "STRING", "mode": "REQUIRED"},
-            {"name": "address", "typeooo": "STRING", "mode": "REQUIRED"},
+            {
+                "name": "full_name",
+                "type": "STRING",
+                "mode": "REQUIRED",
+                "someNewProperty": "test-value",
+            },
+            {
+                "name": "age",
+                # Note: This type should be included, too. Avoid client-side
+                # validation, as it could prevent backwards-compatible
+                # evolution of the server-side behavior.
+                "typo": "INTEGER",
+                "mode": "REQUIRED",
+                "anotherNewProperty": "another-test",
+            },
         ]
-        with self.assertRaises(Exception):
-            self._call_fut(schema)
+
+        # Make sure the setter doesn't mutate schema.
+        expected_schema = copy.deepcopy(schema)
+
+        result = self._call_fut(schema)
+
+        for api_repr, field in zip(expected_schema, result):
+            assert field.to_api_repr() == api_repr
 
     def test_valid_mapping_representation(self):
         from google.cloud.bigquery.schema import SchemaField