Merge remote-tracking branch 'origin/main' into b323176126-pandas-gbq

tswast · tswast · commit 2ae1121cfab2 · 2025-01-03T12:54:08.000-06:00
diff --git a/.kokoro/docker/docs/requirements.txt b/.kokoro/docker/docs/requirements.txt
@@ -4,9 +4,9 @@
 #
 #    pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in
 #
-argcomplete==3.5.2 \
-    --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \
-    --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb
+argcomplete==3.5.3 \
+    --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \
+    --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392
     # via nox
 colorlog==6.9.0 \
     --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \
diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py
@@ -16,8 +16,9 @@
 
 import collections
 import enum
-from typing import Any, Dict, Iterable, Optional, Union, cast
+from typing import Any, cast, Dict, Iterable, Optional, Union
 
+from google.cloud.bigquery import _helpers
 from google.cloud.bigquery import standard_sql
 from google.cloud.bigquery.enums import StandardSqlTypeNames
 
@@ -203,15 +204,8 @@ def __init__(
             self._properties["rangeElementType"] = {"type": range_element_type}
         if isinstance(range_element_type, FieldElementType):
             self._properties["rangeElementType"] = range_element_type.to_api_repr()
-
-        self._fields = tuple(fields)
-
-    @staticmethod
-    def __get_int(api_repr, name):
-        v = api_repr.get(name, _DEFAULT_VALUE)
-        if v is not _DEFAULT_VALUE:
-            v = int(v)
-        return v
+        if fields:  # Don't set the property if it's not set.
+            self._properties["fields"] = [field.to_api_repr() for field in fields]
 
     @classmethod
     def from_api_repr(cls, api_repr: dict) -> "SchemaField":
@@ -225,43 +219,19 @@ def from_api_repr(cls, api_repr: dict) -> "SchemaField":
         Returns:
             google.cloud.bigquery.schema.SchemaField: The ``SchemaField`` object.
         """
-        field_type = api_repr["type"].upper()
-
-        # Handle optional properties with default values
-        mode = api_repr.get("mode", "NULLABLE")
-        description = api_repr.get("description", _DEFAULT_VALUE)
-        fields = api_repr.get("fields", ())
-        policy_tags = api_repr.get("policyTags", _DEFAULT_VALUE)
+        placeholder = cls("this_will_be_replaced", "PLACEHOLDER")
 
-        default_value_expression = api_repr.get("defaultValueExpression", None)
+        # Note: we don't make a copy of api_repr because this can cause
+        # unnecessary slowdowns, especially on deeply nested STRUCT / RECORD
+        # fields. See https://github.com/googleapis/python-bigquery/issues/6
+        placeholder._properties = api_repr
 
-        if policy_tags is not None and policy_tags is not _DEFAULT_VALUE:
-            policy_tags = PolicyTagList.from_api_repr(policy_tags)
-
-        if api_repr.get("rangeElementType"):
-            range_element_type = cast(dict, api_repr.get("rangeElementType"))
-            element_type = range_element_type.get("type")
-        else:
-            element_type = None
-
-        return cls(
-            field_type=field_type,
-            fields=[cls.from_api_repr(f) for f in fields],
-            mode=mode.upper(),
-            default_value_expression=default_value_expression,
-            description=description,
-            name=api_repr["name"],
-            policy_tags=policy_tags,
-            precision=cls.__get_int(api_repr, "precision"),
-            scale=cls.__get_int(api_repr, "scale"),
-            max_length=cls.__get_int(api_repr, "maxLength"),
-            range_element_type=element_type,
-        )
+        return placeholder
 
     @property
     def name(self):
         """str: The name of the field."""
-        return self._properties["name"]
+        return self._properties.get("name", "")
 
     @property
     def field_type(self):
@@ -270,7 +240,10 @@ def field_type(self):
         See:
         https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableFieldSchema.FIELDS.type
         """
-        return self._properties["type"]
+        type_ = self._properties.get("type")
+        if type_ is None:  # Shouldn't happen, but some unit tests do this.
+            return None
+        return cast(str, type_).upper()
 
     @property
     def mode(self):
@@ -279,7 +252,7 @@ def mode(self):
         See:
         https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableFieldSchema.FIELDS.mode
         """
-        return self._properties.get("mode")
+        return cast(str, self._properties.get("mode", "NULLABLE")).upper()
 
     @property
     def is_nullable(self):
@@ -299,17 +272,17 @@ def description(self):
     @property
     def precision(self):
         """Optional[int]: Precision (number of digits) for the NUMERIC field."""
-        return self._properties.get("precision")
+        return _helpers._int_or_none(self._properties.get("precision"))
 
     @property
     def scale(self):
         """Optional[int]: Scale (digits after decimal) for the NUMERIC field."""
-        return self._properties.get("scale")
+        return _helpers._int_or_none(self._properties.get("scale"))
 
     @property
     def max_length(self):
         """Optional[int]: Maximum length for the STRING or BYTES field."""
-        return self._properties.get("maxLength")
+        return _helpers._int_or_none(self._properties.get("maxLength"))
 
     @property
     def range_element_type(self):
@@ -329,7 +302,7 @@ def fields(self):
 
         Must be empty unset if ``field_type`` is not 'RECORD'.
         """
-        return self._fields
+        return tuple(_to_schema_fields(self._properties.get("fields", [])))
 
     @property
     def policy_tags(self):
@@ -345,15 +318,10 @@ def to_api_repr(self) -> dict:
         Returns:
             Dict: A dictionary representing the SchemaField in a serialized form.
         """
-        answer = self._properties.copy()
-
-        # If this is a RECORD type, then sub-fields are also included,
-        # add this to the serialized representation.
-        if self.field_type.upper() in _STRUCT_TYPES:
-            answer["fields"] = [f.to_api_repr() for f in self.fields]
-
-        # Done; return the serialized dictionary.
-        return answer
+        # Note: we don't make a copy of _properties because this can cause
+        # unnecessary slowdowns, especially on deeply nested STRUCT / RECORD
+        # fields. See https://github.com/googleapis/python-bigquery/issues/6
+        return self._properties
 
     def _key(self):
         """A tuple key that uniquely describes this field.
@@ -389,7 +357,7 @@ def _key(self):
             self.mode.upper(),  # pytype: disable=attribute-error
             self.default_value_expression,
             self.description,
-            self._fields,
+            self.fields,
             policy_tags,
         )
 
diff --git a/samples/desktopapp/requirements-test.txt b/samples/desktopapp/requirements-test.txt
@@ -1,4 +1,4 @@
-google-cloud-testutils==1.4.0
+google-cloud-testutils==1.5.0
 pytest===7.4.4; python_version == '3.7'
-pytest==8.3.3; python_version >= '3.8'
+pytest==8.3.4; python_version >= '3.8'
 mock==5.1.0
diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt
@@ -1,3 +1,3 @@
 pytest===7.4.4; python_version == '3.7'
-pytest==8.3.3; python_version >= '3.8'
+pytest==8.3.4; python_version >= '3.8'
 mock==5.1.0
diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt
@@ -1,21 +1,22 @@
-attrs==24.2.0
-certifi==2024.8.30
+attrs===24.2.0; python_version == '3.7'
+attrs==24.3.0; python_version >= '3.8'
+certifi==2024.12.14
 cffi===1.15.1; python_version == '3.7'
 cffi==1.17.1; python_version >= '3.8'
-charset-normalizer==3.4.0
-click==8.1.7
+charset-normalizer==3.4.1
+click==8.1.8
 click-plugins==1.1.1
 cligj==0.7.2
 dataclasses==0.8; python_version < '3.7'
 db-dtypes==1.3.1
 Fiona===1.9.6; python_version == '3.7'
 Fiona==1.10.1; python_version >= '3.8'
-geojson==3.1.0
+geojson==3.2.0
 geopandas===0.10.2; python_version == '3.7'
 geopandas===0.13.2; python_version == '3.8'
 geopandas==1.0.1; python_version >= '3.9'
-google-api-core==2.23.0
-google-auth==2.36.0
+google-api-core==2.24.0
+google-auth==2.37.0
 google-cloud-bigquery==3.27.0
 google-cloud-bigquery-storage==2.27.0
 google-cloud-core==2.4.1
@@ -24,7 +25,7 @@ google-crc32c==1.6.0; python_version >= '3.9'
 google-resumable-media==2.7.2
 googleapis-common-protos==1.66.0
 grpcio===1.62.2; python_version == '3.7'
-grpcio==1.68.0; python_version >= '3.8'
+grpcio==1.68.1; python_version >= '3.8'
 idna==3.10
 munch==4.0.0
 mypy-extensions==1.0.0
@@ -44,7 +45,7 @@ pyasn1-modules==0.4.1; python_version >= '3.8'
 pycparser===2.21; python_version == '3.7'
 pycparser==2.22; python_version >= '3.8'
 pyparsing===3.1.4; python_version < '3.9'
-pyparsing==3.2.0; python_version >= '3.9'
+pyparsing==3.2.1; python_version >= '3.9'
 python-dateutil==2.9.0.post0
 pytz==2024.2
 PyYAML===6.0.1; python_version == '3.7'
@@ -53,9 +54,10 @@ requests==2.31.0; python_version == '3.7'
 requests==2.32.3; python_version >= '3.8'
 rsa==4.9
 Shapely==2.0.6
-six==1.16.0
+six==1.17.0
 typing-extensions===4.7.1; python_version == '3.7'
 typing-extensions==4.12.2; python_version >= '3.8'
 typing-inspect==0.9.0
 urllib3===1.26.18; python_version == '3.7'
-urllib3==2.2.3; python_version >= '3.8'
+urllib3===2.2.3; python_version == '3.8'
+urllib3==2.3.0; python_version >= '3.9'
diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt
@@ -1,4 +1,4 @@
-google-cloud-testutils==1.4.0
+google-cloud-testutils==1.5.0
 pytest===7.4.4; python_version == '3.7'
-pytest==8.3.3; python_version >= '3.8'
+pytest==8.3.4; python_version >= '3.8'
 mock==5.1.0
diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt
@@ -1,4 +1,4 @@
-bigquery_magics==0.4.0
+bigquery_magics==0.5.0
 db-dtypes==1.3.1
 google.cloud.bigquery==3.27.0
 google-cloud-bigquery-storage==2.27.0
diff --git a/samples/notebooks/requirements-test.txt b/samples/notebooks/requirements-test.txt
@@ -1,4 +1,4 @@
-google-cloud-testutils==1.4.0
+google-cloud-testutils==1.5.0
 pytest===7.4.4; python_version == '3.7'
-pytest==8.3.3; python_version >= '3.8'
+pytest==8.3.4; python_version >= '3.8'
 mock==5.1.0
diff --git a/samples/notebooks/requirements.txt b/samples/notebooks/requirements.txt
@@ -1,4 +1,4 @@
-bigquery-magics==0.4.0
+bigquery-magics==0.5.0
 db-dtypes==1.3.1
 google-cloud-bigquery==3.27.0
 google-cloud-bigquery-storage==2.27.0
@@ -7,7 +7,8 @@ ipython===8.0.1; python_version == '3.8'
 ipython===8.18.1; python_version >= '3.9'
 matplotlib===3.5.3; python_version == '3.7'
 matplotlib===3.7.4; python_version == '3.8'
-matplotlib==3.9.2; python_version >= '3.9'
+matplotlib===3.9.2; python_version == '3.9'
+matplotlib==3.10.0; python_version >= '3.10'
 pandas===1.3.5; python_version == '3.7'
 pandas===2.0.3; python_version == '3.8'
 pandas==2.2.3; python_version >= '3.9'
diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt
@@ -1,5 +1,5 @@
 # samples/snippets should be runnable with no "extras"
-google-cloud-testutils==1.4.0
+google-cloud-testutils==1.5.0
 pytest===7.4.4; python_version == '3.7'
-pytest==8.3.3; python_version >= '3.8'
+pytest==8.3.4; python_version >= '3.8'
 mock==5.1.0
diff --git a/tests/unit/job/test_load_config.py b/tests/unit/job/test_load_config.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import warnings
 
 import pytest
@@ -571,16 +572,34 @@ def test_schema_setter_valid_mappings_list(self):
             config._properties["load"]["schema"], {"fields": [full_name_repr, age_repr]}
         )
 
-    def test_schema_setter_invalid_mappings_list(self):
+    def test_schema_setter_allows_unknown_properties(self):
         config = self._get_target_class()()
 
         schema = [
-            {"name": "full_name", "type": "STRING", "mode": "REQUIRED"},
-            {"name": "age", "typeoo": "INTEGER", "mode": "REQUIRED"},
+            {
+                "name": "full_name",
+                "type": "STRING",
+                "mode": "REQUIRED",
+                "someNewProperty": "test-value",
+            },
+            {
+                "name": "age",
+                # Note: This type should be included, too. Avoid client-side
+                # validation, as it could prevent backwards-compatible
+                # evolution of the server-side behavior.
+                "typo": "INTEGER",
+                "mode": "REQUIRED",
+                "anotherNewProperty": "another-test",
+            },
         ]
 
-        with self.assertRaises(Exception):
-            config.schema = schema
+        # Make sure the setter doesn't mutate schema.
+        expected_schema = copy.deepcopy(schema)
+
+        config.schema = schema
+
+        # _properties should include all fields, including unknown ones.
+        assert config._properties["load"]["schema"]["fields"] == expected_schema
 
     def test_schema_setter_unsetting_schema(self):
         from google.cloud.bigquery.schema import SchemaField
diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py
@@ -12,14 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from google.cloud import bigquery
-from google.cloud.bigquery.standard_sql import StandardSqlStructType
-from google.cloud.bigquery.schema import PolicyTagList
+import copy
 import unittest
 from unittest import mock
 
 import pytest
 
+from google.cloud import bigquery
+from google.cloud.bigquery.standard_sql import StandardSqlStructType
+from google.cloud.bigquery.schema import PolicyTagList
+
 
 class TestSchemaField(unittest.TestCase):
     @staticmethod
@@ -821,13 +823,32 @@ def test_schema_fields_sequence(self):
         result = self._call_fut(schema)
         self.assertEqual(result, schema)
 
-    def test_invalid_mapping_representation(self):
+    def test_unknown_properties(self):
         schema = [
-            {"name": "full_name", "type": "STRING", "mode": "REQUIRED"},
-            {"name": "address", "typeooo": "STRING", "mode": "REQUIRED"},
+            {
+                "name": "full_name",
+                "type": "STRING",
+                "mode": "REQUIRED",
+                "someNewProperty": "test-value",
+            },
+            {
+                "name": "age",
+                # Note: This type should be included, too. Avoid client-side
+                # validation, as it could prevent backwards-compatible
+                # evolution of the server-side behavior.
+                "typo": "INTEGER",
+                "mode": "REQUIRED",
+                "anotherNewProperty": "another-test",
+            },
         ]
-        with self.assertRaises(Exception):
-            self._call_fut(schema)
+
+        # Make sure the setter doesn't mutate schema.
+        expected_schema = copy.deepcopy(schema)
+
+        result = self._call_fut(schema)
+
+        for api_repr, field in zip(expected_schema, result):
+            assert field.to_api_repr() == api_repr
 
     def test_valid_mapping_representation(self):
         from google.cloud.bigquery.schema import SchemaField
diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py

Original file line number	Diff line number	Diff line change
`@@ -4,9 +4,9 @@`
`4`	`4`	`#`
`5`	`5`	`# pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in`
`6`	`6`	`#`
`7`		`-argcomplete==3.5.2 \`
`8`		`- --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \`
`9`		`- --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb`
	`7`	`+argcomplete==3.5.3 \`
	`8`	`+ --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \`
	`9`	`+ --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392`
`10`	`10`	`# via nox`
`11`	`11`	`colorlog==6.9.0 \`
`12`	`12`	`--hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-bigquery_magics==0.4.0`
	`1`	`+bigquery_magics==0.5.0`
`2`	`2`	`db-dtypes==1.3.1`
`3`	`3`	`google.cloud.bigquery==3.27.0`
`4`	`4`	`google-cloud-bigquery-storage==2.27.0`