Most recent round of tweaks and experiments

Author: chalmerlowe

google/cloud/bigquery/_helpers.py

@@ -1040,7 +1040,7 @@ def _isinstance_or_raise(
     raise TypeError(msg)
 
 
-def _from_api_repr(obj, resource: dict):
+def _from_api_repr(cls, resource: dict):
     """Factory: constructs an instance of the class (cls)
     given its API representation.
 
@@ -1051,6 +1051,6 @@ def _from_api_repr(obj, resource: dict):
     Returns:
         An instance of the class initialized with data from 'resource'.
     """
-    config = obj
+    config = cls
     config._properties = copy.deepcopy(resource)
     return config

google/cloud/bigquery/dataset.py

@@ -23,6 +23,7 @@
 import google.cloud._helpers  # type: ignore
 
 from google.cloud.bigquery import _helpers
+from google.cloud.bigquery._helpers import _isinstance_or_raise, _get_sub_prop
 from google.cloud.bigquery.model import ModelReference
 from google.cloud.bigquery.routine import Routine, RoutineReference
 from google.cloud.bigquery.table import Table, TableReference
@@ -946,17 +947,28 @@ def external_catalog_dataset_options(self):
         BigQuery catalog. Contains metadata of open source database, schema
         or namespace represented by the current dataset."""
 
-        return self._properties.get("externalCatalogDatasetOptions")
+        prop = _helpers._get_sub_prop(
+            self._properties, ["externalCatalogDatasetOptions"]
+        )
+        # self._PROPERTY_TO_API_FIELD["external_catalog_dataset_options"]
+        # )
+
+        if prop is not None:
+            prop = ExternalCatalogDatasetOptions().from_api_repr(prop)
+            print("DINOSAUR dataset.py prop: ", prop, type(prop))
+        return prop
+
+        # prop = self._get_sub_prop("destinationEncryptionConfiguration")
+        # if prop is not None:
+        #     prop = EncryptionConfiguration.from_api_repr(prop)
+        # return prop
 
     @external_catalog_dataset_options.setter
     def external_catalog_dataset_options(self, value):
-        if not isinstance(value, ExternalCatalogDatasetOptions) and value is not None:
-            raise ValueError(
-                "external_catalog_dataset_options must be an "
-                "ExternalCatalogDatasetOptions object or None. "
-                f"Got {repr(value)}."
-            )
-        self._properties["externalCatalogDatasetOptions"] = value.to_api_repr()
+        value = _isinstance_or_raise(value, ExternalCatalogDatasetOptions)
+        self._properties[
+            self._PROPERTY_TO_API_FIELD["external_catalog_dataset_options"]
+        ] = value.to_api_repr()
 
     table = _get_table_reference
 

google/cloud/bigquery/external_config.py

@@ -1066,6 +1066,7 @@ def to_api_repr(self) -> dict:
         config = copy.deepcopy(self._properties)
         return config
 
+    @classmethod
     def from_api_repr(self, resource):
         return _from_api_repr(self, resource)
 
@@ -1151,5 +1152,18 @@ def to_api_repr(self) -> dict:
         config = copy.deepcopy(self._properties)
         return config
 
-    def from_api_repr(self, resource):
-        return _from_api_repr(self, resource)
+    @classmethod
+    def from_api_repr(cls, resource: dict) -> "TODO":
+        """Factory: constructs an instance of the class (cls)
+        given its API representation.
+
+        Args:
+            resource (Dict[str, Any]):
+                API representation of the object to be instantiated.
+
+        Returns:
+            An instance of the class initialized with data from 'resource'.
+        """
+        config = cls()
+        config._properties = copy.deepcopy(resource)
+        return config

google/cloud/bigquery/table.py

@@ -59,6 +59,7 @@
 
 import google.cloud._helpers  # type: ignore
 from google.cloud.bigquery import _helpers
+from google.cloud.bigquery._helpers import _isinstance_or_raise
 from google.cloud.bigquery import _pandas_helpers
 from google.cloud.bigquery import _versions_helpers
 from google.cloud.bigquery import exceptions as bq_exceptions
@@ -407,6 +408,7 @@ class Table(_TableBase):
         "view_query": "view",
         "require_partition_filter": "requirePartitionFilter",
         "table_constraints": "tableConstraints",
+        "external_catalog_table_options": "externalCatalogTableOptions",
     }
 
     def __init__(self, table_ref, schema=None) -> None:
@@ -1006,17 +1008,21 @@ def external_catalog_table_options(self):
         BigQuery catalog. Contains metadata of open source database, schema
         or namespace represented by the current dataset."""
 
-        return self._properties.get("externalCatalogTableOptions")
+        prop = self._properties.get(
+            self._PROPERTY_TO_API_FIELD["external_catalog_table_options"]
+        )
+        if prop is not None:
+            prop = ExternalCatalogTableOptions.from_api_repr(prop)
+        return prop
 
     @external_catalog_table_options.setter
     def external_catalog_table_options(self, value):
-        if not isinstance(value, ExternalCatalogTableOptions) and value is not None:
-            raise ValueError(
-                "external_catalog_table_options must be an "
-                "ExternalCatalogTableOptions object or None. "
-                f"Got {repr(value)}."
-            )
-        self._properties["externalCatalogTableOptions"] = value.to_api_repr()
+        value = _isinstance_or_raise(
+            value, ExternalCatalogTableOptions, none_allowed=False
+        )
+        self._properties[
+            self._PROPERTY_TO_API_FIELD["external_catalog_table_options"]
+        ] = value.to_api_repr()
 
     @classmethod
     def from_string(cls, full_table_id: str) -> "Table":

tests/unit/Untitled-2.py

@@ -24,7 +24,10 @@
 from unittest import mock
 
 import google.api_core
-from google.cloud.bigquery._helpers import _isinstance_or_raise
+from google.cloud.bigquery._helpers import (
+    _isinstance_or_raise,
+    _from_api_repr,
+)
 
 
 @pytest.mark.skipif(
@@ -1696,3 +1699,21 @@ def test__invalid_isinstance_or_raise(self, value, dtype, none_allowed, expected
             result = _isinstance_or_raise(value, dtype, none_allowed=none_allowed)
 
             assert result == e
+
+
+class _MockClass:
+    def __init__(self):
+        self._properties = {}
+
+
+@pytest.fixture
+def mock_class():
+    return _MockClass
+
+
+class Test__from_api_repr:
+    def test_from_api_repr(self, mock_class):
+        resource = {"foo": "bar", "baz": {"qux": 1}}
+        config = _from_api_repr(mock_class, resource)
+        assert config._properties == resource
+        assert config._properties is not resource

tests/unit/test__helpers.py

@@ -1014,6 +1014,55 @@ def test_from_string_legacy_string(self):
         with self.assertRaises(ValueError):
             cls.from_string("string-project:string_dataset")
 
+    API_REPR = {
+        "datasetReference": {"projectId": "project", "datasetId": "dataset-id"},
+        "labels": {},
+        "externalCatalogDatasetOptions": {
+            "defaultStorageLocationUri": "gs://test-bucket/test-path",
+            "parameters": {"key": "value"},
+        },
+    }
+
+    def test_external_catalog_dataset_options_setter(self):
+        from google.cloud.bigquery.external_config import ExternalCatalogDatasetOptions
+
+        dataset = self._make_one(self.DS_REF)
+        external_dataset_catalog_options = ExternalCatalogDatasetOptions(
+            default_storage_location_uri="gs://test-bucket/test-path",
+            parameters={"key": "value"},
+        )
+
+        # test the setter
+        dataset.external_catalog_dataset_options = external_dataset_catalog_options
+        expected = self.API_REPR
+        result = dataset.to_api_repr()
+        assert result == expected
+
+    def test_external_catalog_dataset_options_getter(self):
+        from google.cloud.bigquery.external_config import ExternalCatalogDatasetOptions
+
+        dataset = self._make_one(self.DS_REF)
+        external_dataset_catalog_options = ExternalCatalogDatasetOptions(
+            default_storage_location_uri="gs://test-bucket/test-path",
+            parameters={"key": "value"},
+        )
+        dataset.external_catalog_dataset_options = external_dataset_catalog_options
+        print("DINOSAUR test_dataset.py dataset: ", dataset, type(dataset))
+        expected = external_dataset_catalog_options
+        result = dataset.external_catalog_dataset_options
+
+        assert result == expected
+
+    def test_external_catalog_dataset_options_from_api_repr(self):
+        from google.cloud.bigquery.external_config import ExternalCatalogDatasetOptions
+
+        resource = self.API_REPR
+
+        dataset = self._make_one(self.DS_REF)
+        dataset = dataset.from_api_repr(resource)
+        result = dataset.external_catalog_dataset_options
+        assert result == resource["externalCatalogDatasetOptions"]
+
     def test__build_resource_w_custom_field(self):
         dataset = self._make_one(self.DS_REF)
         dataset._properties["newAlphaProperty"] = "unreleased property"

tests/unit/test_dataset.py

@@ -955,6 +955,14 @@ def test_to_api_repr(self):
         assert resource["defaultStorageLocationUri"] == default_storage_location_uri
         assert resource["parameters"] == parameters
 
+    def test_from_api_repr(self):
+        instance = self._make_one()
+        resource = {
+            "defaultStorageLocationUri": "gs://test-bucket/test-path",
+            "parameters": {"key": "value"},
+        }
+        instance.from_api_repr(resource)
+
 
 class TestExternalCatalogTableOptions:
     @staticmethod
@@ -1041,3 +1049,12 @@ def test_ctor_invalid_input(
             )
 
         assert "Pass" in str(e.value)
+
+    def test_from_api_repr(self):
+        instance = self._make_one()
+        resource = {
+            "connectionId": "connection123",
+            "parameters": {"key": "value"},
+            "storageDescriptor": "placeholder",
+        }
+        instance.from_api_repr(resource)

tests/unit/test_external_config.py

@@ -5822,3 +5822,95 @@ def test_table_reference_to_bqstorage_v1_stable(table_path):
     for klass in (mut.TableReference, mut.Table, mut.TableListItem):
         got = klass.from_string(table_path).to_bqstorage()
         assert got == expected
+
+
+@pytest.fixture(scope="class")
+def external_catalog_table_options():
+    from google.cloud.bigquery.external_config import ExternalCatalogTableOptions
+
+    return ExternalCatalogTableOptions(
+        connection_id="connection123",
+        parameters={"key": "value"},
+        storage_descriptor="placeholder",
+    )
+
+
+class TestExternalCatalogTableOptions:
+    PROJECT = "project_id"
+    DS_ID = "dataset_id"
+    TABLE_NAME = "table_name"
+
+    @staticmethod
+    def _get_target_class():
+        from google.cloud.bigquery.table import Table
+
+        return Table
+
+    @classmethod
+    def _make_one(self, *args, **kw):
+        return self._get_target_class()(*args, **kw)
+
+    def test_external_catalog_table_options_getter(
+        self, external_catalog_table_options
+    ):
+        from google.cloud.bigquery.external_config import ExternalCatalogTableOptions
+
+        # create objects for the test
+        dataset = DatasetReference(self.PROJECT, self.DS_ID)
+        table_ref = dataset.table(self.TABLE_NAME)
+        table = self._make_one(table_ref)
+
+        # Confirm that external catalog table options have not been set
+        assert table.external_catalog_table_options is None
+
+        # Add an ExternalCatalogTableOptions object to the table.
+        table._properties[
+            "externalCatalogTableOptions"
+        ] = external_catalog_table_options
+        table_repr = table.to_api_repr()
+
+        # Extract the ecto object.
+        ecto_output = table_repr["externalCatalogTableOptions"]
+
+        # Confirm that external catalog table options are an
+        # ExternalCatalogTableOptions object
+        assert isinstance(ecto_output, ExternalCatalogTableOptions)
+
+        expected = {
+            "connectionId": "connection123",
+            "parameters": {"key": "value"},
+            "storageDescriptor": "placeholder",
+        }
+        result = ecto_output.to_api_repr()
+
+        # Confirm that the api_repr of the ecto_output matches the inputs
+        assert result == expected
+
+    def test_external_catalog_table_options_setter(
+        self, external_catalog_table_options
+    ):
+        # from google.cloud.bigquery.external_config import ExternalCatalogTableOptions
+
+        # create objects for the test
+        dataset = DatasetReference(self.PROJECT, self.DS_ID)
+        table_ref = dataset.table(self.TABLE_NAME)
+        table = self._make_one(table_ref)
+
+        # Add an ExternalCatalogTableOptions object to the table.
+        table.external_catalog_table_options = external_catalog_table_options
+        expected = {
+            "tableReference": {
+                "projectId": "project_id",
+                "datasetId": "dataset_id",
+                "tableId": "table_name",
+            },
+            "labels": {},
+            "externalCatalogTableOptions": {
+                "connectionId": "connection123",
+                "parameters": {"key": "value"},
+                "storageDescriptor": "placeholder",
+            },
+        }
+        # Confirm that the api_repr of the ecto_output matches the inputs
+        result = table.to_api_repr()
+        assert result == expected