Merge branch 'add-pangea-classes' into pangea-v1alpha

Creates a development branch focused on pangea features.

Author: chalmerlowe

google/cloud/bigquery/_helpers.py

@@ -15,14 +15,15 @@
 """Shared helper functions for BigQuery API classes."""
 
 import base64
+import copy
 import datetime
 import decimal
 import json
 import math
 import re
 import os
 import warnings
-from typing import Optional, Union
+from typing import Optional, Union, Any, Tuple, Type
 
 from dateutil import relativedelta
 from google.cloud._helpers import UTC  # type: ignore
@@ -1004,3 +1005,52 @@ def _verify_job_config_type(job_config, expected_type, param_name="job_config"):
                 job_config=job_config,
             )
         )
+
+
+def _isinstance_or_raise(
+    value: Any,
+    dtype: Union[Type, Tuple[Type, ...]],
+    none_allowed: Optional[bool] = False,
+) -> Any:
+    """Determine whether a value type matches a given datatype or None.
+
+    Args:
+        value (Any): Value to be checked.
+        dtype (type): Expected data type or tuple of data types.
+        none_allowed Optional(bool): whether value is allowed to be None. Default
+           is False.
+
+    Returns:
+        Any: Returns the input value if the type check is successful.
+
+    Raises:
+        TypeError: If the input value's type does not match the expected data type(s).
+    """
+    if none_allowed and value is None:
+        return value
+
+    if isinstance(value, dtype):
+        return value
+
+    or_none = ""
+    if none_allowed:
+        or_none = " (or None)"
+
+    msg = f"Pass {value} as a '{dtype}'{or_none}. Got {type(value)}."
+    raise TypeError(msg)
+
+
+def _from_api_repr(cls, resource: dict):
+    """Factory: constructs an instance of the class (cls)
+    given its API representation.
+
+    Args:
+        resource (Dict[str, Any]):
+            API representation of the object to be instantiated.
+
+    Returns:
+        An instance of the class initialized with data from 'resource'.
+    """
+    config = cls
+    config._properties = copy.deepcopy(resource)
+    return config

google/cloud/bigquery/dataset.py

@@ -23,10 +23,13 @@
 import google.cloud._helpers  # type: ignore
 
 from google.cloud.bigquery import _helpers
+from google.cloud.bigquery._helpers import _isinstance_or_raise
 from google.cloud.bigquery.model import ModelReference
 from google.cloud.bigquery.routine import Routine, RoutineReference
 from google.cloud.bigquery.table import Table, TableReference
 from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
+from google.cloud.bigquery.external_config import ExternalCatalogDatasetOptions
+
 
 from typing import Optional, List, Dict, Any, Union
 
@@ -530,6 +533,7 @@ class Dataset(object):
         "storage_billing_model": "storageBillingModel",
         "max_time_travel_hours": "maxTimeTravelHours",
         "default_rounding_mode": "defaultRoundingMode",
+        "external_catalog_dataset_options": "externalCatalogDatasetOptions",
     }
 
     def __init__(self, dataset_ref) -> None:
@@ -937,10 +941,31 @@ def _build_resource(self, filter_fields):
         """Generate a resource for ``update``."""
         return _helpers._build_resource_from_properties(self, filter_fields)
 
-    table = _get_table_reference
+    @property
+    def external_catalog_dataset_options(self):
+        """Options defining open source compatible datasets living in the
+        BigQuery catalog. Contains metadata of open source database, schema
+        or namespace represented by the current dataset."""
 
-    model = _get_model_reference
+        prop = _helpers._get_sub_prop(
+            self._properties, ["externalCatalogDatasetOptions"]
+        )
 
+        if prop is not None:
+            prop = ExternalCatalogDatasetOptions().from_api_repr(prop)
+        return prop
+
+    @external_catalog_dataset_options.setter
+    def external_catalog_dataset_options(self, value):
+        value = _isinstance_or_raise(
+            value, ExternalCatalogDatasetOptions, none_allowed=True
+        )
+        self._properties[
+            self._PROPERTY_TO_API_FIELD["external_catalog_dataset_options"]
+        ] = value.to_api_repr()
+
+    table = _get_table_reference
+    model = _get_model_reference
     routine = _get_routine_reference
 
     def __repr__(self):

google/cloud/bigquery/external_config.py

@@ -18,18 +18,22 @@
    Job.configuration.query.tableDefinitions.
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, annotations
 
 import base64
 import copy
 from typing import Any, Dict, FrozenSet, Iterable, Optional, Union
 
-from google.cloud.bigquery._helpers import _to_bytes
-from google.cloud.bigquery._helpers import _bytes_to_json
-from google.cloud.bigquery._helpers import _int_or_none
-from google.cloud.bigquery._helpers import _str_or_none
+from google.cloud.bigquery._helpers import (
+    _to_bytes,
+    _bytes_to_json,
+    _int_or_none,
+    _str_or_none,
+    _isinstance_or_raise,
+    _get_sub_prop,
+)
 from google.cloud.bigquery.format_options import AvroOptions, ParquetOptions
-from google.cloud.bigquery.schema import SchemaField
+from google.cloud.bigquery.schema import SchemaField, StorageDescriptor
 
 
 class ExternalSourceFormat(object):
@@ -1003,3 +1007,182 @@ def from_api_repr(cls, resource: dict) -> "ExternalConfig":
         config = cls(resource["sourceFormat"])
         config._properties = copy.deepcopy(resource)
         return config
+
+
+class ExternalCatalogDatasetOptions:
+    """Options defining open source compatible datasets living in the BigQuery catalog.
+    Contains metadata of open source database, schema or namespace represented
+    by the current dataset.
+
+    Args:
+        default_storage_location_uri (Optional[str]): The storage location URI for all
+            tables in the dataset. Equivalent to hive metastore's database
+            locationUri. Maximum length of 1024 characters. (str)
+        parameters (Optional[dict[str, Any]]): A map of key value pairs defining the parameters
+            and properties of the open source schema. Maximum size of 2Mib.
+    """
+
+    def __init__(
+        self,
+        default_storage_location_uri: Optional[str] = None,
+        parameters: Optional[Dict[str, Any]] = None,
+    ):
+        self._properties = {}
+        self.default_storage_location_uri = default_storage_location_uri
+        self.parameters = parameters
+
+    @property
+    def default_storage_location_uri(self) -> Any:
+        """Optional. The storage location URI for all tables in the dataset.
+        Equivalent to hive metastore's database locationUri. Maximum length of
+        1024 characters."""
+
+        return self._properties.get("defaultStorageLocationUri")
+
+    @default_storage_location_uri.setter
+    def default_storage_location_uri(self, value: str):
+        value = _isinstance_or_raise(value, str, none_allowed=True)
+        self._properties["defaultStorageLocationUri"] = value
+
+    @property
+    def parameters(self) -> Any:
+        """Optional. A map of key value pairs defining the parameters and
+        properties of the open source schema. Maximum size of 2Mib."""
+
+        return self._properties.get("parameters")
+
+    @parameters.setter
+    def parameters(self, value: dict[str, Any]):
+        value = _isinstance_or_raise(value, dict, none_allowed=True)
+        self._properties["parameters"] = value
+
+    def to_api_repr(self) -> dict:
+        """Build an API representation of this object.
+
+        Returns:
+            Dict[str, Any]:
+                A dictionary in the format used by the BigQuery API.
+        """
+        config = copy.deepcopy(self._properties)
+        return config
+
+    @classmethod
+    def from_api_repr(cls, resource: dict) -> ExternalCatalogDatasetOptions:
+        """Factory: constructs an instance of the class (cls)
+        given its API representation.
+
+        Args:
+            resource (Dict[str, Any]):
+                API representation of the object to be instantiated.
+
+        Returns:
+            An instance of the class initialized with data from 'resource'.
+        """
+        config = cls()
+        config._properties = copy.deepcopy(resource)
+        return config
+
+
+class ExternalCatalogTableOptions:
+    """Metadata about open source compatible table. The fields contained in these
+    options correspond to hive metastore's table level properties.
+
+    Args:
+        connection_id  (Optional[str]): The connection specifying the credentials to be
+            used to read external storage, such as Azure Blob, Cloud Storage, or
+            S3. The connection is needed to read the open source table from
+            BigQuery Engine. The connection_id can have the form `..` or
+            `projects//locations//connections/`.
+        parameters (Union[Dict[str, Any], None]): A map of key value pairs defining the parameters
+            and properties of the open source table. Corresponds with hive meta
+            store table parameters. Maximum size of 4Mib.
+        storage_descriptor (Optional[StorageDescriptor]): A storage descriptor containing information
+            about the physical storage of this table.
+    """
+
+    def __init__(
+        self,
+        connection_id: Optional[str] = None,
+        parameters: Union[Dict[str, Any], None] = None,
+        storage_descriptor: Optional[
+            StorageDescriptor
+        ] = None,  # TODO implement StorageDescriptor, then correct this type hint
+    ):
+        self._properties = {}  # type: Dict[str, Any]
+        self.connection_id = connection_id
+        self.parameters = parameters
+        self.storage_descriptor = storage_descriptor
+
+    @property
+    def connection_id(self):
+        """Optional. The connection specifying the credentials to be
+        used to read external storage, such as Azure Blob, Cloud Storage, or
+        S3. The connection is needed to read the open source table from
+        BigQuery Engine. The connection_id can have the form `..` or
+        `projects//locations//connections/`. (str)
+        """
+        return self._properties.get("connectionId")
+
+    @connection_id.setter
+    def connection_id(self, value: Optional[str]):
+        value = _isinstance_or_raise(value, str, none_allowed=True)
+        self._properties["connectionId"] = value
+
+    @property
+    def parameters(self) -> Any:
+        """Optional. A map of key value pairs defining the parameters and
+        properties of the open source table. Corresponds with hive meta
+        store table parameters. Maximum size of 4Mib.
+        """
+
+        return self._properties.get("parameters")
+
+    @parameters.setter
+    def parameters(self, value: Union[Dict[str, Any], None]):
+        value = _isinstance_or_raise(value, dict, none_allowed=True)
+        self._properties["parameters"] = value
+
+    @property
+    def storage_descriptor(self) -> Any:
+        """Optional. A storage descriptor containing information about the
+        physical storage of this table."""
+
+        prop = _get_sub_prop(self._properties, ["storageDescriptor"])
+
+        if prop is not None:
+            prop = StorageDescriptor().from_api_repr(prop)
+        return prop
+
+    @storage_descriptor.setter
+    def storage_descriptor(self, value):
+        value = _isinstance_or_raise(value, StorageDescriptor, none_allowed=True)
+        if value is not None:
+            self._properties["storageDescriptor"] = value.to_api_repr()
+        else:
+            self._properties["storageDescriptor"] = value
+
+    def to_api_repr(self) -> dict:
+        """Build an API representation of this object.
+
+        Returns:
+            Dict[str, Any]:
+                A dictionary in the format used by the BigQuery API.
+        """
+        config = copy.deepcopy(self._properties)
+        return config
+
+    @classmethod
+    def from_api_repr(cls, resource: dict) -> ExternalCatalogTableOptions:
+        """Factory: constructs an instance of the class (cls)
+        given its API representation.
+
+        Args:
+            resource (Dict[str, Any]):
+                API representation of the object to be instantiated.
+
+        Returns:
+            An instance of the class initialized with data from 'resource'.
+        """
+        config = cls()
+        config._properties = copy.deepcopy(resource)
+        return config

google/cloud/bigquery/query.py

@@ -1003,7 +1003,6 @@ def __init__(
     ):
         self.name = name
         self.range_element_type = self._parse_range_element_type(range_element_type)
-        print(self.range_element_type.type_._type)
         self.start = start
         self.end = end