Merge branch 'main' into fix-dataset-output-only-docs-final

Author: chalmerlowe

CHANGELOG.md

@@ -5,6 +5,24 @@
 [1]: https://pypi.org/project/google-cloud-bigquery/#history
 
 
+## [3.34.0](https://github.com/googleapis/python-bigquery/compare/v3.33.0...v3.34.0) (2025-05-27)
+
+
+### Features
+
+* Job creation mode GA ([#2190](https://github.com/googleapis/python-bigquery/issues/2190)) ([64cd39f](https://github.com/googleapis/python-bigquery/commit/64cd39fb395c4a03ef6d2ec8261e1709477b2186))
+
+
+### Bug Fixes
+
+* **deps:** Update all dependencies ([#2184](https://github.com/googleapis/python-bigquery/issues/2184)) ([12490f2](https://github.com/googleapis/python-bigquery/commit/12490f2f03681516465fc34217dcdf57000f6fdd))
+
+
+### Documentation
+
+* Update query.py ([#2192](https://github.com/googleapis/python-bigquery/issues/2192)) ([9b5ee78](https://github.com/googleapis/python-bigquery/commit/9b5ee78f046d9ca3f758eeca6244b8485fe35875))
+* Use query_and_wait in the array parameters sample ([#2202](https://github.com/googleapis/python-bigquery/issues/2202)) ([28a9994](https://github.com/googleapis/python-bigquery/commit/28a9994792ec90a6a4d16835faf2137c09c0fb02))
+
 ## [3.33.0](https://github.com/googleapis/python-bigquery/compare/v3.32.0...v3.33.0) (2025-05-19)
 
 

google/cloud/bigquery/_job_helpers.py

@@ -37,7 +37,6 @@
 
 import copy
 import functools
-import os
 import uuid
 import textwrap
 from typing import Any, Dict, Optional, TYPE_CHECKING, Union
@@ -400,12 +399,6 @@ def query_and_wait(
 ) -> table.RowIterator:
     """Run the query, wait for it to finish, and return the results.
 
-    While ``jobCreationMode=JOB_CREATION_OPTIONAL`` is in preview in the
-    ``jobs.query`` REST API, use the default ``jobCreationMode`` unless
-    the environment variable ``QUERY_PREVIEW_ENABLED=true``. After
-    ``jobCreationMode`` is GA, this method will always use
-    ``jobCreationMode=JOB_CREATION_OPTIONAL``. See:
-    https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query
 
     Args:
         client:
@@ -500,9 +493,8 @@ def query_and_wait(
         request_body["maxResults"] = min(page_size, max_results)
     elif page_size is not None or max_results is not None:
         request_body["maxResults"] = page_size or max_results
-
-    if os.getenv("QUERY_PREVIEW_ENABLED", "").casefold() == "true":
-        request_body["jobCreationMode"] = "JOB_CREATION_OPTIONAL"
+    if client.default_job_creation_mode:
+        request_body["jobCreationMode"] = client.default_job_creation_mode
 
     def do_query():
         request_body["requestId"] = make_job_id()

google/cloud/bigquery/client.py

@@ -90,7 +90,8 @@
 from google.cloud.bigquery.dataset import Dataset
 from google.cloud.bigquery.dataset import DatasetListItem
 from google.cloud.bigquery.dataset import DatasetReference
-from google.cloud.bigquery.enums import AutoRowIDs
+
+from google.cloud.bigquery.enums import AutoRowIDs, DatasetView, UpdateMode
 from google.cloud.bigquery.format_options import ParquetOptions
 from google.cloud.bigquery.job import (
     CopyJob,
@@ -221,6 +222,10 @@ class Client(ClientWithProject):
         client_options (Optional[Union[google.api_core.client_options.ClientOptions, Dict]]):
             Client options used to set user options on the client. API Endpoint
             should be set through client_options.
+        default_job_creation_mode (Optional[str]):
+            Sets the default job creation mode used by query methods such as
+            query_and_wait().  For lightweight queries, JOB_CREATION_OPTIONAL is
+            generally recommended.
 
     Raises:
         google.auth.exceptions.DefaultCredentialsError:
@@ -243,6 +248,7 @@ def __init__(
         client_options: Optional[
             Union[google.api_core.client_options.ClientOptions, Dict[str, Any]]
         ] = None,
+        default_job_creation_mode: Optional[str] = None,
     ) -> None:
         if client_options is None:
             client_options = {}
@@ -277,6 +283,7 @@ def __init__(
         self._connection = Connection(self, **kw_args)
         self._location = location
         self._default_load_job_config = copy.deepcopy(default_load_job_config)
+        self.default_job_creation_mode = default_job_creation_mode
 
         # Use property setter so validation can run.
         self.default_query_job_config = default_query_job_config
@@ -286,6 +293,15 @@ def location(self):
         """Default location for jobs / datasets / tables."""
         return self._location
 
+    @property
+    def default_job_creation_mode(self):
+        """Default job creation mode used for query execution."""
+        return self._default_job_creation_mode
+
+    @default_job_creation_mode.setter
+    def default_job_creation_mode(self, value: Optional[str]):
+        self._default_job_creation_mode = value
+
     @property
     def default_query_job_config(self) -> Optional[QueryJobConfig]:
         """Default ``QueryJobConfig`` or ``None``.
@@ -849,6 +865,7 @@ def get_dataset(
         dataset_ref: Union[DatasetReference, str],
         retry: retries.Retry = DEFAULT_RETRY,
         timeout: TimeoutType = DEFAULT_TIMEOUT,
+        dataset_view: Optional[DatasetView] = None,
     ) -> Dataset:
         """Fetch the dataset referenced by ``dataset_ref``
 
@@ -866,7 +883,21 @@ def get_dataset(
             timeout (Optional[float]):
                 The number of seconds to wait for the underlying HTTP transport
                 before using ``retry``.
-
+            dataset_view (Optional[google.cloud.bigquery.enums.DatasetView]):
+                Specifies the view that determines which dataset information is
+                returned. By default, dataset metadata (e.g. friendlyName, description,
+                labels, etc) and ACL information are returned. This argument can
+                take on the following possible enum values.
+
+                * :attr:`~google.cloud.bigquery.enums.DatasetView.ACL`:
+                    Includes dataset metadata and the ACL.
+                * :attr:`~google.cloud.bigquery.enums.DatasetView.FULL`:
+                    Includes all dataset metadata, including the ACL and table metadata.
+                    This view is not supported by the `datasets.list` API method.
+                * :attr:`~google.cloud.bigquery.enums.DatasetView.METADATA`:
+                    Includes basic dataset metadata, but not the ACL.
+                * :attr:`~google.cloud.bigquery.enums.DatasetView.DATASET_VIEW_UNSPECIFIED`:
+                    The server will decide which view to use. Currently defaults to FULL.
         Returns:
             google.cloud.bigquery.dataset.Dataset:
                 A ``Dataset`` instance.
@@ -876,6 +907,12 @@ def get_dataset(
                 dataset_ref, default_project=self.project
             )
         path = dataset_ref.path
+
+        if dataset_view:
+            query_params = {"datasetView": dataset_view.value}
+        else:
+            query_params = {}
+
         span_attributes = {"path": path}
         api_response = self._call_api(
             retry,
@@ -884,6 +921,7 @@ def get_dataset(
             method="GET",
             path=path,
             timeout=timeout,
+            query_params=query_params,
         )
         return Dataset.from_api_repr(api_response)
 
@@ -1183,6 +1221,7 @@ def update_dataset(
         fields: Sequence[str],
         retry: retries.Retry = DEFAULT_RETRY,
         timeout: TimeoutType = DEFAULT_TIMEOUT,
+        update_mode: Optional[UpdateMode] = None,
     ) -> Dataset:
         """Change some fields of a dataset.
 
@@ -1222,6 +1261,20 @@ def update_dataset(
             timeout (Optional[float]):
                 The number of seconds to wait for the underlying HTTP transport
                 before using ``retry``.
+            update_mode (Optional[google.cloud.bigquery.enums.UpdateMode]):
+                Specifies the kind of information to update in a dataset.
+                By default, dataset metadata (e.g. friendlyName, description,
+                labels, etc) and ACL information are updated. This argument can
+                take on the following possible enum values.
+
+                * :attr:`~google.cloud.bigquery.enums.UPDATE_MODE_UNSPECIFIED`:
+                    The default value. Behavior defaults to UPDATE_FULL.
+                * :attr:`~google.cloud.bigquery.enums.UpdateMode.UPDATE_METADATA`:
+                    Includes metadata information for the dataset, such as friendlyName, description, labels, etc.
+                * :attr:`~google.cloud.bigquery.enums.UpdateMode.UPDATE_ACL`:
+                    Includes ACL information for the dataset, which defines dataset access for one or more entities.
+                * :attr:`~google.cloud.bigquery.enums.UpdateMode.UPDATE_FULL`:
+                    Includes both dataset metadata and ACL information.
 
         Returns:
             google.cloud.bigquery.dataset.Dataset:
@@ -1235,6 +1288,11 @@ def update_dataset(
         path = dataset.path
         span_attributes = {"path": path, "fields": fields}
 
+        if update_mode:
+            query_params = {"updateMode": update_mode.value}
+        else:
+            query_params = {}
+
         api_response = self._call_api(
             retry,
             span_name="BigQuery.updateDataset",
@@ -1244,6 +1302,7 @@ def update_dataset(
             data=partial,
             headers=headers,
             timeout=timeout,
+            query_params=query_params,
         )
         return Dataset.from_api_repr(api_response)
 
@@ -1986,6 +2045,7 @@ def _get_query_results(
         location: Optional[str] = None,
         timeout: TimeoutType = DEFAULT_TIMEOUT,
         page_size: int = 0,
+        start_index: Optional[int] = None,
     ) -> _QueryResults:
         """Get the query results object for a query job.
 
@@ -2004,9 +2064,12 @@ def _get_query_results(
                 before using ``retry``. If set, this connection timeout may be
                 increased to a minimum value. This prevents retries on what
                 would otherwise be a successful response.
-            page_size (int):
+            page_size (Optional[int]):
                 Maximum number of rows in a single response. See maxResults in
                 the jobs.getQueryResults REST API.
+            start_index (Optional[int]):
+                Zero-based index of the starting row. See startIndex in the
+                jobs.getQueryResults REST API.
 
         Returns:
             google.cloud.bigquery.query._QueryResults:
@@ -2036,6 +2099,9 @@ def _get_query_results(
         if location is not None:
             extra_params["location"] = location
 
+        if start_index is not None:
+            extra_params["startIndex"] = start_index
+
         path = "/projects/{}/queries/{}".format(project, job_id)
 
         # This call is typically made in a polling loop that checks whether the
@@ -3532,13 +3598,6 @@ def query_and_wait(
     ) -> RowIterator:
         """Run the query, wait for it to finish, and return the results.
 
-        While ``jobCreationMode=JOB_CREATION_OPTIONAL`` is in preview in the
-        ``jobs.query`` REST API, use the default ``jobCreationMode`` unless
-        the environment variable ``QUERY_PREVIEW_ENABLED=true``. After
-        ``jobCreationMode`` is GA, this method will always use
-        ``jobCreationMode=JOB_CREATION_OPTIONAL``. See:
-        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query
-
         Args:
             query (str):
                 SQL query to be executed. Defaults to the standard SQL

google/cloud/bigquery/enums.py

@@ -80,6 +80,24 @@ class CreateDisposition(object):
     returned in the job result."""
 
 
+class DatasetView(enum.Enum):
+    """DatasetView specifies which dataset information is returned."""
+
+    DATASET_VIEW_UNSPECIFIED = "DATASET_VIEW_UNSPECIFIED"
+    """The default value. Currently maps to the FULL view."""
+
+    METADATA = "METADATA"
+    """View metadata information for the dataset, such as friendlyName,
+    description, labels, etc."""
+
+    ACL = "ACL"
+    """View ACL information for the dataset, which defines dataset access
+    for one or more entities."""
+
+    FULL = "FULL"
+    """View both dataset metadata and ACL information."""
+
+
 class DefaultPandasDTypes(enum.Enum):
     """Default Pandas DataFrem DTypes to convert BigQuery data. These
     Sentinel values are used instead of None to maintain backward compatibility,
@@ -407,3 +425,40 @@ class BigLakeTableFormat(object):
 
     ICEBERG = "ICEBERG"
     """Apache Iceberg format."""
+
+
+class UpdateMode(enum.Enum):
+    """Specifies the kind of information to update in a dataset."""
+
+    UPDATE_MODE_UNSPECIFIED = "UPDATE_MODE_UNSPECIFIED"
+    """The default value. Behavior defaults to UPDATE_FULL."""
+
+    UPDATE_METADATA = "UPDATE_METADATA"
+    """Includes metadata information for the dataset, such as friendlyName,
+    description, labels, etc."""
+
+    UPDATE_ACL = "UPDATE_ACL"
+    """Includes ACL information for the dataset, which defines dataset access
+    for one or more entities."""
+
+    UPDATE_FULL = "UPDATE_FULL"
+    """Includes both dataset metadata and ACL information."""
+
+
+class JobCreationMode(object):
+    """Documented values for Job Creation Mode."""
+
+    JOB_CREATION_MODE_UNSPECIFIED = "JOB_CREATION_MODE_UNSPECIFIED"
+    """Job creation mode is unspecified."""
+
+    JOB_CREATION_REQUIRED = "JOB_CREATION_REQUIRED"
+    """Job creation is always required."""
+
+    JOB_CREATION_OPTIONAL = "JOB_CREATION_OPTIONAL"
+    """Job creation is optional.
+
+    Returning immediate results is prioritized.
+    BigQuery will automatically determine if a Job needs to be created.
+    The conditions under which BigQuery can decide to not create a Job are
+    subject to change.
+    """

google/cloud/bigquery/external_config.py

@@ -22,6 +22,7 @@
 
 import base64
 import copy
+import typing
 from typing import Any, Dict, FrozenSet, Iterable, Optional, Union
 
 from google.cloud.bigquery._helpers import _to_bytes
@@ -835,10 +836,10 @@ def schema(self):
         See
         https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.schema
         """
-        # TODO: The typehinting for this needs work. Setting this pragma to temporarily
-        # manage a pytype issue that came up in another PR. See Issue: #2132
-        prop = self._properties.get("schema", {})  # type: ignore
-        return [SchemaField.from_api_repr(field) for field in prop.get("fields", [])]  # type: ignore
+        prop: Dict[str, Any] = typing.cast(
+            Dict[str, Any], self._properties.get("schema", {})
+        )
+        return [SchemaField.from_api_repr(field) for field in prop.get("fields", [])]
 
     @schema.setter
     def schema(self, value):

google/cloud/bigquery/job/base.py

@@ -435,9 +435,7 @@ def __init__(self, job_id, client):
     @property
     def configuration(self) -> _JobConfig:
         """Job-type specific configurtion."""
-        # TODO: The typehinting for this needs work. Setting this pragma to temporarily
-        # manage a pytype issue that came up in another PR. See Issue: #2132
-        configuration = self._CONFIG_CLASS()  # pytype: disable=not-callable
+        configuration: _JobConfig = self._CONFIG_CLASS()  # pytype: disable=not-callable
         configuration._properties = self._properties.setdefault("configuration", {})
         return configuration
 

google/cloud/bigquery/job/query.py

@@ -1409,6 +1409,7 @@ def _reload_query_results(
         retry: "retries.Retry" = DEFAULT_RETRY,
         timeout: Optional[float] = None,
         page_size: int = 0,
+        start_index: Optional[int] = None,
     ):
         """Refresh the cached query results unless already cached and complete.
 
@@ -1421,6 +1422,9 @@ def _reload_query_results(
             page_size (int):
                 Maximum number of rows in a single response. See maxResults in
                 the jobs.getQueryResults REST API.
+            start_index (Optional[int]):
+                Zero-based index of the starting row. See startIndex in the
+                jobs.getQueryResults REST API.
         """
         # Optimization: avoid a call to jobs.getQueryResults if it's already
         # been fetched, e.g. from jobs.query first page of results.
@@ -1468,6 +1472,7 @@ def _reload_query_results(
             location=self.location,
             timeout=transport_timeout,
             page_size=page_size,
+            start_index=start_index,
         )
 
     def result(  # type: ignore  # (incompatible with supertype)
@@ -1570,6 +1575,9 @@ def result(  # type: ignore  # (incompatible with supertype)
         if page_size is not None:
             reload_query_results_kwargs["page_size"] = page_size
 
+        if start_index is not None:
+            reload_query_results_kwargs["start_index"] = start_index
+
         try:
             retry_do_query = getattr(self, "_retry_do_query", None)
             if retry_do_query is not None: