From 28349404fbfbe05e4edf3f8b188f60fb06a1a98b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 5 Nov 2025 15:15:53 -0800 Subject: [PATCH 1/4] chore(tests): re-enable pipeline system tests on kokoro --- tests/system/test__helpers.py | 2 -- tests/system/test_pipeline_acceptance.py | 8 +------- tests/system/test_system.py | 10 ---------- tests/system/test_system_async.py | 10 ---------- 4 files changed, 1 insertion(+), 29 deletions(-) diff --git a/tests/system/test__helpers.py b/tests/system/test__helpers.py index 74b12b7c3..434e1c875 100644 --- a/tests/system/test__helpers.py +++ b/tests/system/test__helpers.py @@ -20,5 +20,3 @@ # run all tests against default database, and a named database TEST_DATABASES = [None, FIRESTORE_OTHER_DB] TEST_DATABASES_W_ENTERPRISE = TEST_DATABASES + [FIRESTORE_ENTERPRISE_DB] -# TODO remove when kokoro fully supports enterprise mode/pipelines -IS_KOKORO_TEST = os.getenv("KOKORO_JOB_NAME") is not None diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index fd74d9458..232d350e5 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -33,16 +33,10 @@ from google.cloud.firestore import Client, AsyncClient -from test__helpers import FIRESTORE_ENTERPRISE_DB, IS_KOKORO_TEST +from test__helpers import FIRESTORE_ENTERPRISE_DB FIRESTORE_PROJECT = os.environ.get("GCLOUD_PROJECT") -# TODO: enable kokoro tests when internal test project is whitelisted -pytestmark = pytest.mark.skipif( - condition=IS_KOKORO_TEST, - reason="Pipeline tests are currently not supported by kokoro", -) - test_dir_name = os.path.dirname(__file__) id_format = ( diff --git a/tests/system/test_system.py b/tests/system/test_system.py index ce4f64b32..46d28c569 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -45,7 +45,6 @@ ENTERPRISE_MODE_ERROR, TEST_DATABASES, TEST_DATABASES_W_ENTERPRISE, - IS_KOKORO_TEST, ) @@ -67,10 +66,6 @@ def _get_credentials_and_project(): def database(request): from test__helpers import FIRESTORE_ENTERPRISE_DB - # enterprise mode currently does not support RunQuery calls in prod on kokoro test project - # TODO: remove skip when kokoro test project supports full enterprise mode - if request.param == FIRESTORE_ENTERPRISE_DB and IS_KOKORO_TEST: - pytest.skip("enterprise mode does not support RunQuery on kokoro") return request.param @@ -99,11 +94,6 @@ def verify_pipeline(query): """ from google.cloud.firestore_v1.base_aggregation import BaseAggregationQuery - # return early on kokoro. Test project doesn't currently support pipelines - # TODO: enable pipeline verification when kokoro test project is whitelisted - if IS_KOKORO_TEST: - pytest.skip("skipping pipeline verification on kokoro") - def _clean_results(results): if isinstance(results, dict): return {k: _clean_results(v) for k, v in results.items()} diff --git a/tests/system/test_system_async.py b/tests/system/test_system_async.py index ed679402a..c412c2125 100644 --- a/tests/system/test_system_async.py +++ b/tests/system/test_system_async.py @@ -56,7 +56,6 @@ ENTERPRISE_MODE_ERROR, TEST_DATABASES, TEST_DATABASES_W_ENTERPRISE, - IS_KOKORO_TEST, ) RETRIES = retries.AsyncRetry( @@ -145,10 +144,6 @@ def _verify_explain_metrics_analyze_false(explain_metrics): def database(request): from test__helpers import FIRESTORE_ENTERPRISE_DB - # enterprise mode currently does not support RunQuery calls in prod on kokoro test project - # TODO: remove skip when kokoro test project supports full enterprise mode - if request.param == FIRESTORE_ENTERPRISE_DB and IS_KOKORO_TEST: - pytest.skip("enterprise mode does not support RunQuery on kokoro") return request.param @@ -179,11 +174,6 @@ async def verify_pipeline(query): """ from google.cloud.firestore_v1.base_aggregation import BaseAggregationQuery - # return early on kokoro. Test project doesn't currently support pipelines - # TODO: enable pipeline verification when kokoro test project is whitelisted - if IS_KOKORO_TEST: - pytest.skip("skipping pipeline verification on kokoro") - def _clean_results(results): if isinstance(results, dict): return {k: _clean_results(v) for k, v in results.items()} From 716e5211ee35f99c0bb1c88cc66760149d2b4709 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 9 Jan 2026 13:53:51 -0800 Subject: [PATCH 2/4] chore: import main back into pipeline_preview branch --- .github/.OwlBot.lock.yaml | 17 - .github/.OwlBot.yaml | 33 - .github/auto-approve.yml | 3 - .github/release-please.yml | 12 - .github/release-trigger.yml | 2 - .github/sync-repo-settings.yaml | 47 - .github/workflows/lint.yml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/system_emulated.yml | 2 +- .github/workflows/unittest.yml | 2 +- .kokoro/presubmit/system-3.14.cfg | 7 - .kokoro/presubmit/system-3.7.cfg | 7 - .kokoro/presubmit/system-3.9.cfg | 7 - .kokoro/presubmit/system.cfg | 2 +- .kokoro/samples/python3.14/common.cfg | 40 + .kokoro/samples/python3.14/continuous.cfg | 6 + .kokoro/samples/python3.14/periodic-head.cfg | 11 + .kokoro/samples/python3.14/periodic.cfg | 6 + .kokoro/samples/python3.14/presubmit.cfg | 6 + .../generator-input/.repo-metadata.json | 18 + .../generator-input/librarian.py | 82 +- .librarian/generator-input/noxfile.py | 584 ++++++++++ .librarian/generator-input/setup.py | 95 ++ .librarian/state.yaml | 49 + CHANGELOG.md | 16 + CONTRIBUTING.rst | 1 + README.rst | 2 +- docs/README.rst | 198 +++- google/cloud/firestore/gapic_version.py | 2 +- .../firestore_admin_v1/gapic_metadata.json | 15 + .../cloud/firestore_admin_v1/gapic_version.py | 4 +- .../services/firestore_admin/async_client.py | 137 +++ .../services/firestore_admin/client.py | 181 ++- .../firestore_admin/transports/base.py | 19 +- .../firestore_admin/transports/grpc.py | 52 +- .../transports/grpc_asyncio.py | 59 +- .../firestore_admin/transports/rest.py | 221 +++- .../firestore_admin/transports/rest_base.py | 57 + .../firestore_admin_v1/types/__init__.py | 8 + .../firestore_admin_v1/types/database.py | 6 +- .../types/firestore_admin.py | 70 +- .../firestore_admin_v1/types/operation.py | 56 + .../firestore_admin_v1/types/snapshot.py | 67 ++ google/cloud/firestore_bundle/__init__.py | 104 ++ .../cloud/firestore_bundle/gapic_version.py | 4 +- google/cloud/firestore_v1/aggregation.py | 18 +- .../cloud/firestore_v1/async_aggregation.py | 12 +- google/cloud/firestore_v1/async_batch.py | 3 +- google/cloud/firestore_v1/async_client.py | 17 +- google/cloud/firestore_v1/async_collection.py | 5 +- google/cloud/firestore_v1/base_batch.py | 8 +- google/cloud/firestore_v1/base_client.py | 6 +- google/cloud/firestore_v1/base_collection.py | 3 +- google/cloud/firestore_v1/base_document.py | 6 +- google/cloud/firestore_v1/client.py | 7 +- google/cloud/firestore_v1/document.py | 2 +- google/cloud/firestore_v1/gapic_metadata.json | 15 + google/cloud/firestore_v1/gapic_version.py | 4 +- .../firestore_v1/services/firestore/client.py | 46 +- .../services/firestore/transports/base.py | 19 +- .../services/firestore/transports/grpc.py | 8 +- .../firestore/transports/grpc_asyncio.py | 22 +- .../services/firestore/transports/rest.py | 106 +- .../firestore/transports/rest_base.py | 57 + google/cloud/firestore_v1/stream_generator.py | 2 +- google/cloud/firestore_v1/types/document.py | 20 +- .../cloud/firestore_v1/types/explain_stats.py | 10 +- google/cloud/firestore_v1/types/firestore.py | 42 +- google/cloud/firestore_v1/types/pipeline.py | 2 +- google/cloud/firestore_v1/types/query.py | 176 +-- librarian.py | 118 ++ noxfile.py | 10 +- scripts/fixup_firestore_admin_v1_keywords.py | 212 ---- scripts/fixup_firestore_v1_keywords.py | 197 ---- setup.py | 8 +- .../test_firestore_admin.py | 1006 ++++++++++++++++- .../unit/gapic/firestore_v1/test_firestore.py | 257 ++++- tests/unit/gapic/v1/__init__.py | 0 tests/unit/v1/test_async_client.py | 3 + 79 files changed, 3806 insertions(+), 942 deletions(-) delete mode 100644 .github/.OwlBot.lock.yaml delete mode 100644 .github/.OwlBot.yaml delete mode 100644 .github/auto-approve.yml delete mode 100644 .github/release-please.yml delete mode 100644 .github/release-trigger.yml delete mode 100644 .github/sync-repo-settings.yaml delete mode 100644 .kokoro/presubmit/system-3.14.cfg delete mode 100644 .kokoro/presubmit/system-3.7.cfg delete mode 100644 .kokoro/presubmit/system-3.9.cfg create mode 100644 .kokoro/samples/python3.14/common.cfg create mode 100644 .kokoro/samples/python3.14/continuous.cfg create mode 100644 .kokoro/samples/python3.14/periodic-head.cfg create mode 100644 .kokoro/samples/python3.14/periodic.cfg create mode 100644 .kokoro/samples/python3.14/presubmit.cfg create mode 100644 .librarian/generator-input/.repo-metadata.json rename owlbot.py => .librarian/generator-input/librarian.py (56%) create mode 100644 .librarian/generator-input/noxfile.py create mode 100644 .librarian/generator-input/setup.py create mode 100644 .librarian/state.yaml mode change 120000 => 100644 docs/README.rst create mode 100644 google/cloud/firestore_admin_v1/types/snapshot.py create mode 100644 librarian.py delete mode 100644 scripts/fixup_firestore_admin_v1_keywords.py delete mode 100644 scripts/fixup_firestore_v1_keywords.py delete mode 100644 tests/unit/gapic/v1/__init__.py diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml deleted file mode 100644 index 9a7846675..000000000 --- a/.github/.OwlBot.lock.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:4a9e5d44b98e8672e2037ee22bc6b4f8e844a2d75fcb78ea8a4b38510112abc6 -# created: 2025-10-07 diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml deleted file mode 100644 index b720d256a..000000000 --- a/.github/.OwlBot.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - -deep-remove-regex: - - /owl-bot-staging - -deep-preserve-regex: - - /owl-bot-staging/firestore/v1beta1 - -deep-copy-regex: - - source: /google/firestore/(v.*)/.*-py/(.*) - dest: /owl-bot-staging/firestore/$1/$2 - - source: /google/firestore/admin/(v.*)/.*-py/(.*) - dest: /owl-bot-staging/firestore_admin/$1/$2 - - source: /google/firestore/bundle/(.*-py)/(.*) - dest: /owl-bot-staging/firestore_bundle/$1/$2 - -begin-after-commit-hash: 107ed1217b5e87048263f52cd3911d5f851aca7e - diff --git a/.github/auto-approve.yml b/.github/auto-approve.yml deleted file mode 100644 index 311ebbb85..000000000 --- a/.github/auto-approve.yml +++ /dev/null @@ -1,3 +0,0 @@ -# https://github.com/googleapis/repo-automation-bots/tree/main/packages/auto-approve -processes: - - "OwlBotTemplateChanges" diff --git a/.github/release-please.yml b/.github/release-please.yml deleted file mode 100644 index fe749ff6b..000000000 --- a/.github/release-please.yml +++ /dev/null @@ -1,12 +0,0 @@ -releaseType: python -handleGHRelease: true -manifest: true -# NOTE: this section is generated by synthtool.languages.python -# See https://github.com/googleapis/synthtool/blob/master/synthtool/languages/python.py -branches: -- branch: v1 - handleGHRelease: true - releaseType: python -- branch: v0 - handleGHRelease: true - releaseType: python diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml deleted file mode 100644 index 95896588a..000000000 --- a/.github/release-trigger.yml +++ /dev/null @@ -1,2 +0,0 @@ -enabled: true -multiScmName: python-firestore diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml deleted file mode 100644 index 389c3747c..000000000 --- a/.github/sync-repo-settings.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Rules for main branch protection -branchProtectionRules: -# Identifies the protection rule pattern. Name of the branch to be protected. -# Defaults to `main` -- pattern: main - # Can admins overwrite branch protection. - # Defaults to `true` - isAdminEnforced: true - # Number of approving reviews required to update matching branches. - # Defaults to `1` - requiredApprovingReviewCount: 1 - # Are reviews from code owners required to update matching branches. - # Defaults to `false` - requiresCodeOwnerReviews: true - # Require up to date branches - requiresStrictStatusChecks: true - # List of required status check contexts that must pass for commits to be accepted to matching branches. - requiredStatusCheckContexts: - - 'Kokoro' - - 'Kokoro system' - - 'cla/google' - - 'OwlBot Post Processor' - - 'docs' - - 'docfx' - - 'lint' - - 'unit (3.9)' - - 'unit (3.10)' - - 'unit (3.11)' - - 'unit (3.12)' - - 'unit (3.13)' - - 'unit (3.14)' - - 'cover' - - 'run-systests' -# List of explicit permissions to add (additive only) -permissionRules: - # Team slug to add to repository permissions - - team: yoshi-admins - # Access level required, one of push|pull|admin|maintain|triage - permission: admin - # Team slug to add to repository permissions - - team: yoshi-python-admins - # Access level required, one of push|pull|admin|maintain|triage - permission: admin - # Team slug to add to repository permissions - - team: yoshi-python - # Access level required, one of push|pull|admin|maintain|triage - permission: push diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9a0598202..3ed755f00 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.14" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 27075146a..4997affc7 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.14" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/.github/workflows/system_emulated.yml b/.github/workflows/system_emulated.yml index bb7986a0a..62a879072 100644 --- a/.github/workflows/system_emulated.yml +++ b/.github/workflows/system_emulated.yml @@ -17,7 +17,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: '3.14' # firestore emulator requires java 21+ - name: Setup Java diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 494bb568f..cc6fe2b2f 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -45,7 +45,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.14" - name: Install coverage run: | python -m pip install --upgrade setuptools pip wheel diff --git a/.kokoro/presubmit/system-3.14.cfg b/.kokoro/presubmit/system-3.14.cfg deleted file mode 100644 index 86e7c5d77..000000000 --- a/.kokoro/presubmit/system-3.14.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "system-3.14" -} \ No newline at end of file diff --git a/.kokoro/presubmit/system-3.7.cfg b/.kokoro/presubmit/system-3.7.cfg deleted file mode 100644 index 461537b3f..000000000 --- a/.kokoro/presubmit/system-3.7.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "system-3.7" -} \ No newline at end of file diff --git a/.kokoro/presubmit/system-3.9.cfg b/.kokoro/presubmit/system-3.9.cfg deleted file mode 100644 index b8ae66b37..000000000 --- a/.kokoro/presubmit/system-3.9.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "system-3.9" -} \ No newline at end of file diff --git a/.kokoro/presubmit/system.cfg b/.kokoro/presubmit/system.cfg index bd1fb514b..73904141b 100644 --- a/.kokoro/presubmit/system.cfg +++ b/.kokoro/presubmit/system.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "system-3.9" + value: "system-3.14" } diff --git a/.kokoro/samples/python3.14/common.cfg b/.kokoro/samples/python3.14/common.cfg new file mode 100644 index 000000000..4e07d3590 --- /dev/null +++ b/.kokoro/samples/python3.14/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.14" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-314" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-firestore/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-firestore/.kokoro/trampoline_v2.sh" diff --git a/.kokoro/samples/python3.14/continuous.cfg b/.kokoro/samples/python3.14/continuous.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.14/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.14/periodic-head.cfg b/.kokoro/samples/python3.14/periodic-head.cfg new file mode 100644 index 000000000..21998d090 --- /dev/null +++ b/.kokoro/samples/python3.14/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-firestore/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.14/periodic.cfg b/.kokoro/samples/python3.14/periodic.cfg new file mode 100644 index 000000000..71cd1e597 --- /dev/null +++ b/.kokoro/samples/python3.14/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.14/presubmit.cfg b/.kokoro/samples/python3.14/presubmit.cfg new file mode 100644 index 000000000..a1c8d9759 --- /dev/null +++ b/.kokoro/samples/python3.14/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.librarian/generator-input/.repo-metadata.json b/.librarian/generator-input/.repo-metadata.json new file mode 100644 index 000000000..670bbc0e4 --- /dev/null +++ b/.librarian/generator-input/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "name": "firestore", + "name_pretty": "Cloud Firestore", + "product_documentation": "https://cloud.google.com/firestore", + "client_documentation": "https://cloud.google.com/python/docs/reference/firestore/latest", + "issue_tracker": "https://issuetracker.google.com/savedsearches/5337669", + "release_level": "stable", + "language": "python", + "library_type": "GAPIC_COMBO", + "repo": "googleapis/python-firestore", + "distribution_name": "google-cloud-firestore", + "api_id": "firestore.googleapis.com", + "requires_billing": true, + "default_version": "v1", + "codeowner_team": "@googleapis/api-firestore @googleapis/api-firestore-partners", + "api_shortname": "firestore", + "api_description": "is a fully-managed NoSQL document database for mobile, web, and server development from Firebase and Google Cloud Platform. It's backed by a multi-region replicated database that ensures once data is committed, it's durable even in the face of unexpected disasters. Not only that, but despite being a distributed database, it's also strongly consistent and offers seamless integration with other Firebase and Google Cloud Platform products, including Google Cloud Functions." +} diff --git a/owlbot.py b/.librarian/generator-input/librarian.py similarity index 56% rename from owlbot.py rename to .librarian/generator-input/librarian.py index a9323ce3c..ec92a9345 100644 --- a/owlbot.py +++ b/.librarian/generator-input/librarian.py @@ -28,50 +28,10 @@ firestore_default_version = "v1" firestore_admin_default_version = "v1" -# This is a customized version of the s.get_staging_dirs() function from synthtool to -# cater for copying 3 different folders from googleapis-gen -# which are firestore, firestore/admin and firestore/bundle. -# Source https://github.com/googleapis/synthtool/blob/master/synthtool/transforms.py#L280 -def get_staging_dirs( - default_version: Optional[str] = None, sub_directory: Optional[str] = None -) -> List[Path]: - """Returns the list of directories, one per version, copied from - https://github.com/googleapis/googleapis-gen. Will return in lexical sorting - order with the exception of the default_version which will be last (if specified). - - Args: - default_version (str): the default version of the API. The directory for this version - will be the last item in the returned list if specified. - sub_directory (str): if a `sub_directory` is provided, only the directories within the - specified `sub_directory` will be returned. - - Returns: the empty list if no file were copied. - """ - - staging = Path("owl-bot-staging") - - if sub_directory: - staging /= sub_directory - - if staging.is_dir(): - # Collect the subdirectories of the staging directory. - versions = [v.name for v in staging.iterdir() if v.is_dir()] - # Reorder the versions so the default version always comes last. - versions = [v for v in versions if v != default_version] - versions.sort() - if default_version is not None: - versions += [default_version] - dirs = [staging / v for v in versions] - for dir in dirs: - s._tracked_paths.add(dir) - return dirs - else: - return [] - -def update_fixup_scripts(library): +def update_fixup_scripts(path): # Add message for missing 'libcst' dependency s.replace( - library / "scripts/fixup*.py", + library / "scripts" / path, """import libcst as cst""", """try: import libcst as cst @@ -82,19 +42,21 @@ def update_fixup_scripts(library): """, ) -for library in get_staging_dirs(default_version=firestore_default_version, sub_directory="firestore"): - s.move(library / f"google/cloud/firestore_{library.name}", excludes=[f"__init__.py", "**/gapic_version.py", "noxfile.py"]) +for library in s.get_staging_dirs(default_version=firestore_default_version): + s.move(library / f"google/cloud/firestore_{library.name}", excludes=[f"__init__.py", "noxfile.py"]) s.move(library / f"tests/", f"tests") - update_fixup_scripts(library) - s.move(library / "scripts") + fixup_script_path = "fixup_firestore_v1_keywords.py" + update_fixup_scripts(fixup_script_path) + s.move(library / "scripts" / fixup_script_path) -for library in get_staging_dirs(default_version=firestore_admin_default_version, sub_directory="firestore_admin"): - s.move(library / f"google/cloud/firestore_admin_{library.name}", excludes=[f"__init__.py", "**/gapic_version.py", "noxfile.py"]) +for library in s.get_staging_dirs(default_version=firestore_admin_default_version): + s.move(library / f"google/cloud/firestore_admin_{library.name}", excludes=[f"__init__.py", "noxfile.py"]) s.move(library / f"tests", f"tests") - update_fixup_scripts(library) - s.move(library / "scripts") + fixup_script_path = "fixup_firestore_admin_v1_keywords.py" + update_fixup_scripts(fixup_script_path) + s.move(library / "scripts" / fixup_script_path) -for library in get_staging_dirs(sub_directory="firestore_bundle"): +for library in s.get_staging_dirs(): s.replace( library / "google/cloud/bundle/types/bundle.py", "from google.firestore.v1 import document_pb2 # type: ignore\n" @@ -127,7 +89,7 @@ def update_fixup_scripts(library): s.move( library / f"google/cloud/bundle", f"google/cloud/firestore_bundle", - excludes=["**/gapic_version.py", "noxfile.py"], + excludes=["noxfile.py"], ) s.move(library / f"tests", f"tests") @@ -143,24 +105,14 @@ def update_fixup_scripts(library): microgenerator=True, cov_level=100, split_system_tests=True, - default_python_version="3.13", - system_test_python_versions=["3.9", "3.14"], + default_python_version="3.14", + system_test_python_versions=["3.14"], unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"], ) s.move(templated_files, - excludes=[".github/release-please.yml", "renovate.json"]) + excludes=[".github/**", ".kokoro/**", "renovate.json"]) python.py_samples(skip_readmes=True) s.shell.run(["nox", "-s", "blacken"], hide_output=False) - -s.replace( - ".kokoro/build.sh", - "# Setup service account credentials.", - """\ -# Setup firestore account credentials -export FIRESTORE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/firebase-credentials.json - -# Setup service account credentials.""", -) diff --git a/.librarian/generator-input/noxfile.py b/.librarian/generator-input/noxfile.py new file mode 100644 index 000000000..4fb209cbc --- /dev/null +++ b/.librarian/generator-input/noxfile.py @@ -0,0 +1,584 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! + +from __future__ import absolute_import + +import os +import pathlib +import re +import shutil +from typing import Dict, List +import warnings + +import nox + +FLAKE8_VERSION = "flake8==6.1.0" +PYTYPE_VERSION = "pytype==2020.7.24" +BLACK_VERSION = "black[jupyter]==23.7.0" +ISORT_VERSION = "isort==5.11.0" +LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] + +DEFAULT_PYTHON_VERSION = "3.14" + +UNIT_TEST_PYTHON_VERSIONS: List[str] = [ + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", + "3.14", +] +UNIT_TEST_STANDARD_DEPENDENCIES = [ + "mock", + "asyncmock", + "pytest", + "pytest-cov", + "pytest-asyncio==0.21.2", +] +UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ + "aiounittest", + "six", + "freezegun", +] +UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [] +UNIT_TEST_DEPENDENCIES: List[str] = [] +UNIT_TEST_EXTRAS: List[str] = [] +UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} + +SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.14"] +SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ + "mock", + "pytest", + "google-cloud-testutils", +] +SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [ + "pytest-asyncio==0.21.2", + "six", +] +SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_EXTRAS: List[str] = [] +SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +nox.options.sessions = [ + "unit-3.9", + "unit-3.10", + "unit-3.11", + "unit-3.12", + "unit-3.13", + "unit-3.14", + "system_emulated", + "system", + "mypy", + "cover", + "lint", + "lint_setup_py", + "blacken", + "docs", + "docfx", + "format", +] + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install(FLAKE8_VERSION, BLACK_VERSION) + session.run( + "black", + "--check", + *LINT_PATHS, + ) + session.run("flake8", "google", "tests") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run( + "isort", + "--fss", + *LINT_PATHS, + ) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python="3.7") +def pytype(session): + """Verify type hints are pytype compatible.""" + session.install(PYTYPE_VERSION) + session.run( + "pytype", + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def mypy(session): + """Verify type hints are mypy compatible.""" + session.install("-e", ".") + session.install("mypy", "types-setuptools", "types-protobuf") + session.run( + "mypy", + "-p", + "google.cloud.firestore_v1", + "--no-incremental", + "--check-untyped-defs", + "--exclude", + "services", + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install("setuptools", "docutils", "pygments") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + +def install_unittest_dependencies(session, *constraints): + standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES + session.install(*standard_deps, *constraints) + + if UNIT_TEST_EXTERNAL_DEPENDENCIES: + warnings.warn( + "'unit_test_external_dependencies' is deprecated. Instead, please " + "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", + DeprecationWarning, + ) + session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_LOCAL_DEPENDENCIES: + session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_EXTRAS_BY_PYTHON: + extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif UNIT_TEST_EXTRAS: + extras = UNIT_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@nox.parametrize( + "protobuf_implementation", + ["python", "upb", "cpp"], +) +def unit(session, protobuf_implementation): + # Install all test dependencies, then install this package in-place. + + py_version = tuple([int(v) for v in session.python.split(".")]) + if protobuf_implementation == "cpp" and py_version >= (3, 11): + session.skip("cpp implementation is not supported in python 3.11+") + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + install_unittest_dependencies(session, "-c", constraints_path) + + # TODO(https://github.com/googleapis/synthtool/issues/1976): + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") + + # Run py.test against the unit tests. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=google", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + +def install_systemtest_dependencies(session, *constraints): + # Use pre-release gRPC for system tests. + # Exclude version 1.52.0rc1 which has a known issue. + # See https://github.com/grpc/grpc/issues/32163 + session.install("--pre", "grpcio!=1.52.0rc1") + + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: + session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_LOCAL_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTRAS_BY_PYTHON: + extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif SYSTEM_TEST_EXTRAS: + extras = SYSTEM_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system_emulated(session): + import subprocess + import signal + + try: + # https://github.com/googleapis/python-firestore/issues/472 + # Kokoro image doesn't have java installed, don't attempt to run emulator. + subprocess.call(["java", "--version"]) + except OSError: + session.skip("java not found but required for emulator support") + + try: + subprocess.call(["gcloud", "--version"]) + except OSError: + session.skip("gcloud not found but required for emulator support") + + # Currently, CI/CD doesn't have beta component of gcloud. + subprocess.call( + [ + "gcloud", + "components", + "install", + "beta", + "cloud-firestore-emulator", + ] + ) + + hostport = "localhost:8789" + session.env["FIRESTORE_EMULATOR_HOST"] = hostport + + p = subprocess.Popen( + [ + "gcloud", + "--quiet", + "beta", + "emulators", + "firestore", + "start", + "--host-port", + hostport, + ] + ) + + try: + system(session) + finally: + # Stop Emulator + os.killpg(os.getpgid(p.pid), signal.SIGKILL) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system(session): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + install_systemtest_dependencies(session, "-c", constraints_path) + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if system_test_folder_exists: + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run( + "coverage", + "report", + "--show-missing", + "--fail-under=100", + "--omit=tests/*", + ) + + session.run("coverage", "erase") + + +@nox.session(python="3.10") +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "sphinx==4.5.0", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python="3.10") +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "gcp-sphinx-docfx-yaml", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.parametrize( + "protobuf_implementation", + ["python", "upb", "cpp"], +) +def prerelease_deps(session, protobuf_implementation): + """Run all tests with prerelease versions of dependencies installed.""" + + py_version = tuple([int(v) for v in session.python.split(".")]) + if protobuf_implementation == "cpp" and py_version >= (3, 11): + session.skip("cpp implementation is not supported in python 3.11+") + + # Install all dependencies + session.install("-e", ".[all, tests, tracing]") + unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES + session.install(*unit_deps_all) + system_deps_all = ( + SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES + ) + session.install(*system_deps_all) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + session.install(*constraints_deps) + + prerel_deps = [ + "protobuf", + # dependency of grpc + "six", + "grpc-google-iam-v1", + "googleapis-common-protos", + "grpcio", + "grpcio-status", + "google-api-core", + "google-auth", + "proto-plus", + "google-cloud-testutils", + # dependencies of google-cloud-testutils" + "click", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + ] + session.install(*other_deps) + + # Print out prerelease package versions + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run("python", "-c", "import grpc; print(grpc.__version__)") + session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + + session.run( + "py.test", + "tests/unit", + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Only run system tests if found. + if os.path.exists(system_test_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + if os.path.exists(system_test_folder_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) diff --git a/.librarian/generator-input/setup.py b/.librarian/generator-input/setup.py new file mode 100644 index 000000000..28d6faf51 --- /dev/null +++ b/.librarian/generator-input/setup.py @@ -0,0 +1,95 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os + +import setuptools + +# Package metadata. + +name = "google-cloud-firestore" +description = "Google Cloud Firestore API client library" + +package_root = os.path.abspath(os.path.dirname(__file__)) + +version = {} +with open(os.path.join(package_root, "google/cloud/firestore/gapic_version.py")) as fp: + exec(fp.read(), version) +version = version["__version__"] +release_status = "Development Status :: 5 - Production/Stable" +dependencies = [ + "google-api-core[grpc] >= 1.34.0, <3.0.0,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", + # Exclude incompatible versions of `google-auth` + # See https://github.com/googleapis/google-cloud-python/issues/12364 + "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0", + "google-cloud-core >= 1.4.1, <3.0.0", + "proto-plus >= 1.22.0, <2.0.0", + "proto-plus >= 1.22.2, <2.0.0; python_version>='3.11'", + "proto-plus >= 1.25.0, <2.0.0; python_version>='3.13'", + "protobuf>=3.20.2,<7.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", +] +extras = {} + + +# Setup boilerplate below this line. + +package_root = os.path.abspath(os.path.dirname(__file__)) +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + +# Only include packages under the 'google' namespace. Do not include tests, +# benchmarks, etc. +packages = [ + package + for package in setuptools.find_namespace_packages() + if package.startswith("google") +] + +setuptools.setup( + name=name, + version=version, + description=description, + long_description=readme, + author="Google LLC", + author_email="googleapis-packages@google.com", + license="Apache 2.0", + url="https://github.com/googleapis/python-firestore", + classifiers=[ + release_status, + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Operating System :: OS Independent", + "Topic :: Internet", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + platforms="Posix; MacOS X; Windows", + packages=packages, + install_requires=dependencies, + extras_require=extras, + python_requires=">=3.7", + include_package_data=True, + zip_safe=False, +) diff --git a/.librarian/state.yaml b/.librarian/state.yaml new file mode 100644 index 000000000..7715edb27 --- /dev/null +++ b/.librarian/state.yaml @@ -0,0 +1,49 @@ +image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:b8058df4c45e9a6e07f6b4d65b458d0d059241dd34c814f151c8bf6b89211209 +libraries: + - id: google-cloud-firestore + version: 2.22.0 + last_generated_commit: 1a9d00bed77e6db82ff67764ffe14e3b5209f5cd + apis: + - path: google/firestore/v1 + service_config: firestore_v1.yaml + - path: google/firestore/admin/v1 + service_config: firestore_v1.yaml + - path: google/firestore/bundle + service_config: "" + source_roots: + - . + preserve_regex: [] + remove_regex: + - ^google/cloud/firestore_v1/services + - ^google/cloud/firestore_v1/types + - ^google/cloud/firestore_v1/gapic + - ^google/cloud/firestore_v1/py.typed + - ^google/cloud/firestore_admin_v1/services + - ^google/cloud/firestore_admin_v1/types + - ^google/cloud/firestore_admin_v1/gapic + - ^google/cloud/firestore_admin_v1/py.typed + - ^google/cloud/firestore_bundle/services + - ^google/cloud/firestore_bundle/types + - ^google/cloud/firestore_bundle/__init__.py + - ^google/cloud/firestore_bundle/gapic + - ^google/cloud/firestore_bundle/py.typed + - ^tests/unit/gapic + - ^tests/__init__.py + - ^tests/unit/__init__.py + - ^.pre-commit-config.yaml + - ^.repo-metadata.json + - ^.trampolinerc + - ^.coveragerc + - ^SECURITY.md + - ^noxfile.py + - ^owlbot.py + - ^samples/AUTHORING_GUIDE.md + - ^samples/CONTRIBUTING.md + - ^samples/generated_samples + - ^scripts/fixup_firestore_v1_keywords.py + - ^scripts/fixup_firestore_admin_v1_keywords.py + - ^setup.py + - ^README.rst + - ^docs/README.rst + - ^docs/summary_overview.md + tag_format: v{version} diff --git a/CHANGELOG.md b/CHANGELOG.md index 893a01297..ee59f43a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ [1]: https://pypi.org/project/google-cloud-firestore/#history +## [2.22.0](https://github.com/googleapis/python-firestore/compare/v2.21.0...v2.22.0) (2025-12-16) + + +### Features + +* support mTLS certificates when available (#1140) ([403afb08109c8271eddd97d6172136271cc0a8a9](https://github.com/googleapis/python-firestore/commit/403afb08109c8271eddd97d6172136271cc0a8a9)) +* Add support for Python 3.14 (#1110) ([52b2055d01ab5d2c34e00f8861e29990f89cd3d8](https://github.com/googleapis/python-firestore/commit/52b2055d01ab5d2c34e00f8861e29990f89cd3d8)) +* Expose tags field in Database and RestoreDatabaseRequest public protos (#1074) ([49836391dc712bd482781a26ccd3c8a8408c473b](https://github.com/googleapis/python-firestore/commit/49836391dc712bd482781a26ccd3c8a8408c473b)) +* Added read_time as a parameter to various calls (synchronous/base classes) (#1050) ([d8e3af1f9dbdfaf5df0d993a0a7e28883472c621](https://github.com/googleapis/python-firestore/commit/d8e3af1f9dbdfaf5df0d993a0a7e28883472c621)) + + +### Bug Fixes + +* improve typing (#1136) ([d1c730d9eef867d16d7818a75f7d58439a942c1d](https://github.com/googleapis/python-firestore/commit/d1c730d9eef867d16d7818a75f7d58439a942c1d)) +* update the async transactional types (#1066) ([210a14a4b758d70aad05940665ed2a2a21ae2a8b](https://github.com/googleapis/python-firestore/commit/210a14a4b758d70aad05940665ed2a2a21ae2a8b)) + ## [2.21.0](https://github.com/googleapis/python-firestore/compare/v2.20.2...v2.21.0) (2025-05-23) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c91768524..b59294006 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -246,6 +246,7 @@ We support: .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ .. _Python 3.12: https://docs.python.org/3.12/ +.. _Python 3.13: https://docs.python.org/3.13/ .. _Python 3.14: https://docs.python.org/3.14/ diff --git a/README.rst b/README.rst index e349bf783..71250f4f7 100644 --- a/README.rst +++ b/README.rst @@ -61,7 +61,7 @@ Supported Python Versions Our client libraries are compatible with all current `active`_ and `maintenance`_ versions of Python. -Python >= 3.7 +Python >= 3.7, including 3.14 .. _active: https://devguide.python.org/devcycle/#in-development-main-branch .. _maintenance: https://devguide.python.org/devcycle/#maintenance-branches diff --git a/docs/README.rst b/docs/README.rst deleted file mode 120000 index 89a010694..000000000 --- a/docs/README.rst +++ /dev/null @@ -1 +0,0 @@ -../README.rst \ No newline at end of file diff --git a/docs/README.rst b/docs/README.rst new file mode 100644 index 000000000..71250f4f7 --- /dev/null +++ b/docs/README.rst @@ -0,0 +1,197 @@ +Python Client for Cloud Firestore API +===================================== + +|stable| |pypi| |versions| + +`Cloud Firestore API`_: is a fully-managed NoSQL document database for mobile, web, and server development from Firebase and Google Cloud Platform. It's backed by a multi-region replicated database that ensures once data is committed, it's durable even in the face of unexpected disasters. Not only that, but despite being a distributed database, it's also strongly consistent and offers seamless integration with other Firebase and Google Cloud Platform products, including Google Cloud Functions. + +- `Client Library Documentation`_ +- `Product Documentation`_ + +.. |stable| image:: https://img.shields.io/badge/support-stable-gold.svg + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels +.. |pypi| image:: https://img.shields.io/pypi/v/google-cloud-firestore.svg + :target: https://pypi.org/project/google-cloud-firestore/ +.. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-firestore.svg + :target: https://pypi.org/project/google-cloud-firestore/ +.. _Cloud Firestore API: https://cloud.google.com/firestore +.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/firestore/latest/summary_overview +.. _Product Documentation: https://cloud.google.com/firestore + +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. `Enable the Cloud Firestore API.`_ +4. `Set up Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Enable the Cloud Firestore API.: https://cloud.google.com/firestore +.. _Set up Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html + +Installation +~~~~~~~~~~~~ + +Install this library in a virtual environment using `venv`_. `venv`_ is a tool that +creates isolated Python environments. These isolated environments can have separate +versions of Python packages, which allows you to isolate one project's dependencies +from the dependencies of other projects. + +With `venv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. + +.. _`venv`: https://docs.python.org/3/library/venv.html + + +Code samples and snippets +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Code samples and snippets live in the `samples/`_ folder. + +.. _samples/: https://github.com/googleapis/python-firestore/tree/main/samples + + +Supported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^ +Our client libraries are compatible with all current `active`_ and `maintenance`_ versions of +Python. + +Python >= 3.7, including 3.14 + +.. _active: https://devguide.python.org/devcycle/#in-development-main-branch +.. _maintenance: https://devguide.python.org/devcycle/#maintenance-branches + +Unsupported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Python <= 3.6 + +If you are using an `end-of-life`_ +version of Python, we recommend that you update as soon as possible to an actively supported version. + +.. _end-of-life: https://devguide.python.org/devcycle/#end-of-life-branches + +Mac/Linux +^^^^^^^^^ + +.. code-block:: console + + python3 -m venv + source /bin/activate + pip install google-cloud-firestore + + +Windows +^^^^^^^ + +.. code-block:: console + + py -m venv + .\\Scripts\activate + pip install google-cloud-firestore + +Next Steps +~~~~~~~~~~ + +- Read the `Client Library Documentation`_ for Cloud Firestore API + to see other available methods on the client. +- Read the `Cloud Firestore API Product documentation`_ to learn + more about the product and see How-to Guides. +- View this `README`_ to see the full list of Cloud + APIs that we cover. + +.. _Cloud Firestore API Product documentation: https://cloud.google.com/firestore +.. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst + +Logging +------- + +This library uses the standard Python :code:`logging` functionality to log some RPC events that could be of interest for debugging and monitoring purposes. +Note the following: + +#. Logs may contain sensitive information. Take care to **restrict access to the logs** if they are saved, whether it be on local storage or on Google Cloud Logging. +#. Google may refine the occurrence, level, and content of various log messages in this library without flagging such changes as breaking. **Do not depend on immutability of the logging events**. +#. By default, the logging events from this library are not handled. You must **explicitly configure log handling** using one of the mechanisms below. + +Simple, environment-based configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To enable logging for this library without any changes in your code, set the :code:`GOOGLE_SDK_PYTHON_LOGGING_SCOPE` environment variable to a valid Google +logging scope. This configures handling of logging events (at level :code:`logging.DEBUG` or higher) from this library in a default manner, emitting the logged +messages in a structured format. It does not currently allow customizing the logging levels captured nor the handlers, formatters, etc. used for any logging +event. + +A logging scope is a period-separated namespace that begins with :code:`google`, identifying the Python module or package to log. + +- Valid logging scopes: :code:`google`, :code:`google.cloud.asset.v1`, :code:`google.api`, :code:`google.auth`, etc. +- Invalid logging scopes: :code:`foo`, :code:`123`, etc. + +**NOTE**: If the logging scope is invalid, the library does not set up any logging handlers. + +Environment-Based Examples +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Enabling the default handler for all Google-based loggers + +.. code-block:: console + + export GOOGLE_SDK_PYTHON_LOGGING_SCOPE=google + +- Enabling the default handler for a specific Google module (for a client library called :code:`library_v1`): + +.. code-block:: console + + export GOOGLE_SDK_PYTHON_LOGGING_SCOPE=google.cloud.library_v1 + + +Advanced, code-based configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also configure a valid logging scope using Python's standard `logging` mechanism. + +Code-Based Examples +^^^^^^^^^^^^^^^^^^^ + +- Configuring a handler for all Google-based loggers + +.. code-block:: python + + import logging + + from google.cloud import library_v1 + + base_logger = logging.getLogger("google") + base_logger.addHandler(logging.StreamHandler()) + base_logger.setLevel(logging.DEBUG) + +- Configuring a handler for a specific Google module (for a client library called :code:`library_v1`): + +.. code-block:: python + + import logging + + from google.cloud import library_v1 + + base_logger = logging.getLogger("google.cloud.library_v1") + base_logger.addHandler(logging.StreamHandler()) + base_logger.setLevel(logging.DEBUG) + +Logging details +~~~~~~~~~~~~~~~ + +#. Regardless of which of the mechanisms above you use to configure logging for this library, by default logging events are not propagated up to the root + logger from the `google`-level logger. If you need the events to be propagated to the root logger, you must explicitly set + :code:`logging.getLogger("google").propagate = True` in your code. +#. You can mix the different logging configurations above for different Google modules. For example, you may want use a code-based logging configuration for + one library, but decide you need to also set up environment-based logging configuration for another library. + + #. If you attempt to use both code-based and environment-based configuration for the same module, the environment-based configuration will be ineffectual + if the code -based configuration gets applied first. + +#. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get + executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured. + (This is the reason for 2.i. above.) diff --git a/google/cloud/firestore/gapic_version.py b/google/cloud/firestore/gapic_version.py index e546bae05..03d6d0200 100644 --- a/google/cloud/firestore/gapic_version.py +++ b/google/cloud/firestore/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.21.0" # {x-release-please-version} +__version__ = "2.22.0" # {x-release-please-version} diff --git a/google/cloud/firestore_admin_v1/gapic_metadata.json b/google/cloud/firestore_admin_v1/gapic_metadata.json index e2c91bdb5..b8d4cb298 100644 --- a/google/cloud/firestore_admin_v1/gapic_metadata.json +++ b/google/cloud/firestore_admin_v1/gapic_metadata.json @@ -15,6 +15,11 @@ "bulk_delete_documents" ] }, + "CloneDatabase": { + "methods": [ + "clone_database" + ] + }, "CreateBackupSchedule": { "methods": [ "create_backup_schedule" @@ -175,6 +180,11 @@ "bulk_delete_documents" ] }, + "CloneDatabase": { + "methods": [ + "clone_database" + ] + }, "CreateBackupSchedule": { "methods": [ "create_backup_schedule" @@ -335,6 +345,11 @@ "bulk_delete_documents" ] }, + "CloneDatabase": { + "methods": [ + "clone_database" + ] + }, "CreateBackupSchedule": { "methods": [ "create_backup_schedule" diff --git a/google/cloud/firestore_admin_v1/gapic_version.py b/google/cloud/firestore_admin_v1/gapic_version.py index e546bae05..ced4e0faf 100644 --- a/google/cloud/firestore_admin_v1/gapic_version.py +++ b/google/cloud/firestore_admin_v1/gapic_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.21.0" # {x-release-please-version} +__version__ = "2.22.0" # {x-release-please-version} diff --git a/google/cloud/firestore_admin_v1/services/firestore_admin/async_client.py b/google/cloud/firestore_admin_v1/services/firestore_admin/async_client.py index 56531fa29..a2800e34e 100644 --- a/google/cloud/firestore_admin_v1/services/firestore_admin/async_client.py +++ b/google/cloud/firestore_admin_v1/services/firestore_admin/async_client.py @@ -4111,6 +4111,143 @@ async def sample_delete_backup_schedule(): metadata=metadata, ) + async def clone_database( + self, + request: Optional[Union[firestore_admin.CloneDatabaseRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> operation_async.AsyncOperation: + r"""Creates a new database by cloning an existing one. + + The new database must be in the same cloud region or + multi-region location as the existing database. This behaves + similar to + [FirestoreAdmin.CreateDatabase][google.firestore.admin.v1.FirestoreAdmin.CreateDatabase] + except instead of creating a new empty database, a new database + is created with the database type, index configuration, and + documents from an existing database. + + The [long-running operation][google.longrunning.Operation] can + be used to track the progress of the clone, with the Operation's + [metadata][google.longrunning.Operation.metadata] field type + being the + [CloneDatabaseMetadata][google.firestore.admin.v1.CloneDatabaseMetadata]. + The [response][google.longrunning.Operation.response] type is + the [Database][google.firestore.admin.v1.Database] if the clone + was successful. The new database is not readable or writeable + until the LRO has completed. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import firestore_admin_v1 + + async def sample_clone_database(): + # Create a client + client = firestore_admin_v1.FirestoreAdminAsyncClient() + + # Initialize request argument(s) + pitr_snapshot = firestore_admin_v1.PitrSnapshot() + pitr_snapshot.database = "database_value" + + request = firestore_admin_v1.CloneDatabaseRequest( + parent="parent_value", + database_id="database_id_value", + pitr_snapshot=pitr_snapshot, + ) + + # Make the request + operation = client.clone_database(request=request) + + print("Waiting for operation to complete...") + + response = (await operation).result() + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.firestore_admin_v1.types.CloneDatabaseRequest, dict]]): + The request object. The request message for + [FirestoreAdmin.CloneDatabase][google.firestore.admin.v1.FirestoreAdmin.CloneDatabase]. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + google.api_core.operation_async.AsyncOperation: + An object representing a long-running operation. + + The result type for the operation will be + :class:`google.cloud.firestore_admin_v1.types.Database` + A Cloud Firestore Database. + + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, firestore_admin.CloneDatabaseRequest): + request = firestore_admin.CloneDatabaseRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[ + self._client._transport.clone_database + ] + + header_params = {} + + routing_param_regex = re.compile("^projects/(?P[^/]+)(?:/.*)?$") + regex_match = routing_param_regex.match(request.pitr_snapshot.database) + if regex_match and regex_match.group("project_id"): + header_params["project_id"] = regex_match.group("project_id") + + routing_param_regex = re.compile( + "^projects/[^/]+/databases/(?P[^/]+)(?:/.*)?$" + ) + regex_match = routing_param_regex.match(request.pitr_snapshot.database) + if regex_match and regex_match.group("database_id"): + header_params["database_id"] = regex_match.group("database_id") + + if header_params: + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(header_params), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Wrap the response in an operation future. + response = operation_async.from_gapic( + response, + self._client._transport.operations_client, + database.Database, + metadata_type=gfa_operation.CloneDatabaseMetadata, + ) + + # Done; return the response. + return response + async def list_operations( self, request: Optional[operations_pb2.ListOperationsRequest] = None, diff --git a/google/cloud/firestore_admin_v1/services/firestore_admin/client.py b/google/cloud/firestore_admin_v1/services/firestore_admin/client.py index d05b82787..b55c157cf 100644 --- a/google/cloud/firestore_admin_v1/services/firestore_admin/client.py +++ b/google/cloud/firestore_admin_v1/services/firestore_admin/client.py @@ -198,6 +198,34 @@ def _get_default_mtls_endpoint(api_endpoint): _DEFAULT_ENDPOINT_TEMPLATE = "firestore.{UNIVERSE_DOMAIN}" _DEFAULT_UNIVERSE = "googleapis.com" + @staticmethod + def _use_client_cert_effective(): + """Returns whether client certificate should be used for mTLS if the + google-auth version supports should_use_client_cert automatic mTLS enablement. + + Alternatively, read from the GOOGLE_API_USE_CLIENT_CERTIFICATE env var. + + Returns: + bool: whether client certificate should be used for mTLS + Raises: + ValueError: (If using a version of google-auth without should_use_client_cert and + GOOGLE_API_USE_CLIENT_CERTIFICATE is set to an unexpected value.) + """ + # check if google-auth version supports should_use_client_cert for automatic mTLS enablement + if hasattr(mtls, "should_use_client_cert"): # pragma: NO COVER + return mtls.should_use_client_cert() + else: # pragma: NO COVER + # if unsupported, fallback to reading from env var + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + @classmethod def from_service_account_info(cls, info: dict, *args, **kwargs): """Creates an instance of this client using the provided credentials @@ -555,12 +583,8 @@ def get_mtls_endpoint_and_cert_source( ) if client_options is None: client_options = client_options_lib.ClientOptions() - use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_client_cert = FirestoreAdminClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" @@ -568,7 +592,7 @@ def get_mtls_endpoint_and_cert_source( # Figure out the client cert source to use. client_cert_source = None - if use_client_cert == "true": + if use_client_cert: if client_options.client_cert_source: client_cert_source = client_options.client_cert_source elif mtls.has_default_client_cert_source(): @@ -600,20 +624,14 @@ def _read_environment_variables(): google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT is not any of ["auto", "never", "always"]. """ - use_client_cert = os.getenv( - "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" - ).lower() + use_client_cert = FirestoreAdminClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + return use_client_cert, use_mtls_endpoint, universe_domain_env @staticmethod def _get_client_cert_source(provided_cert_source, use_cert_flag): @@ -4591,6 +4609,141 @@ def sample_delete_backup_schedule(): metadata=metadata, ) + def clone_database( + self, + request: Optional[Union[firestore_admin.CloneDatabaseRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> gac_operation.Operation: + r"""Creates a new database by cloning an existing one. + + The new database must be in the same cloud region or + multi-region location as the existing database. This behaves + similar to + [FirestoreAdmin.CreateDatabase][google.firestore.admin.v1.FirestoreAdmin.CreateDatabase] + except instead of creating a new empty database, a new database + is created with the database type, index configuration, and + documents from an existing database. + + The [long-running operation][google.longrunning.Operation] can + be used to track the progress of the clone, with the Operation's + [metadata][google.longrunning.Operation.metadata] field type + being the + [CloneDatabaseMetadata][google.firestore.admin.v1.CloneDatabaseMetadata]. + The [response][google.longrunning.Operation.response] type is + the [Database][google.firestore.admin.v1.Database] if the clone + was successful. The new database is not readable or writeable + until the LRO has completed. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import firestore_admin_v1 + + def sample_clone_database(): + # Create a client + client = firestore_admin_v1.FirestoreAdminClient() + + # Initialize request argument(s) + pitr_snapshot = firestore_admin_v1.PitrSnapshot() + pitr_snapshot.database = "database_value" + + request = firestore_admin_v1.CloneDatabaseRequest( + parent="parent_value", + database_id="database_id_value", + pitr_snapshot=pitr_snapshot, + ) + + # Make the request + operation = client.clone_database(request=request) + + print("Waiting for operation to complete...") + + response = operation.result() + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.firestore_admin_v1.types.CloneDatabaseRequest, dict]): + The request object. The request message for + [FirestoreAdmin.CloneDatabase][google.firestore.admin.v1.FirestoreAdmin.CloneDatabase]. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + google.api_core.operation.Operation: + An object representing a long-running operation. + + The result type for the operation will be + :class:`google.cloud.firestore_admin_v1.types.Database` + A Cloud Firestore Database. + + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, firestore_admin.CloneDatabaseRequest): + request = firestore_admin.CloneDatabaseRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.clone_database] + + header_params = {} + + routing_param_regex = re.compile("^projects/(?P[^/]+)(?:/.*)?$") + regex_match = routing_param_regex.match(request.pitr_snapshot.database) + if regex_match and regex_match.group("project_id"): + header_params["project_id"] = regex_match.group("project_id") + + routing_param_regex = re.compile( + "^projects/[^/]+/databases/(?P[^/]+)(?:/.*)?$" + ) + regex_match = routing_param_regex.match(request.pitr_snapshot.database) + if regex_match and regex_match.group("database_id"): + header_params["database_id"] = regex_match.group("database_id") + + if header_params: + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(header_params), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Wrap the response in an operation future. + response = gac_operation.from_gapic( + response, + self._transport.operations_client, + database.Database, + metadata_type=gfa_operation.CloneDatabaseMetadata, + ) + + # Done; return the response. + return response + def __enter__(self) -> "FirestoreAdminClient": return self diff --git a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/base.py b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/base.py index f290fcbfe..7d582d9b5 100644 --- a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/base.py +++ b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/base.py @@ -81,9 +81,10 @@ def __init__( credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. + This argument is mutually exclusive with credentials. This argument will be + removed in the next major version of this library. scopes (Optional[Sequence[str]]): A list of scopes. quota_project_id (Optional[str]): An optional project to use for billing and quota. @@ -357,6 +358,11 @@ def _prep_wrapped_messages(self, client_info): default_timeout=None, client_info=client_info, ), + self.clone_database: gapic_v1.method.wrap_method( + self.clone_database, + default_timeout=120.0, + client_info=client_info, + ), self.cancel_operation: gapic_v1.method.wrap_method( self.cancel_operation, default_timeout=None, @@ -688,6 +694,15 @@ def delete_backup_schedule( ]: raise NotImplementedError() + @property + def clone_database( + self, + ) -> Callable[ + [firestore_admin.CloneDatabaseRequest], + Union[operations_pb2.Operation, Awaitable[operations_pb2.Operation]], + ]: + raise NotImplementedError() + @property def list_operations( self, diff --git a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/grpc.py b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/grpc.py index c6e7824c2..f6531a190 100644 --- a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/grpc.py +++ b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/grpc.py @@ -192,9 +192,10 @@ def __init__( are specified, the client will attempt to ascertain the credentials from the environment. This argument is ignored if a ``channel`` instance is provided. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if a ``channel`` instance is provided. + This argument will be removed in the next major version of this library. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if a ``channel`` instance is provided. channel (Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]]): @@ -328,9 +329,10 @@ def create_channel( credentials identify this application to the service. If none are specified, the client will attempt to ascertain the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. + This argument is mutually exclusive with credentials. This argument will be + removed in the next major version of this library. scopes (Optional[Sequence[str]]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. @@ -1279,6 +1281,50 @@ def delete_backup_schedule( ) return self._stubs["delete_backup_schedule"] + @property + def clone_database( + self, + ) -> Callable[[firestore_admin.CloneDatabaseRequest], operations_pb2.Operation]: + r"""Return a callable for the clone database method over gRPC. + + Creates a new database by cloning an existing one. + + The new database must be in the same cloud region or + multi-region location as the existing database. This behaves + similar to + [FirestoreAdmin.CreateDatabase][google.firestore.admin.v1.FirestoreAdmin.CreateDatabase] + except instead of creating a new empty database, a new database + is created with the database type, index configuration, and + documents from an existing database. + + The [long-running operation][google.longrunning.Operation] can + be used to track the progress of the clone, with the Operation's + [metadata][google.longrunning.Operation.metadata] field type + being the + [CloneDatabaseMetadata][google.firestore.admin.v1.CloneDatabaseMetadata]. + The [response][google.longrunning.Operation.response] type is + the [Database][google.firestore.admin.v1.Database] if the clone + was successful. The new database is not readable or writeable + until the LRO has completed. + + Returns: + Callable[[~.CloneDatabaseRequest], + ~.Operation]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "clone_database" not in self._stubs: + self._stubs["clone_database"] = self._logged_channel.unary_unary( + "/google.firestore.admin.v1.FirestoreAdmin/CloneDatabase", + request_serializer=firestore_admin.CloneDatabaseRequest.serialize, + response_deserializer=operations_pb2.Operation.FromString, + ) + return self._stubs["clone_database"] + def close(self): self._logged_channel.close() diff --git a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/grpc_asyncio.py b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/grpc_asyncio.py index 9dd9d6155..117707853 100644 --- a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/grpc_asyncio.py +++ b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/grpc_asyncio.py @@ -189,8 +189,9 @@ def create_channel( credentials identify this application to the service. If none are specified, the client will attempt to ascertain the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. + credentials_file (Optional[str]): Deprecated. A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. This argument will be + removed in the next major version of this library. scopes (Optional[Sequence[str]]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. @@ -241,9 +242,10 @@ def __init__( are specified, the client will attempt to ascertain the credentials from the environment. This argument is ignored if a ``channel`` instance is provided. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if a ``channel`` instance is provided. + This argument will be removed in the next major version of this library. scopes (Optional[Sequence[str]]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. @@ -1331,6 +1333,52 @@ def delete_backup_schedule( ) return self._stubs["delete_backup_schedule"] + @property + def clone_database( + self, + ) -> Callable[ + [firestore_admin.CloneDatabaseRequest], Awaitable[operations_pb2.Operation] + ]: + r"""Return a callable for the clone database method over gRPC. + + Creates a new database by cloning an existing one. + + The new database must be in the same cloud region or + multi-region location as the existing database. This behaves + similar to + [FirestoreAdmin.CreateDatabase][google.firestore.admin.v1.FirestoreAdmin.CreateDatabase] + except instead of creating a new empty database, a new database + is created with the database type, index configuration, and + documents from an existing database. + + The [long-running operation][google.longrunning.Operation] can + be used to track the progress of the clone, with the Operation's + [metadata][google.longrunning.Operation.metadata] field type + being the + [CloneDatabaseMetadata][google.firestore.admin.v1.CloneDatabaseMetadata]. + The [response][google.longrunning.Operation.response] type is + the [Database][google.firestore.admin.v1.Database] if the clone + was successful. The new database is not readable or writeable + until the LRO has completed. + + Returns: + Callable[[~.CloneDatabaseRequest], + Awaitable[~.Operation]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "clone_database" not in self._stubs: + self._stubs["clone_database"] = self._logged_channel.unary_unary( + "/google.firestore.admin.v1.FirestoreAdmin/CloneDatabase", + request_serializer=firestore_admin.CloneDatabaseRequest.serialize, + response_deserializer=operations_pb2.Operation.FromString, + ) + return self._stubs["clone_database"] + def _prep_wrapped_messages(self, client_info): """Precompute the wrapped methods, overriding the base class method to use async wrappers.""" self._wrapped_methods = { @@ -1544,6 +1592,11 @@ def _prep_wrapped_messages(self, client_info): default_timeout=None, client_info=client_info, ), + self.clone_database: self._wrap_method( + self.clone_database, + default_timeout=120.0, + client_info=client_info, + ), self.cancel_operation: self._wrap_method( self.cancel_operation, default_timeout=None, diff --git a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/rest.py b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/rest.py index c96be2e32..41e819c87 100644 --- a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/rest.py +++ b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/rest.py @@ -97,6 +97,14 @@ def post_bulk_delete_documents(self, response): logging.log(f"Received response: {response}") return response + def pre_clone_database(self, request, metadata): + logging.log(f"Received request: {request}") + return request, metadata + + def post_clone_database(self, response): + logging.log(f"Received response: {response}") + return response + def pre_create_backup_schedule(self, request, metadata): logging.log(f"Received request: {request}") return request, metadata @@ -376,6 +384,54 @@ def post_bulk_delete_documents_with_metadata( """ return response, metadata + def pre_clone_database( + self, + request: firestore_admin.CloneDatabaseRequest, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[ + firestore_admin.CloneDatabaseRequest, Sequence[Tuple[str, Union[str, bytes]]] + ]: + """Pre-rpc interceptor for clone_database + + Override in a subclass to manipulate the request or metadata + before they are sent to the FirestoreAdmin server. + """ + return request, metadata + + def post_clone_database( + self, response: operations_pb2.Operation + ) -> operations_pb2.Operation: + """Post-rpc interceptor for clone_database + + DEPRECATED. Please use the `post_clone_database_with_metadata` + interceptor instead. + + Override in a subclass to read or manipulate the response + after it is returned by the FirestoreAdmin server but before + it is returned to user code. This `post_clone_database` interceptor runs + before the `post_clone_database_with_metadata` interceptor. + """ + return response + + def post_clone_database_with_metadata( + self, + response: operations_pb2.Operation, + metadata: Sequence[Tuple[str, Union[str, bytes]]], + ) -> Tuple[operations_pb2.Operation, Sequence[Tuple[str, Union[str, bytes]]]]: + """Post-rpc interceptor for clone_database + + Override in a subclass to read or manipulate the response or metadata after it + is returned by the FirestoreAdmin server but before it is returned to user code. + + We recommend only using this `post_clone_database_with_metadata` + interceptor in new development instead of the `post_clone_database` interceptor. + When both interceptors are used, this `post_clone_database_with_metadata` interceptor runs after the + `post_clone_database` interceptor. The (possibly modified) response returned by + `post_clone_database` will be passed to + `post_clone_database_with_metadata`. + """ + return response, metadata + def pre_create_backup_schedule( self, request: firestore_admin.CreateBackupScheduleRequest, @@ -1857,9 +1913,10 @@ def __init__( are specified, the client will attempt to ascertain the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. + This argument is ignored if ``channel`` is provided. This argument will be + removed in the next major version of this library. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if ``channel`` is provided. client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client @@ -2115,6 +2172,158 @@ def __call__( ) return resp + class _CloneDatabase( + _BaseFirestoreAdminRestTransport._BaseCloneDatabase, FirestoreAdminRestStub + ): + def __hash__(self): + return hash("FirestoreAdminRestTransport.CloneDatabase") + + @staticmethod + def _get_response( + host, + metadata, + query_params, + session, + timeout, + transcoded_request, + body=None, + ): + uri = transcoded_request["uri"] + method = transcoded_request["method"] + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(session, method)( + "{host}{uri}".format(host=host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params, strict=True), + data=body, + ) + return response + + def __call__( + self, + request: firestore_admin.CloneDatabaseRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = (), + ) -> operations_pb2.Operation: + r"""Call the clone database method over HTTP. + + Args: + request (~.firestore_admin.CloneDatabaseRequest): + The request object. The request message for + [FirestoreAdmin.CloneDatabase][google.firestore.admin.v1.FirestoreAdmin.CloneDatabase]. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be + sent along with the request as metadata. Normally, each value must be of type `str`, + but for metadata keys ending with the suffix `-bin`, the corresponding values must + be of type `bytes`. + + Returns: + ~.operations_pb2.Operation: + This resource represents a + long-running operation that is the + result of a network API call. + + """ + + http_options = ( + _BaseFirestoreAdminRestTransport._BaseCloneDatabase._get_http_options() + ) + + request, metadata = self._interceptor.pre_clone_database(request, metadata) + transcoded_request = _BaseFirestoreAdminRestTransport._BaseCloneDatabase._get_transcoded_request( + http_options, request + ) + + body = _BaseFirestoreAdminRestTransport._BaseCloneDatabase._get_request_body_json( + transcoded_request + ) + + # Jsonify the query params + query_params = _BaseFirestoreAdminRestTransport._BaseCloneDatabase._get_query_params_json( + transcoded_request + ) + + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + request_url = "{host}{uri}".format( + host=self._host, uri=transcoded_request["uri"] + ) + method = transcoded_request["method"] + try: + request_payload = json_format.MessageToJson(request) + except: + request_payload = None + http_request = { + "payload": request_payload, + "requestMethod": method, + "requestUrl": request_url, + "headers": dict(metadata), + } + _LOGGER.debug( + f"Sending request for google.firestore.admin_v1.FirestoreAdminClient.CloneDatabase", + extra={ + "serviceName": "google.firestore.admin.v1.FirestoreAdmin", + "rpcName": "CloneDatabase", + "httpRequest": http_request, + "metadata": http_request["headers"], + }, + ) + + # Send the request + response = FirestoreAdminRestTransport._CloneDatabase._get_response( + self._host, + metadata, + query_params, + self._session, + timeout, + transcoded_request, + body, + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + resp = operations_pb2.Operation() + json_format.Parse(response.content, resp, ignore_unknown_fields=True) + + resp = self._interceptor.post_clone_database(resp) + response_metadata = [(k, str(v)) for k, v in response.headers.items()] + resp, _ = self._interceptor.post_clone_database_with_metadata( + resp, response_metadata + ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + try: + response_payload = json_format.MessageToJson(resp) + except: + response_payload = None + http_response = { + "payload": response_payload, + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.firestore.admin_v1.FirestoreAdminClient.clone_database", + extra={ + "serviceName": "google.firestore.admin.v1.FirestoreAdmin", + "rpcName": "CloneDatabase", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) + return resp + class _CreateBackupSchedule( _BaseFirestoreAdminRestTransport._BaseCreateBackupSchedule, FirestoreAdminRestStub, @@ -6507,6 +6716,14 @@ def bulk_delete_documents( # In C++ this would require a dynamic_cast return self._BulkDeleteDocuments(self._session, self._host, self._interceptor) # type: ignore + @property + def clone_database( + self, + ) -> Callable[[firestore_admin.CloneDatabaseRequest], operations_pb2.Operation]: + # The return type is fine, but mypy isn't sophisticated enough to determine what's going on here. + # In C++ this would require a dynamic_cast + return self._CloneDatabase(self._session, self._host, self._interceptor) # type: ignore + @property def create_backup_schedule( self, diff --git a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/rest_base.py b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/rest_base.py index 19a0c9856..56b6ce93f 100644 --- a/google/cloud/firestore_admin_v1/services/firestore_admin/transports/rest_base.py +++ b/google/cloud/firestore_admin_v1/services/firestore_admin/transports/rest_base.py @@ -156,6 +156,63 @@ def _get_query_params_json(transcoded_request): query_params["$alt"] = "json;enum-encoding=int" return query_params + class _BaseCloneDatabase: + def __hash__(self): # pragma: NO COVER + return NotImplementedError("__hash__ must be implemented.") + + __REQUIRED_FIELDS_DEFAULT_VALUES: Dict[str, Any] = {} + + @classmethod + def _get_unset_required_fields(cls, message_dict): + return { + k: v + for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() + if k not in message_dict + } + + @staticmethod + def _get_http_options(): + http_options: List[Dict[str, str]] = [ + { + "method": "post", + "uri": "/v1/{parent=projects/*}/databases:clone", + "body": "*", + }, + ] + return http_options + + @staticmethod + def _get_transcoded_request(http_options, request): + pb_request = firestore_admin.CloneDatabaseRequest.pb(request) + transcoded_request = path_template.transcode(http_options, pb_request) + return transcoded_request + + @staticmethod + def _get_request_body_json(transcoded_request): + # Jsonify the request body + + body = json_format.MessageToJson( + transcoded_request["body"], use_integers_for_enums=True + ) + return body + + @staticmethod + def _get_query_params_json(transcoded_request): + query_params = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=True, + ) + ) + query_params.update( + _BaseFirestoreAdminRestTransport._BaseCloneDatabase._get_unset_required_fields( + query_params + ) + ) + + query_params["$alt"] = "json;enum-encoding=int" + return query_params + class _BaseCreateBackupSchedule: def __hash__(self): # pragma: NO COVER return NotImplementedError("__hash__ must be implemented.") diff --git a/google/cloud/firestore_admin_v1/types/__init__.py b/google/cloud/firestore_admin_v1/types/__init__.py index 249147d52..c76372e5d 100644 --- a/google/cloud/firestore_admin_v1/types/__init__.py +++ b/google/cloud/firestore_admin_v1/types/__init__.py @@ -25,6 +25,7 @@ from .firestore_admin import ( BulkDeleteDocumentsRequest, BulkDeleteDocumentsResponse, + CloneDatabaseRequest, CreateBackupScheduleRequest, CreateDatabaseMetadata, CreateDatabaseRequest, @@ -73,6 +74,7 @@ ) from .operation import ( BulkDeleteDocumentsMetadata, + CloneDatabaseMetadata, ExportDocumentsMetadata, ExportDocumentsResponse, FieldOperationMetadata, @@ -87,6 +89,9 @@ DailyRecurrence, WeeklyRecurrence, ) +from .snapshot import ( + PitrSnapshot, +) from .user_creds import ( UserCreds, ) @@ -97,6 +102,7 @@ "Field", "BulkDeleteDocumentsRequest", "BulkDeleteDocumentsResponse", + "CloneDatabaseRequest", "CreateBackupScheduleRequest", "CreateDatabaseMetadata", "CreateDatabaseRequest", @@ -139,6 +145,7 @@ "Index", "LocationMetadata", "BulkDeleteDocumentsMetadata", + "CloneDatabaseMetadata", "ExportDocumentsMetadata", "ExportDocumentsResponse", "FieldOperationMetadata", @@ -150,5 +157,6 @@ "BackupSchedule", "DailyRecurrence", "WeeklyRecurrence", + "PitrSnapshot", "UserCreds", ) diff --git a/google/cloud/firestore_admin_v1/types/database.py b/google/cloud/firestore_admin_v1/types/database.py index eafa21df1..f46bede62 100644 --- a/google/cloud/firestore_admin_v1/types/database.py +++ b/google/cloud/firestore_admin_v1/types/database.py @@ -213,9 +213,9 @@ class PointInTimeRecoveryEnablement(proto.Enum): Reads are supported on selected versions of the data from within the past 7 days: - - Reads against any timestamp within the past hour - - Reads against 1-minute snapshots beyond 1 hour and within - 7 days + - Reads against any timestamp within the past hour + - Reads against 1-minute snapshots beyond 1 hour and within + 7 days ``version_retention_period`` and ``earliest_version_time`` can be used to determine the supported versions. diff --git a/google/cloud/firestore_admin_v1/types/firestore_admin.py b/google/cloud/firestore_admin_v1/types/firestore_admin.py index a4b577b78..9ede35cac 100644 --- a/google/cloud/firestore_admin_v1/types/firestore_admin.py +++ b/google/cloud/firestore_admin_v1/types/firestore_admin.py @@ -24,6 +24,7 @@ from google.cloud.firestore_admin_v1.types import field as gfa_field from google.cloud.firestore_admin_v1.types import index as gfa_index from google.cloud.firestore_admin_v1.types import schedule +from google.cloud.firestore_admin_v1.types import snapshot from google.cloud.firestore_admin_v1.types import user_creds as gfa_user_creds from google.protobuf import field_mask_pb2 # type: ignore from google.protobuf import timestamp_pb2 # type: ignore @@ -73,6 +74,7 @@ "ListBackupsResponse", "DeleteBackupRequest", "RestoreDatabaseRequest", + "CloneDatabaseRequest", }, ) @@ -951,7 +953,7 @@ class ListBackupsRequest(proto.Message): [Backup][google.firestore.admin.v1.Backup] are eligible for filtering: - - ``database_uid`` (supports ``=`` only) + - ``database_uid`` (supports ``=`` only) """ parent: str = proto.Field( @@ -1079,4 +1081,70 @@ class RestoreDatabaseRequest(proto.Message): ) +class CloneDatabaseRequest(proto.Message): + r"""The request message for + [FirestoreAdmin.CloneDatabase][google.firestore.admin.v1.FirestoreAdmin.CloneDatabase]. + + Attributes: + parent (str): + Required. The project to clone the database in. Format is + ``projects/{project_id}``. + database_id (str): + Required. The ID to use for the database, which will become + the final component of the database's resource name. This + database ID must not be associated with an existing + database. + + This value should be 4-63 characters. Valid characters are + /[a-z][0-9]-/ with first character a letter and the last a + letter or a number. Must not be UUID-like + /[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}/. + + "(default)" database ID is also valid. + pitr_snapshot (google.cloud.firestore_admin_v1.types.PitrSnapshot): + Required. Specification of the PITR data to + clone from. The source database must exist. + + The cloned database will be created in the same + location as the source database. + encryption_config (google.cloud.firestore_admin_v1.types.Database.EncryptionConfig): + Optional. Encryption configuration for the cloned database. + + If this field is not specified, the cloned database will use + the same encryption configuration as the source database, + namely + [use_source_encryption][google.firestore.admin.v1.Database.EncryptionConfig.use_source_encryption]. + tags (MutableMapping[str, str]): + Optional. Immutable. Tags to be bound to the cloned + database. + + The tags should be provided in the format of + ``tagKeys/{tag_key_id} -> tagValues/{tag_value_id}``. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + database_id: str = proto.Field( + proto.STRING, + number=2, + ) + pitr_snapshot: snapshot.PitrSnapshot = proto.Field( + proto.MESSAGE, + number=6, + message=snapshot.PitrSnapshot, + ) + encryption_config: gfa_database.Database.EncryptionConfig = proto.Field( + proto.MESSAGE, + number=4, + message=gfa_database.Database.EncryptionConfig, + ) + tags: MutableMapping[str, str] = proto.MapField( + proto.STRING, + proto.STRING, + number=5, + ) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/firestore_admin_v1/types/operation.py b/google/cloud/firestore_admin_v1/types/operation.py index c58f24273..c50455693 100644 --- a/google/cloud/firestore_admin_v1/types/operation.py +++ b/google/cloud/firestore_admin_v1/types/operation.py @@ -20,6 +20,7 @@ import proto # type: ignore from google.cloud.firestore_admin_v1.types import index as gfa_index +from google.cloud.firestore_admin_v1.types import snapshot from google.protobuf import timestamp_pb2 # type: ignore @@ -34,6 +35,7 @@ "BulkDeleteDocumentsMetadata", "ExportDocumentsResponse", "RestoreDatabaseMetadata", + "CloneDatabaseMetadata", "Progress", }, ) @@ -558,6 +560,60 @@ class RestoreDatabaseMetadata(proto.Message): ) +class CloneDatabaseMetadata(proto.Message): + r"""Metadata for the [long-running + operation][google.longrunning.Operation] from the + [CloneDatabase][google.firestore.admin.v1.CloneDatabase] request. + + Attributes: + start_time (google.protobuf.timestamp_pb2.Timestamp): + The time the clone was started. + end_time (google.protobuf.timestamp_pb2.Timestamp): + The time the clone finished, unset for + ongoing clones. + operation_state (google.cloud.firestore_admin_v1.types.OperationState): + The operation state of the clone. + database (str): + The name of the database being cloned to. + pitr_snapshot (google.cloud.firestore_admin_v1.types.PitrSnapshot): + The snapshot from which this database was + cloned. + progress_percentage (google.cloud.firestore_admin_v1.types.Progress): + How far along the clone is as an estimated + percentage of remaining time. + """ + + start_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=1, + message=timestamp_pb2.Timestamp, + ) + end_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=2, + message=timestamp_pb2.Timestamp, + ) + operation_state: "OperationState" = proto.Field( + proto.ENUM, + number=3, + enum="OperationState", + ) + database: str = proto.Field( + proto.STRING, + number=4, + ) + pitr_snapshot: snapshot.PitrSnapshot = proto.Field( + proto.MESSAGE, + number=7, + message=snapshot.PitrSnapshot, + ) + progress_percentage: "Progress" = proto.Field( + proto.MESSAGE, + number=6, + message="Progress", + ) + + class Progress(proto.Message): r"""Describes the progress of the operation. Unit of work is generic and must be interpreted based on where diff --git a/google/cloud/firestore_admin_v1/types/snapshot.py b/google/cloud/firestore_admin_v1/types/snapshot.py new file mode 100644 index 000000000..e56a125f5 --- /dev/null +++ b/google/cloud/firestore_admin_v1/types/snapshot.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.protobuf import timestamp_pb2 # type: ignore + + +__protobuf__ = proto.module( + package="google.firestore.admin.v1", + manifest={ + "PitrSnapshot", + }, +) + + +class PitrSnapshot(proto.Message): + r"""A consistent snapshot of a database at a specific point in + time. A PITR (Point-in-time recovery) snapshot with previous + versions of a database's data is available for every minute up + to the associated database's data retention period. If the PITR + feature is enabled, the retention period is 7 days; otherwise, + it is one hour. + + Attributes: + database (str): + Required. The name of the database that this was a snapshot + of. Format: ``projects/{project}/databases/{database}``. + database_uid (bytes): + Output only. Public UUID of the database the + snapshot was associated with. + snapshot_time (google.protobuf.timestamp_pb2.Timestamp): + Required. Snapshot time of the database. + """ + + database: str = proto.Field( + proto.STRING, + number=1, + ) + database_uid: bytes = proto.Field( + proto.BYTES, + number=2, + ) + snapshot_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=3, + message=timestamp_pb2.Timestamp, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/firestore_bundle/__init__.py b/google/cloud/firestore_bundle/__init__.py index 1b6469437..30faafe58 100644 --- a/google/cloud/firestore_bundle/__init__.py +++ b/google/cloud/firestore_bundle/__init__.py @@ -15,8 +15,18 @@ # from google.cloud.firestore_bundle import gapic_version as package_version +import google.api_core as api_core +import sys + __version__ = package_version.__version__ +if sys.version_info >= (3, 8): # pragma: NO COVER + from importlib import metadata +else: # pragma: NO COVER + # TODO(https://github.com/googleapis/python-api-core/issues/835): Remove + # this code path once we drop support for Python 3.7 + import importlib_metadata as metadata + from .types.bundle import BundledDocumentMetadata from .types.bundle import BundledQuery @@ -26,6 +36,100 @@ from .bundle import FirestoreBundle +if hasattr(api_core, "check_python_version") and hasattr( + api_core, "check_dependency_versions" +): # pragma: NO COVER + api_core.check_python_version("google.cloud.bundle") # type: ignore + api_core.check_dependency_versions("google.cloud.bundle") # type: ignore +else: # pragma: NO COVER + # An older version of api_core is installed which does not define the + # functions above. We do equivalent checks manually. + try: + import warnings + import sys + + _py_version_str = sys.version.split()[0] + _package_label = "google.cloud.bundle" + if sys.version_info < (3, 9): + warnings.warn( + "You are using a non-supported Python version " + + f"({_py_version_str}). Google will not post any further " + + f"updates to {_package_label} supporting this Python version. " + + "Please upgrade to the latest Python version, or at " + + f"least to Python 3.9, and then update {_package_label}.", + FutureWarning, + ) + if sys.version_info[:2] == (3, 9): + warnings.warn( + f"You are using a Python version ({_py_version_str}) " + + f"which Google will stop supporting in {_package_label} in " + + "January 2026. Please " + + "upgrade to the latest Python version, or at " + + "least to Python 3.10, before then, and " + + f"then update {_package_label}.", + FutureWarning, + ) + + def parse_version_to_tuple(version_string: str): + """Safely converts a semantic version string to a comparable tuple of integers. + Example: "4.25.8" -> (4, 25, 8) + Ignores non-numeric parts and handles common version formats. + Args: + version_string: Version string in the format "x.y.z" or "x.y.z" + Returns: + Tuple of integers for the parsed version string. + """ + parts = [] + for part in version_string.split("."): + try: + parts.append(int(part)) + except ValueError: + # If it's a non-numeric part (e.g., '1.0.0b1' -> 'b1'), stop here. + # This is a simplification compared to 'packaging.parse_version', but sufficient + # for comparing strictly numeric semantic versions. + break + return tuple(parts) + + def _get_version(dependency_name): + try: + version_string: str = metadata.version(dependency_name) + parsed_version = parse_version_to_tuple(version_string) + return (parsed_version, version_string) + except Exception: + # Catch exceptions from metadata.version() (e.g., PackageNotFoundError) + # or errors during parse_version_to_tuple + return (None, "--") + + _dependency_package = "google.protobuf" + _next_supported_version = "4.25.8" + _next_supported_version_tuple = (4, 25, 8) + _recommendation = " (we recommend 6.x)" + (_version_used, _version_used_string) = _get_version(_dependency_package) + if _version_used and _version_used < _next_supported_version_tuple: + warnings.warn( + f"Package {_package_label} depends on " + + f"{_dependency_package}, currently installed at version " + + f"{_version_used_string}. Future updates to " + + f"{_package_label} will require {_dependency_package} at " + + f"version {_next_supported_version} or higher{_recommendation}." + + " Please ensure " + + "that either (a) your Python environment doesn't pin the " + + f"version of {_dependency_package}, so that updates to " + + f"{_package_label} can require the higher version, or " + + "(b) you manually update your Python environment to use at " + + f"least version {_next_supported_version} of " + + f"{_dependency_package}.", + FutureWarning, + ) + except Exception: + warnings.warn( + "Could not determine the version of Python " + + "currently being used. To continue receiving " + + "updates for {_package_label}, ensure you are " + + "using a supported version of Python; see " + + "https://devguide.python.org/versions/" + ) + __all__ = ( "BundleElement", "BundleMetadata", diff --git a/google/cloud/firestore_bundle/gapic_version.py b/google/cloud/firestore_bundle/gapic_version.py index e546bae05..ced4e0faf 100644 --- a/google/cloud/firestore_bundle/gapic_version.py +++ b/google/cloud/firestore_bundle/gapic_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.21.0" # {x-release-please-version} +__version__ = "2.22.0" # {x-release-please-version} diff --git a/google/cloud/firestore_v1/aggregation.py b/google/cloud/firestore_v1/aggregation.py index 4070cd22b..69c4dc6bd 100644 --- a/google/cloud/firestore_v1/aggregation.py +++ b/google/cloud/firestore_v1/aggregation.py @@ -67,8 +67,7 @@ def get( messages. Args: - transaction - (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]): + transaction (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]): An existing transaction that this query will run in. If a ``transaction`` is used and it already has write operations added, this method cannot be used (i.e. read-after-write is not @@ -77,8 +76,7 @@ def get( should be retried. Defaults to a system-specified policy. timeout (float): The timeout for this request. Defaults to a system-specified value. - explain_options - (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): + explain_options (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): Options to enable query profiling for this query. When set, explain_metrics will be available on the returned generator. read_time (Optional[datetime.datetime]): If set, reads documents as they were at the given @@ -155,16 +153,14 @@ def _make_stream( this method cannot be used (i.e. read-after-write is not allowed). Args: - transaction - (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]): + transaction (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]): An existing transaction that this query will run in. retry (Optional[google.api_core.retry.Retry]): Designation of what errors, if any, should be retried. Defaults to a system-specified policy. timeout (Optional[float]): The timeout for this request. Defaults to a system-specified value. - explain_options - (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): + explain_options (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): Options to enable query profiling for this query. When set, explain_metrics will be available on the returned generator. read_time (Optional[datetime.datetime]): If set, reads documents as they were at the given @@ -237,16 +233,14 @@ def stream( this method cannot be used (i.e. read-after-write is not allowed). Args: - transaction - (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]): + transaction (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]): An existing transaction that this query will run in. retry (Optional[google.api_core.retry.Retry]): Designation of what errors, if any, should be retried. Defaults to a system-specified policy. timeout (Optinal[float]): The timeout for this request. Defaults to a system-specified value. - explain_options - (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): + explain_options (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): Options to enable query profiling for this query. When set, explain_metrics will be available on the returned generator. read_time (Optional[datetime.datetime]): If set, reads documents as they were at the given diff --git a/google/cloud/firestore_v1/async_aggregation.py b/google/cloud/firestore_v1/async_aggregation.py index e273f514a..5825a06d8 100644 --- a/google/cloud/firestore_v1/async_aggregation.py +++ b/google/cloud/firestore_v1/async_aggregation.py @@ -63,8 +63,7 @@ async def get( This sends a ``RunAggregationQuery`` RPC and returns a list of aggregation results in the stream of ``RunAggregationQueryResponse`` messages. Args: - transaction - (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]): + transaction (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]): An existing transaction that this query will run in. If a ``transaction`` is used and it already has write operations added, this method cannot be used (i.e. read-after-write is not @@ -73,8 +72,7 @@ async def get( should be retried. Defaults to a system-specified policy. timeout (float): The timeout for this request. Defaults to a system-specified value. - explain_options - (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): + explain_options (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): Options to enable query profiling for this query. When set, explain_metrics will be available on the returned generator. read_time (Optional[datetime.datetime]): If set, reads documents as they were at the given @@ -134,8 +132,7 @@ async def _make_stream( system-specified policy. timeout (Optional[float]): The timeout for this request. Defaults to a system-specified value. - explain_options - (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): + explain_options (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): Options to enable query profiling for this query. When set, explain_metrics will be available on the returned generator. read_time (Optional[datetime.datetime]): If set, reads documents as they were at the given @@ -200,8 +197,7 @@ def stream( system-specified policy. timeout (Optional[float]): The timeout for this request. Defaults to a system-specified value. - explain_options - (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): + explain_options (Optional[:class:`~google.cloud.firestore_v1.query_profile.ExplainOptions`]): Options to enable query profiling for this query. When set, explain_metrics will be available on the returned generator. read_time (Optional[datetime.datetime]): If set, reads documents as they were at the given diff --git a/google/cloud/firestore_v1/async_batch.py b/google/cloud/firestore_v1/async_batch.py index 689753fe9..f74ccacea 100644 --- a/google/cloud/firestore_v1/async_batch.py +++ b/google/cloud/firestore_v1/async_batch.py @@ -19,6 +19,7 @@ from google.api_core import retry_async as retries from google.cloud.firestore_v1.base_batch import BaseWriteBatch +from google.cloud.firestore_v1.types.write import WriteResult class AsyncWriteBatch(BaseWriteBatch): @@ -40,7 +41,7 @@ async def commit( self, retry: retries.AsyncRetry | object | None = gapic_v1.method.DEFAULT, timeout: float | None = None, - ) -> list: + ) -> list[WriteResult]: """Commit the changes accumulated in this batch. Args: diff --git a/google/cloud/firestore_v1/async_client.py b/google/cloud/firestore_v1/async_client.py index 3acbedc76..efc4a47c0 100644 --- a/google/cloud/firestore_v1/async_client.py +++ b/google/cloud/firestore_v1/async_client.py @@ -25,7 +25,15 @@ """ from __future__ import annotations -from typing import TYPE_CHECKING, Any, AsyncGenerator, Iterable, List, Optional, Union +from typing import ( + TYPE_CHECKING, + Any, + AsyncGenerator, + Iterable, + List, + Optional, + Union, +) from google.api_core import gapic_v1 from google.api_core import retry_async as retries @@ -40,6 +48,7 @@ from google.cloud.firestore_v1.async_transaction import AsyncTransaction from google.cloud.firestore_v1.base_client import _parse_batch_get # type: ignore from google.cloud.firestore_v1.base_client import _CLIENT_INFO, BaseClient, _path_helper +from google.cloud.firestore_v1.base_transaction import MAX_ATTEMPTS from google.cloud.firestore_v1.field_path import FieldPath from google.cloud.firestore_v1.services.firestore import ( async_client as firestore_client, @@ -412,7 +421,9 @@ def batch(self) -> AsyncWriteBatch: """ return AsyncWriteBatch(self) - def transaction(self, **kwargs) -> AsyncTransaction: + def transaction( + self, max_attempts: int = MAX_ATTEMPTS, read_only: bool = False + ) -> AsyncTransaction: """Get a transaction that uses this client. See :class:`~google.cloud.firestore_v1.async_transaction.AsyncTransaction` for @@ -428,7 +439,7 @@ def transaction(self, **kwargs) -> AsyncTransaction: :class:`~google.cloud.firestore_v1.async_transaction.AsyncTransaction`: A transaction attached to this client. """ - return AsyncTransaction(self, **kwargs) + return AsyncTransaction(self, max_attempts=max_attempts, read_only=read_only) @property def _pipeline_cls(self): diff --git a/google/cloud/firestore_v1/async_collection.py b/google/cloud/firestore_v1/async_collection.py index cc99aa460..561111163 100644 --- a/google/cloud/firestore_v1/async_collection.py +++ b/google/cloud/firestore_v1/async_collection.py @@ -15,7 +15,7 @@ """Classes for representing collections for the Google Cloud Firestore API.""" from __future__ import annotations -from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Tuple +from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Tuple, cast from google.api_core import gapic_v1 from google.api_core import retry_async as retries @@ -153,7 +153,8 @@ def document(self, document_id: str | None = None) -> AsyncDocumentReference: :class:`~google.cloud.firestore_v1.document.async_document.AsyncDocumentReference`: The child document. """ - return super(AsyncCollectionReference, self).document(document_id) + doc = super(AsyncCollectionReference, self).document(document_id) + return cast("AsyncDocumentReference", doc) async def list_documents( self, diff --git a/google/cloud/firestore_v1/base_batch.py b/google/cloud/firestore_v1/base_batch.py index b0d50f1f4..851c7849f 100644 --- a/google/cloud/firestore_v1/base_batch.py +++ b/google/cloud/firestore_v1/base_batch.py @@ -15,7 +15,7 @@ """Helpers for batch requests to the Google Cloud Firestore API.""" from __future__ import annotations import abc -from typing import Dict, Union +from typing import Any, Dict, Union # Types needed only for Type Hints from google.api_core import retry as retries @@ -67,7 +67,9 @@ def commit(self): write depend on the implementing class.""" raise NotImplementedError() - def create(self, reference: BaseDocumentReference, document_data: dict) -> None: + def create( + self, reference: BaseDocumentReference, document_data: dict[str, Any] + ) -> None: """Add a "change" to this batch to create a document. If the document given by ``reference`` already exists, then this @@ -120,7 +122,7 @@ def set( def update( self, reference: BaseDocumentReference, - field_updates: dict, + field_updates: dict[str, Any], option: _helpers.WriteOption | None = None, ) -> None: """Add a "change" to update a document. diff --git a/google/cloud/firestore_v1/base_client.py b/google/cloud/firestore_v1/base_client.py index 8c8b9532d..4ba8a7c06 100644 --- a/google/cloud/firestore_v1/base_client.py +++ b/google/cloud/firestore_v1/base_client.py @@ -58,7 +58,7 @@ DocumentSnapshot, ) from google.cloud.firestore_v1.base_query import BaseQuery -from google.cloud.firestore_v1.base_transaction import BaseTransaction +from google.cloud.firestore_v1.base_transaction import MAX_ATTEMPTS, BaseTransaction from google.cloud.firestore_v1.bulk_writer import BulkWriter, BulkWriterOptions from google.cloud.firestore_v1.field_path import render_field_path from google.cloud.firestore_v1.services.firestore import client as firestore_client @@ -500,7 +500,9 @@ def collections( def batch(self) -> BaseWriteBatch: raise NotImplementedError - def transaction(self, **kwargs) -> BaseTransaction: + def transaction( + self, max_attempts: int = MAX_ATTEMPTS, read_only: bool = False + ) -> BaseTransaction: raise NotImplementedError def pipeline(self) -> PipelineSource: diff --git a/google/cloud/firestore_v1/base_collection.py b/google/cloud/firestore_v1/base_collection.py index 567fe4d8a..070e54cc4 100644 --- a/google/cloud/firestore_v1/base_collection.py +++ b/google/cloud/firestore_v1/base_collection.py @@ -35,6 +35,7 @@ from google.api_core import retry as retries from google.cloud.firestore_v1 import _helpers +from google.cloud.firestore_v1.base_document import BaseDocumentReference from google.cloud.firestore_v1.base_query import QueryType if TYPE_CHECKING: # pragma: NO COVER @@ -134,7 +135,7 @@ def _aggregation_query(self) -> BaseAggregationQuery: def _vector_query(self) -> BaseVectorQuery: raise NotImplementedError - def document(self, document_id: Optional[str] = None): + def document(self, document_id: Optional[str] = None) -> BaseDocumentReference: """Create a sub-document underneath the current collection. Args: diff --git a/google/cloud/firestore_v1/base_document.py b/google/cloud/firestore_v1/base_document.py index 517db20d3..fe6113bfc 100644 --- a/google/cloud/firestore_v1/base_document.py +++ b/google/cloud/firestore_v1/base_document.py @@ -418,7 +418,7 @@ def _client(self): return self._reference._client @property - def exists(self): + def exists(self) -> bool: """Existence flag. Indicates if the document existed at the time this snapshot @@ -430,7 +430,7 @@ def exists(self): return self._exists @property - def id(self): + def id(self) -> str: """The document identifier (within its collection). Returns: @@ -439,7 +439,7 @@ def id(self): return self._reference.id @property - def reference(self): + def reference(self) -> BaseDocumentReference: """Document reference corresponding to document that owns this data. Returns: diff --git a/google/cloud/firestore_v1/client.py b/google/cloud/firestore_v1/client.py index c23943b24..ba2ca176d 100644 --- a/google/cloud/firestore_v1/client.py +++ b/google/cloud/firestore_v1/client.py @@ -39,6 +39,7 @@ # Types needed only for Type Hints from google.cloud.firestore_v1.base_document import DocumentSnapshot +from google.cloud.firestore_v1.base_transaction import MAX_ATTEMPTS from google.cloud.firestore_v1.batch import WriteBatch from google.cloud.firestore_v1.collection import CollectionReference from google.cloud.firestore_v1.document import DocumentReference @@ -393,7 +394,9 @@ def batch(self) -> WriteBatch: """ return WriteBatch(self) - def transaction(self, **kwargs) -> Transaction: + def transaction( + self, max_attempts: int = MAX_ATTEMPTS, read_only: bool = False + ) -> Transaction: """Get a transaction that uses this client. See :class:`~google.cloud.firestore_v1.transaction.Transaction` for @@ -409,7 +412,7 @@ def transaction(self, **kwargs) -> Transaction: :class:`~google.cloud.firestore_v1.transaction.Transaction`: A transaction attached to this client. """ - return Transaction(self, **kwargs) + return Transaction(self, max_attempts=max_attempts, read_only=read_only) @property def _pipeline_cls(self): diff --git a/google/cloud/firestore_v1/document.py b/google/cloud/firestore_v1/document.py index 4e0132e49..4bb6399a7 100644 --- a/google/cloud/firestore_v1/document.py +++ b/google/cloud/firestore_v1/document.py @@ -169,7 +169,7 @@ def set( def update( self, - field_updates: dict, + field_updates: dict[str, Any], option: _helpers.WriteOption | None = None, retry: retries.Retry | object | None = gapic_v1.method.DEFAULT, timeout: float | None = None, diff --git a/google/cloud/firestore_v1/gapic_metadata.json b/google/cloud/firestore_v1/gapic_metadata.json index d0462f964..03a6e428b 100644 --- a/google/cloud/firestore_v1/gapic_metadata.json +++ b/google/cloud/firestore_v1/gapic_metadata.json @@ -40,6 +40,11 @@ "delete_document" ] }, + "ExecutePipeline": { + "methods": [ + "execute_pipeline" + ] + }, "GetDocument": { "methods": [ "get_document" @@ -125,6 +130,11 @@ "delete_document" ] }, + "ExecutePipeline": { + "methods": [ + "execute_pipeline" + ] + }, "GetDocument": { "methods": [ "get_document" @@ -210,6 +220,11 @@ "delete_document" ] }, + "ExecutePipeline": { + "methods": [ + "execute_pipeline" + ] + }, "GetDocument": { "methods": [ "get_document" diff --git a/google/cloud/firestore_v1/gapic_version.py b/google/cloud/firestore_v1/gapic_version.py index e546bae05..ced4e0faf 100644 --- a/google/cloud/firestore_v1/gapic_version.py +++ b/google/cloud/firestore_v1/gapic_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.21.0" # {x-release-please-version} +__version__ = "2.22.0" # {x-release-please-version} diff --git a/google/cloud/firestore_v1/services/firestore/client.py b/google/cloud/firestore_v1/services/firestore/client.py index 49ea18d2a..e362896af 100644 --- a/google/cloud/firestore_v1/services/firestore/client.py +++ b/google/cloud/firestore_v1/services/firestore/client.py @@ -169,6 +169,34 @@ def _get_default_mtls_endpoint(api_endpoint): _DEFAULT_ENDPOINT_TEMPLATE = "firestore.{UNIVERSE_DOMAIN}" _DEFAULT_UNIVERSE = "googleapis.com" + @staticmethod + def _use_client_cert_effective(): + """Returns whether client certificate should be used for mTLS if the + google-auth version supports should_use_client_cert automatic mTLS enablement. + + Alternatively, read from the GOOGLE_API_USE_CLIENT_CERTIFICATE env var. + + Returns: + bool: whether client certificate should be used for mTLS + Raises: + ValueError: (If using a version of google-auth without should_use_client_cert and + GOOGLE_API_USE_CLIENT_CERTIFICATE is set to an unexpected value.) + """ + # check if google-auth version supports should_use_client_cert for automatic mTLS enablement + if hasattr(mtls, "should_use_client_cert"): # pragma: NO COVER + return mtls.should_use_client_cert() + else: # pragma: NO COVER + # if unsupported, fallback to reading from env var + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + @classmethod def from_service_account_info(cls, info: dict, *args, **kwargs): """Creates an instance of this client using the provided credentials @@ -334,12 +362,8 @@ def get_mtls_endpoint_and_cert_source( ) if client_options is None: client_options = client_options_lib.ClientOptions() - use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_client_cert = FirestoreClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" @@ -347,7 +371,7 @@ def get_mtls_endpoint_and_cert_source( # Figure out the client cert source to use. client_cert_source = None - if use_client_cert == "true": + if use_client_cert: if client_options.client_cert_source: client_cert_source = client_options.client_cert_source elif mtls.has_default_client_cert_source(): @@ -379,20 +403,14 @@ def _read_environment_variables(): google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT is not any of ["auto", "never", "always"]. """ - use_client_cert = os.getenv( - "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" - ).lower() + use_client_cert = FirestoreClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + return use_client_cert, use_mtls_endpoint, universe_domain_env @staticmethod def _get_client_cert_source(provided_cert_source, use_cert_flag): diff --git a/google/cloud/firestore_v1/services/firestore/transports/base.py b/google/cloud/firestore_v1/services/firestore/transports/base.py index ffccd7f0d..905dded09 100644 --- a/google/cloud/firestore_v1/services/firestore/transports/base.py +++ b/google/cloud/firestore_v1/services/firestore/transports/base.py @@ -75,9 +75,10 @@ def __init__( credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. + This argument is mutually exclusive with credentials. This argument will be + removed in the next major version of this library. scopes (Optional[Sequence[str]]): A list of scopes. quota_project_id (Optional[str]): An optional project to use for billing and quota. @@ -292,7 +293,19 @@ def _prep_wrapped_messages(self, client_info): ), self.execute_pipeline: gapic_v1.method.wrap_method( self.execute_pipeline, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.InternalServerError, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=300.0, + ), + default_timeout=300.0, client_info=client_info, ), self.run_aggregation_query: gapic_v1.method.wrap_method( diff --git a/google/cloud/firestore_v1/services/firestore/transports/grpc.py b/google/cloud/firestore_v1/services/firestore/transports/grpc.py index 2a8f4caf9..f057d16e3 100644 --- a/google/cloud/firestore_v1/services/firestore/transports/grpc.py +++ b/google/cloud/firestore_v1/services/firestore/transports/grpc.py @@ -164,9 +164,10 @@ def __init__( are specified, the client will attempt to ascertain the credentials from the environment. This argument is ignored if a ``channel`` instance is provided. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if a ``channel`` instance is provided. + This argument will be removed in the next major version of this library. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if a ``channel`` instance is provided. channel (Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]]): @@ -299,9 +300,10 @@ def create_channel( credentials identify this application to the service. If none are specified, the client will attempt to ascertain the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. + This argument is mutually exclusive with credentials. This argument will be + removed in the next major version of this library. scopes (Optional[Sequence[str]]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. diff --git a/google/cloud/firestore_v1/services/firestore/transports/grpc_asyncio.py b/google/cloud/firestore_v1/services/firestore/transports/grpc_asyncio.py index 8801dc45a..cf6006672 100644 --- a/google/cloud/firestore_v1/services/firestore/transports/grpc_asyncio.py +++ b/google/cloud/firestore_v1/services/firestore/transports/grpc_asyncio.py @@ -161,8 +161,9 @@ def create_channel( credentials identify this application to the service. If none are specified, the client will attempt to ascertain the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. + credentials_file (Optional[str]): Deprecated. A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. This argument will be + removed in the next major version of this library. scopes (Optional[Sequence[str]]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. @@ -213,9 +214,10 @@ def __init__( are specified, the client will attempt to ascertain the credentials from the environment. This argument is ignored if a ``channel`` instance is provided. - credentials_file (Optional[str]): A file with credentials that can + credentials_file (Optional[str]): Deprecated. A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if a ``channel`` instance is provided. + This argument will be removed in the next major version of this library. scopes (Optional[Sequence[str]]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. @@ -992,7 +994,19 @@ def _prep_wrapped_messages(self, client_info): ), self.execute_pipeline: self._wrap_method( self.execute_pipeline, - default_timeout=None, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.InternalServerError, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=300.0, + ), + default_timeout=300.0, client_info=client_info, ), self.run_aggregation_query: self._wrap_method( diff --git a/google/cloud/firestore_v1/services/firestore/transports/rest.py b/google/cloud/firestore_v1/services/firestore/transports/rest.py index 121aa7386..07e2cdbca 100644 --- a/google/cloud/firestore_v1/services/firestore/transports/rest.py +++ b/google/cloud/firestore_v1/services/firestore/transports/rest.py @@ -1006,27 +1006,27 @@ def __init__( credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. - - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional(Sequence[str])): A list of scopes. This argument is - ignored if ``channel`` is provided. - client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client - certificate to configure mutual TLS HTTP channel. It is ignored - if ``channel`` is provided. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you are developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - url_scheme: the protocol scheme for the API endpoint. Normally - "https", but for testing or local servers, - "http" can be specified. + credentials_file (Optional[str]): Deprecated. A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. This argument will be + removed in the next major version of this library. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if ``channel`` is provided. + client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client + certificate to configure mutual TLS HTTP channel. It is ignored + if ``channel`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you are developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + url_scheme: the protocol scheme for the API endpoint. Normally + "https", but for testing or local servers, + "http" can be specified. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. @@ -1180,6 +1180,22 @@ def __call__( resp, _ = self._interceptor.post_batch_get_documents_with_metadata( resp, response_metadata ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + http_response = { + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.firestore_v1.FirestoreClient.batch_get_documents", + extra={ + "serviceName": "google.firestore.v1.Firestore", + "rpcName": "BatchGetDocuments", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) return resp class _BatchWrite(_BaseFirestoreRestTransport._BaseBatchWrite, FirestoreRestStub): @@ -2052,6 +2068,22 @@ def __call__( resp, _ = self._interceptor.post_execute_pipeline_with_metadata( resp, response_metadata ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + http_response = { + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.firestore_v1.FirestoreClient.execute_pipeline", + extra={ + "serviceName": "google.firestore.v1.Firestore", + "rpcName": "ExecutePipeline", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) return resp class _GetDocument(_BaseFirestoreRestTransport._BaseGetDocument, FirestoreRestStub): @@ -2934,6 +2966,22 @@ def __call__( resp, _ = self._interceptor.post_run_aggregation_query_with_metadata( resp, response_metadata ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + http_response = { + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.firestore_v1.FirestoreClient.run_aggregation_query", + extra={ + "serviceName": "google.firestore.v1.Firestore", + "rpcName": "RunAggregationQuery", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) return resp class _RunQuery(_BaseFirestoreRestTransport._BaseRunQuery, FirestoreRestStub): @@ -3064,6 +3112,22 @@ def __call__( resp, _ = self._interceptor.post_run_query_with_metadata( resp, response_metadata ) + if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor( + logging.DEBUG + ): # pragma: NO COVER + http_response = { + "headers": dict(response.headers), + "status": response.status_code, + } + _LOGGER.debug( + "Received response for google.firestore_v1.FirestoreClient.run_query", + extra={ + "serviceName": "google.firestore.v1.Firestore", + "rpcName": "RunQuery", + "metadata": http_response["headers"], + "httpResponse": http_response, + }, + ) return resp class _UpdateDocument( diff --git a/google/cloud/firestore_v1/services/firestore/transports/rest_base.py b/google/cloud/firestore_v1/services/firestore/transports/rest_base.py index 721f0792f..7d0c52f94 100644 --- a/google/cloud/firestore_v1/services/firestore/transports/rest_base.py +++ b/google/cloud/firestore_v1/services/firestore/transports/rest_base.py @@ -476,6 +476,63 @@ def _get_query_params_json(transcoded_request): return query_params + class _BaseExecutePipeline: + def __hash__(self): # pragma: NO COVER + return NotImplementedError("__hash__ must be implemented.") + + __REQUIRED_FIELDS_DEFAULT_VALUES: Dict[str, Any] = {} + + @classmethod + def _get_unset_required_fields(cls, message_dict): + return { + k: v + for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() + if k not in message_dict + } + + @staticmethod + def _get_http_options(): + http_options: List[Dict[str, str]] = [ + { + "method": "post", + "uri": "/v1/{database=projects/*/databases/*}/documents:executePipeline", + "body": "*", + }, + ] + return http_options + + @staticmethod + def _get_transcoded_request(http_options, request): + pb_request = firestore.ExecutePipelineRequest.pb(request) + transcoded_request = path_template.transcode(http_options, pb_request) + return transcoded_request + + @staticmethod + def _get_request_body_json(transcoded_request): + # Jsonify the request body + + body = json_format.MessageToJson( + transcoded_request["body"], use_integers_for_enums=True + ) + return body + + @staticmethod + def _get_query_params_json(transcoded_request): + query_params = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=True, + ) + ) + query_params.update( + _BaseFirestoreRestTransport._BaseExecutePipeline._get_unset_required_fields( + query_params + ) + ) + + query_params["$alt"] = "json;enum-encoding=int" + return query_params + class _BaseGetDocument: def __hash__(self): # pragma: NO COVER return NotImplementedError("__hash__ must be implemented.") diff --git a/google/cloud/firestore_v1/stream_generator.py b/google/cloud/firestore_v1/stream_generator.py index 7e39a3fba..b4b585601 100644 --- a/google/cloud/firestore_v1/stream_generator.py +++ b/google/cloud/firestore_v1/stream_generator.py @@ -50,7 +50,7 @@ def __init__( self._explain_options = explain_options self._explain_metrics = None - def __iter__(self) -> StreamGenerator: + def __iter__(self) -> StreamGenerator[T]: return self def __next__(self) -> T: diff --git a/google/cloud/firestore_v1/types/document.py b/google/cloud/firestore_v1/types/document.py index 1757571b1..8073ad97a 100644 --- a/google/cloud/firestore_v1/types/document.py +++ b/google/cloud/firestore_v1/types/document.py @@ -74,7 +74,7 @@ class Document(proto.Message): may contain any character. Some characters, including :literal:`\``, must be escaped using a ``\``. For example, :literal:`\`x&y\`` represents ``x&y`` and - :literal:`\`bak\`tik\`` represents :literal:`bak`tik`. + :literal:`\`bak\\`tik\`` represents :literal:`bak`tik`. create_time (google.protobuf.timestamp_pb2.Timestamp): Output only. The time at which the document was created. @@ -195,10 +195,10 @@ class Value(proto.Message): **Requires:** - - Must follow [field reference][FieldReference.field_path] - limitations. + - Must follow [field reference][FieldReference.field_path] + limitations. - - Not allowed to be used when writing documents. + - Not allowed to be used when writing documents. This field is a member of `oneof`_ ``value_type``. function_value (google.cloud.firestore_v1.types.Function): @@ -206,7 +206,7 @@ class Value(proto.Message): **Requires:** - - Not allowed to be used when writing documents. + - Not allowed to be used when writing documents. This field is a member of `oneof`_ ``value_type``. pipeline_value (google.cloud.firestore_v1.types.Pipeline): @@ -214,7 +214,7 @@ class Value(proto.Message): **Requires:** - - Not allowed to be used when writing documents. + - Not allowed to be used when writing documents. This field is a member of `oneof`_ ``value_type``. """ @@ -353,8 +353,8 @@ class Function(proto.Message): **Requires:** - - must be in snake case (lower case with underscore - separator). + - must be in snake case (lower case with underscore + separator). args (MutableSequence[google.cloud.firestore_v1.types.Value]): Optional. Ordered list of arguments the given function expects. @@ -417,8 +417,8 @@ class Stage(proto.Message): **Requires:** - - must be in snake case (lower case with underscore - separator). + - must be in snake case (lower case with underscore + separator). args (MutableSequence[google.cloud.firestore_v1.types.Value]): Optional. Ordered list of arguments the given stage expects. diff --git a/google/cloud/firestore_v1/types/explain_stats.py b/google/cloud/firestore_v1/types/explain_stats.py index 1fda228b6..b0f9421ba 100644 --- a/google/cloud/firestore_v1/types/explain_stats.py +++ b/google/cloud/firestore_v1/types/explain_stats.py @@ -31,16 +31,18 @@ class ExplainStats(proto.Message): - r"""Explain stats for an RPC request, includes both the optimized - plan and execution stats. + r"""Pipeline explain stats. + + Depending on the explain options in the original request, this + can contain the optimized plan and / or execution stats. Attributes: data (google.protobuf.any_pb2.Any): The format depends on the ``output_format`` options in the request. - The only option today is ``TEXT``, which is a - ``google.protobuf.StringValue``. + Currently there are two supported options: ``TEXT`` and + ``JSON``. Both supply a ``google.protobuf.StringValue``. """ data: any_pb2.Any = proto.Field( diff --git a/google/cloud/firestore_v1/types/firestore.py b/google/cloud/firestore_v1/types/firestore.py index f1753c92f..4e53ba313 100644 --- a/google/cloud/firestore_v1/types/firestore.py +++ b/google/cloud/firestore_v1/types/firestore.py @@ -932,21 +932,23 @@ class ExecutePipelineResponse(proto.Message): only a partial progress message is returned. The fields present in the returned documents are only those - that were explicitly requested in the pipeline, this include - those like [``__name__``][google.firestore.v1.Document.name] - & + that were explicitly requested in the pipeline, this + includes those like + [``__name__``][google.firestore.v1.Document.name] and [``__update_time__``][google.firestore.v1.Document.update_time]. This is explicitly a divergence from ``Firestore.RunQuery`` / ``Firestore.GetDocument`` RPCs which always return such fields even when they are not specified in the [``mask``][google.firestore.v1.DocumentMask]. execution_time (google.protobuf.timestamp_pb2.Timestamp): - The time at which the document(s) were read. + The time at which the results are valid. - This may be monotonically increasing; in this case, the - previous documents in the result stream are guaranteed not - to have changed between their ``execution_time`` and this - one. + This is a (not strictly) monotonically increasing value + across multiple responses in the same stream. The API + guarantees that all previously returned results are still + valid at the latest ``execution_time``. This allows the API + consumer to treat the query if it ran at the latest + ``execution_time`` returned. If the query returns no results, a response with ``execution_time`` and no ``results`` will be sent, and this @@ -954,9 +956,11 @@ class ExecutePipelineResponse(proto.Message): explain_stats (google.cloud.firestore_v1.types.ExplainStats): Query explain stats. - Contains all metadata related to pipeline - planning and execution, specific contents depend - on the supplied pipeline options. + This is present on the **last** response if the request + configured explain to run in 'analyze' or 'explain' mode in + the pipeline options. If the query does not return any + results, a response with ``explain_stats`` and no + ``results`` will still be sent. """ transaction: bytes = proto.Field( @@ -1162,8 +1166,8 @@ class PartitionQueryRequest(proto.Message): For example, two subsequent calls using a page_token may return: - - cursor B, cursor M, cursor Q - - cursor A, cursor U, cursor W + - cursor B, cursor M, cursor Q + - cursor A, cursor U, cursor W To obtain a complete result set ordered with respect to the results of the query supplied to PartitionQuery, the results @@ -1237,9 +1241,9 @@ class PartitionQueryResponse(proto.Message): cursors A and B, running the following three queries will return the entire result set of the original query: - - query, end_at A - - query, start_at A, end_at B - - query, start_at B + - query, end_at A + - query, start_at A, end_at B + - query, start_at B An empty result may indicate that the query has too few results to be partitioned, or that the query is not yet @@ -1561,9 +1565,9 @@ class Target(proto.Message): Note that if the client sends multiple ``AddTarget`` requests without an ID, the order of IDs returned in - ``TargetChage.target_ids`` are undefined. Therefore, clients - should provide a target ID instead of relying on the server - to assign one. + ``TargetChange.target_ids`` are undefined. Therefore, + clients should provide a target ID instead of relying on the + server to assign one. If ``target_id`` is non-zero, there must not be an existing active target on this stream with the same ID. diff --git a/google/cloud/firestore_v1/types/pipeline.py b/google/cloud/firestore_v1/types/pipeline.py index 29fbe884b..07688dda7 100644 --- a/google/cloud/firestore_v1/types/pipeline.py +++ b/google/cloud/firestore_v1/types/pipeline.py @@ -34,7 +34,7 @@ class StructuredPipeline(proto.Message): r"""A Firestore query represented as an ordered list of operations / stages. - This is considered the top-level function which plans & executes a + This is considered the top-level function which plans and executes a query. It is logically equivalent to ``query(stages, options)``, but prevents the client from having to build a function wrapper. diff --git a/google/cloud/firestore_v1/types/query.py b/google/cloud/firestore_v1/types/query.py index 9aa8977dd..d50742785 100644 --- a/google/cloud/firestore_v1/types/query.py +++ b/google/cloud/firestore_v1/types/query.py @@ -66,25 +66,25 @@ class StructuredQuery(proto.Message): Firestore guarantees a stable ordering through the following rules: - - The ``order_by`` is required to reference all fields used - with an inequality filter. - - All fields that are required to be in the ``order_by`` - but are not already present are appended in - lexicographical ordering of the field name. - - If an order on ``__name__`` is not specified, it is - appended by default. + - The ``order_by`` is required to reference all fields used + with an inequality filter. + - All fields that are required to be in the ``order_by`` but + are not already present are appended in lexicographical + ordering of the field name. + - If an order on ``__name__`` is not specified, it is + appended by default. Fields are appended with the same sort direction as the last order specified, or 'ASCENDING' if no order was specified. For example: - - ``ORDER BY a`` becomes ``ORDER BY a ASC, __name__ ASC`` - - ``ORDER BY a DESC`` becomes - ``ORDER BY a DESC, __name__ DESC`` - - ``WHERE a > 1`` becomes - ``WHERE a > 1 ORDER BY a ASC, __name__ ASC`` - - ``WHERE __name__ > ... AND a > 1`` becomes - ``WHERE __name__ > ... AND a > 1 ORDER BY a ASC, __name__ ASC`` + - ``ORDER BY a`` becomes ``ORDER BY a ASC, __name__ ASC`` + - ``ORDER BY a DESC`` becomes + ``ORDER BY a DESC, __name__ DESC`` + - ``WHERE a > 1`` becomes + ``WHERE a > 1 ORDER BY a ASC, __name__ ASC`` + - ``WHERE __name__ > ... AND a > 1`` becomes + ``WHERE __name__ > ... AND a > 1 ORDER BY a ASC, __name__ ASC`` start_at (google.cloud.firestore_v1.types.Cursor): A potential prefix of a position in the result set to start the query at. @@ -106,10 +106,10 @@ class StructuredQuery(proto.Message): Continuing off the example above, attaching the following start cursors will have varying impact: - - ``START BEFORE (2, /k/123)``: start the query right - before ``a = 1 AND b > 2 AND __name__ > /k/123``. - - ``START AFTER (10)``: start the query right after - ``a = 1 AND b > 10``. + - ``START BEFORE (2, /k/123)``: start the query right before + ``a = 1 AND b > 2 AND __name__ > /k/123``. + - ``START AFTER (10)``: start the query right after + ``a = 1 AND b > 10``. Unlike ``OFFSET`` which requires scanning over the first N results to skip, a start cursor allows the query to begin at @@ -119,8 +119,8 @@ class StructuredQuery(proto.Message): Requires: - - The number of values cannot be greater than the number of - fields specified in the ``ORDER BY`` clause. + - The number of values cannot be greater than the number of + fields specified in the ``ORDER BY`` clause. end_at (google.cloud.firestore_v1.types.Cursor): A potential prefix of a position in the result set to end the query at. @@ -130,8 +130,8 @@ class StructuredQuery(proto.Message): Requires: - - The number of values cannot be greater than the number of - fields specified in the ``ORDER BY`` clause. + - The number of values cannot be greater than the number of + fields specified in the ``ORDER BY`` clause. offset (int): The number of documents to skip before returning the first result. @@ -142,8 +142,8 @@ class StructuredQuery(proto.Message): Requires: - - The value must be greater than or equal to zero if - specified. + - The value must be greater than or equal to zero if + specified. limit (google.protobuf.wrappers_pb2.Int32Value): The maximum number of results to return. @@ -151,8 +151,8 @@ class StructuredQuery(proto.Message): Requires: - - The value must be greater than or equal to zero if - specified. + - The value must be greater than or equal to zero if + specified. find_nearest (google.cloud.firestore_v1.types.StructuredQuery.FindNearest): Optional. A potential nearest neighbors search. @@ -256,7 +256,7 @@ class CompositeFilter(proto.Message): Requires: - - At least one filter is present. + - At least one filter is present. """ class Operator(proto.Enum): @@ -310,27 +310,27 @@ class Operator(proto.Enum): Requires: - - That ``field`` come first in ``order_by``. + - That ``field`` come first in ``order_by``. LESS_THAN_OR_EQUAL (2): The given ``field`` is less than or equal to the given ``value``. Requires: - - That ``field`` come first in ``order_by``. + - That ``field`` come first in ``order_by``. GREATER_THAN (3): The given ``field`` is greater than the given ``value``. Requires: - - That ``field`` come first in ``order_by``. + - That ``field`` come first in ``order_by``. GREATER_THAN_OR_EQUAL (4): The given ``field`` is greater than or equal to the given ``value``. Requires: - - That ``field`` come first in ``order_by``. + - That ``field`` come first in ``order_by``. EQUAL (5): The given ``field`` is equal to the given ``value``. NOT_EQUAL (6): @@ -338,9 +338,9 @@ class Operator(proto.Enum): Requires: - - No other ``NOT_EQUAL``, ``NOT_IN``, ``IS_NOT_NULL``, or - ``IS_NOT_NAN``. - - That ``field`` comes first in the ``order_by``. + - No other ``NOT_EQUAL``, ``NOT_IN``, ``IS_NOT_NULL``, or + ``IS_NOT_NAN``. + - That ``field`` comes first in the ``order_by``. ARRAY_CONTAINS (7): The given ``field`` is an array that contains the given ``value``. @@ -350,31 +350,31 @@ class Operator(proto.Enum): Requires: - - That ``value`` is a non-empty ``ArrayValue``, subject to - disjunction limits. - - No ``NOT_IN`` filters in the same query. + - That ``value`` is a non-empty ``ArrayValue``, subject to + disjunction limits. + - No ``NOT_IN`` filters in the same query. ARRAY_CONTAINS_ANY (9): The given ``field`` is an array that contains any of the values in the given array. Requires: - - That ``value`` is a non-empty ``ArrayValue``, subject to - disjunction limits. - - No other ``ARRAY_CONTAINS_ANY`` filters within the same - disjunction. - - No ``NOT_IN`` filters in the same query. + - That ``value`` is a non-empty ``ArrayValue``, subject to + disjunction limits. + - No other ``ARRAY_CONTAINS_ANY`` filters within the same + disjunction. + - No ``NOT_IN`` filters in the same query. NOT_IN (10): The value of the ``field`` is not in the given array. Requires: - - That ``value`` is a non-empty ``ArrayValue`` with at most - 10 values. - - No other ``OR``, ``IN``, ``ARRAY_CONTAINS_ANY``, - ``NOT_IN``, ``NOT_EQUAL``, ``IS_NOT_NULL``, or - ``IS_NOT_NAN``. - - That ``field`` comes first in the ``order_by``. + - That ``value`` is a non-empty ``ArrayValue`` with at most + 10 values. + - No other ``OR``, ``IN``, ``ARRAY_CONTAINS_ANY``, + ``NOT_IN``, ``NOT_EQUAL``, ``IS_NOT_NULL``, or + ``IS_NOT_NAN``. + - That ``field`` comes first in the ``order_by``. """ OPERATOR_UNSPECIFIED = 0 LESS_THAN = 1 @@ -433,17 +433,17 @@ class Operator(proto.Enum): Requires: - - No other ``NOT_EQUAL``, ``NOT_IN``, ``IS_NOT_NULL``, or - ``IS_NOT_NAN``. - - That ``field`` comes first in the ``order_by``. + - No other ``NOT_EQUAL``, ``NOT_IN``, ``IS_NOT_NULL``, or + ``IS_NOT_NAN``. + - That ``field`` comes first in the ``order_by``. IS_NOT_NULL (5): The given ``field`` is not equal to ``NULL``. Requires: - - A single ``NOT_EQUAL``, ``NOT_IN``, ``IS_NOT_NULL``, or - ``IS_NOT_NAN``. - - That ``field`` comes first in the ``order_by``. + - A single ``NOT_EQUAL``, ``NOT_IN``, ``IS_NOT_NULL``, or + ``IS_NOT_NAN``. + - That ``field`` comes first in the ``order_by``. """ OPERATOR_UNSPECIFIED = 0 IS_NAN = 2 @@ -493,9 +493,9 @@ class FieldReference(proto.Message): Requires: - - MUST be a dot-delimited (``.``) string of segments, where - each segment conforms to [document field - name][google.firestore.v1.Document.fields] limitations. + - MUST be a dot-delimited (``.``) string of segments, where + each segment conforms to [document field + name][google.firestore.v1.Document.fields] limitations. """ field_path: str = proto.Field( @@ -555,9 +555,9 @@ class FindNearest(proto.Message): when the vectors are more similar, the comparison is inverted. - - For EUCLIDEAN, COSINE: WHERE distance <= - distance_threshold - - For DOT_PRODUCT: WHERE distance >= distance_threshold + - For EUCLIDEAN, COSINE: + ``WHERE distance <= distance_threshold`` + - For DOT_PRODUCT: ``WHERE distance >= distance_threshold`` """ class DistanceMeasure(proto.Enum): @@ -688,8 +688,8 @@ class StructuredAggregationQuery(proto.Message): Requires: - - A minimum of one and maximum of five aggregations per - query. + - A minimum of one and maximum of five aggregations per + query. """ class Aggregation(proto.Message): @@ -749,9 +749,9 @@ class Aggregation(proto.Message): Requires: - - Must be unique across all aggregation aliases. - - Conform to [document field - name][google.firestore.v1.Document.fields] limitations. + - Must be unique across all aggregation aliases. + - Conform to [document field + name][google.firestore.v1.Document.fields] limitations. """ class Count(proto.Message): @@ -778,7 +778,7 @@ class Count(proto.Message): Requires: - - Must be greater than zero when present. + - Must be greater than zero when present. """ up_to: wrappers_pb2.Int64Value = proto.Field( @@ -790,26 +790,26 @@ class Count(proto.Message): class Sum(proto.Message): r"""Sum of the values of the requested field. - - Only numeric values will be aggregated. All non-numeric values - including ``NULL`` are skipped. + - Only numeric values will be aggregated. All non-numeric values + including ``NULL`` are skipped. - - If the aggregated values contain ``NaN``, returns ``NaN``. - Infinity math follows IEEE-754 standards. + - If the aggregated values contain ``NaN``, returns ``NaN``. + Infinity math follows IEEE-754 standards. - - If the aggregated value set is empty, returns 0. + - If the aggregated value set is empty, returns 0. - - Returns a 64-bit integer if all aggregated numbers are integers - and the sum result does not overflow. Otherwise, the result is - returned as a double. Note that even if all the aggregated values - are integers, the result is returned as a double if it cannot fit - within a 64-bit signed integer. When this occurs, the returned - value will lose precision. + - Returns a 64-bit integer if all aggregated numbers are integers + and the sum result does not overflow. Otherwise, the result is + returned as a double. Note that even if all the aggregated values + are integers, the result is returned as a double if it cannot fit + within a 64-bit signed integer. When this occurs, the returned + value will lose precision. - - When underflow occurs, floating-point aggregation is - non-deterministic. This means that running the same query - repeatedly without any changes to the underlying values could - produce slightly different results each time. In those cases, - values should be stored as integers over floating-point numbers. + - When underflow occurs, floating-point aggregation is + non-deterministic. This means that running the same query + repeatedly without any changes to the underlying values could + produce slightly different results each time. In those cases, + values should be stored as integers over floating-point numbers. Attributes: field (google.cloud.firestore_v1.types.StructuredQuery.FieldReference): @@ -825,15 +825,15 @@ class Sum(proto.Message): class Avg(proto.Message): r"""Average of the values of the requested field. - - Only numeric values will be aggregated. All non-numeric values - including ``NULL`` are skipped. + - Only numeric values will be aggregated. All non-numeric values + including ``NULL`` are skipped. - - If the aggregated values contain ``NaN``, returns ``NaN``. - Infinity math follows IEEE-754 standards. + - If the aggregated values contain ``NaN``, returns ``NaN``. + Infinity math follows IEEE-754 standards. - - If the aggregated value set is empty, returns ``NULL``. + - If the aggregated value set is empty, returns ``NULL``. - - Always returns the result as a double. + - Always returns the result as a double. Attributes: field (google.cloud.firestore_v1.types.StructuredQuery.FieldReference): diff --git a/librarian.py b/librarian.py new file mode 100644 index 000000000..ec92a9345 --- /dev/null +++ b/librarian.py @@ -0,0 +1,118 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated parts of this library.""" +from pathlib import Path +from typing import List, Optional + +import synthtool as s +from synthtool import gcp +from synthtool.languages import python + +common = gcp.CommonTemplates() + +# This library ships clients for 3 different APIs, +# firestore, firestore_admin and firestore_bundle. +# firestore_bundle is not versioned +firestore_default_version = "v1" +firestore_admin_default_version = "v1" + +def update_fixup_scripts(path): + # Add message for missing 'libcst' dependency + s.replace( + library / "scripts" / path, + """import libcst as cst""", + """try: + import libcst as cst +except ImportError: + raise ImportError('Run `python -m pip install "libcst >= 0.2.5"` to install libcst.') + + + """, + ) + +for library in s.get_staging_dirs(default_version=firestore_default_version): + s.move(library / f"google/cloud/firestore_{library.name}", excludes=[f"__init__.py", "noxfile.py"]) + s.move(library / f"tests/", f"tests") + fixup_script_path = "fixup_firestore_v1_keywords.py" + update_fixup_scripts(fixup_script_path) + s.move(library / "scripts" / fixup_script_path) + +for library in s.get_staging_dirs(default_version=firestore_admin_default_version): + s.move(library / f"google/cloud/firestore_admin_{library.name}", excludes=[f"__init__.py", "noxfile.py"]) + s.move(library / f"tests", f"tests") + fixup_script_path = "fixup_firestore_admin_v1_keywords.py" + update_fixup_scripts(fixup_script_path) + s.move(library / "scripts" / fixup_script_path) + +for library in s.get_staging_dirs(): + s.replace( + library / "google/cloud/bundle/types/bundle.py", + "from google.firestore.v1 import document_pb2 # type: ignore\n" + "from google.firestore.v1 import query_pb2 # type: ignore", + "from google.cloud.firestore_v1.types import document as document_pb2 # type: ignore\n" + "from google.cloud.firestore_v1.types import query as query_pb2 # type: ignore" + ) + + s.replace( + library / "google/cloud/bundle/__init__.py", + "from .types.bundle import BundleMetadata\n" + "from .types.bundle import NamedQuery\n", + "from .types.bundle import BundleMetadata\n" + "from .types.bundle import NamedQuery\n" + "\n" + "from .bundle import FirestoreBundle\n", + ) + + s.replace( + library / "google/cloud/bundle/__init__.py", + "from google.cloud.bundle import gapic_version as package_version\n", + "from google.cloud.firestore_bundle import gapic_version as package_version\n", + ) + + s.replace( + library / "google/cloud/bundle/__init__.py", + "\'BundledQuery\',", + "\"BundledQuery\",\n\"FirestoreBundle\",",) + + s.move( + library / f"google/cloud/bundle", + f"google/cloud/firestore_bundle", + excludes=["noxfile.py"], + ) + s.move(library / f"tests", f"tests") + +s.remove_staging_dirs() + +# ---------------------------------------------------------------------------- +# Add templated files +# ---------------------------------------------------------------------------- +templated_files = common.py_library( + samples=False, # set to True only if there are samples + unit_test_external_dependencies=["aiounittest", "six", "freezegun"], + system_test_external_dependencies=["pytest-asyncio", "six"], + microgenerator=True, + cov_level=100, + split_system_tests=True, + default_python_version="3.14", + system_test_python_versions=["3.14"], + unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"], +) + +s.move(templated_files, + excludes=[".github/**", ".kokoro/**", "renovate.json"]) + +python.py_samples(skip_readmes=True) + +s.shell.run(["nox", "-s", "blacken"], hide_output=False) diff --git a/noxfile.py b/noxfile.py index 1e7039284..a1bce0822 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,6 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# DO NOT EDIT THIS FILE OUTSIDE OF `.librarian/generator-input` +# The source of truth for this file is `.librarian/generator-input` + + # Generated by synthtool. DO NOT EDIT! from __future__ import absolute_import @@ -33,7 +37,7 @@ ISORT_VERSION = "isort==5.11.0" LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] -DEFAULT_PYTHON_VERSION = "3.13" +DEFAULT_PYTHON_VERSION = "3.14" UNIT_TEST_PYTHON_VERSIONS: List[str] = [ "3.7", @@ -62,7 +66,7 @@ UNIT_TEST_EXTRAS: List[str] = [] UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.9", "3.14"] +SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.14"] SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ "mock", "pytest", @@ -475,7 +479,7 @@ def docfx(session): ) -@nox.session(python="3.13") +@nox.session(python=DEFAULT_PYTHON_VERSION) @nox.parametrize( "protobuf_implementation", ["python", "upb", "cpp"], diff --git a/scripts/fixup_firestore_admin_v1_keywords.py b/scripts/fixup_firestore_admin_v1_keywords.py deleted file mode 100644 index 0920ce408..000000000 --- a/scripts/fixup_firestore_admin_v1_keywords.py +++ /dev/null @@ -1,212 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -try: - import libcst as cst -except ImportError: - raise ImportError('Run `python -m pip install "libcst >= 0.2.5"` to install libcst.') - - - -import pathlib -import sys -from typing import (Any, Callable, Dict, List, Sequence, Tuple) - - -def partition( - predicate: Callable[[Any], bool], - iterator: Sequence[Any] -) -> Tuple[List[Any], List[Any]]: - """A stable, out-of-place partition.""" - results = ([], []) - - for i in iterator: - results[int(predicate(i))].append(i) - - # Returns trueList, falseList - return results[1], results[0] - - -class firestore_adminCallTransformer(cst.CSTTransformer): - CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') - METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'bulk_delete_documents': ('name', 'collection_ids', 'namespace_ids', ), - 'create_backup_schedule': ('parent', 'backup_schedule', ), - 'create_database': ('parent', 'database', 'database_id', ), - 'create_index': ('parent', 'index', ), - 'create_user_creds': ('parent', 'user_creds', 'user_creds_id', ), - 'delete_backup': ('name', ), - 'delete_backup_schedule': ('name', ), - 'delete_database': ('name', 'etag', ), - 'delete_index': ('name', ), - 'delete_user_creds': ('name', ), - 'disable_user_creds': ('name', ), - 'enable_user_creds': ('name', ), - 'export_documents': ('name', 'collection_ids', 'output_uri_prefix', 'namespace_ids', 'snapshot_time', ), - 'get_backup': ('name', ), - 'get_backup_schedule': ('name', ), - 'get_database': ('name', ), - 'get_field': ('name', ), - 'get_index': ('name', ), - 'get_user_creds': ('name', ), - 'import_documents': ('name', 'collection_ids', 'input_uri_prefix', 'namespace_ids', ), - 'list_backups': ('parent', 'filter', ), - 'list_backup_schedules': ('parent', ), - 'list_databases': ('parent', 'show_deleted', ), - 'list_fields': ('parent', 'filter', 'page_size', 'page_token', ), - 'list_indexes': ('parent', 'filter', 'page_size', 'page_token', ), - 'list_user_creds': ('parent', ), - 'reset_user_password': ('name', ), - 'restore_database': ('parent', 'database_id', 'backup', 'encryption_config', 'tags', ), - 'update_backup_schedule': ('backup_schedule', 'update_mask', ), - 'update_database': ('database', 'update_mask', ), - 'update_field': ('field', 'update_mask', ), - } - - def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: - try: - key = original.func.attr.value - kword_params = self.METHOD_TO_PARAMS[key] - except (AttributeError, KeyError): - # Either not a method from the API or too convoluted to be sure. - return updated - - # If the existing code is valid, keyword args come after positional args. - # Therefore, all positional args must map to the first parameters. - args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) - if any(k.keyword.value == "request" for k in kwargs): - # We've already fixed this file, don't fix it again. - return updated - - kwargs, ctrl_kwargs = partition( - lambda a: a.keyword.value not in self.CTRL_PARAMS, - kwargs - ) - - args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] - ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) - for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) - - request_arg = cst.Arg( - value=cst.Dict([ - cst.DictElement( - cst.SimpleString("'{}'".format(name)), -cst.Element(value=arg.value) - ) - # Note: the args + kwargs looks silly, but keep in mind that - # the control parameters had to be stripped out, and that - # those could have been passed positionally or by keyword. - for name, arg in zip(kword_params, args + kwargs)]), - keyword=cst.Name("request") - ) - - return updated.with_changes( - args=[request_arg] + ctrl_kwargs - ) - - -def fix_files( - in_dir: pathlib.Path, - out_dir: pathlib.Path, - *, - transformer=firestore_adminCallTransformer(), -): - """Duplicate the input dir to the output dir, fixing file method calls. - - Preconditions: - * in_dir is a real directory - * out_dir is a real, empty directory - """ - pyfile_gen = ( - pathlib.Path(os.path.join(root, f)) - for root, _, files in os.walk(in_dir) - for f in files if os.path.splitext(f)[1] == ".py" - ) - - for fpath in pyfile_gen: - with open(fpath, 'r') as f: - src = f.read() - - # Parse the code and insert method call fixes. - tree = cst.parse_module(src) - updated = tree.visit(transformer) - - # Create the path and directory structure for the new file. - updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) - updated_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate the updated source file at the corresponding path. - with open(updated_path, 'w') as f: - f.write(updated.code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="""Fix up source that uses the firestore_admin client library. - -The existing sources are NOT overwritten but are copied to output_dir with changes made. - -Note: This tool operates at a best-effort level at converting positional - parameters in client method calls to keyword based parameters. - Cases where it WILL FAIL include - A) * or ** expansion in a method call. - B) Calls via function or method alias (includes free function calls) - C) Indirect or dispatched calls (e.g. the method is looked up dynamically) - - These all constitute false negatives. The tool will also detect false - positives when an API method shares a name with another method. -""") - parser.add_argument( - '-d', - '--input-directory', - required=True, - dest='input_dir', - help='the input directory to walk for python files to fix up', - ) - parser.add_argument( - '-o', - '--output-directory', - required=True, - dest='output_dir', - help='the directory to output files fixed via un-flattening', - ) - args = parser.parse_args() - input_dir = pathlib.Path(args.input_dir) - output_dir = pathlib.Path(args.output_dir) - if not input_dir.is_dir(): - print( - f"input directory '{input_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if not output_dir.is_dir(): - print( - f"output directory '{output_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if os.listdir(output_dir): - print( - f"output directory '{output_dir}' is not empty", - file=sys.stderr, - ) - sys.exit(-1) - - fix_files(input_dir, output_dir) diff --git a/scripts/fixup_firestore_v1_keywords.py b/scripts/fixup_firestore_v1_keywords.py deleted file mode 100644 index 6481e76bb..000000000 --- a/scripts/fixup_firestore_v1_keywords.py +++ /dev/null @@ -1,197 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -try: - import libcst as cst -except ImportError: - raise ImportError('Run `python -m pip install "libcst >= 0.2.5"` to install libcst.') - - - -import pathlib -import sys -from typing import (Any, Callable, Dict, List, Sequence, Tuple) - - -def partition( - predicate: Callable[[Any], bool], - iterator: Sequence[Any] -) -> Tuple[List[Any], List[Any]]: - """A stable, out-of-place partition.""" - results = ([], []) - - for i in iterator: - results[int(predicate(i))].append(i) - - # Returns trueList, falseList - return results[1], results[0] - - -class firestoreCallTransformer(cst.CSTTransformer): - CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') - METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'batch_get_documents': ('database', 'documents', 'mask', 'transaction', 'new_transaction', 'read_time', ), - 'batch_write': ('database', 'writes', 'labels', ), - 'begin_transaction': ('database', 'options', ), - 'commit': ('database', 'writes', 'transaction', ), - 'create_document': ('parent', 'collection_id', 'document', 'document_id', 'mask', ), - 'delete_document': ('name', 'current_document', ), - 'get_document': ('name', 'mask', 'transaction', 'read_time', ), - 'list_collection_ids': ('parent', 'page_size', 'page_token', 'read_time', ), - 'list_documents': ('parent', 'collection_id', 'page_size', 'page_token', 'order_by', 'mask', 'transaction', 'read_time', 'show_missing', ), - 'listen': ('database', 'add_target', 'remove_target', 'labels', ), - 'partition_query': ('parent', 'structured_query', 'partition_count', 'page_token', 'page_size', 'read_time', ), - 'rollback': ('database', 'transaction', ), - 'run_aggregation_query': ('parent', 'structured_aggregation_query', 'transaction', 'new_transaction', 'read_time', 'explain_options', ), - 'run_query': ('parent', 'structured_query', 'transaction', 'new_transaction', 'read_time', 'explain_options', ), - 'update_document': ('document', 'update_mask', 'mask', 'current_document', ), - 'write': ('database', 'stream_id', 'writes', 'stream_token', 'labels', ), - } - - def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: - try: - key = original.func.attr.value - kword_params = self.METHOD_TO_PARAMS[key] - except (AttributeError, KeyError): - # Either not a method from the API or too convoluted to be sure. - return updated - - # If the existing code is valid, keyword args come after positional args. - # Therefore, all positional args must map to the first parameters. - args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) - if any(k.keyword.value == "request" for k in kwargs): - # We've already fixed this file, don't fix it again. - return updated - - kwargs, ctrl_kwargs = partition( - lambda a: a.keyword.value not in self.CTRL_PARAMS, - kwargs - ) - - args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] - ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) - for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) - - request_arg = cst.Arg( - value=cst.Dict([ - cst.DictElement( - cst.SimpleString("'{}'".format(name)), -cst.Element(value=arg.value) - ) - # Note: the args + kwargs looks silly, but keep in mind that - # the control parameters had to be stripped out, and that - # those could have been passed positionally or by keyword. - for name, arg in zip(kword_params, args + kwargs)]), - keyword=cst.Name("request") - ) - - return updated.with_changes( - args=[request_arg] + ctrl_kwargs - ) - - -def fix_files( - in_dir: pathlib.Path, - out_dir: pathlib.Path, - *, - transformer=firestoreCallTransformer(), -): - """Duplicate the input dir to the output dir, fixing file method calls. - - Preconditions: - * in_dir is a real directory - * out_dir is a real, empty directory - """ - pyfile_gen = ( - pathlib.Path(os.path.join(root, f)) - for root, _, files in os.walk(in_dir) - for f in files if os.path.splitext(f)[1] == ".py" - ) - - for fpath in pyfile_gen: - with open(fpath, 'r') as f: - src = f.read() - - # Parse the code and insert method call fixes. - tree = cst.parse_module(src) - updated = tree.visit(transformer) - - # Create the path and directory structure for the new file. - updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) - updated_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate the updated source file at the corresponding path. - with open(updated_path, 'w') as f: - f.write(updated.code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="""Fix up source that uses the firestore client library. - -The existing sources are NOT overwritten but are copied to output_dir with changes made. - -Note: This tool operates at a best-effort level at converting positional - parameters in client method calls to keyword based parameters. - Cases where it WILL FAIL include - A) * or ** expansion in a method call. - B) Calls via function or method alias (includes free function calls) - C) Indirect or dispatched calls (e.g. the method is looked up dynamically) - - These all constitute false negatives. The tool will also detect false - positives when an API method shares a name with another method. -""") - parser.add_argument( - '-d', - '--input-directory', - required=True, - dest='input_dir', - help='the input directory to walk for python files to fix up', - ) - parser.add_argument( - '-o', - '--output-directory', - required=True, - dest='output_dir', - help='the directory to output files fixed via un-flattening', - ) - args = parser.parse_args() - input_dir = pathlib.Path(args.input_dir) - output_dir = pathlib.Path(args.output_dir) - if not input_dir.is_dir(): - print( - f"input directory '{input_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if not output_dir.is_dir(): - print( - f"output directory '{output_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if os.listdir(output_dir): - print( - f"output directory '{output_dir}' is not empty", - file=sys.stderr, - ) - sys.exit(-1) - - fix_files(input_dir, output_dir) diff --git a/setup.py b/setup.py index 8625abce9..72a6f53bd 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# DO NOT EDIT THIS FILE OUTSIDE OF `.librarian/generator-input` +# The source of truth for this file is `.librarian/generator-input` + + import io import os @@ -90,10 +94,6 @@ install_requires=dependencies, extras_require=extras, python_requires=">=3.7", - scripts=[ - "scripts/fixup_firestore_v1_keywords.py", - "scripts/fixup_firestore_admin_v1_keywords.py", - ], include_package_data=True, zip_safe=False, ) diff --git a/tests/unit/gapic/firestore_admin_v1/test_firestore_admin.py b/tests/unit/gapic/firestore_admin_v1/test_firestore_admin.py index db8276d57..7ef138d15 100644 --- a/tests/unit/gapic/firestore_admin_v1/test_firestore_admin.py +++ b/tests/unit/gapic/firestore_admin_v1/test_firestore_admin.py @@ -75,6 +75,7 @@ from google.cloud.firestore_admin_v1.types import index as gfa_index from google.cloud.firestore_admin_v1.types import operation as gfa_operation from google.cloud.firestore_admin_v1.types import schedule +from google.cloud.firestore_admin_v1.types import snapshot from google.cloud.firestore_admin_v1.types import user_creds from google.cloud.firestore_admin_v1.types import user_creds as gfa_user_creds from google.cloud.location import locations_pb2 @@ -185,12 +186,19 @@ def test__read_environment_variables(): with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): - with pytest.raises(ValueError) as excinfo: - FirestoreAdminClient._read_environment_variables() - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with pytest.raises(ValueError) as excinfo: + FirestoreAdminClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + else: + assert FirestoreAdminClient._read_environment_variables() == ( + False, + "auto", + None, + ) with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): assert FirestoreAdminClient._read_environment_variables() == ( @@ -229,6 +237,105 @@ def test__read_environment_variables(): ) +def test_use_client_cert_effective(): + # Test case 1: Test when `should_use_client_cert` returns True. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=True + ): + assert FirestoreAdminClient._use_client_cert_effective() is True + + # Test case 2: Test when `should_use_client_cert` returns False. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should NOT be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=False + ): + assert FirestoreAdminClient._use_client_cert_effective() is False + + # Test case 3: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "true". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert FirestoreAdminClient._use_client_cert_effective() is True + + # Test case 4: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert FirestoreAdminClient._use_client_cert_effective() is False + + # Test case 5: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "True". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "True"}): + assert FirestoreAdminClient._use_client_cert_effective() is True + + # Test case 6: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "False". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "False"} + ): + assert FirestoreAdminClient._use_client_cert_effective() is False + + # Test case 7: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "TRUE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "TRUE"}): + assert FirestoreAdminClient._use_client_cert_effective() is True + + # Test case 8: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "FALSE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "FALSE"} + ): + assert FirestoreAdminClient._use_client_cert_effective() is False + + # Test case 9: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not set. + # In this case, the method should return False, which is the default value. + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, clear=True): + assert FirestoreAdminClient._use_client_cert_effective() is False + + # Test case 10: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should raise a ValueError as the environment variable must be either + # "true" or "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + with pytest.raises(ValueError): + FirestoreAdminClient._use_client_cert_effective() + + # Test case 11: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should return False as the environment variable is set to an invalid value. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + assert FirestoreAdminClient._use_client_cert_effective() is False + + # Test case 12: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is unset. Also, + # the GOOGLE_API_CONFIG environment variable is unset. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": ""}): + with mock.patch.dict(os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": ""}): + assert FirestoreAdminClient._use_client_cert_effective() is False + + def test__get_client_cert_source(): mock_provided_cert_source = mock.Mock() mock_default_cert_source = mock.Mock() @@ -594,17 +701,6 @@ def test_firestore_admin_client_client_options( == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client = client_class(transport=transport_name) - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: @@ -820,6 +916,119 @@ def test_firestore_admin_client_get_mtls_endpoint_and_cert_source(client_class): assert api_endpoint == mock_api_endpoint assert cert_source is None + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "Unsupported". + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset. + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", None) + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset(empty). + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() @@ -870,18 +1079,6 @@ def test_firestore_admin_client_get_mtls_endpoint_and_cert_source(client_class): == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client_class.get_mtls_endpoint_and_cert_source() - - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - @pytest.mark.parametrize( "client_class", [FirestoreAdminClient, FirestoreAdminAsyncClient] @@ -11859,6 +12056,192 @@ async def test_delete_backup_schedule_flattened_error_async(): ) +@pytest.mark.parametrize( + "request_type", + [ + firestore_admin.CloneDatabaseRequest, + dict, + ], +) +def test_clone_database(request_type, transport: str = "grpc"): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = operations_pb2.Operation(name="operations/spam") + response = client.clone_database(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = firestore_admin.CloneDatabaseRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, future.Future) + + +def test_clone_database_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = firestore_admin.CloneDatabaseRequest( + parent="parent_value", + database_id="database_id_value", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + call.return_value.name = ( + "foo" # operation_request.operation in compute client(s) expect a string. + ) + client.clone_database(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == firestore_admin.CloneDatabaseRequest( + parent="parent_value", + database_id="database_id_value", + ) + + +def test_clone_database_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.clone_database in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = ( + "foo" # operation_request.operation in compute client(s) expect a string. + ) + client._transport._wrapped_methods[client._transport.clone_database] = mock_rpc + request = {} + client.clone_database(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + # Operation methods call wrapper_fn to build a cached + # client._transport.operations_client instance on first rpc call. + # Subsequent calls should use the cached wrapper + wrapper_fn.reset_mock() + + client.clone_database(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + + +@pytest.mark.asyncio +async def test_clone_database_async_use_cached_wrapped_rpc( + transport: str = "grpc_asyncio", +): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = FirestoreAdminAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert ( + client._client._transport.clone_database + in client._client._transport._wrapped_methods + ) + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[ + client._client._transport.clone_database + ] = mock_rpc + + request = {} + await client.clone_database(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + # Operation methods call wrapper_fn to build a cached + # client._transport.operations_client instance on first rpc call. + # Subsequent calls should use the cached wrapper + wrapper_fn.reset_mock() + + await client.clone_database(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + + +@pytest.mark.asyncio +async def test_clone_database_async( + transport: str = "grpc_asyncio", request_type=firestore_admin.CloneDatabaseRequest +): + client = FirestoreAdminAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/spam") + ) + response = await client.clone_database(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = firestore_admin.CloneDatabaseRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, future.Future) + + +@pytest.mark.asyncio +async def test_clone_database_async_from_dict(): + await test_clone_database_async(request_type=dict) + + def test_create_index_rest_use_cached_wrapped_rpc(): # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, # instead of constructing them on each call @@ -17625,31 +18008,166 @@ def test_delete_backup_schedule_rest_flattened_error(transport: str = "rest"): ) -def test_credentials_transport_error(): - # It is an error to provide credentials and a transport instance. - transport = transports.FirestoreAdminGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): +def test_clone_database_rest_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: client = FirestoreAdminClient( credentials=ga_credentials.AnonymousCredentials(), - transport=transport, + transport="rest", ) - # It is an error to provide a credentials file and a transport instance. - transport = transports.FirestoreAdminGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = FirestoreAdminClient( - client_options={"credentials_file": "credentials.json"}, - transport=transport, + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.clone_database in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = ( + "foo" # operation_request.operation in compute client(s) expect a string. ) + client._transport._wrapped_methods[client._transport.clone_database] = mock_rpc - # It is an error to provide an api_key and a transport instance. - transport = transports.FirestoreAdminGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) + request = {} + client.clone_database(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + # Operation methods build a cached wrapper on first rpc call + # subsequent calls should use the cached wrapper + wrapper_fn.reset_mock() + + client.clone_database(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + + +def test_clone_database_rest_required_fields( + request_type=firestore_admin.CloneDatabaseRequest, +): + transport_class = transports.FirestoreAdminRestTransport + + request_init = {} + request_init["parent"] = "" + request_init["database_id"] = "" + request = request_type(**request_init) + pb_request = request_type.pb(request) + jsonified_request = json.loads( + json_format.MessageToJson(pb_request, use_integers_for_enums=False) + ) + + # verify fields with default values are dropped + + unset_fields = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ).clone_database._get_unset_required_fields(jsonified_request) + jsonified_request.update(unset_fields) + + # verify required fields with default values are now present + + jsonified_request["parent"] = "parent_value" + jsonified_request["databaseId"] = "database_id_value" + + unset_fields = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ).clone_database._get_unset_required_fields(jsonified_request) + jsonified_request.update(unset_fields) + + # verify required fields with non-default values are left alone + assert "parent" in jsonified_request + assert jsonified_request["parent"] == "parent_value" + assert "databaseId" in jsonified_request + assert jsonified_request["databaseId"] == "database_id_value" + + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="rest", + ) + request = request_type(**request_init) + + # Designate an appropriate value for the returned response. + return_value = operations_pb2.Operation(name="operations/spam") + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # We need to mock transcode() because providing default values + # for required fields will fail the real version if the http_options + # expect actual values for those fields. + with mock.patch.object(path_template, "transcode") as transcode: + # A uri without fields and an empty body will force all the + # request fields to show up in the query_params. + pb_request = request_type.pb(request) + transcode_result = { + "uri": "v1/sample_method", + "method": "post", + "query_params": pb_request, + } + transcode_result["body"] = pb_request + transcode.return_value = transcode_result + + response_value = Response() + response_value.status_code = 200 + json_return_value = json_format.MessageToJson(return_value) + + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} + + response = client.clone_database(request) + + expected_params = [("$alt", "json;enum-encoding=int")] + actual_params = req.call_args.kwargs["params"] + assert expected_params == actual_params + + +def test_clone_database_rest_unset_required_fields(): + transport = transports.FirestoreAdminRestTransport( + credentials=ga_credentials.AnonymousCredentials + ) + + unset_fields = transport.clone_database._get_unset_required_fields({}) + assert set(unset_fields) == ( + set(()) + & set( + ( + "parent", + "databaseId", + "pitrSnapshot", + ) + ) + ) + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.FirestoreAdminGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.FirestoreAdminGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = FirestoreAdminClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide an api_key and a transport instance. + transport = transports.FirestoreAdminGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) options = client_options.ClientOptions() options.api_key = "api_key" with pytest.raises(ValueError): @@ -18404,6 +18922,91 @@ def test_delete_backup_schedule_empty_call_grpc(): assert args[0] == request_msg +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_clone_database_empty_call_grpc(): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + call.return_value = operations_pb2.Operation(name="operations/op") + client.clone_database(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest() + + assert args[0] == request_msg + + +def test_clone_database_routing_parameters_request_1_grpc(): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + call.return_value = operations_pb2.Operation(name="operations/op") + client.clone_database( + request={"pitr_snapshot": {"database": "projects/sample1/sample2"}} + ) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, kw = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest( + **{"pitr_snapshot": {"database": "projects/sample1/sample2"}} + ) + + assert args[0] == request_msg + + expected_headers = {"project_id": "sample1"} + assert ( + gapic_v1.routing_header.to_grpc_metadata(expected_headers) in kw["metadata"] + ) + + +def test_clone_database_routing_parameters_request_2_grpc(): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + call.return_value = operations_pb2.Operation(name="operations/op") + client.clone_database( + request={ + "pitr_snapshot": { + "database": "projects/sample1/databases/sample2/sample3" + } + } + ) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, kw = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest( + **{ + "pitr_snapshot": { + "database": "projects/sample1/databases/sample2/sample3" + } + } + ) + + assert args[0] == request_msg + + expected_headers = {"project_id": "sample1", "database_id": "sample2"} + assert ( + gapic_v1.routing_header.to_grpc_metadata(expected_headers) in kw["metadata"] + ) + + def test_transport_kind_grpc_asyncio(): transport = FirestoreAdminAsyncClient.get_transport_class("grpc_asyncio")( credentials=async_anonymous_credentials() @@ -19270,6 +19873,103 @@ async def test_delete_backup_schedule_empty_call_grpc_asyncio(): assert args[0] == request_msg +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_clone_database_empty_call_grpc_asyncio(): + client = FirestoreAdminAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/spam") + ) + await client.clone_database(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest() + + assert args[0] == request_msg + + +@pytest.mark.asyncio +async def test_clone_database_routing_parameters_request_1_grpc_asyncio(): + client = FirestoreAdminAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/spam") + ) + await client.clone_database( + request={"pitr_snapshot": {"database": "projects/sample1/sample2"}} + ) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, kw = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest( + **{"pitr_snapshot": {"database": "projects/sample1/sample2"}} + ) + + assert args[0] == request_msg + + expected_headers = {"project_id": "sample1"} + assert ( + gapic_v1.routing_header.to_grpc_metadata(expected_headers) in kw["metadata"] + ) + + +@pytest.mark.asyncio +async def test_clone_database_routing_parameters_request_2_grpc_asyncio(): + client = FirestoreAdminAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/spam") + ) + await client.clone_database( + request={ + "pitr_snapshot": { + "database": "projects/sample1/databases/sample2/sample3" + } + } + ) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, kw = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest( + **{ + "pitr_snapshot": { + "database": "projects/sample1/databases/sample2/sample3" + } + } + ) + + assert args[0] == request_msg + + expected_headers = {"project_id": "sample1", "database_id": "sample2"} + assert ( + gapic_v1.routing_header.to_grpc_metadata(expected_headers) in kw["metadata"] + ) + + def test_transport_kind_rest(): transport = FirestoreAdminClient.get_transport_class("rest")( credentials=ga_credentials.AnonymousCredentials() @@ -23839,6 +24539,129 @@ def test_delete_backup_schedule_rest_interceptors(null_interceptor): pre.assert_called_once() +def test_clone_database_rest_bad_request( + request_type=firestore_admin.CloneDatabaseRequest, +): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="rest" + ) + # send a request that will satisfy transcoding + request_init = {"parent": "projects/sample1"} + request = request_type(**request_init) + + # Mock the http request call within the method and fake a BadRequest error. + with mock.patch.object(Session, "request") as req, pytest.raises( + core_exceptions.BadRequest + ): + # Wrap the value into a proper Response obj + response_value = mock.Mock() + json_return_value = "" + response_value.json = mock.Mock(return_value={}) + response_value.status_code = 400 + response_value.request = mock.Mock() + req.return_value = response_value + req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} + client.clone_database(request) + + +@pytest.mark.parametrize( + "request_type", + [ + firestore_admin.CloneDatabaseRequest, + dict, + ], +) +def test_clone_database_rest_call_success(request_type): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="rest" + ) + + # send a request that will satisfy transcoding + request_init = {"parent": "projects/sample1"} + request = request_type(**request_init) + + # Mock the http request call within the method and fake a response. + with mock.patch.object(type(client.transport._session), "request") as req: + # Designate an appropriate value for the returned response. + return_value = operations_pb2.Operation(name="operations/spam") + + # Wrap the value into a proper Response obj + response_value = mock.Mock() + response_value.status_code = 200 + json_return_value = json_format.MessageToJson(return_value) + response_value.content = json_return_value.encode("UTF-8") + req.return_value = response_value + req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} + response = client.clone_database(request) + + # Establish that the response is the type that we expect. + json_return_value = json_format.MessageToJson(return_value) + + +@pytest.mark.parametrize("null_interceptor", [True, False]) +def test_clone_database_rest_interceptors(null_interceptor): + transport = transports.FirestoreAdminRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + interceptor=None + if null_interceptor + else transports.FirestoreAdminRestInterceptor(), + ) + client = FirestoreAdminClient(transport=transport) + + with mock.patch.object( + type(client.transport._session), "request" + ) as req, mock.patch.object( + path_template, "transcode" + ) as transcode, mock.patch.object( + operation.Operation, "_set_result_from_operation" + ), mock.patch.object( + transports.FirestoreAdminRestInterceptor, "post_clone_database" + ) as post, mock.patch.object( + transports.FirestoreAdminRestInterceptor, "post_clone_database_with_metadata" + ) as post_with_metadata, mock.patch.object( + transports.FirestoreAdminRestInterceptor, "pre_clone_database" + ) as pre: + pre.assert_not_called() + post.assert_not_called() + post_with_metadata.assert_not_called() + pb_message = firestore_admin.CloneDatabaseRequest.pb( + firestore_admin.CloneDatabaseRequest() + ) + transcode.return_value = { + "method": "post", + "uri": "my_uri", + "body": pb_message, + "query_params": pb_message, + } + + req.return_value = mock.Mock() + req.return_value.status_code = 200 + req.return_value.headers = {"header-1": "value-1", "header-2": "value-2"} + return_value = json_format.MessageToJson(operations_pb2.Operation()) + req.return_value.content = return_value + + request = firestore_admin.CloneDatabaseRequest() + metadata = [ + ("key", "val"), + ("cephalopod", "squid"), + ] + pre.return_value = request, metadata + post.return_value = operations_pb2.Operation() + post_with_metadata.return_value = operations_pb2.Operation(), metadata + + client.clone_database( + request, + metadata=[ + ("key", "val"), + ("cephalopod", "squid"), + ], + ) + + pre.assert_called_once() + post.assert_called_once() + post_with_metadata.assert_called_once() + + def test_cancel_operation_rest_bad_request( request_type=operations_pb2.CancelOperationRequest, ): @@ -24736,6 +25559,88 @@ def test_delete_backup_schedule_empty_call_rest(): assert args[0] == request_msg +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_clone_database_empty_call_rest(): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="rest", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + client.clone_database(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest() + + assert args[0] == request_msg + + +def test_clone_database_routing_parameters_request_1_rest(): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="rest", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + client.clone_database( + request={"pitr_snapshot": {"database": "projects/sample1/sample2"}} + ) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, kw = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest( + **{"pitr_snapshot": {"database": "projects/sample1/sample2"}} + ) + + assert args[0] == request_msg + + expected_headers = {"project_id": "sample1"} + assert ( + gapic_v1.routing_header.to_grpc_metadata(expected_headers) in kw["metadata"] + ) + + +def test_clone_database_routing_parameters_request_2_rest(): + client = FirestoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="rest", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object(type(client.transport.clone_database), "__call__") as call: + client.clone_database( + request={ + "pitr_snapshot": { + "database": "projects/sample1/databases/sample2/sample3" + } + } + ) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, kw = call.mock_calls[0] + request_msg = firestore_admin.CloneDatabaseRequest( + **{ + "pitr_snapshot": { + "database": "projects/sample1/databases/sample2/sample3" + } + } + ) + + assert args[0] == request_msg + + expected_headers = {"project_id": "sample1", "database_id": "sample2"} + assert ( + gapic_v1.routing_header.to_grpc_metadata(expected_headers) in kw["metadata"] + ) + + def test_firestore_admin_rest_lro_client(): client = FirestoreAdminClient( credentials=ga_credentials.AnonymousCredentials(), @@ -24817,6 +25722,7 @@ def test_firestore_admin_base_transport(): "list_backup_schedules", "update_backup_schedule", "delete_backup_schedule", + "clone_database", "get_operation", "cancel_operation", "delete_operation", @@ -25189,6 +26095,9 @@ def test_firestore_admin_client_transport_session_collision(transport_name): session1 = client1.transport.delete_backup_schedule._session session2 = client2.transport.delete_backup_schedule._session assert session1 != session2 + session1 = client1.transport.clone_database._session + session2 = client2.transport.clone_database._session + assert session1 != session2 def test_firestore_admin_grpc_transport_channel(): @@ -25219,6 +26128,7 @@ def test_firestore_admin_grpc_asyncio_transport_channel(): # Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are # removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.filterwarnings("ignore::FutureWarning") @pytest.mark.parametrize( "transport_class", [ diff --git a/tests/unit/gapic/firestore_v1/test_firestore.py b/tests/unit/gapic/firestore_v1/test_firestore.py index d91e91c96..af45e4326 100644 --- a/tests/unit/gapic/firestore_v1/test_firestore.py +++ b/tests/unit/gapic/firestore_v1/test_firestore.py @@ -162,12 +162,19 @@ def test__read_environment_variables(): with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): - with pytest.raises(ValueError) as excinfo: - FirestoreClient._read_environment_variables() - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with pytest.raises(ValueError) as excinfo: + FirestoreClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + else: + assert FirestoreClient._read_environment_variables() == ( + False, + "auto", + None, + ) with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): assert FirestoreClient._read_environment_variables() == (False, "never", None) @@ -194,6 +201,105 @@ def test__read_environment_variables(): ) +def test_use_client_cert_effective(): + # Test case 1: Test when `should_use_client_cert` returns True. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=True + ): + assert FirestoreClient._use_client_cert_effective() is True + + # Test case 2: Test when `should_use_client_cert` returns False. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should NOT be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=False + ): + assert FirestoreClient._use_client_cert_effective() is False + + # Test case 3: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "true". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert FirestoreClient._use_client_cert_effective() is True + + # Test case 4: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert FirestoreClient._use_client_cert_effective() is False + + # Test case 5: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "True". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "True"}): + assert FirestoreClient._use_client_cert_effective() is True + + # Test case 6: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "False". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "False"} + ): + assert FirestoreClient._use_client_cert_effective() is False + + # Test case 7: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "TRUE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "TRUE"}): + assert FirestoreClient._use_client_cert_effective() is True + + # Test case 8: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "FALSE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "FALSE"} + ): + assert FirestoreClient._use_client_cert_effective() is False + + # Test case 9: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not set. + # In this case, the method should return False, which is the default value. + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, clear=True): + assert FirestoreClient._use_client_cert_effective() is False + + # Test case 10: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should raise a ValueError as the environment variable must be either + # "true" or "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + with pytest.raises(ValueError): + FirestoreClient._use_client_cert_effective() + + # Test case 11: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should return False as the environment variable is set to an invalid value. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + assert FirestoreClient._use_client_cert_effective() is False + + # Test case 12: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is unset. Also, + # the GOOGLE_API_CONFIG environment variable is unset. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": ""}): + with mock.patch.dict(os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": ""}): + assert FirestoreClient._use_client_cert_effective() is False + + def test__get_client_cert_source(): mock_provided_cert_source = mock.Mock() mock_default_cert_source = mock.Mock() @@ -557,17 +663,6 @@ def test_firestore_client_client_options(client_class, transport_class, transpor == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client = client_class(transport=transport_name) - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: @@ -779,6 +874,119 @@ def test_firestore_client_get_mtls_endpoint_and_cert_source(client_class): assert api_endpoint == mock_api_endpoint assert cert_source is None + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "Unsupported". + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset. + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", None) + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset(empty). + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() @@ -829,18 +1037,6 @@ def test_firestore_client_get_mtls_endpoint_and_cert_source(client_class): == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client_class.get_mtls_endpoint_and_cert_source() - - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - @pytest.mark.parametrize("client_class", [FirestoreClient, FirestoreAsyncClient]) @mock.patch.object( @@ -7703,7 +7899,7 @@ def test_execute_pipeline_rest_required_fields( iter_content.return_value = iter(json_return_value) response = client.execute_pipeline(request) - expected_params = [] + expected_params = [("$alt", "json;enum-encoding=int")] actual_params = req.call_args.kwargs["params"] assert expected_params == actual_params @@ -12590,6 +12786,7 @@ def test_firestore_grpc_asyncio_transport_channel(): # Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are # removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.filterwarnings("ignore::FutureWarning") @pytest.mark.parametrize( "transport_class", [transports.FirestoreGrpcTransport, transports.FirestoreGrpcAsyncIOTransport], diff --git a/tests/unit/gapic/v1/__init__.py b/tests/unit/gapic/v1/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/v1/test_async_client.py b/tests/unit/v1/test_async_client.py index 210aae88d..3aeef8f9f 100644 --- a/tests/unit/v1/test_async_client.py +++ b/tests/unit/v1/test_async_client.py @@ -374,6 +374,9 @@ async def test_asyncclient_get_all_read_time(): @pytest.mark.asyncio +@pytest.mark.filterwarnings( + "ignore:coroutine method 'aclose' of 'AsyncIter' was never awaited:RuntimeWarning" +) async def test_asyncclient_get_all_unknown_result(): from google.cloud.firestore_v1.base_client import _BAD_DOC_TEMPLATE From b16722ba30cf7a052df6e2f7ac2ecf62bb5f6f69 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 9 Jan 2026 14:58:27 -0800 Subject: [PATCH 3/4] updated filtered warnings --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 308d1b494..7cd904ecc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -24,3 +24,4 @@ filterwarnings = ignore:.*\'asyncio.iscoroutinefunction\' is deprecated.*:DeprecationWarning ignore:.*\'asyncio.get_event_loop_policy\' is deprecated.*:DeprecationWarning ignore:.*Please upgrade to the latest Python version.*:FutureWarning + ignore:(?s).*using a Python version.*past its end of life.*:FutureWarning From afedeb86f6189e5fd9b4ff55a7a32960c49009b7 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 9 Jan 2026 16:16:15 -0800 Subject: [PATCH 4/4] removed duplicate _BaseExecutePipeline --- .../firestore/transports/rest_base.py | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/google/cloud/firestore_v1/services/firestore/transports/rest_base.py b/google/cloud/firestore_v1/services/firestore/transports/rest_base.py index 7d0c52f94..4b5ce5319 100644 --- a/google/cloud/firestore_v1/services/firestore/transports/rest_base.py +++ b/google/cloud/firestore_v1/services/firestore/transports/rest_base.py @@ -474,62 +474,6 @@ def _get_query_params_json(transcoded_request): ) ) - return query_params - - class _BaseExecutePipeline: - def __hash__(self): # pragma: NO COVER - return NotImplementedError("__hash__ must be implemented.") - - __REQUIRED_FIELDS_DEFAULT_VALUES: Dict[str, Any] = {} - - @classmethod - def _get_unset_required_fields(cls, message_dict): - return { - k: v - for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() - if k not in message_dict - } - - @staticmethod - def _get_http_options(): - http_options: List[Dict[str, str]] = [ - { - "method": "post", - "uri": "/v1/{database=projects/*/databases/*}/documents:executePipeline", - "body": "*", - }, - ] - return http_options - - @staticmethod - def _get_transcoded_request(http_options, request): - pb_request = firestore.ExecutePipelineRequest.pb(request) - transcoded_request = path_template.transcode(http_options, pb_request) - return transcoded_request - - @staticmethod - def _get_request_body_json(transcoded_request): - # Jsonify the request body - - body = json_format.MessageToJson( - transcoded_request["body"], use_integers_for_enums=True - ) - return body - - @staticmethod - def _get_query_params_json(transcoded_request): - query_params = json.loads( - json_format.MessageToJson( - transcoded_request["query_params"], - use_integers_for_enums=True, - ) - ) - query_params.update( - _BaseFirestoreRestTransport._BaseExecutePipeline._get_unset_required_fields( - query_params - ) - ) - query_params["$alt"] = "json;enum-encoding=int" return query_params