Merge branch 'master' of github.com:mongodb/mongo-python-driver

Author: blink1073

.evergreen/config.yml

@@ -991,6 +991,30 @@ task_groups:
     tasks:
       - oidc-auth-test-azure-latest
 
+  - name: testgcpoidc_task_group
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - func: fix absolute paths
+      - func: make files executable
+      - command: subprocess.exec
+        params:
+          binary: bash
+          env:
+            GCPOIDC_VMNAME_PREFIX: "PYTHON_DRIVER"
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/setup.sh
+    teardown_task:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    tasks:
+      - oidc-auth-test-gcp-latest
+
   - name: testoidc_task_group
     setup_group:
       - func: fetch source
@@ -1966,6 +1990,25 @@ tasks:
             export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh"
             bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh
 
+    - name: "oidc-auth-test-gcp-latest"
+      commands:
+      - command: shell.exec
+        params:
+          shell: bash
+          script: |-
+            set -o errexit
+            ${PREPARE_SHELL}
+            cd src
+            git add .
+            git commit -m "add files"
+            export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-python-driver.tgz
+            git archive -o $GCPOIDC_DRIVERS_TAR_FILE HEAD
+            # Define the command to run on the VM.
+            # Ensure that we source the environment file created for us, set up any other variables we need,
+            # and then run our test suite on the vm.
+            export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh"
+            bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh
+
     - name: "test-fips-standalone"
       tags: ["fips"]
       commands:
@@ -2995,18 +3038,25 @@ buildvariants:
 - matrix_name: "oidc-auth-test"
   matrix_spec:
     platform: [ rhel8, macos-1100, windows-64-vsMulti-small ]
-  display_name: "MONGODB-OIDC Auth ${platform}"
+  display_name: "OIDC Auth ${platform}"
   tasks:
     - name: testoidc_task_group
       batchtime: 20160 # 14 days
 
 - name: testazureoidc-variant
-  display_name: "Azure OIDC"
-  run_on: ubuntu2004-small
+  display_name: "OIDC Auth Azure"
+  run_on: ubuntu2204-small
   tasks:
     - name: testazureoidc_task_group
       batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
 
+- name: testgcpoidc-variant
+  display_name: "OIDC Auth GCP"
+  run_on: ubuntu2204-small
+  tasks:
+    - name: testgcpoidc_task_group
+      batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
+
 - matrix_name: "aws-auth-test"
   matrix_spec:
     platform: [ubuntu-20.04]

.evergreen/run-mongodb-oidc-test.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 set +x          # Disable debug trace
-set -o errexit  # Exit the script with error if any of the commands fail
+set -eu
 
 echo "Running MONGODB-OIDC authentication tests"
 
@@ -18,6 +18,9 @@ if [ $OIDC_ENV == "test" ]; then
 elif [ $OIDC_ENV == "azure" ]; then
     source ./env.sh
 
+elif [ $OIDC_ENV == "gcp" ]; then
+    source ./secrets-export.sh
+
 else
     echo "Unrecognized OIDC_ENV $OIDC_ENV"
     exit 1

doc/changelog.rst

@@ -41,6 +41,8 @@ PyMongo 4.7 brings a number of improvements including:
   :attr:`pymongo.monitoring.ConnectionReadyEvent.duration` properties.
 - Added the ``type`` and ``kwargs`` arguments to :class:`~pymongo.operations.SearchIndexModel` to enable
   creating vector search indexes in MongoDB Atlas.
+- Fixed a bug where ``read_concern`` and ``write_concern`` were improperly added to
+  :meth:`~pymongo.collection.Collection.list_search_indexes` queries.
 
 
 Unavoidable breaking changes

pymongo/_gcp_helpers.py

@@ -0,0 +1,39 @@
+# Copyright 2024-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""GCP helpers."""
+from __future__ import annotations
+
+from typing import Any
+from urllib.request import Request, urlopen
+
+
+def _get_gcp_response(resource: str, timeout: float = 5) -> dict[str, Any]:
+    url = "http://metadata/computeMetadata/v1/instance/service-accounts/default/identity"
+    url += f"?audience={resource}"
+    headers = {"Metadata-Flavor": "Google", "Accept": "application/json"}
+    request = Request(url, headers=headers)  # noqa: S310
+    try:
+        with urlopen(request, timeout=timeout) as response:  # noqa: S310
+            status = response.status
+            body = response.read().decode("utf8")
+    except Exception as e:
+        msg = "Failed to acquire IMDS access token: %s" % e
+        raise ValueError(msg) from None
+
+    if status != 200:
+        msg = "Failed to acquire IMDS access token."
+        raise ValueError(msg)
+
+    return dict(access_token=body)

pymongo/auth.py

@@ -41,6 +41,7 @@
     _authenticate_oidc,
     _get_authenticator,
     _OIDCAzureCallback,
+    _OIDCGCPCallback,
     _OIDCProperties,
     _OIDCTestCallback,
 )
@@ -207,6 +208,13 @@ def _build_credentials_tuple(
                         "Azure environment for MONGODB-OIDC requires a TOKEN_RESOURCE auth mechanism property"
                     )
                 callback = _OIDCAzureCallback(token_resource)
+            elif environ == "gcp":
+                passwd = None
+                if not token_resource:
+                    raise ConfigurationError(
+                        "GCP provider for MONGODB-OIDC requires a TOKEN_RESOURCE auth mechanism property"
+                    )
+                callback = _OIDCGCPCallback(token_resource)
             else:
                 raise ConfigurationError(f"unrecognized ENVIRONMENT for MONGODB-OIDC: {environ}")
         else:

pymongo/auth_oidc.py

@@ -26,7 +26,9 @@
 from bson.binary import Binary
 from pymongo._azure_helpers import _get_azure_response
 from pymongo._csot import remaining
+from pymongo._gcp_helpers import _get_gcp_response
 from pymongo.errors import ConfigurationError, OperationFailure
+from pymongo.helpers import _AUTHENTICATION_FAILURE_CODE
 
 if TYPE_CHECKING:
     from pymongo.auth import MongoCredential
@@ -36,7 +38,7 @@
 @dataclass
 class OIDCIdPInfo:
     issuer: str
-    clientId: str
+    clientId: Optional[str] = field(default=None)
     requestScopes: Optional[list[str]] = field(default=None)
 
 
@@ -133,6 +135,15 @@ def fetch(self, context: OIDCCallbackContext) -> OIDCCallbackResult:
         )
 
 
+class _OIDCGCPCallback(OIDCCallback):
+    def __init__(self, token_resource: str) -> None:
+        self.token_resource = token_resource
+
+    def fetch(self, context: OIDCCallbackContext) -> OIDCCallbackResult:
+        resp = _get_gcp_response(self.token_resource, context.timeout_seconds)
+        return OIDCCallbackResult(access_token=resp["access_token"])
+
+
 @dataclass
 class _OIDCAuthenticator:
     username: str
@@ -179,30 +190,43 @@ def get_spec_auth_cmd(self) -> Optional[MutableMapping[str, Any]]:
 
     def _authenticate_machine(self, conn: Connection) -> Mapping[str, Any]:
         # If there is a cached access token, try to authenticate with it. If
-        # authentication fails, it's possible the cached access token is expired. In
-        # that case, invalidate the access token, fetch a new access token, and try
-        # to authenticate again.
+        # authentication fails with error code 18, invalidate the access token,
+        # fetch a new access token, and try to authenticate again. If authentication
+        # fails for any other reason, raise the error to the user.
         if self.access_token:
             try:
                 return self._sasl_start_jwt(conn)
-            except Exception:  # noqa: S110
-                pass
+            except OperationFailure as e:
+                if self._is_auth_error(e):
+                    return self._authenticate_machine(conn)
+                raise
         return self._sasl_start_jwt(conn)
 
     def _authenticate_human(self, conn: Connection) -> Optional[Mapping[str, Any]]:
         # If we have a cached access token, try a JwtStepRequest.
+        # authentication fails with error code 18, invalidate the access token,
+        # and try to authenticate again.  If authentication fails for any other
+        # reason, raise the error to the user.
         if self.access_token:
             try:
                 return self._sasl_start_jwt(conn)
-            except Exception:  # noqa: S110
-                pass
+            except OperationFailure as e:
+                if self._is_auth_error(e):
+                    return self._authenticate_human(conn)
+                raise
 
         # If we have a cached refresh token, try a JwtStepRequest with that.
+        # If authentication fails with error code 18, invalidate the access and
+        # refresh tokens, and try to authenticate again. If authentication fails for
+        # any other reason, raise the error to the user.
         if self.refresh_token:
             try:
                 return self._sasl_start_jwt(conn)
-            except Exception:  # noqa: S110
-                pass
+            except OperationFailure as e:
+                if self._is_auth_error(e):
+                    self.refresh_token = None
+                    return self._authenticate_human(conn)
+                raise
 
         # Start a new Two-Step SASL conversation.
         # Run a PrincipalStepRequest to get the IdpInfo.
@@ -270,10 +294,16 @@ def _get_access_token(self) -> Optional[str]:
     def _run_command(self, conn: Connection, cmd: MutableMapping[str, Any]) -> Mapping[str, Any]:
         try:
             return conn.command("$external", cmd, no_reauth=True)  # type: ignore[call-arg]
-        except OperationFailure:
-            self._invalidate(conn)
+        except OperationFailure as e:
+            if self._is_auth_error(e):
+                self._invalidate(conn)
             raise
 
+    def _is_auth_error(self, err: Exception) -> bool:
+        if not isinstance(err, OperationFailure):
+            return False
+        return err.code == _AUTHENTICATION_FAILURE_CODE
+
     def _invalidate(self, conn: Connection) -> None:
         # Ignore the invalidation if a token gen id is given and is less than our
         # current token gen id.

pymongo/collection.py

@@ -72,6 +72,7 @@
     _IndexList,
     _Op,
 )
+from pymongo.read_concern import DEFAULT_READ_CONCERN, ReadConcern
 from pymongo.read_preferences import ReadPreference, _ServerMode
 from pymongo.results import (
     BulkWriteResult,
@@ -81,7 +82,7 @@
     UpdateResult,
 )
 from pymongo.typings import _CollationIn, _DocumentType, _DocumentTypeArg, _Pipeline
-from pymongo.write_concern import WriteConcern, validate_boolean
+from pymongo.write_concern import DEFAULT_WRITE_CONCERN, WriteConcern, validate_boolean
 
 T = TypeVar("T")
 
@@ -119,7 +120,6 @@ class ReturnDocument:
     from pymongo.collation import Collation
     from pymongo.database import Database
     from pymongo.pool import Connection
-    from pymongo.read_concern import ReadConcern
     from pymongo.server import Server
 
 
@@ -2364,7 +2364,10 @@ def list_search_indexes(
             pipeline = [{"$listSearchIndexes": {"name": name}}]
 
         coll = self.with_options(
-            codec_options=DEFAULT_CODEC_OPTIONS, read_preference=ReadPreference.PRIMARY
+            codec_options=DEFAULT_CODEC_OPTIONS,
+            read_preference=ReadPreference.PRIMARY,
+            write_concern=DEFAULT_WRITE_CONCERN,
+            read_concern=DEFAULT_READ_CONCERN,
         )
         cmd = _CollectionAggregationCommand(
             coll,

pymongo/common.py

@@ -426,7 +426,6 @@ def validate_read_preference_tags(name: str, value: Any) -> list[dict[str, str]]
         "AWS_SESSION_TOKEN",
         "ENVIRONMENT",
         "TOKEN_RESOURCE",
-        "ALLOWED_HOSTS",
     ]
 )
 

pymongo/helpers.py

@@ -90,6 +90,9 @@
 # Server code raised when re-authentication is required
 _REAUTHENTICATION_REQUIRED_CODE: int = 391
 
+# Server code raised when authentication fails.
+_AUTHENTICATION_FAILURE_CODE: int = 18
+
 
 def _gen_index_name(keys: _IndexList) -> str:
     """Generate an index name from the set of fields it is over."""

pymongo/monitor.py

@@ -335,8 +335,12 @@ def __init__(self, topology: Topology, topology_settings: TopologySettings):
         self._seedlist = self._settings._seeds
         assert isinstance(self._settings.fqdn, str)
         self._fqdn: str = self._settings.fqdn
+        self._startup_time = time.monotonic()
 
     def _run(self) -> None:
+        # Don't poll right after creation, wait 60 seconds first
+        if time.monotonic() < self._startup_time + common.MIN_SRV_RESCAN_INTERVAL:
+            return
         seedlist = self._get_seedlist()
         if seedlist:
             self._seedlist = seedlist