Additional cosmos diagnostics logging support (#41012)

* Update _base.py

* update diagnostics to capture all cosmos exceptions

This addition to cosmos logging will capture exceptions even if they get retried and succeed, it will also capture servicerequest and serviceresponse errors.

* update cosmos diagnostics

* Update diagnostics

These changes include using on exception to capture logs that were not thrown with a response. This includes 500s, service request and service response errors. This also fixes issues with hiechical loggers and making sure we check all filters to see if we should add the log attributes to all request and response.

* update typos and check result[1]

* update diagnostics logs

* small updates

* update tests and mypy and pylint fixes

* Update _cosmos_http_logging_policy.py

* update pylint and test fixes

* Update CHANGELOG.md

Author: bambriz

sdk/cosmos/azure-cosmos/CHANGELOG.md

@@ -11,10 +11,12 @@
 * Fixed session container session token logic. The SDK will now only send the relevant partition-local session tokens for read document requests and write requests when multi-region writes are enabled, as opposed to the entire compound session token for the container for every document request. See [PR 41678](https://github.com/Azure/azure-sdk-for-python/pull/41678).
 * Write requests for single-write region accounts will no longer send session tokens when using session consistency. See [PR 41678](https://github.com/Azure/azure-sdk-for-python/pull/41678).
 * Fixed bug where container cache was not being properly updated resulting in unnecessary extra requests. See [PR 42143](https://github.com/Azure/azure-sdk-for-python/pull/42143).
+* Fixed bug where the Filters on Parent loggers or handlers of Cosmos Diagnostics loggers were not being applied. See [PR 41012](https://github.com/Azure/azure-sdk-for-python/pull/41012)
 
 #### Other Changes
 * Changed to include client id in headers for all requests. See [PR 42104](https://github.com/Azure/azure-sdk-for-python/pull/42104).
 * Added an option to override AAD audience scope through environment variable. See [PR 42228](https://github.com/Azure/azure-sdk-for-python/pull/42228).
+* Diagnostics logging will now log ServiceRequestError Exceptions, ServiceResponseError Exceptions, and Cosmos exceptions with a status code of 500 or greater. See [PR 41012](https://github.com/Azure/azure-sdk-for-python/pull/41012)
 
 ### 4.14.0b1 (2025-07-14)
 

sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_client_connection.py

@@ -207,7 +207,7 @@ def __init__( # pylint: disable=too-many-statements
             else:
                 scope = base.create_scope_from_url(self.url_connection)
             credentials_policy = CosmosBearerTokenCredentialPolicy(self.aad_credentials, scope)
-
+        self._enable_diagnostics_logging = kwargs.pop("enable_diagnostics_logging", False)
         policies = [
             HeadersPolicy(**kwargs),
             ProxyPolicy(proxies=proxies),
@@ -220,7 +220,7 @@ def __init__( # pylint: disable=too-many-statements
             DistributedTracingPolicy(**kwargs),
             CosmosHttpLoggingPolicy(
                 logger=kwargs.pop("logger", None),
-                enable_diagnostics_logging=kwargs.pop("enable_diagnostics_logging", False),
+                enable_diagnostics_logging=self._enable_diagnostics_logging,
                 global_endpoint_manager=self._global_endpoint_manager,
                 **kwargs
             ),

sdk/cosmos/azure-cosmos/azure/cosmos/_cosmos_http_logging_policy.py

@@ -23,6 +23,7 @@
 """
 import json
 import time
+import logging
 from typing import Optional
 
 from azure.core.exceptions import AzureError, ClientAuthenticationError, ServiceRequestError, ServiceResponseError
@@ -41,6 +42,7 @@
 from .documents import _OperationType
 from .exceptions import CosmosHttpResponseError
 from .http_constants import HttpHeaders, StatusCodes, SubStatusCodes, ResourceType
+from ._cosmos_http_logging_policy import _log_diagnostics_error
 
 
 # pylint: disable=protected-access, disable=too-many-lines, disable=too-many-statements, disable=too-many-branches
@@ -94,6 +96,8 @@ def Execute(client, global_endpoint_manager, function, *args, **kwargs): # pylin
     service_request_retry_policy = _service_request_retry_policy.ServiceRequestRetryPolicy(
         client.connection_policy, global_endpoint_manager, pk_range_wrapper, *args,
     )
+    # Get logger
+    logger = kwargs.get("logger", logging.getLogger("azure.cosmos._retry_utility"))
     # HttpRequest we would need to modify for Container Recreate Retry Policy
     request = None
     if args and len(args) > 3:
@@ -136,10 +140,22 @@ def Execute(client, global_endpoint_manager, function, *args, **kwargs): # pylin
                 response = exceptions.InternalException(status_code=StatusCodes.NOT_FOUND,
                                                         headers={HttpHeaders.SubStatus:
                                                                      SubStatusCodes.THROUGHPUT_OFFER_NOT_FOUND})
-                raise exceptions.CosmosResourceNotFoundError(
+                e_offer = exceptions.CosmosResourceNotFoundError(
                     status_code=StatusCodes.NOT_FOUND,
                     message="Could not find ThroughputProperties for container " + link,
                     response=response)
+
+                response_headers = result[1] if len(result) > 1 else {}
+                logger_attributes = {
+                    "duration": time.time() - start_time,
+                    "verb": request.method,
+                    "status_code": e_offer.status_code,
+                    "sub_status_code": e_offer.sub_status,
+                }
+                _log_diagnostics_error(client._enable_diagnostics_logging, request, response_headers, e_offer,
+                                           logger_attributes, global_endpoint_manager, logger=logger)
+                raise e_offer
+
             return result
         except exceptions.CosmosHttpResponseError as e:
             if request:

sdk/cosmos/azure-cosmos/azure/cosmos/_retry_utility.py

@@ -217,7 +217,7 @@ def __init__( # pylint: disable=too-many-statements
             else:
                 scope = base.create_scope_from_url(self.url_connection)
             credentials_policy = AsyncCosmosBearerTokenCredentialPolicy(self.aad_credentials, scope)
-
+        self._enable_diagnostics_logging = kwargs.pop("enable_diagnostics_logging", False)
         policies = [
             HeadersPolicy(**kwargs),
             ProxyPolicy(proxies=proxies),
@@ -230,7 +230,7 @@ def __init__( # pylint: disable=too-many-statements
             DistributedTracingPolicy(**kwargs),
             CosmosHttpLoggingPolicy(
                 logger=kwargs.pop("logger", None),
-                enable_diagnostics_logging=kwargs.pop("enable_diagnostics_logging", False),
+                enable_diagnostics_logging=self._enable_diagnostics_logging,
                 global_endpoint_manager=self._global_endpoint_manager,
                 **kwargs
             ),

sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client_connection_async.py

@@ -24,6 +24,7 @@
 import asyncio  # pylint: disable=do-not-import-asyncio
 import json
 import time
+import logging
 
 from azure.core.exceptions import AzureError, ClientAuthenticationError, ServiceRequestError, ServiceResponseError
 from azure.core.pipeline.policies import AsyncRetryPolicy
@@ -42,6 +43,7 @@
                               _has_database_account_header)
 from ..exceptions import CosmosHttpResponseError
 from ..http_constants import HttpHeaders, StatusCodes, SubStatusCodes
+from .._cosmos_http_logging_policy import _log_diagnostics_error
 
 
 # pylint: disable=protected-access, disable=too-many-lines, disable=too-many-statements, disable=too-many-branches
@@ -93,6 +95,8 @@ async def ExecuteAsync(client, global_endpoint_manager, function, *args, **kwarg
     service_request_retry_policy = _service_request_retry_policy.ServiceRequestRetryPolicy(
         client.connection_policy, global_endpoint_manager, pk_range_wrapper, *args,
     )
+    # Get Logger
+    logger = kwargs.get("logger", logging.getLogger("azure.cosmos._retry_utility_async"))
     # HttpRequest we would need to modify for Container Recreate Retry Policy
     request = None
     if args and len(args) > 3:
@@ -135,10 +139,22 @@ async def ExecuteAsync(client, global_endpoint_manager, function, *args, **kwarg
                 response = exceptions.InternalException(status_code=StatusCodes.NOT_FOUND,
                                                         headers={HttpHeaders.SubStatus:
                                                                      SubStatusCodes.THROUGHPUT_OFFER_NOT_FOUND})
-                raise exceptions.CosmosResourceNotFoundError(
+                e_offer = exceptions.CosmosResourceNotFoundError(
                     status_code=StatusCodes.NOT_FOUND,
                     message="Could not find ThroughputProperties for container " + link,
                     response=response)
+
+                response_headers = result[1] if len(result) > 1 else {}
+                logger_attributes = {
+                    "duration": time.time() - start_time,
+                    "verb": request.method,
+                    "status_code": e_offer.status_code,
+                    "sub_status_code": e_offer.sub_status,
+                }
+                _log_diagnostics_error(client._enable_diagnostics_logging, request, response_headers, e_offer,
+                                       logger_attributes, global_endpoint_manager, logger=logger)
+                raise e_offer
+
             return result
         except exceptions.CosmosHttpResponseError as e:
             if request:

sdk/cosmos/azure-cosmos/azure/cosmos/aio/_retry_utility_async.py

@@ -467,3 +467,24 @@ class ResourceType:
     Topology = "topology"
     DatabaseAccount = "databaseaccount"
     PartitionKey = "partitionkey"
+
+    @staticmethod
+    def IsCollectionChild(resourceType: str) -> bool:
+        return resourceType in (
+            ResourceType.Document,
+            ResourceType.Attachment,
+            ResourceType.Conflict,
+            ResourceType.Schema,
+            ResourceType.UserDefinedFunction,
+            ResourceType.Trigger,
+            ResourceType.StoredProcedure,
+            ResourceType.PartitionKey,
+        )
+
+# The list of headers we do not want to log, it needs to be updated if any new headers should not be logged
+_cosmos_disallow_list = ["Authorization", "ProxyAuthorization", "TransferEncoding"]
+_cosmos_allow_list = set(
+    v.lower()
+    for k, v in HttpHeaders.__dict__.items()
+    if not k.startswith("_") and k not in _cosmos_disallow_list
+)

sdk/cosmos/azure-cosmos/azure/cosmos/http_constants.py

@@ -11,6 +11,7 @@
 
 import azure.cosmos.cosmos_client as cosmos_client
 from azure.cosmos import PartitionKey, ContainerProxy
+from azure.cosmos.exceptions import CosmosHttpResponseError
 import test_config
 from _fault_injection_transport import FaultInjectionTransport
 from test_fault_injection_transport import TestFaultInjectionTransport
@@ -101,6 +102,25 @@ def setUpClass(cls):
                                                                     logger=cls.logger_filtered_diagnostic,
                                                                     enable_diagnostics_logging=True)
 
+        # Create a root logger with a mock handler and a filter for status codes above 400
+        cls.root_mock_handler = test_config.MockHandler()
+        cls.root_mock_handler.addFilter(FilterStatusCode())
+        cls.root_logger = create_logger("rootLogger", cls.root_mock_handler)
+
+        # Create child loggers
+        cls.root_logger_child = logging.getLogger("rootLogger.child")
+        cls.root_logger_grandchild = logging.getLogger("rootLogger.child.grandchild")
+
+        # Use the grandchild logger for the Cosmos client
+        cls.client_grandchild_logger = cosmos_client.CosmosClient(
+            cls.host,
+            cls.masterKey,
+            consistency_level="Session",
+            connection_policy=cls.connectionPolicy,
+            logger=cls.root_logger_grandchild,
+            enable_diagnostics_logging=True
+        )
+
     def test_default_http_logging_policy(self):
         # Test if we can log into from creating a database
         database_id = "database_test-" + str(uuid.uuid4())
@@ -327,6 +347,78 @@ def test_activity_id_logging_policy(self):
         self.client_activity_id.delete_database(database_id)
         self.mock_handler_activity_id.reset()
 
+    def test_logging_exceptions_with_no_response(self):
+        # Create a mock handler and logger for capturing logs
+        mock_handler = test_config.MockHandler()
+        logger = create_logger("test_logger_fault_injection", mock_handler)
+
+        # Set up FaultInjectionTransport to inject a 502 error
+        id_value = str(uuid.uuid4())
+        document_definition = {'id': id_value,
+                               'pk': id_value,
+                               'name': 'sample document',
+                               'key': 'value'}
+        custom_transport = FaultInjectionTransport()
+        predicate: Callable[[HttpRequest], bool] = lambda r: FaultInjectionTransport.predicate_req_for_document_with_id(
+            r, id_value)
+        custom_transport.add_fault(predicate, lambda r: FaultInjectionTransport.error_after_delay(
+            1000,
+            CosmosHttpResponseError(
+                status_code=502,
+                message="Some random reverse proxy error.")))
+
+
+        # Initialize the client with the custom transport and logger
+        initialized_objects = TestFaultInjectionTransport.setup_method_with_custom_transport(
+            custom_transport,
+            default_endpoint=CONFIG.host,
+            key=CONFIG.masterKey,
+            database_id=CONFIG.TEST_DATABASE_ID,
+            container_id=CONFIG.TEST_SINGLE_PARTITION_CONTAINER_ID,
+            preferred_locations=[L1, L2],
+            excluded_locations=[],
+            multiple_write_locations=True,
+            custom_logger=logger
+        )
+        mock_handler.reset()
+
+        # Attempt to create an item, which should trigger the injected 502 error
+        container: ContainerProxy = initialized_objects["col"]
+        try:
+            container.create_item(body=document_definition)
+            pytest.fail("Expected exception not thrown")
+        except CosmosHttpResponseError as cosmosError:
+            # Verify that the logger captured the 502 error and was called from on_exception
+            assert any(m.status_code == 502 and "on_exception" in m.funcName for m in mock_handler.messages)
+
+    def test_hierarchical_logger_with_filter(self):
+        # Reset the mock handler before the test
+        self.root_mock_handler.reset()
+
+        # Attempt to read a nonexistent item
+        database_id = "database_test_hierarchical_logger_" + str(uuid.uuid4())
+        container_id = "container_test_hierarchical_logger_" + str(uuid.uuid4())
+        database = self.client_grandchild_logger.create_database(id=database_id)
+        container = database.create_container(id=container_id, partition_key=PartitionKey(path="/pk"))
+
+        try:
+            container.read_item(item="nonexistent_item", partition_key="nonexistent_pk")
+        except:
+            pass
+
+        # Verify that the error was logged by the root logger's mock handler
+        assert len(self.root_mock_handler.messages) == 2
+        log_record = self.root_mock_handler.messages[0]
+        assert hasattr(log_record, "status_code")
+        assert log_record.status_code == 404
+        assert log_record.name == "rootLogger.child.grandchild"
+        assert not bool(self.root_logger_grandchild.filters)
+        assert not bool(self.root_logger_child.filters)
+        assert bool(self.root_mock_handler.filters)
+
+        # Clean up
+        self.client_grandchild_logger.delete_database(database_id)
+        self.root_mock_handler.reset()
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()