Add logging_format configuration option in api (Azure#40057)

Author: lzchen

sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Features Added
 
+- Add `logging_format` as configuration option in api
+    ([#40057](https://github.com/Azure/azure-sdk-for-python/pull/40057))
 - Enable Azure AI Agents instrumentation
     ([#40043](https://github.com/Azure/azure-sdk-for-python/pull/40043))
 

sdk/monitor/azure-monitor-opentelemetry/README.md

@@ -61,6 +61,7 @@ You can use `configure_azure_monitor` to set up instrumentation for your app to
 |-------------------|----------------------------------------------------|----------------------|
 | `connection_string` | The [connection string][connection_string_doc] for your Application Insights resource. The connection string will be automatically populated from the `APPLICATIONINSIGHTS_CONNECTION_STRING` environment variable if not explicitly passed in. | `APPLICATIONINSIGHTS_CONNECTION_STRING` |
 | `enable_live_metrics` | Enable [live metrics][application_insights_live_metrics] feature. Defaults to `False`. | `N/A` |
+| `logging_formatter` | A Python logging [formatter][python_logging_formatter] that will be used to format collected logs. | `N/A` |
 | `logger_name` | The name of the [Python logger][python_logger] under which telemetry is collected. Setting this value is imperative so logs created from the SDK itself are not tracked. | `N/A` |
 | `instrumentation_options` | A nested dictionary that determines which instrumentations to enable or disable. Instrumentations are referred to by their [Library Names](#officially-supported-instrumentations). For example, `{"azure_sdk": {"enabled": False}, "flask": {"enabled": False}, "django": {"enabled": True}}` will disable Azure Core Tracing and the Flask instrumentation but leave Django and the other default instrumentations enabled. The `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS` environment variable explained below can also be used to disable instrumentations. | `N/A` |
 | `resource` | Specifies the OpenTelemetry [Resource][ot_spec_resource] associated with your application. Passed in [Resource Attributes][ot_spec_resource_attributes] take priority over default attributes and those from [Resource Detectors][ot_python_resource_detectors]. | [OTEL_SERVICE_NAME][ot_spec_service_name], [OTEL_RESOURCE_ATTRIBUTES][ot_spec_resource_attributes], [OTEL_EXPERIMENTAL_RESOURCE_DETECTORS][ot_python_resource_detectors] |
@@ -256,6 +257,7 @@ contact [[email protected]](mailto:[email protected]) with any additio
 [pypi_urllib3]: https://pypi.org/project/urllib3/
 [python]: https://www.python.org/downloads/
 [python_logger]: https://docs.python.org/3/library/logging.html#logger-objects
+[python_logging_formatter]: https://docs.python.org/3/library/logging.html#formatter-objects
 [python_logging_level]: https://docs.python.org/3/library/logging.html#levels
 [samples]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples
 [samples_manual]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/manually_instrumented.py

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_configure.py

@@ -4,7 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 from functools import cached_property
-from logging import getLogger
+from logging import getLogger, Formatter
 from typing import Dict, List, cast
 
 from opentelemetry._events import _set_event_logger_provider
@@ -42,6 +42,7 @@
     DISABLE_TRACING_ARG,
     ENABLE_LIVE_METRICS_ARG,
     LOGGER_NAME_ARG,
+    LOGGING_FORMATTER_ARG,
     RESOURCE_ARG,
     SAMPLING_RATIO_ARG,
     SPAN_PROCESSORS_ARG,
@@ -172,11 +173,20 @@ def _setup_logging(configurations: Dict[str, ConfigurationValue]):
     logger_provider.add_log_record_processor(log_record_processor)
     set_logger_provider(logger_provider)
     logger_name: str = configurations[LOGGER_NAME_ARG]  # type: ignore
+    logging_formatter: Formatter = configurations[LOGGING_FORMATTER_ARG]  # type: ignore
     logger = getLogger(logger_name)
     # Only add OpenTelemetry LoggingHandler if logger does not already have the handler
     # This is to prevent most duplicate logging telemetry
     if not any(isinstance(handler, LoggingHandler) for handler in logger.handlers):
         handler = LoggingHandler(logger_provider=logger_provider)
+        if logging_formatter:
+            try:
+                handler.setFormatter(logging_formatter)
+            except Exception as ex:  # pylint: disable=broad-except
+                _logger.warning(
+                    "Exception occurred when adding logging Formatter: %s.",
+                    ex,
+                )
         logger.addHandler(handler)
 
     # Setup EventLoggerProvider

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py

@@ -18,6 +18,7 @@
 DISABLE_TRACING_ARG = "disable_tracing"
 DISTRO_VERSION_ARG = _AZURE_MONITOR_DISTRO_VERSION_ARG
 LOGGER_NAME_ARG = "logger_name"
+LOGGING_FORMATTER_ARG = "logging_formatter"
 INSTRUMENTATION_OPTIONS_ARG = "instrumentation_options"
 RESOURCE_ARG = "resource"
 SAMPLING_RATIO_ARG = "sampling_ratio"

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_utils/configurations.py

@@ -4,7 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from logging import getLogger
+from logging import getLogger, Formatter
 from os import environ
 from typing import Dict
 
@@ -34,6 +34,7 @@
     ENABLE_LIVE_METRICS_ARG,
     INSTRUMENTATION_OPTIONS_ARG,
     LOGGER_NAME_ARG,
+    LOGGING_FORMATTER_ARG,
     RESOURCE_ARG,
     SAMPLING_RATIO_ARG,
     SPAN_PROCESSORS_ARG,
@@ -66,6 +67,7 @@ def _get_configurations(**kwargs) -> Dict[str, ConfigurationValue]:
     _default_disable_metrics(configurations)
     _default_disable_tracing(configurations)
     _default_logger_name(configurations)
+    _default_logging_formatter(configurations)
     _default_resource(configurations)
     _default_sampling_ratio(configurations)
     _default_instrumentation_options(configurations)
@@ -104,6 +106,12 @@ def _default_logger_name(configurations):
     configurations.setdefault(LOGGER_NAME_ARG, "")
 
 
+def _default_logging_formatter(configurations):
+    formatter = configurations.get(LOGGING_FORMATTER_ARG)
+    if not isinstance(formatter, Formatter):
+        configurations[LOGGING_FORMATTER_ARG] = None
+
+
 def _default_resource(configurations):
     environ.setdefault(OTEL_EXPERIMENTAL_RESOURCE_DETECTORS, ",".join(_SUPPORTED_RESOURCE_DETECTORS))
     if RESOURCE_ARG not in configurations:

sdk/monitor/azure-monitor-opentelemetry/samples/README.md

@@ -17,14 +17,17 @@ For guidance on the samples README, visit the [sample guide](https://github.com/
 
 |**File Name**|**Description**|
 |----------------|-------------|
+|[logging/basic.py][logging_basic] | Produce logs using basic logging configurations |
 |[logging/correlated_logs.py][correlated_logs] | Produce logs correlated with spans |
+|[logging/custom_event.py][custom_event] | Produce custom events using logs |
 |[logging/custom_properties.py][custom_properties] | Add custom propterties to logs |
 |[logging/exception_logs.py][exception_logs] | Produce exception logs |
 |[logging/logs_with_traces.py][logs_with_traces] | Produce correlated logs inside an instrumented http library's distributed tracing |
-|[logging/basic.py][logging_basic] | Produce logs |
 |[metrics/attributes.py][attributes] | Add attributes to custom metrics counters |
 |[metrics/instruments.py][instruments] | Create observable instruments |
 |[metrics/live_metrics.py][live_metrics] | Live metrics feature |
+|[tracing/azure_ai_inference.py][azure_ai_inference] | Instrument an app using Azure AI inference SDK |
+|[tracing/azure_blob_storage.py][azure_blob_storage] | Instrument an app using Azure Blob storage SDK |
 |[tracing/django/sample/manage.py][django] | Instrument a django app |
 |[tracing/db_psycopg2.py][db_psycopg2] | Instrument the PsycoPG2 library |
 |[tracing/http_fastapi.py][http_fastapi] | Instrument a FastAPI app |
@@ -34,6 +37,7 @@ For guidance on the samples README, visit the [sample guide](https://github.com/
 |[tracing/http_urllib3.py][http_urllib3] | Instrument the URLLib library |
 |[tracing/instrumentation_options.py][instrumentation_options] | Enable and disable instrumentations |
 |[tracing/manually_instrumented.py][manual] | Manually add instrumentation |
+|[tracing/modify_spans.py][modify_spans] | Modify spans with span processors |
 |[tracing/sampling.py][sampling] | Sample distributed tracing telemetry |
 |[tracing/tracing_simple.py][tracing_simple] | Produce manual spans |
 
@@ -63,13 +67,16 @@ To learn more, see the [Azure Monitor OpenTelemetry Distro documentation][distro
 [distro_docs]: https://learn.microsoft.com/azure/azure-monitor/app/opentelemetry-enable?tabs=python
 [otel_docs]: https://opentelemetry.io/docs/
 [correlated_logs]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/logging/correlated_logs.py
+[custom_event]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/logging/custom_event.py
 [custom_properties]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/logging/custom_properties.py
 [exception_logs]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/logging/exception_logs.py
 [logs_with_traces]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/logging/logs_with_traces.py
 [logging_basic]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/logging/basic.py
 [attributes]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/metrics/attributes.py
 [instruments]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/metrics/instruments.py
 [instruments]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/metrics/live_metrics.py
+[azure_ai_inference]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/azure_ai_inference.py
+[azure_blob_storage]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/azure_blob_storage.py
 [django]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/django/sample/manage.py
 [db_psycopg2]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/db_psycopg2.py
 [http_fastapi]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/http_fastapi.py
@@ -78,6 +85,7 @@ To learn more, see the [Azure Monitor OpenTelemetry Distro documentation][distro
 [http_urllib]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/http_urllib.py
 [http_urllib3]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/http_urllib3.py
 [instrumentation_options]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/instrumentation_options.py
+[modify_spans]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/modify_spans.py
 [manual]: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/manually_instrumented.py
 [sampling]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/sampling.py
 [tracing_simple]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples/tracing/simple.py

sdk/monitor/azure-monitor-opentelemetry/samples/logging/basic.py

@@ -4,26 +4,31 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from logging import INFO, getLogger
+from logging import INFO, Formatter, getLogger
 
 from azure.monitor.opentelemetry import configure_azure_monitor
 
+# logging.basicConfig and logging.config will be overwritten by the SDK
+# We recommend using logger specific configuration or the below apis to configure logging
+
 configure_azure_monitor(
     # Set logger_name to the name of the logger you want to capture logging telemetry with
     # This is imperative so you do not collect logging telemetry from the SDK itself.
-    logger_name="my_application_logger",
+    logger_name="my_app_logger",
+    # You can specify the logging format of your collected logs by passing in a logging.Formatter
+    logging_formatter=Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 )
 
 # Logging telemetry will be collected from logging calls made with this logger and all of it's children loggers.
-logger = getLogger("my_application_logger")
+logger = getLogger("my_app_logger")
 logger.setLevel(INFO)
 
 # Logging calls with any logger that is a child logger will also be tracked
-logger_child = getLogger("my-application_logger.module")
+logger_child = getLogger("my_app_logger.module")
 logger_child.setLevel(INFO)
 
 # Logging calls with this logger will not be tracked
-logger_not_tracked = getLogger("not_my_application_logger")
+logger_not_tracked = getLogger("not_my_app_logger")
 logger_not_tracked.setLevel(INFO)
 
 logger.info("info log")

sdk/monitor/azure-monitor-opentelemetry/tests/test_configure.py

@@ -354,11 +354,13 @@ def test_setup_logging(
         logger_mock = Mock()
         logger_mock.handlers = []
         get_logger_mock.return_value = logger_mock
+        formatter_init_mock = Mock()
 
         configurations = {
             "connection_string": "test_cs",
             "logger_name": "test",
             "resource": TEST_RESOURCE,
+            "logging_formatter": formatter_init_mock
         }
         _setup_logging(configurations)
 
@@ -370,6 +372,7 @@ def test_setup_logging(
         )
         lp_init_mock.add_log_record_processor.assert_called_once_with(blrp_init_mock)
         logging_handler_mock.assert_called_once_with(logger_provider=lp_init_mock)
+        logging_handler_init_mock.setFormatter.assert_called_once_with(formatter_init_mock)
         get_logger_mock.assert_called_once_with("test")
         logger_mock.addHandler.assert_called_once_with(logging_handler_init_mock)
         elp_mock.assert_called_once_with(lp_init_mock)
@@ -414,6 +417,7 @@ def test_setup_logging_duplicate_logger(
             "connection_string": "test_cs",
             "logger_name": "test",
             "resource": TEST_RESOURCE,
+            "logging_formatter": None,
         }
         _setup_logging(configurations)
 
@@ -424,7 +428,6 @@ def test_setup_logging_duplicate_logger(
             log_exp_init_mock,
         )
         lp_init_mock.add_log_record_processor.assert_called_once_with(blrp_init_mock)
-        # logging_handler_mock.assert_not_called()
         get_logger_mock.assert_called_once_with("test")
         logger_mock.addHandler.assert_not_called()
 
@@ -660,26 +663,46 @@ def test_setup_instrumentations_disabled(
         logger_mock.debug.assert_called_once()
 
     @patch("azure.monitor.opentelemetry._configure.AzureDiagnosticLogging")
+    @patch("azure.monitor.opentelemetry._configure._is_on_functions")
     @patch("azure.monitor.opentelemetry._configure._is_attach_enabled")
     def test_send_attach_warning_true(
         self,
         is_attach_enabled_mock,
+        is_on_functions_mock,
         mock_diagnostics,
     ):
         is_attach_enabled_mock.return_value = True
+        is_on_functions_mock.return_value = False
         _send_attach_warning()
         mock_diagnostics.warning.assert_called_once_with(
             "Distro detected that automatic attach may have occurred. Check your data to ensure that telemetry is not being duplicated. This may impact your cost.",
             _DISTRO_DETECTS_ATTACH,
         )
 
     @patch("azure.monitor.opentelemetry._configure.AzureDiagnosticLogging")
+    @patch("azure.monitor.opentelemetry._configure._is_on_functions")
     @patch("azure.monitor.opentelemetry._configure._is_attach_enabled")
     def test_send_attach_warning_false(
         self,
         is_attach_enabled_mock,
+        is_on_functions_mock,
         mock_diagnostics,
     ):
         is_attach_enabled_mock.return_value = False
+        is_on_functions_mock.return_value = False
+        _send_attach_warning()
+        mock_diagnostics.warning.assert_not_called()
+
+    @patch("azure.monitor.opentelemetry._configure.AzureDiagnosticLogging")
+    @patch("azure.monitor.opentelemetry._configure._is_on_functions")
+    @patch("azure.monitor.opentelemetry._configure._is_attach_enabled")
+    def test_send_attach_warning_false_on_functions(
+        self,
+        is_attach_enabled_mock,
+        is_on_functions_mock,
+        mock_diagnostics,
+    ):
+        is_attach_enabled_mock.return_value = True
+        is_on_functions_mock.return_value = True
         _send_attach_warning()
         mock_diagnostics.warning.assert_not_called()