diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py index 188a25842..0ac1f69de 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py @@ -65,6 +65,7 @@ DEFAULT_METRIC_EXPORT_INTERVAL = 60000.0 AWS_LAMBDA_FUNCTION_NAME_CONFIG = "AWS_LAMBDA_FUNCTION_NAME" AWS_XRAY_DAEMON_ADDRESS_CONFIG = "AWS_XRAY_DAEMON_ADDRESS" +OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG = "OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED" _logger: Logger = getLogger(__name__) @@ -85,6 +86,11 @@ class AwsOpenTelemetryConfigurator(_OTelSDKConfigurator): # pylint: disable=no-self-use @override def _configure(self, **kwargs): + if _is_defer_to_workers_enabled() and _is_wsgi_master_process(): + _logger.info( + "Skipping ADOT initialization since deferral to worker is enabled, and this is a master process." + ) + return _initialize_components() @@ -156,6 +162,27 @@ def _init_tracing( # END The OpenTelemetry Authors code +def _is_defer_to_workers_enabled(): + return os.environ.get(OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG, "false").strip().lower() == "true" + + +def _is_wsgi_master_process(): + # Since the auto-instrumentation loads whenever a process is created and due to known issues with instrumenting + # WSGI apps using OTel, we want to skip the instrumentation of master process. + # This function is used to identify if the current process is a WSGI server's master process or not. + # Typically, a WSGI fork process model server spawns a single master process and multiple worker processes. + # When the master process starts, we use an environment variable as a marker. Since child worker processes inherit + # the master process environment, checking this marker in worker will tell that master process has been seen. + # Note: calling this function more than once in the same master process will return incorrect result. + # So use carefully. + if os.environ.get("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", "false").lower() == "true": + _logger.info("pid %s identified as a worker process", str(os.getpid())) + return False + os.environ["IS_WSGI_MASTER_PROCESS_ALREADY_SEEN"] = "true" + _logger.info("pid %s identified as a master process", str(os.getpid())) + return True + + def _exclude_urls_for_instrumentations(): urls_to_exclude_instr = "SamplingTargets,GetSamplingRules" requests_excluded_urls = os.environ.pop("OTEL_PYTHON_REQUESTS_EXCLUDED_URLS", "") diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py index 080ee7c7e..d8bfc5756 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py @@ -16,6 +16,8 @@ _customize_sampler, _customize_span_processors, _is_application_signals_enabled, + _is_defer_to_workers_enabled, + _is_wsgi_master_process, ) from amazon.opentelemetry.distro.aws_opentelemetry_distro import AwsOpenTelemetryDistro from amazon.opentelemetry.distro.aws_span_metrics_processor import AwsSpanMetricsProcessor @@ -305,6 +307,50 @@ def test_application_signals_exporter_provider(self): self.assertEqual("127.0.0.1:2000", exporter._udp_exporter._endpoint) os.environ.pop("AWS_LAMBDA_FUNCTION_NAME", None) + def test_is_defer_to_workers_enabled(self): + os.environ.setdefault("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", "True") + self.assertTrue(_is_defer_to_workers_enabled()) + os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None) + + os.environ.setdefault("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", "False") + self.assertFalse(_is_defer_to_workers_enabled()) + os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None) + self.assertFalse(_is_defer_to_workers_enabled()) + + def test_is_wsgi_master_process_first_time(self): + self.assertTrue(_is_wsgi_master_process()) + self.assertEqual(os.environ["IS_WSGI_MASTER_PROCESS_ALREADY_SEEN"], "true") + os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None) + + @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator._initialize_components") + def test_initialize_components_skipped_in_master_when_deferred_enabled(self, mock_initialize_components): + os.environ.setdefault("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", "True") + os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None) + self.assertTrue(_is_defer_to_workers_enabled()) + AwsOpenTelemetryConfigurator()._configure() + mock_initialize_components.assert_not_called() + os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None) + os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None) + + @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator._initialize_components") + def test_initialize_components_called_in_worker_when_deferred_enabled(self, mock_initialize_components): + os.environ.setdefault("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", "True") + os.environ.setdefault("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", "true") + self.assertTrue(_is_defer_to_workers_enabled()) + self.assertFalse(_is_wsgi_master_process()) + AwsOpenTelemetryConfigurator()._configure() + mock_initialize_components.assert_called_once() + os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None) + os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None) + + @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator._initialize_components") + def test_initialize_components_called_when_deferred_disabled(self, mock_initialize_components): + os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None) + self.assertFalse(_is_defer_to_workers_enabled()) + AwsOpenTelemetryConfigurator()._configure() + mock_initialize_components.assert_called_once() + os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None) + def validate_distro_environ(): tc: TestCase = TestCase()