distro: add cloud resource detectors (#198)

* distro: add cloud resource detectors

This add some code to dinamically filter the available resource
detectors in order to avoid any eventual costly detection.

* Remove container resource detector that has not been published

* Update distributoin default settings

* Update README.md

* Update dev-requirements with the dependencies

Author: xrmx

README.md

@@ -78,9 +78,14 @@ This distribution sets the following defaults:
 
 - `OTEL_TRACES_EXPORTER`: `otlp`
 - `OTEL_METRICS_EXPORTER`: `otlp`
+- `OTEL_LOGS_EXPORTER`: `otlp`
 - `OTEL_EXPORTER_OTLP_PROTOCOL`: `grpc`
-- `OTEL_EXPERIMENTAL_RESOURCE_DETECTORS`: `process_runtime,os,otel,telemetry_distro`
+- `OTEL_EXPERIMENTAL_RESOURCE_DETECTORS`: `process_runtime,os,otel,telemetry_distro,_gcp,aws_ec2,aws_ecs,aws_elastic_beanstalk,azure_app_service,azure_vm`
 - `OTEL_METRICS_EXEMPLAR_FILTER`: `always_off`
+- `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE`: `DELTA`
+
+> [!NOTE]
+> `OTEL_EXPERIMENTAL_RESOURCE_DETECTORS` cloud resource detectors are dynamically set. When running in a Kubernetes Pod it will be set to `process_runtime,os,otel,telemetry_distro,_gcp,aws_eks`.
 
 ### Distribution specific configuration variables
 

dev-requirements.txt

@@ -43,6 +43,7 @@ opentelemetry-api==1.28.2
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-system-metrics
+    #   opentelemetry-resourcedetector-gcp
     #   opentelemetry-sdk
     #   opentelemetry-semantic-conventions
     #   oteltest
@@ -68,11 +69,20 @@ opentelemetry-proto==1.28.2
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
     #   oteltest
+opentelemetry-resource-detector-azure==0.1.5
+    # via elastic-opentelemetry (pyproject.toml)
+opentelemetry-resourcedetector-gcp==1.7.0a0
+    # via elastic-opentelemetry (pyproject.toml)
 opentelemetry-sdk==1.28.2
     # via
     #   elastic-opentelemetry (pyproject.toml)
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-resource-detector-azure
+    #   opentelemetry-resourcedetector-gcp
+    #   opentelemetry-sdk-extension-aws
+opentelemetry-sdk-extension-aws==2.0.2
+    # via elastic-opentelemetry (pyproject.toml)
 opentelemetry-semantic-conventions==0.49b2
     # via
     #   elastic-opentelemetry (pyproject.toml)
@@ -83,6 +93,7 @@ oteltest==0.24.0
 packaging==24.2
     # via
     #   build
+    #   elastic-opentelemetry (pyproject.toml)
     #   opentelemetry-instrumentation
     #   pytest
 pip-tools==7.4.1
@@ -103,14 +114,18 @@ pyproject-hooks==1.2.0
 pytest==8.3.4
     # via elastic-opentelemetry (pyproject.toml)
 requests==2.32.3
-    # via opentelemetry-exporter-otlp-proto-http
+    # via
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-resourcedetector-gcp
 tomli==2.2.1
     # via
     #   build
     #   pip-tools
     #   pytest
 typing-extensions==4.12.2
-    # via opentelemetry-sdk
+    # via
+    #   opentelemetry-resourcedetector-gcp
+    #   opentelemetry-sdk
 urllib3==2.2.3
     # via requests
 wheel==0.45.1

pyproject.toml

@@ -31,8 +31,11 @@ dependencies = [
     "opentelemetry-exporter-otlp == 1.28.2",
     "opentelemetry-instrumentation == 0.49b2",
     "opentelemetry-instrumentation-system-metrics == 0.49b2",
-    "opentelemetry-semantic-conventions == 0.49b2",
+    "opentelemetry-resourcedetector-gcp ~= 1.7.0a0",
+    "opentelemetry-resource-detector-azure ~= 0.1.5",
     "opentelemetry-sdk == 1.28.2",
+    "opentelemetry-sdk-extension-aws ~= 2.0.2",
+    "opentelemetry-semantic-conventions == 0.49b2",
     "packaging",
 ]
 
@@ -48,6 +51,7 @@ distro = "elasticotel.distro:ElasticOpenTelemetryDistro"
 [project.entry-points.opentelemetry_resource_detector]
 process_runtime = "elasticotel.sdk.resources:ProcessRuntimeResourceDetector"
 telemetry_distro = "elasticotel.sdk.resources:TelemetryDistroResourceDetector"
+_gcp = "opentelemetry.resourcedetector.gcp_resource_detector._detector:GoogleCloudResourceDetector"
 
 [project.scripts]
 edot-bootstrap = "elasticotel.instrumentation.bootstrap:run"

src/elasticotel/distro/__init__.py

@@ -56,6 +56,7 @@
 from opentelemetry.util._importlib_metadata import EntryPoint
 
 from elasticotel.distro.environment_variables import ELASTIC_OTEL_SYSTEM_METRICS_ENABLED
+from elasticotel.distro.resource_detectors import get_cloud_resource_detectors
 
 
 logger = logging.getLogger(__name__)
@@ -139,8 +140,11 @@ def _configure(self, **kwargs):
         os.environ.setdefault(OTEL_METRICS_EXPORTER, "otlp")
         os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp")
         os.environ.setdefault(OTEL_EXPORTER_OTLP_PROTOCOL, "grpc")
-        os.environ.setdefault(OTEL_EXPERIMENTAL_RESOURCE_DETECTORS, "process_runtime,os,otel,telemetry_distro")
         # disable exemplars by default for now
         os.environ.setdefault(OTEL_METRICS_EXEMPLAR_FILTER, "always_off")
         # preference to use DELTA temporality as we can handle only this kind of Histograms
         os.environ.setdefault(OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, "DELTA")
+
+        base_resource_detectors = ["process_runtime", "os", "otel", "telemetry_distro"]
+        detectors = base_resource_detectors + get_cloud_resource_detectors()
+        os.environ.setdefault(OTEL_EXPERIMENTAL_RESOURCE_DETECTORS, ",".join(detectors))

src/elasticotel/distro/resource_detectors.py

@@ -0,0 +1,65 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+AWS_LAMBDA_DETECTORS = ["aws_lambda"]
+AZURE_FUNCTIONS_DETECTORS = ["azure_functions"]
+GCP_CLOUD_RUN_DETECTORS = ["_gcp"]
+KUBERNETES_DETECTORS = ["_gcp", "aws_eks"]
+OTHER_CLOUD_DETECTORS = [
+    "_gcp",
+    "aws_ec2",
+    "aws_ecs",
+    "aws_elastic_beanstalk",
+    "azure_app_service",
+    "azure_vm",
+]
+
+
+def _on_aws_lambda():
+    """Cheap check to detect if we are running on AWS lambda"""
+    return "AWS_LAMBDA_FUNCTION_NAME" in os.environ
+
+
+def _on_azure_functions():
+    """Cheap check to detect if we are running on Azure functions"""
+    return "FUNCTIONS_WORKER_RUNTIME" in os.environ
+
+
+def _on_gcp_cloud_run():
+    """Cheap check to detect if we are running inside Google Cloud Run"""
+    return "K_CONFIGURATION" in os.environ
+
+
+def _on_k8s():
+    """Cheap check to detect if we are running inside a Kubernetes pod or not"""
+    return "KUBERNETES_SERVICE_HOST" in os.environ
+
+
+def get_cloud_resource_detectors():
+    """Helper to get a subset of the available cloud resource detectors depending on the environment
+
+    This is done to avoid loading resource detectors doing HTTP requests for metadata that will fail"""
+    if _on_aws_lambda():
+        return AWS_LAMBDA_DETECTORS
+    elif _on_azure_functions():
+        return AZURE_FUNCTIONS_DETECTORS
+    elif _on_gcp_cloud_run():
+        return GCP_CLOUD_RUN_DETECTORS
+    elif _on_k8s():
+        return KUBERNETES_DETECTORS
+    return OTHER_CLOUD_DETECTORS

tests/distro/test_distro.py

@@ -42,7 +42,8 @@ def test_default_configuration(self):
         self.assertEqual("otlp", os.environ.get(OTEL_LOGS_EXPORTER))
         self.assertEqual("grpc", os.environ.get(OTEL_EXPORTER_OTLP_PROTOCOL))
         self.assertEqual(
-            "process_runtime,os,otel,telemetry_distro", os.environ.get(OTEL_EXPERIMENTAL_RESOURCE_DETECTORS)
+            "process_runtime,os,otel,telemetry_distro,_gcp,aws_ec2,aws_ecs,aws_elastic_beanstalk,azure_app_service,azure_vm",
+            os.environ.get(OTEL_EXPERIMENTAL_RESOURCE_DETECTORS),
         )
         self.assertEqual("always_off", os.environ.get(OTEL_METRICS_EXEMPLAR_FILTER))
         self.assertEqual("DELTA", os.environ.get(OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE))

tests/distro/test_resource_detectors.py

@@ -0,0 +1,49 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from unittest import TestCase, mock
+
+from elasticotel.distro.resource_detectors import get_cloud_resource_detectors
+
+
+class TestGetCloudResourceDetectors(TestCase):
+    @mock.patch.dict("os.environ", {"AWS_LAMBDA_FUNCTION_NAME": "lambda"}, clear=True)
+    def test_aws_lambda(self):
+        resource_detectors = get_cloud_resource_detectors()
+        self.assertEqual(resource_detectors, ["aws_lambda"])
+
+    @mock.patch.dict("os.environ", {"FUNCTIONS_WORKER_RUNTIME": "azure"}, clear=True)
+    def test_azure_functions(self):
+        resource_detectors = get_cloud_resource_detectors()
+        self.assertEqual(resource_detectors, ["azure_functions"])
+
+    @mock.patch.dict("os.environ", {"K_CONFIGURATION": "cloudrun"}, clear=True)
+    def test_gcp_cloud_run(self):
+        resource_detectors = get_cloud_resource_detectors()
+        self.assertEqual(resource_detectors, ["_gcp"])
+
+    @mock.patch.dict("os.environ", {"KUBERNETES_SERVICE_HOST": "k8s"}, clear=True)
+    def test_kubernetes_pod(self):
+        resource_detectors = get_cloud_resource_detectors()
+        self.assertEqual(resource_detectors, ["_gcp", "aws_eks"])
+
+    @mock.patch.dict("os.environ", {}, clear=True)
+    def test_other_cloud_detectors(self):
+        resource_detectors = get_cloud_resource_detectors()
+        self.assertEqual(
+            resource_detectors,
+            ["_gcp", "aws_ec2", "aws_ecs", "aws_elastic_beanstalk", "azure_app_service", "azure_vm"],
+        )