Refactor resource mapping logic into resource detection package (#235)

and update the exporters to use the shared code.

Author: aabmass

e2e-test-server/Dockerfile

@@ -28,6 +28,7 @@ FROM python-base as build-base
 # copy local dependencies
 COPY opentelemetry-exporter-gcp-trace opentelemetry-exporter-gcp-trace
 COPY opentelemetry-propagator-gcp opentelemetry-propagator-gcp
+COPY opentelemetry-resourcedetector-gcp opentelemetry-resourcedetector-gcp
 WORKDIR $SRC/e2e-test-server
 # copy requirements/constraints
 COPY e2e-test-server/requirements.txt e2e-test-server/constraints.txt ./

e2e-test-server/constraints.txt

@@ -31,7 +31,7 @@ PyYAML==5.4.1
 requests==2.25.1
 rsa==4.7.2
 six==1.15.0
-typing-extensions==3.10.0.0
+typing-extensions==4.5.0
 typing-inspect==0.6.0
 urllib3==1.26.4
 waitress==2.0.0

e2e-test-server/requirements.txt

@@ -1,6 +1,7 @@
 -c constraints.txt
 ../opentelemetry-exporter-gcp-trace
 ../opentelemetry-propagator-gcp
+../opentelemetry-resourcedetector-gcp
 opentelemetry-sdk
 opentelemetry-api
 Flask

opentelemetry-exporter-gcp-monitoring/setup.cfg

@@ -28,6 +28,7 @@ install_requires =
     google-cloud-monitoring ~= 2.0
     opentelemetry-api ~= 1.0
     opentelemetry-sdk ~= 1.0
+    opentelemetry-resourcedetector-gcp >= 1.5.0dev0, == 1.*
 
 [options.packages.find]
 where = src

opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py

@@ -24,6 +24,7 @@
 from google.api.label_pb2 import LabelDescriptor
 from google.api.metric_pb2 import Metric as GMetric
 from google.api.metric_pb2 import MetricDescriptor
+from google.api.monitored_resource_pb2 import MonitoredResource
 from google.cloud.monitoring_v3 import (
     CreateMetricDescriptorRequest,
     CreateTimeSeriesRequest,
@@ -39,10 +40,10 @@
 
 # pylint: disable=no-name-in-module
 from google.protobuf.timestamp_pb2 import Timestamp
-from opentelemetry.exporter.cloud_monitoring._resource import (
+from opentelemetry.exporter.cloud_monitoring.version import __version__
+from opentelemetry.resourcedetector.gcp_resource_detector._mapping import (
     get_monitored_resource,
 )
-from opentelemetry.exporter.cloud_monitoring.version import __version__
 from opentelemetry.sdk.metrics.export import (
     Gauge,
     Histogram,
@@ -296,9 +297,19 @@ def export(
         all_series = []
 
         for resource_metric in metrics_data.resource_metrics:
-            monitored_resource = get_monitored_resource(
+            monitored_resource_data = get_monitored_resource(
                 resource_metric.resource
             )
+            # convert it to proto
+            monitored_resource = (
+                MonitoredResource(
+                    type=monitored_resource_data.type,
+                    labels=monitored_resource_data.labels,
+                )
+                if monitored_resource_data
+                else None
+            )
+
             for scope_metric in resource_metric.scope_metrics:
                 for metric in scope_metric.metrics:
                     # Convert all data_points to Sequences, see

opentelemetry-exporter-gcp-trace/setup.cfg

@@ -28,6 +28,7 @@ install_requires =
     google-cloud-trace ~= 1.1
     opentelemetry-api ~= 1.0
     opentelemetry-sdk ~= 1.0
+    opentelemetry-resourcedetector-gcp >= 1.5.0dev0, == 1.*
 
 [options.packages.find]
 where = src

opentelemetry-exporter-gcp-trace/src/opentelemetry/exporter/cloud_trace/__init__.py

@@ -102,6 +102,12 @@
     OTEL_EXPORTER_GCP_TRACE_RESOURCE_REGEX,
 )
 from opentelemetry.exporter.cloud_trace.version import __version__
+from opentelemetry.resourcedetector.gcp_resource_detector import (
+    _constants as _resource_constants,
+)
+from opentelemetry.resourcedetector.gcp_resource_detector._mapping import (
+    get_monitored_resource,
+)
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import Event
 from opentelemetry.sdk.trace.export import (
@@ -128,6 +134,7 @@
     ("grpc.primary_user_agent", _USER_AGENT),
 ]
 
+
 MAX_NUM_LINKS = 128
 MAX_NUM_EVENTS = 32
 MAX_EVENT_ATTRS = 4
@@ -420,24 +427,6 @@ def _strip_characters(ot_version):
     return "".join(filter(lambda x: x.isdigit() or x == ".", ot_version))
 
 
-OT_RESOURCE_ATTRIBUTE_TO_GCP = {
-    "gce_instance": {
-        "host.id": "instance_id",
-        "cloud.account.id": "project_id",
-        "cloud.zone": "zone",
-    },
-    "gke_container": {
-        "k8s.cluster.name": "cluster_name",
-        "k8s.namespace.name": "namespace_id",
-        "k8s.pod.name": "pod_id",
-        "host.id": "instance_id",
-        "container.name": "container_name",
-        "cloud.account.id": "project_id",
-        "cloud.zone": "zone",
-    },
-}
-
-
 def _extract_resources(
     resource: Resource, resource_regex: Optional[Pattern] = None
 ) -> Dict[str, str]:
@@ -451,24 +440,18 @@ def _extract_resources(
                 if resource_regex.match(k)
             }
         )
-    if resource_attributes.get("cloud.provider") != "gcp":
-        return extracted_attributes
-    resource_type = resource_attributes["gcp.resource_type"]
-    if (
-        not isinstance(resource_type, str)
-        or resource_type not in OT_RESOURCE_ATTRIBUTE_TO_GCP
+    monitored_resource = get_monitored_resource(resource)
+    # Do not map generic_task and generic_node to g.co/r/... span labels.
+    if monitored_resource and monitored_resource.type not in (
+        _resource_constants.GENERIC_NODE,
+        _resource_constants.GENERIC_TASK,
     ):
-        return extracted_attributes
-    extracted_attributes.update(
-        {
-            "g.co/r/{}/{}".format(resource_type, gcp_resource_key): str(
-                resource_attributes[ot_resource_key]
-            )
-            for ot_resource_key, gcp_resource_key in OT_RESOURCE_ATTRIBUTE_TO_GCP[
-                resource_type
-            ].items()
-        }
-    )
+        extracted_attributes.update(
+            {
+                "g.co/r/{}/{}".format(monitored_resource.type, k): v
+                for k, v in monitored_resource.labels.items()
+            }
+        )
     return extracted_attributes
 
 

opentelemetry-exporter-gcp-trace/tests/test_cloud_trace_exporter.py

@@ -124,11 +124,14 @@ def test_constructor_explicit(self):
     def test_export(self):
         resource_info = Resource(
             {
-                "cloud.account.id": 123,
-                "host.id": "host",
-                "cloud.zone": "US",
+                "cloud.account.id": "123",
+                "cloud.platform": "gcp_compute_engine",
                 "cloud.provider": "gcp",
-                "gcp.resource_type": "gce_instance",
+                "cloud.region": "us-east4",
+                "cloud.availability_zone": "us-east4-b",
+                "host.id": "host",
+                "host.name": "fakeName",
+                "host.type": "fakeMachineType",
             }
         )
         span_datas = [
@@ -157,13 +160,12 @@ def test_export(self):
             ),
             "attributes": ProtoSpan.Attributes(
                 attribute_map={
-                    "g.co/r/gce_instance/zone": _format_attribute_value("US"),
+                    "g.co/r/gce_instance/zone": _format_attribute_value(
+                        "us-east4-b"
+                    ),
                     "g.co/r/gce_instance/instance_id": _format_attribute_value(
                         "host"
                     ),
-                    "g.co/r/gce_instance/project_id": _format_attribute_value(
-                        "123"
-                    ),
                     "g.co/agent": self.agent_code,
                     "attr_key": _format_attribute_value("attr_value"),
                 }
@@ -652,27 +654,30 @@ def test_too_many_link_attributes(self):
         )
 
     def test_extract_empty_resources(self):
-        self.assertEqual(_extract_resources(Resource.get_empty()), {})
+        self.assertEqual(
+            _extract_resources(Resource.get_empty()),
+            {},
+        )
 
     def test_extract_resource_attributes_with_regex(self):
         resource_regex = re.compile(r"service\..*")
         resource = Resource(
             attributes={
-                "cloud.account.id": 123,
-                "host.id": "host",
-                "cloud.zone": "US",
+                "cloud.account.id": "123",
+                "cloud.availability_zone": "us-east4-b",
+                "cloud.platform": "gcp_compute_engine",
                 "cloud.provider": "gcp",
-                "extra_info": "extra",
-                "gcp.resource_type": "gce_instance",
-                "not_gcp_resource": "value",
+                "cloud.region": "us-east4",
+                "host.id": "host",
+                "host.name": "fakeName",
+                "host.type": "fakeMachineType",
                 "service.name": "my-app",
                 "service.version": "1",
             }
         )
         expected_extract = {
-            "g.co/r/gce_instance/project_id": "123",
             "g.co/r/gce_instance/instance_id": "host",
-            "g.co/r/gce_instance/zone": "US",
+            "g.co/r/gce_instance/zone": "us-east4-b",
             "service.name": "my-app",
             "service.version": "1",
         }
@@ -684,19 +689,19 @@ def test_non_matching_regex(self):
         resource_regex = re.compile(r"this-regex-matches-nothing")
         resource = Resource(
             attributes={
-                "cloud.account.id": 123,
-                "host.id": "host",
-                "cloud.zone": "US",
+                "cloud.account.id": "123",
+                "cloud.availability_zone": "us-east4-b",
+                "cloud.platform": "gcp_compute_engine",
                 "cloud.provider": "gcp",
-                "extra_info": "extra",
-                "gcp.resource_type": "gce_instance",
-                "not_gcp_resource": "value",
+                "cloud.region": "us-east4",
+                "host.id": "host",
+                "host.name": "fakeName",
+                "host.type": "fakeMachineType",
             }
         )
         expected_extract = {
-            "g.co/r/gce_instance/project_id": "123",
             "g.co/r/gce_instance/instance_id": "host",
-            "g.co/r/gce_instance/zone": "US",
+            "g.co/r/gce_instance/zone": "us-east4-b",
         }
         self.assertEqual(
             _extract_resources(resource, resource_regex), expected_extract
@@ -705,34 +710,22 @@ def test_non_matching_regex(self):
     def test_extract_well_formed_resources(self):
         resource = Resource(
             attributes={
-                "cloud.account.id": 123,
-                "host.id": "host",
-                "cloud.zone": "US",
+                "cloud.account.id": "123",
+                "cloud.availability_zone": "us-east4-b",
+                "cloud.platform": "gcp_compute_engine",
                 "cloud.provider": "gcp",
-                "extra_info": "extra",
-                "gcp.resource_type": "gce_instance",
-                "not_gcp_resource": "value",
+                "cloud.region": "us-east4",
+                "host.id": "host",
+                "host.name": "fakeName",
+                "host.type": "fakeMachineType",
             }
         )
         expected_extract = {
-            "g.co/r/gce_instance/project_id": "123",
             "g.co/r/gce_instance/instance_id": "host",
-            "g.co/r/gce_instance/zone": "US",
+            "g.co/r/gce_instance/zone": "us-east4-b",
         }
         self.assertEqual(_extract_resources(resource), expected_extract)
 
-    def test_extract_malformed_resources(self):
-        # This resource doesn't have all the fields required for a gce_instance
-        # Specifically its missing "host.id", "cloud.zone", "cloud.account.id"
-        resource = Resource(
-            attributes={
-                "gcp.resource_type": "gce_instance",
-                "cloud.provider": "gcp",
-            }
-        )
-        # Should throw when passed a malformed GCP resource dict
-        self.assertRaises(KeyError, _extract_resources, resource)
-
     def test_extract_unsupported_gcp_resources(self):
         # Unsupported gcp resources will be ignored
         resource = Resource(
@@ -741,24 +734,14 @@ def test_extract_unsupported_gcp_resources(self):
                 "host.id": "host",
                 "extra_info": "extra",
                 "not_gcp_resource": "value",
-                "gcp.resource_type": "unsupported_gcp_resource",
+                "cloud.platform": "gcp_some_unsupported_thing",
                 "cloud.provider": "gcp",
             }
         )
-        self.assertEqual(_extract_resources(resource), {})
-
-    def test_extract_unsupported_provider_resources(self):
-        # Resources with currently unsupported providers will be ignored
-        resource = Resource(
-            attributes={
-                "cloud.account.id": "123",
-                "host.id": "host",
-                "extra_info": "extra",
-                "not_gcp_resource": "value",
-                "cloud.provider": "aws",
-            }
+        self.assertEqual(
+            _extract_resources(resource),
+            {},
         )
-        self.assertEqual(_extract_resources(resource), {})
 
     def test_truncate_string(self):
         """Cloud Trace API imposes limits on the length of many things,