Skip to content

Commit 3f342f0

Browse files
aabmassaabmass
authored
Implement OTel resource to monitored resource mapping (#214)
* Implement OTel resource to monitored resource mapping * cleanup * Add clarifying comment
1 parent dbc8f3e commit 3f342f0

File tree

6 files changed

+769
-19
lines changed

6 files changed

+769
-19
lines changed

‎opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@
3636

3737
# pylint: disable=no-name-in-module
3838
from google.protobuf.timestamp_pb2 import Timestamp
39+
from opentelemetry.exporter.cloud_monitoring._resource import (
40+
get_monitored_resource,
41+
)
3942
from opentelemetry.sdk.metrics.export import (
4043
Gauge,
4144
Histogram,
@@ -105,24 +108,6 @@ def __init__(
105108
self._exporter_start_time_nanos,
106109
) = divmod(time_ns(), NANOS_PER_SECOND)
107110

108-
@staticmethod
109-
def _get_monitored_resource(
110-
_: Resource,
111-
) -> Optional[MonitoredResource]:
112-
"""Add Google resource specific information (e.g. instance id, region).
113-
114-
See
115-
https://cloud.google.com/monitoring/custom-metrics/creating-metrics#custom-metric-resources
116-
for supported types
117-
Args:
118-
resource: OTel resource
119-
"""
120-
# TODO: implement new monitored resource mapping spec
121-
return MonitoredResource(
122-
type="generic_node",
123-
labels={"location": "global", "namespace": "", "node_id": ""},
124-
)
125-
126111
def _batch_write(self, series: List[TimeSeries]) -> None:
127112
"""Cloud Monitoring allows writing up to 200 time series at once
128113
@@ -289,7 +274,7 @@ def export(
289274
all_series = []
290275

291276
for resource_metric in metrics_data.resource_metrics:
292-
monitored_resource = self._get_monitored_resource(
277+
monitored_resource = get_monitored_resource(
293278
resource_metric.resource
294279
)
295280
for scope_metric in resource_metric.scope_metrics:

‎opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/_resource.py

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import json
16+
from typing import Dict, Optional, Tuple
17+
18+
from google.api.monitored_resource_pb2 import MonitoredResource
19+
from opentelemetry.sdk.resources import Attributes, Resource
20+
21+
22+
# TODO: use opentelemetry-semantic-conventions package for these constants once it has
23+
# stabilized. Right now, pinning an unstable version would cause dependency conflicts for
24+
# users so these are copied in.
25+
class ResourceAttributes:
26+
AWS_EC2 = "aws_ec2"
27+
CLOUD_ACCOUNT_ID = "cloud.account.id"
28+
CLOUD_AVAILABILITY_ZONE = "cloud.availability_zone"
29+
CLOUD_PLATFORM_KEY = "cloud.platform"
30+
CLOUD_PROVIDER = "cloud.provider"
31+
CLOUD_REGION = "cloud.region"
32+
GCP_COMPUTE_ENGINE = "gcp_compute_engine"
33+
GCP_KUBERNETES_ENGINE = "gcp_kubernetes_engine"
34+
HOST_ID = "host.id"
35+
HOST_NAME = "host.name"
36+
K8S_CLUSTER_NAME = "k8s.cluster.name"
37+
K8S_CONTAINER_NAME = "k8s.container.name"
38+
K8S_NAMESPACE_NAME = "k8s.namespace.name"
39+
K8S_NODE_NAME = "k8s.node.name"
40+
K8S_POD_NAME = "k8s.pod.name"
41+
SERVICE_INSTANCE_ID = "service.instance.id"
42+
SERVICE_NAME = "service.name"
43+
SERVICE_NAMESPACE = "service.namespace"
44+
45+
46+
AWS_ACCOUNT = "aws_account"
47+
AWS_EC2_INSTANCE = "aws_ec2_instance"
48+
CLUSTER_NAME = "cluster_name"
49+
CONTAINER_NAME = "container_name"
50+
GCE_INSTANCE = "gce_instance"
51+
GENERIC_NODE = "generic_node"
52+
GENERIC_TASK = "generic_task"
53+
INSTANCE_ID = "instance_id"
54+
JOB = "job"
55+
K8S_CLUSTER = "k8s_cluster"
56+
K8S_CONTAINER = "k8s_container"
57+
K8S_NODE = "k8s_node"
58+
K8S_POD = "k8s_pod"
59+
LOCATION = "location"
60+
NAMESPACE = "namespace"
61+
NAMESPACE_NAME = "namespace_name"
62+
NODE_ID = "node_id"
63+
NODE_NAME = "node_name"
64+
POD_NAME = "pod_name"
65+
REGION = "region"
66+
TASK_ID = "task_id"
67+
ZONE = "zone"
68+
69+
70+
class MapConfig:
71+
otel_keys: Tuple[str, ...]
72+
"""
73+
OTel resource keys to try and populate the resource label from. For entries with multiple
74+
OTel resource keys, the keys' values will be coalesced in order until there is a non-empty
75+
value.
76+
"""
77+
78+
fallback: str
79+
"""If none of the otelKeys are present in the Resource, fallback to this literal value"""
80+
81+
def __init__(self, *otel_keys: str, fallback: str = ""):
82+
self.otel_keys = otel_keys
83+
self.fallback = fallback
84+
85+
86+
# Mappings of GCM resource label keys onto mapping config from OTel resource for a given
87+
# monitored resource type. Copied from Go impl:
88+
# https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/blob/v1.8.0/internal/resourcemapping/resourcemapping.go#L51
89+
MAPPINGS = {
90+
GCE_INSTANCE: {
91+
ZONE: MapConfig(ResourceAttributes.CLOUD_AVAILABILITY_ZONE),
92+
INSTANCE_ID: MapConfig(ResourceAttributes.HOST_ID),
93+
},
94+
K8S_CONTAINER: {
95+
LOCATION: MapConfig(
96+
ResourceAttributes.CLOUD_AVAILABILITY_ZONE,
97+
ResourceAttributes.CLOUD_REGION,
98+
),
99+
CLUSTER_NAME: MapConfig(ResourceAttributes.K8S_CLUSTER_NAME),
100+
NAMESPACE_NAME: MapConfig(ResourceAttributes.K8S_NAMESPACE_NAME),
101+
POD_NAME: MapConfig(ResourceAttributes.K8S_POD_NAME),
102+
CONTAINER_NAME: MapConfig(ResourceAttributes.K8S_CONTAINER_NAME),
103+
},
104+
K8S_POD: {
105+
LOCATION: MapConfig(
106+
ResourceAttributes.CLOUD_AVAILABILITY_ZONE,
107+
ResourceAttributes.CLOUD_REGION,
108+
),
109+
CLUSTER_NAME: MapConfig(ResourceAttributes.K8S_CLUSTER_NAME),
110+
NAMESPACE_NAME: MapConfig(ResourceAttributes.K8S_NAMESPACE_NAME),
111+
POD_NAME: MapConfig(ResourceAttributes.K8S_POD_NAME),
112+
},
113+
K8S_NODE: {
114+
LOCATION: MapConfig(
115+
ResourceAttributes.CLOUD_AVAILABILITY_ZONE,
116+
ResourceAttributes.CLOUD_REGION,
117+
),
118+
CLUSTER_NAME: MapConfig(ResourceAttributes.K8S_CLUSTER_NAME),
119+
NODE_NAME: MapConfig(ResourceAttributes.K8S_NODE_NAME),
120+
},
121+
K8S_CLUSTER: {
122+
LOCATION: MapConfig(
123+
ResourceAttributes.CLOUD_AVAILABILITY_ZONE,
124+
ResourceAttributes.CLOUD_REGION,
125+
),
126+
CLUSTER_NAME: MapConfig(ResourceAttributes.K8S_CLUSTER_NAME),
127+
},
128+
AWS_EC2_INSTANCE: {
129+
INSTANCE_ID: MapConfig(ResourceAttributes.HOST_ID),
130+
REGION: MapConfig(
131+
ResourceAttributes.CLOUD_AVAILABILITY_ZONE,
132+
ResourceAttributes.CLOUD_REGION,
133+
),
134+
AWS_ACCOUNT: MapConfig(ResourceAttributes.CLOUD_ACCOUNT_ID),
135+
},
136+
GENERIC_TASK: {
137+
LOCATION: MapConfig(
138+
ResourceAttributes.CLOUD_AVAILABILITY_ZONE,
139+
ResourceAttributes.CLOUD_REGION,
140+
fallback="global",
141+
),
142+
NAMESPACE: MapConfig(ResourceAttributes.SERVICE_NAMESPACE),
143+
JOB: MapConfig(ResourceAttributes.SERVICE_NAME),
144+
TASK_ID: MapConfig(ResourceAttributes.SERVICE_INSTANCE_ID),
145+
},
146+
GENERIC_NODE: {
147+
LOCATION: MapConfig(
148+
ResourceAttributes.CLOUD_AVAILABILITY_ZONE,
149+
ResourceAttributes.CLOUD_REGION,
150+
fallback="global",
151+
),
152+
NAMESPACE: MapConfig(ResourceAttributes.SERVICE_NAMESPACE),
153+
NODE_ID: MapConfig(
154+
ResourceAttributes.HOST_ID, ResourceAttributes.HOST_NAME
155+
),
156+
},
157+
}
158+
159+
160+
def get_monitored_resource(
161+
resource: Resource,
162+
) -> Optional[MonitoredResource]:
163+
"""Add Google resource specific information (e.g. instance id, region).
164+
165+
See
166+
https://cloud.google.com/monitoring/custom-metrics/creating-metrics#custom-metric-resources
167+
for supported types
168+
Args:
169+
resource: OTel resource
170+
"""
171+
172+
attrs = resource.attributes
173+
174+
platform = attrs.get(ResourceAttributes.CLOUD_PLATFORM_KEY)
175+
if platform == ResourceAttributes.GCP_COMPUTE_ENGINE:
176+
mr = _create_monitored_resource(GCE_INSTANCE, attrs)
177+
elif platform == ResourceAttributes.GCP_KUBERNETES_ENGINE:
178+
if ResourceAttributes.K8S_CONTAINER_NAME in attrs:
179+
mr = _create_monitored_resource(K8S_CONTAINER, attrs)
180+
elif ResourceAttributes.K8S_POD_NAME in attrs:
181+
mr = _create_monitored_resource(K8S_POD, attrs)
182+
elif ResourceAttributes.K8S_NODE_NAME in attrs:
183+
mr = _create_monitored_resource(K8S_NODE, attrs)
184+
else:
185+
mr = _create_monitored_resource(K8S_CLUSTER, attrs)
186+
elif platform == ResourceAttributes.AWS_EC2:
187+
mr = _create_monitored_resource(AWS_EC2_INSTANCE, attrs)
188+
else:
189+
# fallback to generic_task
190+
if (
191+
ResourceAttributes.SERVICE_NAME in attrs
192+
and ResourceAttributes.SERVICE_INSTANCE_ID in attrs
193+
):
194+
mr = _create_monitored_resource(GENERIC_TASK, attrs)
195+
else:
196+
mr = _create_monitored_resource(GENERIC_NODE, attrs)
197+
198+
return mr
199+
200+
201+
def _create_monitored_resource(
202+
monitored_resource_type: str, resource_attrs: Attributes
203+
) -> MonitoredResource:
204+
mapping = MAPPINGS[monitored_resource_type]
205+
labels: Dict[str, str] = {}
206+
207+
for mr_key, map_config in mapping.items():
208+
mr_value = None
209+
for otel_key in map_config.otel_keys:
210+
if otel_key in resource_attrs:
211+
mr_value = resource_attrs[otel_key]
212+
break
213+
214+
if mr_value is None:
215+
mr_value = map_config.fallback
216+
217+
# OTel attribute values can be any of str, bool, int, float, or Sequence of any of
218+
# them. Encode any non-strings as json string
219+
if not isinstance(mr_value, str):
220+
mr_value = json.dumps(
221+
mr_value, sort_keys=True, indent=None, separators=(",", ":")
222+
)
223+
labels[mr_key] = mr_value
224+
225+
return MonitoredResource(type=monitored_resource_type, labels=labels)

‎opentelemetry-exporter-gcp-monitoring/tests/__snapshots__/test_cloud_monitoring/test_with_resource.json

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
{
2+
"/google.monitoring.v3.MetricService/CreateMetricDescriptor": [
3+
{
4+
"metricDescriptor": {
5+
"description": "foo",
6+
"displayName": "mycounter",
7+
"labels": [
8+
{
9+
"key": "string"
10+
},
11+
{
12+
"key": "int"
13+
},
14+
{
15+
"key": "float"
16+
}
17+
],
18+
"metricKind": "CUMULATIVE",
19+
"type": "workload.googleapis.com/mycounter",
20+
"valueType": "INT64"
21+
},
22+
"name": "projects/fakeproject"
23+
}
24+
],
25+
"/google.monitoring.v3.MetricService/CreateTimeSeries": [
26+
{
27+
"name": "projects/fakeproject",
28+
"timeSeries": [
29+
{
30+
"metric": {
31+
"labels": {
32+
"float": "123.4",
33+
"int": "123",
34+
"string": "string"
35+
},
36+
"type": "workload.googleapis.com/mycounter"
37+
},
38+
"metricKind": "CUMULATIVE",
39+
"points": [
40+
{
41+
"interval": {
42+
"endTime": "str",
43+
"startTime": "str"
44+
},
45+
"value": {
46+
"int64Value": "12"
47+
}
48+
}
49+
],
50+
"resource": {
51+
"labels": {
52+
"cluster_name": "mycluster",
53+
"container_name": "mycontainer",
54+
"location": "myavailzone",
55+
"namespace_name": "myns",
56+
"pod_name": "mypod"
57+
},
58+
"type": "k8s_container"
59+
}
60+
}
61+
]
62+
}
63+
]
64+
}

0 commit comments

Comments
 (0)