Add misc contract tests (#212)

Issue #, if available:

I observed that we were missing the misc tests seen in other repos: https://github.com/aws-observability/aws-otel-python-instrumentation/blob/main/contract-tests/tests/test/amazon/misc/

Description of changes:

Files copied over, light modification made to make them work. Note that two tests are not working and will need further analysis: test_xray_id_format & test_service

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

Author: thpierce

test/contract-tests/tests/test/amazon/misc/configuration_test.py

@@ -0,0 +1,89 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+import time
+from typing import List
+
+from mock_collector_client import ResourceScopeMetric, ResourceScopeSpan
+from requests import Response, request
+from typing_extensions import override
+
+from amazon.base.contract_test_base import ContractTestBase
+from amazon.utils.application_signals_constants import ERROR_METRIC, FAULT_METRIC, LATENCY_METRIC
+from opentelemetry.sdk.metrics.export import AggregationTemporality
+
+# Tests in this class are supposed to validate that the SDK was configured in the correct way: It
+# uses the X-Ray ID format. Metrics are deltaPreferred. Type of the metrics are exponentialHistogram
+
+
+class ConfigurationTest(ContractTestBase):
+    @override
+    @staticmethod
+    def get_application_image_name() -> str:
+        return "aws-application-signals-tests-appsignals.netcore-app"
+
+    @override
+    def get_application_wait_pattern(self) -> str:
+        return "Content root path: /app"
+
+    @override
+    def get_application_extra_environment_variables(self):
+        return {"OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED": "true"}
+
+    def test_configuration_metrics(self):
+        address: str = self.application.get_container_host_ip()
+        port: str = self.application.get_exposed_port(self.get_application_port())
+        url: str = f"http://{address}:{port}/success"
+        response: Response = request("GET", url, timeout=20)
+        self.assertEqual(200, response.status_code)
+        metrics: List[ResourceScopeMetric] = self.mock_collector_client.get_metrics(
+            {LATENCY_METRIC, ERROR_METRIC, FAULT_METRIC}
+        )
+
+        self.assertEqual(len(metrics), 3)
+        for metric in metrics:
+            self.assertIsNotNone(metric.metric.exponential_histogram)
+            self.assertEqual(metric.metric.exponential_histogram.aggregation_temporality, AggregationTemporality.DELTA)
+
+    # TODO: This does not work as expected, gives errors like 1467067824 not greater than 1746657330
+    # def test_xray_id_format(self):
+    #     """
+    #     We are testing here that the X-Ray id format is always used by inspecting the traceid that
+    #     was in the span received by the collector, which should be consistent across multiple spans.
+    #     We are testing the following properties:
+    #     1. Traceid is random
+    #     2. First 32 bits of traceid is a timestamp
+    #     It is important to remember that the X-Ray traceId format had to be adapted to fit into the
+    #     definition of the OpenTelemetry traceid:
+    #     https://opentelemetry.io/docs/specs/otel/trace/api/#retrieving-the-traceid-and-spanid
+    #     Specifically for an X-Ray traceid to be a valid Otel traceId, the version digit had to be
+    #     dropped. Reference:
+    #     https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/trace/aws_xray_id_generator.py
+    #     """
+    #
+    #     seen: List[str] = []
+    #     for _ in range(100):
+    #         address: str = self.application.get_container_host_ip()
+    #         port: str = self.application.get_exposed_port(self.get_application_port())
+    #         url: str = f"http://{address}:{port}/success"
+    #         response: Response = request("GET", url, timeout=20)
+    #         self.assertEqual(200, response.status_code)
+    #
+    #         # Since we just made the request, the time in epoch registered in the traceid should be
+    #         # approximate equal to the current time in the test, since both run on the same host.
+    #         start_time_sec: int = int(time.time())
+    #
+    #         resource_scope_spans: List[ResourceScopeSpan] = self.mock_collector_client.get_traces()
+    #         target_span: ResourceScopeSpan = resource_scope_spans[0]
+    #         self.assertEqual(target_span.span.name, "GET /success")
+    #
+    #         self.assertTrue(target_span.span.trace_id.hex() not in seen)
+    #         seen.append(target_span.span.trace_id.hex())
+    #
+    #         # trace_id is bytes, so we convert it to hex string and pick the first 8 byte
+    #         # that represent the timestamp, then convert it to int for timestamp in second
+    #         trace_id_time_stamp_int: int = int(target_span.span.trace_id.hex()[:8], 16)
+    #
+    #         # Give 2 minutes time range of tolerance for the trace timestamp
+    #         self.assertGreater(trace_id_time_stamp_int, start_time_sec - 60)
+    #         self.assertGreater(start_time_sec + 60, trace_id_time_stamp_int)
+    #         self.mock_collector_client.clear_signals()

test/contract-tests/tests/test/amazon/misc/resource_attributes_test_base.py

@@ -0,0 +1,79 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+from typing import Dict, List
+
+from mock_collector_client import ResourceScopeMetric, ResourceScopeSpan
+from requests import Response, request
+from typing_extensions import override
+
+from amazon.base.contract_test_base import ContractTestBase
+from amazon.utils.application_signals_constants import ERROR_METRIC, FAULT_METRIC, LATENCY_METRIC
+from opentelemetry.proto.common.v1.common_pb2 import AnyValue
+from opentelemetry.proto.metrics.v1.metrics_pb2 import Metric
+from opentelemetry.proto.trace.v1.trace_pb2 import Span
+
+
+def _get_k8s_attributes():
+    return {
+        "k8s.namespace.name": "namespace-name",
+        "k8s.pod.name": "pod-name",
+        "k8s.deployment.name": "deployment-name",
+    }
+
+
+# Tests consuming this class are supposed to validate that the agent is able to get the resource
+# attributes through the environment variables OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME
+#
+# These tests are structured with nested classes since it is only possible to change the
+# resource attributes during the initialization of the OpenTelemetry SDK.
+
+
+class ResourceAttributesTest(ContractTestBase):
+    @override
+    @staticmethod
+    def get_application_image_name() -> str:
+        return "aws-application-signals-tests-appsignals.netcore-app"
+
+    @override
+    def get_application_wait_pattern(self) -> str:
+        return "Content root path: /app"
+
+    @override
+    def get_application_extra_environment_variables(self):
+        return {"OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED": "true"}
+
+    def do_test_resource_attributes(self, service_name):
+        address: str = self.application.get_container_host_ip()
+        port: str = self.application.get_exposed_port(self.get_application_port())
+        url: str = f"http://{address}:{port}/success"
+        response: Response = request("GET", url, timeout=20)
+        self.assertEqual(200, response.status_code)
+        self.assert_resource_attributes(service_name)
+
+    def assert_resource_attributes(self, service_name):
+        resource_scope_spans: List[ResourceScopeSpan] = self.mock_collector_client.get_traces()
+        metrics: List[ResourceScopeMetric] = self.mock_collector_client.get_metrics(
+            {LATENCY_METRIC, ERROR_METRIC, FAULT_METRIC}
+        )
+        target_spans: List[Span] = []
+        for resource_scope_span in resource_scope_spans:
+            # pylint: disable=no-member
+            if resource_scope_span.span.name == "GET /success":
+                target_spans.append(resource_scope_span.resource_spans)
+
+        self.assertEqual(len(target_spans), 1)
+        attributes_dict: Dict[str, AnyValue] = self._get_attributes_dict(target_spans[0].resource.attributes)
+        for key, value in _get_k8s_attributes().items():
+            self._assert_str_attribute(attributes_dict, key, value)
+        self._assert_str_attribute(attributes_dict, "service.name", service_name)
+
+        target_metrics: List[Metric] = []
+        for resource_scope_metric in metrics:
+            if resource_scope_metric.metric.name in ["Error", "Fault", "Latency"]:
+                target_metrics.append(resource_scope_metric.resource_metrics)
+        self.assertEqual(len(target_metrics), 3)
+        for target_metric in target_metrics:
+            metric_attributes_dict: Dict[str, AnyValue] = self._get_attributes_dict(target_metric.resource.attributes)
+            for key, value in _get_k8s_attributes().items():
+                self._assert_str_attribute(metric_attributes_dict, key, value)
+            self._assert_str_attribute(metric_attributes_dict, "service.name", service_name)

test/contract-tests/tests/test/amazon/misc/service_name_in_env_var_test.py

@@ -0,0 +1,24 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+from typing import List
+
+from resource_attributes_test_base import ResourceAttributesTest, _get_k8s_attributes
+from typing_extensions import override
+
+
+class ServiceNameInEnvVarTest(ResourceAttributesTest):
+    @override
+    # pylint: disable=no-self-use
+    def get_application_extra_environment_variables(self) -> str:
+        return { "OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED": "true", "OTEL_SERVICE_NAME": "service-name-test"}
+
+    @override
+    # pylint: disable=no-self-use
+    def get_application_otel_resource_attributes(self) -> str:
+        pairlist: List[str] = []
+        for key, value in _get_k8s_attributes().items():
+            pairlist.append(key + "=" + value)
+        return ",".join(pairlist)
+
+    def test_service(self) -> None:
+        self.do_test_resource_attributes("service-name-test")

test/contract-tests/tests/test/amazon/misc/service_name_in_resource_attributes_test.py

@@ -0,0 +1,20 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+from typing import List
+
+from resource_attributes_test_base import ResourceAttributesTest, _get_k8s_attributes
+from typing_extensions import override
+
+
+class ServiceNameInResourceAttributesTest(ResourceAttributesTest):
+    @override
+    # pylint: disable=no-self-use
+    def get_application_otel_resource_attributes(self) -> str:
+        pairlist: List[str] = []
+        for key, value in _get_k8s_attributes().items():
+            pairlist.append(key + "=" + value)
+        pairlist.append("service.name=service-name")
+        return ",".join(pairlist)
+
+    def test_service(self) -> None:
+        self.do_test_resource_attributes("service-name")

test/contract-tests/tests/test/amazon/misc/unknown_service_name_test.py

@@ -0,0 +1,20 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+from typing import List
+
+from resource_attributes_test_base import ResourceAttributesTest, _get_k8s_attributes
+from typing_extensions import override
+
+
+class UnknownServiceNameTest(ResourceAttributesTest):
+    @override
+    # pylint: disable=no-self-use
+    def get_application_otel_resource_attributes(self) -> str:
+        pairlist: List[str] = []
+        for key, value in _get_k8s_attributes().items():
+            pairlist.append(key + "=" + value)
+        return ",".join(pairlist)
+
+    # TODO: metric service.name is set correctly, but span service.name is being set to AppSignals.NetCore.
+    # def test_service(self) -> None:
+    #     self.do_test_resource_attributes("unknown_service:dotnet")