Merge branch 'main' into update-dockerfile-with-scratch

Author: harrryr

.github/workflows/main_build.yml

@@ -102,7 +102,7 @@ jobs:
   # Application Signals specific e2e tests for ec2
   application-signals-python-e2e-ec2-test:
     needs: [ build ]
-    uses: aws-observability/aws-application-signals-test-framework/.github/workflows/application-signals-python-e2e-ec2-test.yml@main
+    uses: aws-observability/aws-application-signals-test-framework/.github/workflows/application-signals-python-e2e-ec2-default-test.yml@main
     secrets: inherit
     with:
       aws-region: ${{ needs.build.outputs.aws_default_region }}

CONTRIBUTING.md

@@ -23,6 +23,9 @@ Contributions via pull requests are much appreciated. Before sending us a pull r
 1. You are working against the latest source on the *main* branch.
 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
 3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
+4. You are not mixing substantial refactoring changes in with functional changes.
+   1. If refactoring is desirable, publish a separate refactoring PR first, followed by a functional change PR. This will ensure safe and efficient reviews.
+   2. PRs that do not meet these expectations will be rejected.
 
 To send us a pull request, please:
 
@@ -32,6 +35,7 @@ To send us a pull request, please:
 4. Commit to your fork using clear commit messages.
 5. Send us a pull request, answering any default questions in the pull request interface.
 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
+7. Please do not squash commits between revisions, this makes review challenging, as the diff between revisions is harder to find and review.
 
 GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and
 [creating a pull request](https://help.github.com/articles/creating-a-pull-request/).

contract-tests/images/applications/botocore/botocore_server.py

@@ -10,7 +10,7 @@
 import requests
 from botocore.client import BaseClient
 from botocore.config import Config
-from typing_extensions import override
+from typing_extensions import Tuple, override
 
 _PORT: int = 8080
 _ERROR: str = "error"
@@ -253,7 +253,7 @@ def prepare_aws_server() -> None:
 
 def main() -> None:
     prepare_aws_server()
-    server_address: tuple[str, int] = ("0.0.0.0", _PORT)
+    server_address: Tuple[str, int] = ("0.0.0.0", _PORT)
     request_handler_class: type = RequestHandler
     requests_server: ThreadingHTTPServer = ThreadingHTTPServer(server_address, request_handler_class)
     atexit.register(requests_server.shutdown)

contract-tests/images/applications/pymysql/pymysql_server.py

@@ -9,10 +9,11 @@
 import pymysql
 from typing_extensions import override
 
-_PORT: int = 8080
-_SUCCESS: str = "success"
+_CREATE_DATABASE: str = "create_database"
+_DROP_TABLE: str = "drop_table"
 _ERROR: str = "error"
 _FAULT: str = "fault"
+_PORT: int = 8080
 
 _DB_HOST = os.getenv("DB_HOST")
 _DB_USER = os.getenv("DB_USER")
@@ -26,11 +27,17 @@ class RequestHandler(BaseHTTPRequestHandler):
     def do_GET(self):
         status_code: int = 200
         conn = pymysql.connect(host=_DB_HOST, user=_DB_USER, password=_DB_PASS, database=_DB_NAME)
-        if self.in_path(_SUCCESS):
+        conn.autocommit = True  # CREATE DATABASE cannot run in a transaction block
+        if self.in_path(_DROP_TABLE):
             cur = conn.cursor()
             cur.execute("DROP TABLE IF EXISTS test_table")
             cur.close()
             status_code = 200
+        elif self.in_path(_CREATE_DATABASE):
+            cur = conn.cursor()
+            cur.execute("CREATE DATABASE test_database")
+            cur.close()
+            status_code = 200
         elif self.in_path(_FAULT):
             cur = conn.cursor()
             try:

contract-tests/tests/test/amazon/base/contract_test_base.py

@@ -193,7 +193,8 @@ def get_application_extra_environment_variables(self) -> Dict[str, str]:
     def get_application_network_aliases(self) -> List[str]:
         return []
 
-    def get_application_image_name(self) -> str:
+    @staticmethod
+    def get_application_image_name() -> str:
         return None
 
     def get_application_wait_pattern(self) -> str:

contract-tests/tests/test/amazon/base/database_contract_test_base.py

@@ -0,0 +1,154 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+from typing import Dict, List
+
+from mock_collector_client import ResourceScopeMetric, ResourceScopeSpan
+from typing_extensions import override
+
+from amazon.base.contract_test_base import ContractTestBase
+from amazon.utils.application_signals_constants import (
+    AWS_LOCAL_OPERATION,
+    AWS_LOCAL_SERVICE,
+    AWS_REMOTE_OPERATION,
+    AWS_REMOTE_RESOURCE_IDENTIFIER,
+    AWS_REMOTE_RESOURCE_TYPE,
+    AWS_REMOTE_SERVICE,
+    AWS_SPAN_KIND,
+)
+from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
+from opentelemetry.proto.metrics.v1.metrics_pb2 import ExponentialHistogramDataPoint, Metric
+from opentelemetry.proto.trace.v1.trace_pb2 import Span
+from opentelemetry.trace import StatusCode
+
+DATABASE_HOST: str = "mydb"
+DATABASE_USER: str = "root"
+DATABASE_PASSWORD: str = "example"
+DATABASE_NAME: str = "testdb"
+
+
+class DatabaseContractTestBase(ContractTestBase):
+    @staticmethod
+    def get_remote_service() -> str:
+        return None
+
+    @staticmethod
+    def get_database_port() -> int:
+        return None
+
+    def get_remote_resource_identifier(self) -> str:
+        return f"{DATABASE_NAME}|{DATABASE_HOST}|{self.get_database_port()}"
+
+    @override
+    def get_application_extra_environment_variables(self) -> Dict[str, str]:
+        return {
+            "DB_HOST": DATABASE_HOST,
+            "DB_USER": DATABASE_USER,
+            "DB_PASS": DATABASE_PASSWORD,
+            "DB_NAME": DATABASE_NAME,
+        }
+
+    def assert_drop_table_succeeds(self) -> None:
+        self.mock_collector_client.clear_signals()
+        self.do_test_requests("drop_table", "GET", 200, 0, 0, sql_command="DROP TABLE")
+
+    def assert_create_database_succeeds(self) -> None:
+        self.mock_collector_client.clear_signals()
+        self.do_test_requests("create_database", "GET", 200, 0, 0, sql_command="CREATE DATABASE")
+
+    def assert_fault(self) -> None:
+        self.mock_collector_client.clear_signals()
+        self.do_test_requests("fault", "GET", 500, 0, 1, sql_command="SELECT DISTINCT")
+
+    @override
+    def _assert_aws_span_attributes(self, resource_scope_spans: List[ResourceScopeSpan], path: str, **kwargs) -> None:
+        target_spans: List[Span] = []
+        for resource_scope_span in resource_scope_spans:
+            # pylint: disable=no-member
+            if resource_scope_span.span.kind == Span.SPAN_KIND_CLIENT:
+                target_spans.append(resource_scope_span.span)
+
+        self.assertEqual(len(target_spans), 1)
+        self._assert_aws_attributes(target_spans[0].attributes, **kwargs)
+
+    @override
+    def _assert_semantic_conventions_span_attributes(
+        self, resource_scope_spans: List[ResourceScopeSpan], method: str, path: str, status_code: int, **kwargs
+    ) -> None:
+        target_spans: List[Span] = []
+        for resource_scope_span in resource_scope_spans:
+            # pylint: disable=no-member
+            if resource_scope_span.span.kind == Span.SPAN_KIND_CLIENT:
+                target_spans.append(resource_scope_span.span)
+
+        self.assertEqual(target_spans[0].name, kwargs.get("sql_command").split()[0])
+        if status_code == 200:
+            self.assertEqual(target_spans[0].status.code, StatusCode.UNSET.value)
+        else:
+            self.assertEqual(target_spans[0].status.code, StatusCode.ERROR.value)
+
+        self._assert_semantic_conventions_attributes(target_spans[0].attributes, kwargs.get("sql_command"))
+
+    def _assert_semantic_conventions_attributes(self, attributes_list: List[KeyValue], command: str) -> None:
+        attributes_dict: Dict[str, AnyValue] = self._get_attributes_dict(attributes_list)
+        self.assertTrue(attributes_dict.get("db.statement").string_value.startswith(command))
+        self._assert_str_attribute(attributes_dict, "db.system", self.get_remote_service())
+        self._assert_str_attribute(attributes_dict, "db.name", DATABASE_NAME)
+        self._assert_str_attribute(attributes_dict, "net.peer.name", DATABASE_HOST)
+        self._assert_int_attribute(attributes_dict, "net.peer.port", self.get_database_port())
+        self.assertTrue("server.address" not in attributes_dict)
+        self.assertTrue("server.port" not in attributes_dict)
+        self.assertTrue("db.operation" not in attributes_dict)
+
+    @override
+    def _assert_aws_attributes(self, attributes_list: List[KeyValue], **kwargs) -> None:
+        attributes_dict: Dict[str, AnyValue] = self._get_attributes_dict(attributes_list)
+        self._assert_str_attribute(attributes_dict, AWS_LOCAL_SERVICE, self.get_application_otel_service_name())
+        # InternalOperation as OTEL does not instrument the basic server we are using, so the client span is a local
+        # root.
+        self._assert_str_attribute(attributes_dict, AWS_LOCAL_OPERATION, "InternalOperation")
+        self._assert_str_attribute(attributes_dict, AWS_REMOTE_SERVICE, self.get_remote_service())
+        self._assert_str_attribute(attributes_dict, AWS_REMOTE_OPERATION, kwargs.get("sql_command"))
+        self._assert_str_attribute(attributes_dict, AWS_REMOTE_RESOURCE_TYPE, "DB::Connection")
+        self._assert_str_attribute(
+            attributes_dict, AWS_REMOTE_RESOURCE_IDENTIFIER, self.get_remote_resource_identifier()
+        )
+        # See comment above AWS_LOCAL_OPERATION
+        self._assert_str_attribute(attributes_dict, AWS_SPAN_KIND, "LOCAL_ROOT")
+
+    @override
+    def _assert_metric_attributes(
+        self, resource_scope_metrics: List[ResourceScopeMetric], metric_name: str, expected_sum: int, **kwargs
+    ) -> None:
+        target_metrics: List[Metric] = []
+        for resource_scope_metric in resource_scope_metrics:
+            if resource_scope_metric.metric.name.lower() == metric_name.lower():
+                target_metrics.append(resource_scope_metric.metric)
+
+        self.assertEqual(len(target_metrics), 1)
+        target_metric: Metric = target_metrics[0]
+        dp_list: List[ExponentialHistogramDataPoint] = target_metric.exponential_histogram.data_points
+
+        self.assertEqual(len(dp_list), 2)
+        dependency_dp: ExponentialHistogramDataPoint = dp_list[0]
+        service_dp: ExponentialHistogramDataPoint = dp_list[1]
+        if len(dp_list[1].attributes) > len(dp_list[0].attributes):
+            dependency_dp = dp_list[1]
+            service_dp = dp_list[0]
+        attribute_dict: Dict[str, AnyValue] = self._get_attributes_dict(dependency_dp.attributes)
+        self._assert_str_attribute(attribute_dict, AWS_LOCAL_SERVICE, self.get_application_otel_service_name())
+        # See comment on AWS_LOCAL_OPERATION in _assert_aws_attributes
+        self._assert_str_attribute(attribute_dict, AWS_LOCAL_OPERATION, "InternalOperation")
+        self._assert_str_attribute(attribute_dict, AWS_REMOTE_SERVICE, self.get_remote_service())
+        self._assert_str_attribute(attribute_dict, AWS_REMOTE_OPERATION, kwargs.get("sql_command"))
+        self._assert_str_attribute(attribute_dict, AWS_REMOTE_RESOURCE_TYPE, "DB::Connection")
+        self._assert_str_attribute(
+            attribute_dict, AWS_REMOTE_RESOURCE_IDENTIFIER, self.get_remote_resource_identifier()
+        )
+        self._assert_str_attribute(attribute_dict, AWS_SPAN_KIND, "CLIENT")
+        self.check_sum(metric_name, dependency_dp.sum, expected_sum)
+
+        attribute_dict: Dict[str, AnyValue] = self._get_attributes_dict(service_dp.attributes)
+        # See comment on AWS_LOCAL_OPERATION in _assert_aws_attributes
+        self._assert_str_attribute(attribute_dict, AWS_LOCAL_OPERATION, "InternalOperation")
+        self._assert_str_attribute(attribute_dict, AWS_SPAN_KIND, "LOCAL_ROOT")
+        self.check_sum(metric_name, service_dp.sum, expected_sum)

contract-tests/tests/test/amazon/botocore/botocore_test.py

@@ -49,7 +49,8 @@ def get_application_network_aliases(self) -> List[str]:
         return ["error.test", "fault.test"]
 
     @override
-    def get_application_image_name(self) -> str:
+    @staticmethod
+    def get_application_image_name() -> str:
         return "aws-application-signals-tests-botocore-app"
 
     @classmethod

contract-tests/tests/test/amazon/django/django_test.py

@@ -15,7 +15,8 @@
 
 class DjangoTest(ContractTestBase):
     @override
-    def get_application_image_name(self) -> str:
+    @staticmethod
+    def get_application_image_name() -> str:
         return "aws-application-signals-tests-django-app"
 
     @override

contract-tests/tests/test/amazon/misc/configuration_test.py

@@ -17,7 +17,8 @@
 
 class ConfigurationTest(ContractTestBase):
     @override
-    def get_application_image_name(self) -> str:
+    @staticmethod
+    def get_application_image_name() -> str:
         return "aws-application-signals-tests-django-app"
 
     @override

contract-tests/tests/test/amazon/misc/resource_attributes_test_base.py

@@ -30,7 +30,8 @@ def _get_k8s_attributes():
 
 class ResourceAttributesTest(ContractTestBase):
     @override
-    def get_application_image_name(self) -> str:
+    @staticmethod
+    def get_application_image_name() -> str:
         return "aws-application-signals-tests-django-app"
 
     @override