chore: enable new test agent service naming and peer service checks (#6191)

This PR enables service naming and peer.service checks available from the APM Test Agent.

Co-authored-by: Teague Bick <[email protected]>

Author: wconti27

.circleci/config.yml

@@ -15,7 +15,7 @@ mongo_image: &mongo_image mongo:3.6
 httpbin_image: &httpbin_image kennethreitz/httpbin@sha256:2c7abc4803080c22928265744410173b6fea3b898872c01c5fd0f0f9df4a59fb
 vertica_image: &vertica_image sumitchawla/vertica:latest
 rabbitmq_image: &rabbitmq_image rabbitmq:3.7-alpine
-testagent_image: &testagent_image ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.10.1
+testagent_image: &testagent_image ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
 
 parameters:
   coverage:
@@ -128,6 +128,9 @@ commands:
       store_coverage:
         type: boolean
         default: true
+      trace_agent_url:
+        type: string
+        default: "http://localhost:9126"
     steps:
       - attach_workspace:
           at: .
@@ -143,7 +146,7 @@ commands:
                 services: testagent << parameters.docker_services >>
             - run:
                 environment:
-                  DD_TRACE_AGENT_URL: http://localhost:9126
+                  DD_TRACE_AGENT_URL: << parameters.trace_agent_url >> 
                   RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
                 command: |
                   # Sort the hashes to ensure a consistent ordering/division between each node
@@ -161,6 +164,14 @@ commands:
                   - run:
                       name: "Waiting for << parameters.wait >>"
                       command: riot -v run 'wait' << parameters.wait >>
+            - when:
+                condition:
+                  << parameters.trace_agent_url >> != ""
+                steps:
+                  - run:
+                      command: |
+                        echo 'export DD_TRACE_AGENT_URL=<< parameters.trace_agent_url >>' >> "$BASH_ENV"
+                        source "$BASH_ENV"
             - run:
                 environment:
                   RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
@@ -180,6 +191,10 @@ commands:
           path: test-results
       - store_artifacts:
           path: test-results
+      - run:
+          name: Get APM Test Agent Trace Check Results
+          when: always
+          command: bash ./scripts/get-test-agent-results.sh
 
 executors:
   cimg_base:
@@ -223,9 +238,13 @@ postgres_server: &postgres_server
 testagent: &testagent
   image: *testagent_image
   environment:
-    - LOG_LEVEL=WARNING
+    - LOG_LEVEL=DEBUG
     - SNAPSHOT_DIR=/snapshots
+    - PORT=9126
     - SNAPSHOT_CI=1
+    - DD_POOL_TRACE_CHECK_FAILURES=true
+    - DD_DISABLE_ERROR_RESPONSES=true
+    - ENABLED_CHECKS=trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service
 
 jobs:
   pre_check:
@@ -364,12 +383,14 @@ jobs:
     steps:
       - run_test:
           pattern: "tracer"
+          trace_agent_url: ""
 
   ci_visibility:
     <<: *contrib_job
     steps:
       - run_test:
           pattern: "ci_visibility"
+          trace_agent_url: ""
 
   sourcecode:
     <<: *contrib_job
@@ -423,6 +444,7 @@ jobs:
           store_coverage: false
           # We don't want to run the profile-diff venvs
           pattern: 'profile$'
+          trace_agent_url: ""
 
   integration_agent:
     <<: *machine_executor
@@ -449,6 +471,7 @@ jobs:
           snapshot: true
           store_coverage: false
           pattern: 'integration-snapshot'
+          trace_agent_url: ""
 
   vendor:
     <<: *contrib_job_small
@@ -476,6 +499,7 @@ jobs:
       - run_test:
           store_coverage: false
           pattern: 'ddtracerun'
+          trace_agent_url: ""
 
   test_logging:
     <<: *contrib_job
@@ -509,6 +533,7 @@ jobs:
     steps:
       - run_test:
           pattern: 'pylons'
+          trace_agent_url: "http://localhost:8126"
 
   aiohttp:
     <<: *machine_executor
@@ -568,7 +593,15 @@ jobs:
       - image: *ddtrace_dev_image
       - image: redis:4.0-alpine
       - image: *rabbitmq_image
-      - *testagent
+      - image: *testagent_image
+        environment:
+        - LOG_LEVEL=DEBUG
+        - SNAPSHOT_DIR=/snapshots
+        - PORT=9126
+        - SNAPSHOT_CI=1
+        - DD_POOL_TRACE_CHECK_FAILURES=true
+        - DD_DISABLE_ERROR_RESPONSES=true
+        - ENABLED_CHECKS=trace_stall,meta_tracer_version_header,trace_content_length,trace_peer_service,trace_dd_service # disable flaky content length check
     steps:
       - run_test:
           pattern: 'celery'

docker-compose.yml

@@ -137,15 +137,18 @@ services:
         volumes:
           - ddagent:/tmp/ddagent:rw
     testagent:
-        image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.10.1
+        image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
         ports:
             - "127.0.0.1:9126:8126"
         volumes:
             - ./tests/snapshots:/snapshots
         environment:
-            - LOG_LEVEL=WARNING
+            - LOG_LEVEL=DEBUG
             - SNAPSHOT_DIR=/snapshots
             - SNAPSHOT_CI
+            - DD_POOL_TRACE_CHECK_FAILURES=true
+            - DD_DISABLE_ERROR_RESPONSES=true
+            - ENABLED_CHECKS=trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service
     vertica:
         image: sumitchawla/vertica
         environment:

scripts/get-test-agent-results.sh

@@ -0,0 +1,29 @@
+set +e  # Disable exiting from testagent response failure
+SUMMARY_RESPONSE=$(curl -s -w "\n%{http_code}" -o summary_response.txt http://localhost:9126/test/trace_check/summary)
+set -e
+SUMMARY_RESPONSE_CODE=$(echo "$SUMMARY_RESPONSE" | awk 'END {print $NF}')
+
+if [[ SUMMARY_RESPONSE_CODE -eq 200 ]]; then
+    echo "APM Test Agent is running. (HTTP 200)"
+else
+    echo "APM Test Agent is not running and was not used for testing. No checks failed."
+    exit 0
+fi
+
+RESPONSE=$(curl -s -w "\n%{http_code}" -o response.txt http://localhost:9126/test/trace_check/failures)
+RESPONSE_CODE=$(echo "$RESPONSE" | awk 'END {print $NF}')
+    
+if [[ $RESPONSE_CODE -eq 200 ]]; then
+    echo "All APM Test Agent Check Traces returned successful! (HTTP 200)"
+    echo "APM Test Agent Check Traces Summary Results:"
+    cat summary_response.txt | jq '.'
+elif [[ $RESPONSE_CODE -eq 404 ]]; then
+    echo "Real APM Agent running in place of TestAgent, no checks to validate!"
+else
+    echo "APM Test Agent Check Traces failed with response code: $RESPONSE_CODE"
+    echo "Failures:"
+    cat response.txt
+    echo "APM Test Agent Check Traces Summary Results:"
+    cat summary_response.txt | jq '.'
+    exit 1
+fi

tests/contrib/mongoengine/test.py

@@ -193,42 +193,6 @@ def test_analytics_without_rate(self):
             assert len(spans) == 1
             assert spans[0].get_metric(ANALYTICS_SAMPLE_RATE_KEY) == 1.0
 
-    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v0"))
-    def test_span_name_v0_schema(self):
-        """
-        When a user specifies a service for the app
-            The mongoengine integration should not use it.
-        """
-        tracer = self.get_tracer_and_connect()
-        Artist.drop_collection()
-
-        spans = tracer.pop()
-        assert len(spans) == 1
-        assert spans[0].name == "pymongo.cmd"
-
-    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v1"))
-    def test_span_name_v1_schema(self):
-        """
-        When a user specifies a service for the app
-            The mongoengine integration should not use it.
-        """
-        tracer = self.get_tracer_and_connect()
-        Artist.drop_collection()
-
-        spans = tracer.pop()
-        assert len(spans) == 1
-        assert spans[0].name == "mongodb.query"
-
-    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v1"))
-    def test_peer_service_tagging(self):
-        tracer = self.get_tracer_and_connect()
-        Artist.drop_collection()
-
-        spans = tracer.pop()
-        assert len(spans) == 1
-        assert spans[0].get_tag("mongodb.db") == "test"
-        assert spans[0].get_tag("peer.service") == "test"
-
 
 class TestMongoEnginePatchConnectDefault(TracerTestCase, MongoEngineCore):
     """Test suite with a global Pin for the connect function with the default configuration"""
@@ -251,7 +215,7 @@ def get_tracer_and_connect(self):
         return tracer
 
 
-class TestMongoEnginePatchConnectDefaultOnly(TestMongoEnginePatchConnectDefault):
+class TestMongoEnginePatchConnectSchematization(TestMongoEnginePatchConnectDefault):
     @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_SERVICE="mysvc"))
     def test_user_specified_service_default(self):
         """
@@ -337,6 +301,32 @@ def test_unspecified_service_v1(self):
         assert len(spans) == 1
         assert spans[0].service == DEFAULT_SPAN_SERVICE_NAME
 
+    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v0"))
+    def test_span_name_v0_schema(self):
+        """
+        When a user specifies a service for the app
+            The mongoengine integration should not use it.
+        """
+        tracer = self.get_tracer_and_connect()
+        Artist.drop_collection()
+
+        spans = tracer.pop()
+        assert len(spans) == 1
+        assert spans[0].name == "pymongo.cmd"
+
+    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v1"))
+    def test_span_name_v1_schema(self):
+        """
+        When a user specifies a service for the app
+            The mongoengine integration should not use it.
+        """
+        tracer = self.get_tracer_and_connect()
+        Artist.drop_collection()
+
+        spans = tracer.pop()
+        assert len(spans) == 1
+        assert spans[0].name == "mongodb.query"
+
 
 class TestMongoEnginePatchConnect(TestMongoEnginePatchConnectDefault):
     """Test suite with a global Pin for the connect function with custom service"""

tests/contrib/pylibmc/test.py

@@ -286,28 +286,6 @@ def test_disabled(self):
         finally:
             tracer.enabled = True
 
-    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v0"))
-    def test_operation_name_v0_schema(self):
-        """
-        v0 schema: memcached.cmd
-        """
-        client, tracer = self.get_client()
-        client.set("a", "crow")
-        spans = self.get_spans()
-        assert len(spans) == 1
-        assert spans[0].name == "memcached.cmd"
-
-    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v1"))
-    def test_operation_name_v1_schema(self):
-        """
-        v1 schema: memcached.command
-        """
-        client, tracer = self.get_client()
-        client.set("a", "crow")
-        spans = self.get_spans()
-        assert len(spans) == 1
-        assert spans[0].name == "memcached.command"
-
 
 class TestPylibmcLegacy(TracerTestCase, PylibmcCore):
     """Test suite for the tracing of pylibmc with the legacy TracedClient interface"""
@@ -393,7 +371,7 @@ def test_patch_unpatch(self):
         assert len(spans) == 1
 
 
-class TestPylibmcPatchServiceNames(TestPylibmcPatchDefault):
+class TestPylibmcPatchSchematization(TestPylibmcPatchDefault):
     @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_SERVICE="mysvc", DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v0"))
     def test_user_specified_service_v0(self):
         """
@@ -427,3 +405,25 @@ def test_user_specified_service_v1(self):
         spans = self.get_spans()
         assert len(spans) == 1
         assert spans[0].service == "mysvc"
+
+    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v0"))
+    def test_operation_name_v0_schema(self):
+        """
+        v0 schema: memcached.cmd
+        """
+        client, tracer = self.get_client()
+        client.set("a", "crow")
+        spans = self.get_spans()
+        assert len(spans) == 1
+        assert spans[0].name == "memcached.cmd"
+
+    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v1"))
+    def test_operation_name_v1_schema(self):
+        """
+        v1 schema: memcached.command
+        """
+        client, tracer = self.get_client()
+        client.set("a", "crow")
+        spans = self.get_spans()
+        assert len(spans) == 1
+        assert spans[0].name == "memcached.command"

tests/contrib/redis/test_redis.py

@@ -304,30 +304,6 @@ def test_env_user_specified_redis_service_v0(self):
         span = self.get_spans()[0]
         assert span.service == "mysvc", span.service
 
-    @TracerTestCase.run_in_subprocess(
-        env_overrides=dict(DD_REDIS_SERVICE="myredis", DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v1")
-    )
-    def test_env_user_specified_redis_service_v1(self):
-        self.r.get("cheese")
-        span = self.get_spans()[0]
-        assert span.service == "myredis", span.service
-
-        self.reset()
-
-        # Global config
-        with self.override_config("redis", dict(service="cfg-redis")):
-            self.r.get("cheese")
-            span = self.get_spans()[0]
-            assert span.service == "cfg-redis", span.service
-
-        self.reset()
-
-        # Manual override
-        Pin.override(self.r, service="mysvc", tracer=self.tracer)
-        self.r.get("cheese")
-        span = self.get_spans()[0]
-        assert span.service == "mysvc", span.service
-
     @TracerTestCase.run_in_subprocess(
         env_overrides=dict(
             DD_SERVICE="app-svc", DD_REDIS_SERVICE="env-specified-redis-svc", DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v0"
@@ -346,24 +322,6 @@ def test_service_precedence_v0(self):
         span = self.get_spans()[0]
         assert span.service == "override-redis", span.service
 
-    @TracerTestCase.run_in_subprocess(
-        env_overrides=dict(
-            DD_SERVICE="app-svc", DD_REDIS_SERVICE="env-specified-redis-svc", DD_TRACE_SPAN_ATTRIBUTE_SCHEMA="v1"
-        )
-    )
-    def test_service_precedence_v1(self):
-        self.r.get("cheese")
-        span = self.get_spans()[0]
-        assert span.service == "env-specified-redis-svc", span.service
-
-        self.reset()
-
-        # Do a manual override
-        Pin.override(self.r, service="override-redis", tracer=self.tracer)
-        self.r.get("cheese")
-        span = self.get_spans()[0]
-        assert span.service == "override-redis", span.service
-
 
 class TestRedisPatchSnapshot(TracerTestCase):