chore(crashtracking): improve crashtracking tests (#14678)

## Description

This PR updates the name of the test helper `wait_for_crash_messages` to
`get_all_crash_messages`. The reason for this change is because
`wait_for_crash_messages` actually returns all previous crash messages
(telemetry requests originating from `crashtracking`) that the test
client has received up to one second after the helper is called.

This naming issue caused confusion when reasoning about a testing bug
fixed in #14654, that was
introduced with the addition of the `crash_ping` mechanism from the
Crashtracker, where now, we receive more than one message from the
Crashtracker per crash.

This PR also adds a check for the crash ping for each crashtracking
test.

## Testing

Existing updated unit tests

## Risks

<!-- Note any risks associated with this change, or "None" if no risks
-->

## Additional Notes

<!-- Any other information that would be helpful for reviewers -->

Author: gyuheon0h

tests/internal/crashtracker/test_crashtracker.py

@@ -129,7 +129,8 @@ def test_crashtracker_simple():
     # 2. Listens on that port for new connections
     # 3. Starts the crashtracker with the URL set to the port
     # 4. Crashes the process
-    # 5. Verifies that the crashtracker sends a crash report to the server
+    # 5. Verifies that the crashtracker sends a crash ping to the server
+    # 6. Verifies that the crashtracker sends a crash report to the server
     import ctypes
     import os
 
@@ -147,8 +148,10 @@ def test_crashtracker_simple():
             ctypes.string_at(0)
             sys.exit(-1)
 
-        # Part 5
-        # Check to see if the listening socket was triggered, if so accept the connection
+        # Part 5, Check for the crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Part 6, Check to see if the listening socket was triggered, if so accept the connection
         # then check to see if the resulting connection is readable
         report = utils.get_crash_report(client)
         # The crash came from string_at.  Since the over-the-wire format is multipart, chunked HTTP,
@@ -181,7 +184,10 @@ def test_crashtracker_simple_fork():
             ctypes.string_at(0)
             sys.exit(-1)  # just in case
 
-        # Part 5, check
+        # Part 5, check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Part 6, check for crash report
         report = utils.get_crash_report(client)
         assert b"string_at" in report["body"]
 
@@ -235,7 +241,10 @@ def test_crashtracker_simple_sigbus():
                 arr[4095] = b"x"  # sigbus
             sys.exit(-1)  # just in case
 
-        # Part 5, check
+        # Part 5, check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Part 6, check for crash report
         report = utils.get_crash_report(client)
         assert report["body"]
 
@@ -261,7 +270,10 @@ def test_crashtracker_raise_sigsegv():
             os.kill(os.getpid(), signal.SIGSEGV.value)
             sys.exit(-1)
 
-        # Part 5, check
+        # Part 5, check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Part 6, check for crash report
         report = utils.get_crash_report(client)
         assert b"os_kill" in report["body"]
 
@@ -287,7 +299,10 @@ def test_crashtracker_raise_sigbus():
             os.kill(os.getpid(), signal.SIGBUS.value)
             sys.exit(-1)
 
-        # Part 5, check
+        # Part 5, check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Part 6, check for crash report
         report = utils.get_crash_report(client)
         assert b"os_kill" in report["body"]
 
@@ -311,7 +326,10 @@ def test_crashtracker_preload_default(ddtrace_run_python_code_in_subprocess):
         assert not stderr
         assert exitcode == -11  # exit code for SIGSEGV
 
-        # Wait for the connection
+        # Part 5, check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Part 6, check for crash report
         report = utils.get_crash_report(client)
         assert b"string_at" in report["body"]
 
@@ -352,7 +370,10 @@ def test_crashtracker_auto_default(run_python_code_in_subprocess):
         assert not stderr
         assert exitcode == -11
 
-        # Wait for the connection
+        # Part 5, check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Part 6, check for crash report
         report = utils.get_crash_report(client)
         assert b"string_at" in report["body"]
 
@@ -370,6 +391,9 @@ def test_crashtracker_auto_nostack(run_python_code_in_subprocess):
         assert not stderr
         assert exitcode == -11
 
+        # Check for crash ping
+        _ping = utils.get_crash_ping(client)
+
         # Wait for the connection
         report = utils.get_crash_report(client)
         assert b"string_at" not in report["body"]
@@ -413,6 +437,10 @@ def test_crashtracker_tags_required():
             ctypes.string_at(0)
             sys.exit(-1)
 
+        # Check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Check for crash report
         report = utils.get_crash_report(client)
         assert b"string_at" in report["body"]
 
@@ -446,7 +474,10 @@ def test_crashtracker_user_tags_envvar(run_python_code_in_subprocess):
         assert not stderr
         assert exitcode == -11
 
-        # Wait for the connection
+        # Check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Check for crash report
         report = utils.get_crash_report(client)
 
         # Now check for the tags
@@ -466,6 +497,10 @@ def test_crashtracker_set_tag_profiler_config(snapshot_context, run_python_code_
         assert not stderr
         assert exitcode == -11
 
+        # Check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Check for crash report
         report = utils.get_crash_report(client)
         # Now check for the profiler_config tag
         assert b"profiler_config" in report["body"]
@@ -501,6 +536,10 @@ def test_crashtracker_user_tags_profiling():
             ctypes.string_at(0)
             sys.exit(-1)
 
+        # Check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Check for crash report
         report = utils.get_crash_report(client)
         assert b"string_at" in report["body"]
 
@@ -539,6 +578,10 @@ def test_crashtracker_user_tags_core():
             ctypes.string_at(0)
             sys.exit(-1)
 
+        # Check for crash ping
+        _ping = utils.get_crash_ping(client)
+
+        # Check for crash report
         report = utils.get_crash_report(client)
         assert b"string_at" in report["body"]
 

tests/internal/crashtracker/utils.py

@@ -112,7 +112,12 @@ def logs(self):
         return read_files([self.stdout, self.stderr])
 
 
-def wait_for_crash_messages(test_agent_client: TestAgentClient) -> List[TestAgentRequest]:
+def get_all_crash_messages(test_agent_client: TestAgentClient) -> List[TestAgentRequest]:
+    """
+    A test helper to get *all* crash messages is necessary, because crash pings and crash reports
+    are sent through async network requests, so we don't have a guarantee of the order they are received.
+    We differentiate between crash pings and crash reports downstream
+    """
     seen_report_ids = set()
     crash_messages = []
     # 5 iterations * 0.2 second = 1 second total should be enough to get ping + report
@@ -137,7 +142,7 @@ def wait_for_crash_messages(test_agent_client: TestAgentClient) -> List[TestAgen
 
 def get_crash_report(test_agent_client: TestAgentClient) -> TestAgentRequest:
     """Wait for a crash report from the crashtracker listener socket."""
-    crash_messages = wait_for_crash_messages(test_agent_client)
+    crash_messages = get_all_crash_messages(test_agent_client)
     # We want at least the crash report
     assert len(crash_messages) == 2, f"Expected at least 2 messages; got {len(crash_messages)}"
 
@@ -152,6 +157,22 @@ def get_crash_report(test_agent_client: TestAgentClient) -> TestAgentRequest:
     return crash_report
 
 
+def get_crash_ping(test_agent_client: TestAgentClient) -> TestAgentRequest:
+    """Wait for a crash report from the crashtracker listener socket."""
+    crash_messages = get_all_crash_messages(test_agent_client)
+    assert len(crash_messages) == 2, f"Expected at least 2 messages; got {len(crash_messages)}"
+
+    # Find the crash ping (the one with "is_crash_ping":"true")
+    crash_ping = None
+    for message in crash_messages:
+        if b"is_crash_ping:true" in message["body"]:
+            crash_ping = message
+            break
+
+    assert crash_ping is not None, "Could not find crash ping with 'is_crash_ping:true' tag"
+    return crash_ping
+
+
 @contextmanager
 def with_test_agent() -> Generator[TestAgentClient, None, None]:
     base_url = ddtrace.tracer.agent_trace_url or "http://localhost:9126"  # default to local test agent