fix(asgi): ensure error tags are set on request spans [backports #5818 to 1.11] (#5837)

#5818

## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/contributing.html#Release-Note-Guidelines)
are followed.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] OPTIONAL: PR description includes explicit acknowledgement of the
performance implications of the change as reported in the benchmarks PR
comment.

## Reviewer Checklist

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.

Author: mabdinur

ddtrace/contrib/asgi/middleware.py

@@ -213,14 +213,19 @@ async def wrapped_send(message):
                 trace_utils.set_http_meta(
                     span, self.integration_config, status_code=status_code, response_headers=response_headers
                 )
-
                 try:
                     return await send(message)
                 finally:
                     # Per asgi spec, "more_body" is used if there is still data to send
                     # Close the span if "http.response.body" has no more data left to send in the
                     # response.
-                    if message.get("type") == "http.response.body" and not message.get("more_body", False):
+                    if (
+                        message.get("type") == "http.response.body"
+                        and not message.get("more_body", False)
+                        # If the span has an error status code delay finishing the span until the
+                        # traceback and exception message is available
+                        and span.error == 0
+                    ):
                         span.finish()
 
             async def wrapped_blocked_send(message):
@@ -246,7 +251,7 @@ async def wrapped_blocked_send(message):
                 try:
                     return await send(message)
                 finally:
-                    if message.get("type") == "http.response.body":
+                    if message.get("type") == "http.response.body" and span.error == 0:
                         span.finish()
 
             try:
@@ -259,9 +264,6 @@ async def wrapped_blocked_send(message):
                 self.handle_exception_span(exc, span)
                 reraise(exc_type, exc_val, exc_tb)
             finally:
-                try:
-                    del scope["datadog"]["request_span"]
-                except KeyError:
-                    pass
-
+                if span in scope["datadog"]["request_spans"]:
+                    scope["datadog"]["request_spans"].remove(span)
                 span.finish()

releasenotes/notes/fastapi-set-error-tags-before-finishing-span-8fc48f94909251c1.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    asgi: Ensures ``error.message`` and ``error.stack`` tags are set when an exception is raised in a route.

tests/contrib/fastapi/test_fastapi.py

@@ -7,7 +7,6 @@
 
 import ddtrace
 from ddtrace import config
-from ddtrace.constants import ERROR_MSG
 from ddtrace.contrib.fastapi import patch as fastapi_patch
 from ddtrace.contrib.fastapi import unpatch as fastapi_unpatch
 from ddtrace.contrib.starlette.patch import patch as patch_starlette
@@ -341,27 +340,10 @@ def test_invalid_path(client, tracer, test_spans):
     assert request_span.get_tag("span.kind") == "server"
 
 
-def test_500_error_raised(client, tracer, test_spans):
+@snapshot(ignores=["meta.error.stack"])
+def test_500_error_raised(snapshot_client):
     with pytest.raises(RuntimeError):
-        client.get("/500", headers={"X-Token": "DataDog"})
-    spans = test_spans.pop_traces()
-    assert len(spans) == 1
-    assert len(spans[0]) == 1
-
-    request_span = spans[0][0]
-    assert request_span.service == "fastapi"
-    assert request_span.name == "fastapi.request"
-    assert request_span.resource == "GET /500"
-    assert request_span.get_tag("http.route") == "/500"
-    assert request_span.error == 1
-    assert request_span.get_tag("http.method") == "GET"
-    assert request_span.get_tag("http.url") == "http://testserver/500"
-    assert request_span.get_tag("http.status_code") == "500"
-    assert request_span.get_tag(ERROR_MSG) == "Server error"
-    assert request_span.get_tag("error.type") == "builtins.RuntimeError"
-    assert request_span.get_tag("component") == "fastapi"
-    assert request_span.get_tag("span.kind") == "server"
-    assert 'raise RuntimeError("Server error")' in request_span.get_tag("error.stack")
+        snapshot_client.get("/500")
 
 
 def test_streaming_response(client, tracer, test_spans):

tests/snapshots/tests.contrib.fastapi.test_fastapi.test_500_error_raised.json

@@ -0,0 +1,36 @@
+[[
+  {
+    "name": "fastapi.request",
+    "service": "fastapi",
+    "resource": "GET /500",
+    "trace_id": 0,
+    "span_id": 1,
+    "parent_id": 0,
+    "type": "web",
+    "error": 1,
+    "meta": {
+      "_dd.p.dm": "-0",
+      "component": "fastapi",
+      "error.message": "Server error",
+      "error.stack": "Traceback (most recent call last):\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/ddtrace/contrib/asgi/middleware.py\", line 243, in __call__\n    return await self.app(scope, receive, wrapped_send)\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/starlette/middleware/errors.py\", line 181, in __call__\n    raise exc from None\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/starlette/middleware/errors.py\", line 159, in __call__\n    await self.app(scope, receive, _send)\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/starlette/exceptions.py\", line 82, in __call__\n    raise exc from None\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/starlette/exceptions.py\", line 71, in __call__\n    await self.app(scope, receive, sender)\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/starlette/routing.py\", line 566, in __call__\n    await route.handle(scope, receive, send)\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/starlette/routing.py\", line 227, in handle\n    await self.app(scope, receive, send)\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/starlette/routing.py\", line 41, in app\n    response = await func(request)\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/fastapi/routing.py\", line 201, in app\n    raw_response = await run_endpoint_function(\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/.riot/venv_py3913_fastapi~0640/lib/python3.9/site-packages/fastapi/routing.py\", line 148, in run_endpoint_function\n    return await dependant.call(**values)\n  File \"/Users/munirabdinur/go/src/github.com/DataDog/dd-trace-py/tests/contrib/fastapi/app.py\", line 91, in error\n    raise RuntimeError(\"Server error\")\nRuntimeError: Server error\n",
+      "error.type": "builtins.RuntimeError",
+      "http.method": "GET",
+      "http.route": "/500",
+      "http.status_code": "500",
+      "http.url": "http://testserver/500",
+      "http.useragent": "testclient",
+      "http.version": "1.1",
+      "language": "python",
+      "runtime-id": "64d5226c7c7c4f79a067f32ae2c114ef",
+      "span.kind": "server"
+    },
+    "metrics": {
+      "_dd.agent_psr": 1.0,
+      "_dd.top_level": 1,
+      "_dd.tracer_kr": 1.0,
+      "_sampling_priority_v1": 1,
+      "process_id": 24153
+    },
+    "duration": 1430000,
+    "start": 1683658745856878000
+  }]]