fix: avoid prematurely exhausting wrapped generators during stack unwinding [backport 3.9] (#13694)

Backport a4c9ba0 from #13661 to 3.9.

This change resolves an issue wherein `tracer.wrap` on a generator
function that yields another generator when called changed the relative
ordering of generator exhaustion and exception handling calls in the
middle of the stack. This behavior caused applications upgrading from
3.7.0 to 3.9.0 to experience a change in behavior that relied on this
relative ordering.

In short, this bug existed because `for v in g: yield v` was used where
`yield from g` was needed. See PEP380 for explanation of why they're not
the same thing https://peps.python.org/pep-0380/#formal-semantics

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Co-authored-by: Emmett Butler <[email protected]>

Author: github-actions[bot]

ddtrace/_trace/tracer.py

@@ -801,9 +801,7 @@ def func_wrapper(*args, **kwargs):
                 )
 
             with self.trace(span_name, service=service, resource=resource, span_type=span_type):
-                gen = f(*args, **kwargs)
-                for value in gen:
-                    yield value
+                yield from f(*args, **kwargs)
 
         return func_wrapper
 
@@ -820,8 +818,7 @@ def _wrap_generator_async(
         @functools.wraps(f)
         async def func_wrapper(*args, **kwargs):
             with self.trace(span_name, service=service, resource=resource, span_type=span_type):
-                agen = f(*args, **kwargs)
-                async for value in agen:
+                async for value in f(*args, **kwargs):
                     yield value
 
         return func_wrapper

releasenotes/notes/gen-unwind-f0ae0e22900495c1.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    This fix resolves an issue in which traced nested generator functions had their execution order subtly changed
+    in a way that affected the stack unwinding sequence during exception handling. The issue was caused
+    by the tracer's use of simple iteration via ``for v in g: yield v`` during the wrapping of generator functions
+    where full bidrectional communication with the sub-generator via ``yield from g`` was appropriate. See
+    PEP380 for an explanation of how these two generator uses differ.

tests/tracer/test_tracer.py

@@ -313,6 +313,48 @@ def f(tag_name, tag_value):
         assert span.duration is not None
         assert span.duration > 0.03
 
+    def test_tracer_wrap_nested_generators_preserves_stack_order(self):
+        result = {"handled": False}
+
+        def trace_and_call_next(*call_args, **call_kwargs):
+            def _outer_wrapper(func):
+                @self.tracer.wrap("foobar")
+                def _inner_wrapper(*args, **kwargs):
+                    wrapped_generator = func(*args, **kwargs)
+                    try:
+                        yield next(wrapped_generator)
+                    except BaseException as e:
+                        result["handled"] = True
+                        wrapped_generator.throw(e)
+
+                return _inner_wrapper
+
+            return _outer_wrapper(*call_args, **call_kwargs)
+
+        @contextlib.contextmanager
+        @trace_and_call_next
+        def wrapper():
+            try:
+                yield
+            except NotImplementedError:
+                raise ValueError()
+
+        exception_note = (
+            "The expected exception should bubble up from a traced generator-based context manager "
+            "that yields another generator"
+        )
+        try:
+            with wrapper():
+                raise NotImplementedError()
+        except Exception as e:
+            assert isinstance(e, ValueError), exception_note
+        else:
+            assert False, exception_note
+        assert result["handled"], (
+            "Exceptions raised by traced generator-based context managers that yield generators should be "
+            "visible to the caller"
+        )
+
     def test_tracer_disabled(self):
         self.tracer.enabled = True
         with self.trace("foo") as s: