chore(otel): make snapshots less flaky (#13016)

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: mabdinur

tests/opentelemetry/test_context.py

@@ -13,7 +13,7 @@
 from ddtrace.constants import MANUAL_KEEP_KEY
 
 
-@pytest.mark.snapshot
+@pytest.mark.snapshot(wait_for_num_traces=1)
 def test_otel_span_parenting(oteltracer):
     with oteltracer.start_as_current_span("otel-root") as root:
         time.sleep(0.02)
@@ -36,7 +36,7 @@ def test_otel_span_parenting(oteltracer):
         orphan1.end()
 
 
-@pytest.mark.snapshot
+@pytest.mark.snapshot(wait_for_num_traces=1)
 def test_otel_ddtrace_mixed_parenting(oteltracer):
     with oteltracer.start_as_current_span("otel-top-level"):
         with ddtrace.tracer.trace("ddtrace-top-level"):
@@ -51,7 +51,7 @@ def test_otel_ddtrace_mixed_parenting(oteltracer):
                         time.sleep(0.02)
 
 
-@pytest.mark.snapshot
+@pytest.mark.snapshot(wait_for_num_traces=1)
 def test_otel_multithreading(oteltracer):
     def target(parent_context):
         ctx = opentelemetry.trace.set_span_in_context(opentelemetry.trace.NonRecordingSpan(parent_context))
@@ -92,7 +92,7 @@ def _subprocess_task(parent_span_context, errors):
         ot_tracer._tracer.flush()
 
 
-@pytest.mark.snapshot(ignores=["meta.tracestate"])
+@pytest.mark.snapshot(wait_for_num_traces=1, ignores=["meta.tracestate"])
 @pytest.mark.subprocess(env={"DD_TRACE_OTEL_ENABLED": "true"}, ddtrace_run=True, err=None)
 def test_otel_trace_across_fork():
     import multiprocessing
@@ -146,7 +146,7 @@ def test_sampling_decisions_across_processes():
 
 
 @pytest.mark.asyncio
-@pytest.mark.snapshot
+@pytest.mark.snapshot(wait_for_num_traces=1)
 async def test_otel_trace_multiple_coroutines(oteltracer):
     async def coro(i):
         with oteltracer.start_as_current_span("corountine %s" % (i,)):

tests/opentelemetry/test_span.py

@@ -15,7 +15,6 @@
 
 from ddtrace.constants import MANUAL_DROP_KEY
 from ddtrace.internal.opentelemetry.span import Span
-from tests.utils import flaky
 
 
 @pytest.mark.snapshot(wait_for_num_traces=3)
@@ -47,7 +46,7 @@ def test_otel_span_attributes(oteltracer):
         span.set_attribute("should_not_be_set", "attributes can not be added after a span is ended")
 
 
-@pytest.mark.snapshot(wait_for_num_traces=1)
+@pytest.mark.snapshot(wait_for_num_traces=2)
 def test_otel_span_events(oteltracer):
     with oteltracer.start_span("webpage.load") as span1:
         span1.add_event(
@@ -64,7 +63,7 @@ def test_otel_span_events(oteltracer):
     span2.add_event("Event on finished span, event won't be added")
 
 
-@pytest.mark.snapshot
+@pytest.mark.snapshot(wait_for_num_traces=1)
 @pytest.mark.parametrize(
     "override",
     [
@@ -82,8 +81,7 @@ def test_otel_span_attributes_overrides(oteltracer, override):
         span.set_attribute(otel, value)
 
 
-@pytest.mark.snapshot
-@flaky(1741838400, reason="Did not receive expected traces: 'client','server','producer','consumer','internal'")
+@pytest.mark.snapshot(wait_for_num_traces=5)
 def test_otel_span_kind(oteltracer):
     with oteltracer.start_span("otel-client", kind=OtelSpanKind.CLIENT):
         pass

tests/opentelemetry/test_trace.py

@@ -19,7 +19,7 @@ def test_otel_compatible_tracer_is_returned_by_tracer_provider():
     assert isinstance(otel_compatible_tracer, opentelemetry.trace.Tracer)
 
 
-@pytest.mark.snapshot(wait_for_num_traces=1, ignores=["meta.error.stack"])
+@pytest.mark.snapshot(wait_for_num_traces=2, ignores=["meta.error.stack"])
 def test_otel_start_span_record_exception(oteltracer):
     # Avoid mocking time_ns when Span is created. This is a workaround to resolve a rate limit bug.
     raised_span = oteltracer.start_span("test-raised-exception")
@@ -104,21 +104,21 @@ def test_otel_start_span_with_span_links(oteltracer):
         span2.end()
 
 
-@pytest.mark.snapshot(ignores=["meta.error.stack"])
+@pytest.mark.snapshot(wait_for_num_traces=1, ignores=["meta.error.stack"])
 def test_otel_start_span_ignore_exceptions(caplog, oteltracer):
     with pytest.raises(Exception, match="Sorry Otel Span, I failed you"):
         with oteltracer.start_span("otel-error-span", record_exception=False, set_status_on_exception=False):
             raise Exception("Sorry Otel Span, I failed you")
 
 
-@pytest.mark.snapshot
+@pytest.mark.snapshot(wait_for_num_traces=1)
 def test_otel_start_current_span_with_default_args(oteltracer):
     with oteltracer.start_as_current_span("test-start-current-span-defaults") as otel_span:
         assert otel_span.is_recording()
         otel_span.update_name("rename-start-current-span")
 
 
-@pytest.mark.snapshot
+@pytest.mark.snapshot(wait_for_num_traces=1)
 def test_otel_start_current_span_without_default_args(oteltracer):
     with oteltracer.start_as_current_span("root-span") as root:
         with oteltracer.start_as_current_span(
@@ -219,7 +219,10 @@ def otel_flask_app_env(flask_wsgi_application):
         "with_opentelemetry_instrument",
     ],
 )
-@pytest.mark.snapshot(ignores=["metrics.net.peer.port", "meta.traceparent", "meta.tracestate", "meta.flask.version"])
+@pytest.mark.snapshot(
+    wait_for_num_traces=1,
+    ignores=["metrics.net.peer.port", "meta.traceparent", "meta.tracestate", "meta.flask.version"],
+)
 def test_distributed_trace_with_flask_app(flask_client, oteltracer):  # noqa:F811
     with oteltracer.start_as_current_span("test-otel-distributed-trace") as span:
         headers = {}