ci: remove a bunch of flaky marks by fixing tests (#8044)

This change fixes or removes some tests that had been marked as "flaky":

* Removes `test_tracer_safety.py`, which was skipped unconditionally in
its entirety
* Resolves a reliable failure in `test_logging.py` by telling the
`DummyWriter` not to flush its traces to the test agent
* Removes randomization from some `pytest_bdd` tests that expect their
test suites to run in a specific order
* Unskips a `snowflake` test that didn't fail in repeated local runs
* Unskips a `test_utils_http` test that didn't fail in repeated local
runs

It's important to regularly revisit tests marked as `@flaky` and make
sure they're reliable. `@flaky` is a temporary unblocking mechanism, not
a permanent fix.

## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist

- [ ] Title is accurate.
- [ ] No unnecessary changes are introduced.
- [ ] Description motivates each change.
- [ ] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [ ] Testing strategy adequately addresses listed risk(s).
- [ ] Change is maintainable (easy to change, telemetry, documentation).
- [ ] Release note makes sense to a user of the library.
- [ ] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [ ] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
- [ ] If this PR touches code that signs or publishes builds or
packages, or handles credentials of any kind, I've requested a review
from `@DataDog/security-design-and-guidance`.
- [ ] This PR doesn't touch any of that.

Author: emmettbutler

tests/contrib/asyncio/test_tracer_safety.py

@@ -13,7 +13,6 @@
 from ddtrace.internal.constants import MAX_UINT_64BITS
 from ddtrace.vendor import wrapt
 from tests.utils import TracerTestCase
-from tests.utils import flaky
 
 
 logger = logging.getLogger()
@@ -198,8 +197,9 @@ def create_span():
         with self.override_global_config(dict(version="global.version", env="global.env")):
             self._test_logging(create_span=create_span, version="global.version", env="global.env")
 
-    @flaky(until=1704067200)
-    @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_TAGS="service:ddtagservice,env:ddenv,version:ddversion"))
+    @TracerTestCase.run_in_subprocess(
+        env_overrides=dict(DD_TAGS="service:ddtagservice,env:ddenv,version:ddversion", _DD_TEST_TRACE_FLUSH_ENABLED="0")
+    )
     def test_log_DD_TAGS(self):
         def create_span():
             return self.tracer.trace("test.logging")

tests/contrib/logging/test_logging.py

@@ -14,7 +14,6 @@
 from tests.contrib.patch import emit_integration_and_version_to_test_agent
 from tests.utils import DummyCIVisibilityWriter
 from tests.utils import TracerTestCase
-from tests.utils import flaky
 from tests.utils import override_env
 
 
@@ -60,7 +59,6 @@ def test_and_emit_get_version(self):
 
         emit_integration_and_version_to_test_agent("pytest-bdd", version)
 
-    @flaky(1707063167)
     def test_pytest_bdd_scenario_with_parameters(self):
         """Test that pytest-bdd traces scenario with all steps."""
         self.testdir.makefile(
@@ -114,7 +112,7 @@ def check_cfparse(bars):
             """
         )
         file_name = os.path.basename(py_file.strpath)
-        self.inline_run("--ddtrace", file_name)
+        self.inline_run("-p", "no:randomly", "--ddtrace", file_name)
         spans = self.pop_spans()
 
         assert len(spans) == 13  # 3 scenarios + 7 steps + 1 module
@@ -156,7 +154,7 @@ def check():
             """
         )
         file_name = os.path.basename(py_file.strpath)
-        self.inline_run("--ddtrace", file_name)
+        self.inline_run("-p", "no:randomly", "--ddtrace", file_name)
         spans = self.pop_spans()
 
         assert len(spans) == 7
@@ -202,7 +200,7 @@ def check():
             """
         )
         file_name = os.path.basename(py_file.strpath)
-        self.inline_run("--ddtrace", file_name)
+        self.inline_run("-p", "no:randomly", "--ddtrace", file_name)
         spans = self.pop_spans()
 
         assert len(spans) == 7
@@ -225,7 +223,7 @@ def test_simple():
             """
         )
         file_name = os.path.basename(py_file.strpath)
-        self.inline_run("--ddtrace", file_name)
+        self.inline_run("-p", "no:randomly", "--ddtrace", file_name)
         spans = self.pop_spans()
 
         assert len(spans) == 4

tests/contrib/pytest_bdd/test_pytest_bdd.py

@@ -11,7 +11,6 @@
 from ddtrace.contrib.snowflake import patch
 from ddtrace.contrib.snowflake import unpatch
 from tests.opentracer.utils import init_tracer
-from tests.utils import flaky
 from tests.utils import override_config
 from tests.utils import snapshot
 
@@ -176,7 +175,6 @@ def test_snowflake_service_env():
                 assert cur.fetchone() == ("4.30.2",)
 
 
-@flaky(until=1704067200)
 @snapshot()
 @req_mock.activate
 def test_snowflake_pin_override(client):

tests/contrib/snowflake/test_snowflake.py

@@ -2,10 +2,8 @@
 import pytest
 
 from ddtrace.internal.utils.http import connector
-from tests.utils import flaky
 
 
-@flaky(until=1704067201)
 @pytest.mark.parametrize("scheme", ["http", "https"])
 def test_connector(scheme):
     with httpretty.enabled():

tests/internal/test_utils_http.py

@@ -29,6 +29,7 @@
 from ddtrace.internal.encoding import JSONEncoder
 from ddtrace.internal.encoding import MsgpackEncoderV03 as Encoder
 from ddtrace.internal.schema import SCHEMA_VERSION
+from ddtrace.internal.utils.formats import asbool
 from ddtrace.internal.utils.formats import parse_tags_str
 from ddtrace.internal.writer import AgentWriter
 from ddtrace.propagation.http import _DatadogMultiHeader
@@ -570,6 +571,7 @@ class DummyTracer(Tracer):
 
     def __init__(self, *args, **kwargs):
         super(DummyTracer, self).__init__()
+        self._trace_flush_disabled_via_env = not asbool(os.getenv("_DD_TEST_TRACE_FLUSH_ENABLED", True))
         self._trace_flush_enabled = True
         self.configure(*args, **kwargs)
 
@@ -610,7 +612,9 @@ def configure(self, *args, **kwargs):
         if not kwargs.get("writer"):
             # if no writer is present, check if test agent is running to determine if we
             # should emit traces.
-            kwargs["writer"] = DummyWriter(trace_flush_enabled=check_test_agent_status())
+            kwargs["writer"] = DummyWriter(
+                trace_flush_enabled=check_test_agent_status() if not self._trace_flush_disabled_via_env else False
+            )
         super(DummyTracer, self).configure(*args, **kwargs)