chore(ci): coverage and integration test fix [backport #5502, #5521, #5583 to 1.11] (#5634)

This PR backports #5583, #5521, and #5502 to 1.11.

This PR fixes the bash conditional on our `run_test` pipeline jobs so
that when a pipeline is manually triggered with `coverage=true` on
CircleCI, coverage reporting is enabled.


## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/contributing.html#Release-Note-Guidelines)
are followed.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] PR description includes explicit acknowledgement/acceptance of the
performance implications of this PR as reported in the benchmarks PR
comment.

## Reviewer Checklist

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.

---------

Co-authored-by: Brett Langdon <[email protected]>

Author: Yun-Kim

.circleci/config.yml

@@ -16,6 +16,11 @@ httpbin_image: &httpbin_image kennethreitz/httpbin@sha256:2c7abc4803080c22928265
 vertica_image: &vertica_image sumitchawla/vertica:latest
 rabbitmq_image: &rabbitmq_image rabbitmq:3.7-alpine
 
+parameters:
+  coverage:
+    type: boolean
+    default: false
+
 orbs:
   win: circleci/[email protected]
 
@@ -160,10 +165,9 @@ commands:
             - run:
                 environment:
                   DD_TRACE_AGENT_URL: http://localhost:9126
-
                 command: |
                   mv .riot .ddriot
-                  riot list --hash-only '<<parameters.pattern>>' | shuf | circleci tests split | xargs -n 1 ./scripts/ddtest riot -v run --exitfirst --pass-env -s
+                  riot list --hash-only '<<parameters.pattern>>' | shuf | circleci tests split | xargs -n 1 -I {} ./scripts/ddtest riot -v run --exitfirst --pass-env -s {} $([[ << pipeline.parameters.coverage >> == false ]] && echo '--no-cov' )
       - unless:
           condition:
               << parameters.snapshot >>
@@ -176,38 +180,22 @@ commands:
                   - run:
                       name: "Waiting for << parameters.wait >>"
                       command: riot -v run 'wait' << parameters.wait >>
-
             - run:
-                command: "riot list --hash-only '<<parameters.pattern>>' | shuf | circleci tests split | xargs -n 1 riot -v run --exitfirst --pass-env -s"
+                command: |
+                  riot list --hash-only '<<parameters.pattern>>' | shuf | circleci tests split | xargs -n 1 -I {} riot -v run --exitfirst --pass-env -s {} $([[ << pipeline.parameters.coverage >> == false ]] && echo '--no-cov' )
       - save_pip_cache
       - when:
           condition:
-            << parameters.store_coverage >>
+            and:
+              - << pipeline.parameters.coverage >>
+              - << parameters.store_coverage >>
           steps:
             - save_coverage
       - store_test_results:
           path: test-results
       - store_artifacts:
           path: test-results
 
-  run_tox_scenario_with_testagent:
-    description: Run scripts/run-tox-scenario with setup, caching persistence and the testagent
-    parameters:
-      pattern:
-        type: string
-    steps:
-      - checkout
-      - restore_tox_cache
-      - start_docker_services:
-          env: SNAPSHOT_CI=1
-          services: memcached redis testagent
-      - run:
-          name: "Run scripts/run-tox-scenario"
-          environment:
-            DD_TRACE_AGENT_URL: http://localhost:9126
-          command: ./scripts/ddtest scripts/run-tox-scenario '<< parameters.pattern >>'
-      - save_tox_cache
-
   run_tox_scenario:
     description: "Run scripts/run-tox-scenario with setup, caching and persistence"
     parameters:
@@ -224,11 +212,6 @@ commands:
           name: "Run scripts/run-tox-scenario"
           command: scripts/run-tox-scenario '<< parameters.pattern >>'
       - save_tox_cache
-      - when:
-          condition:
-            << parameters.store_coverage >>
-          steps:
-            - save_coverage
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -317,34 +300,44 @@ jobs:
   coverage_report:
     executor: python310
     steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run: pip install coverage codecov diff_cover
-      - run: ls -hal *.coverage
-      # Combine all job coverage reports into one
-      - run: coverage combine *.coverage
-      # Upload coverage report to Codecov
-      # DEV: Do not use the bash uploader, it cannot be trusted
-      - run: codecov
-      # Generate and save xml report
-      # DEV: "--ignore-errors" to skip over files that are missing
-      - run: coverage xml --ignore-errors
-      - store_artifacts:
-          path: coverage.xml
-      # Generate and save JSON report
-      # DEV: "--ignore-errors" to skip over files that are missing
-      - run: coverage json --ignore-errors
-      - store_artifacts:
-          path: coverage.json
-      # Print ddtrace/ report to stdout
-      # DEV: "--ignore-errors" to skip over files that are missing
-      - run: coverage report --ignore-errors --omit=tests/
-      # Print tests/ report to stdout
-      # DEV: "--ignore-errors" to skip over files that are missing
-      - run: coverage report --ignore-errors --omit=ddtrace/
-      # Print diff-cover report to stdout (compares against origin/1.x)
-      - run: diff-cover --compare-branch $(git rev-parse --abbrev-ref origin/HEAD) coverage.xml
+      - when:
+          condition:
+            << pipeline.parameters.coverage >>
+          steps:
+            - checkout
+            - attach_workspace:
+                at: .
+            - run: pip install coverage codecov diff_cover
+            - run: ls -hal *.coverage
+            # Combine all job coverage reports into one
+            - run: coverage combine *.coverage
+            # Upload coverage report to Codecov
+            # DEV: Do not use the bash uploader, it cannot be trusted
+            - run: codecov
+            # Generate and save xml report
+            # DEV: "--ignore-errors" to skip over files that are missing
+            - run: coverage xml --ignore-errors
+            - store_artifacts:
+                path: coverage.xml
+            # Generate and save JSON report
+            # DEV: "--ignore-errors" to skip over files that are missing
+            - run: coverage json --ignore-errors
+            - store_artifacts:
+                path: coverage.json
+            # Print ddtrace/ report to stdout
+            # DEV: "--ignore-errors" to skip over files that are missing
+            - run: coverage report --ignore-errors --omit=tests/
+            # Print tests/ report to stdout
+            # DEV: "--ignore-errors" to skip over files that are missing
+            - run: coverage report --ignore-errors --omit=ddtrace/
+            # Print diff-cover report to stdout (compares against origin/1.x)
+            - run: diff-cover --compare-branch $(git rev-parse --abbrev-ref origin/HEAD) coverage.xml
+      - unless:
+          condition:
+            << pipeline.parameters.coverage >>
+          steps:
+            - run: echo "No coverage data collected. Set run workflow manually with pipeline parameter 'coverage=true'."
+
 
   build_base_venvs:
     resource_class: large
@@ -539,32 +532,26 @@ jobs:
 
   integration_agent:
     <<: *machine_executor
+    parallelism: 2
     steps:
       - attach_workspace:
           at: .
       - checkout
+      - setup_riot
       - start_docker_services:
           services: ddagent
       - run:
           command: |
             mv .riot .ddriot
-            ./scripts/ddtest riot -v run --pass-env -s 'integration-latest'
+            riot list --hash-only 'integration-latest' | shuf | circleci tests split | xargs -n 1 -I {} ./scripts/ddtest riot -v run --pass-env -s {}
 
   integration_testagent:
     <<: *machine_executor
     steps:
-      - attach_workspace:
-          at: .
-      - checkout
-      - start_docker_services:
-          env: SNAPSHOT_CI=1
-          services: testagent
-      - run:
-          environment:
-            DD_TRACE_AGENT_URL: http://localhost:9126
-          command: |
-            mv .riot .ddriot
-            ./scripts/ddtest riot -v run --pass-env -s 'integration-snapshot'
+      - run_test:
+          snapshot: true
+          store_coverage: false
+          pattern: 'integration-snapshot'
 
   vendor:
     <<: *contrib_job_small

.riot/requirements/4797e24.txt

@@ -0,0 +1,36 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --no-annotate .riot/requirements/4797e24.in
+#
+atomicwrites==1.4.1
+attrs==21.4.0
+backports.functools-lru-cache==1.6.4
+configparser==4.0.2
+contextlib2==0.6.0.post1
+coverage==5.5
+enum34==1.1.10
+funcsigs==1.0.2
+futures==3.4.0
+hypothesis==4.57.1
+importlib-metadata==2.1.3
+mock==3.0.5
+more-itertools==5.0.0
+msgpack==1.0.5
+opentracing==2.4.0
+packaging==20.9
+pathlib2==2.3.7.post1
+pluggy==0.13.1
+py==1.11.0
+pyparsing==2.4.7
+pytest-cov==2.12.1
+pytest-mock==2.0.0
+pytest==4.6.0
+scandir==1.10.0
+six==1.16.0
+sortedcontainers==2.4.0
+toml==0.10.2
+typing==3.10.0.0
+wcwidth==0.2.6
+zipp==1.2.0

riotfile.py

@@ -298,6 +298,7 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
         ),
         Venv(
             name="integration",
+            # Enabling coverage for integration tests breaks certain tests in CI
             command="pytest --no-cov {cmdargs} tests/integration/",
             pkgs={"msgpack": [latest]},
             venvs=[
@@ -1576,9 +1577,9 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                     pys=["2.7"],
                     # pytest==4.6 is last to support python 2.7
                     pkgs={
-                        "pytest": ">=4.0,<4.6",
+                        "pytest": ">=4.0,<=4.6",
                         "msgpack": latest,
-                        "pytest-cov": "==2.5.0",
+                        "pytest-cov": "==2.12.1",
                     },
                 ),
                 Venv(

tests/integration/test_integration.py

@@ -771,7 +771,7 @@ def test_regression_logging_in_context(tmpdir, logs_injection, debug_mode, patch
         [sys.executable, "test.py"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=str(tmpdir), env=env
     )
     try:
-        p.wait(timeout=2)
+        p.wait(timeout=10)
     except TypeError:
         # timeout argument added in Python 3.3
         p.wait()