feat(ci_visibility): print report links at the end of the test session (#13545)

This introduces report links to the pytest plugin. At the end of a test
session, ddtrace shows links to the Datadog Test Optimization pages with
the test results for the current commit and for the current CI
job(provided that the CI environment variables with the current job and
pipeline ID are available).

This is a feature that we already have [in
`datadog-ci`](https://github.com/DataDog/datadog-ci/blob/v3.7.0/src/commands/junit/renderer.ts#L50)
for JUnit XML upload , and internally in the dogweb CI. This PR makes it
available in dd-trace-py.

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: vitor-de-araujo

ddtrace/contrib/internal/pytest/_plugin_v2.py

@@ -17,6 +17,7 @@
 from ddtrace.contrib.internal.coverage.utils import _is_coverage_patched
 from ddtrace.contrib.internal.pytest._benchmark_utils import _set_benchmark_data_from_item
 from ddtrace.contrib.internal.pytest._plugin_v1 import _is_pytest_cov_enabled
+from ddtrace.contrib.internal.pytest._report_links import print_test_report_links
 from ddtrace.contrib.internal.pytest._types import _pytest_report_teststatus_return_type
 from ddtrace.contrib.internal.pytest._types import pytest_CallInfo
 from ddtrace.contrib.internal.pytest._types import pytest_Config
@@ -673,6 +674,7 @@ def _pytest_terminal_summary_post_yield(terminalreporter, failed_reports_initial
     quarantine_pytest_terminal_summary_post_yield(terminalreporter)
     attempt_to_fix_pytest_terminal_summary_post_yield(terminalreporter)
 
+    print_test_report_links(terminalreporter)
     return
 
 

ddtrace/contrib/internal/pytest/_report_links.py

@@ -0,0 +1,94 @@
+import os
+import re
+from urllib.parse import quote
+
+from ddtrace.ext import ci
+from ddtrace.internal.ci_visibility import CIVisibility
+
+
+DEFAULT_DATADOG_SITE = "datadoghq.com"
+DEFAULT_DATADOG_SUBDOMAIN = "app"
+
+SAFE_FOR_QUERY = re.compile(r"\A[A-Za-z0-9._-]+\Z")
+
+
+def print_test_report_links(terminalreporter):
+    base_url = _get_base_url(
+        dd_site=os.getenv("DD_SITE", DEFAULT_DATADOG_SITE), dd_subdomain=os.getenv("DD_SUBDOMAIN", "")
+    )
+    ci_tags = CIVisibility.get_ci_tags()
+    settings = CIVisibility.get_session_settings()
+    service = settings.test_service
+    env = CIVisibility.get_dd_env()
+
+    redirect_test_commit_url = _build_test_commit_redirect_url(base_url, ci_tags, service, env)
+    test_runs_url = _build_test_runs_url(base_url, ci_tags)
+
+    if not (redirect_test_commit_url or test_runs_url):
+        return
+
+    terminalreporter.section("Datadog Test Reports", cyan=True, bold=True)
+    terminalreporter.line("View detailed reports in Datadog (they may take a few minutes to become available):")
+
+    if redirect_test_commit_url:
+        terminalreporter.line("")
+        terminalreporter.line("* Commit report:")
+        terminalreporter.line(f"  → {redirect_test_commit_url}")
+
+    if test_runs_url:
+        terminalreporter.line("")
+        terminalreporter.line("* Test runs report:")
+        terminalreporter.line(f"  → {test_runs_url}")
+
+
+def _get_base_url(dd_site, dd_subdomain):
+    # Based on <https://github.com/DataDog/datadog-ci/blob/v3.7.0/src/helpers/app.ts>.
+    subdomain = dd_subdomain or DEFAULT_DATADOG_SUBDOMAIN
+    dd_site_parts = dd_site.split(".")
+    if len(dd_site_parts) == 3:
+        if subdomain == DEFAULT_DATADOG_SUBDOMAIN:
+            return f"https://{dd_site}"
+        else:
+            return f"https://{subdomain}.{dd_site_parts[1]}.{dd_site_parts[2]}"
+    else:
+        return f"https://{subdomain}.{dd_site}"
+
+
+def _build_test_commit_redirect_url(base_url, ci_tags, service, env):
+    params = {
+        "repo_url": ci_tags.get(ci.git.REPOSITORY_URL),
+        "branch": ci_tags.get(ci.git.BRANCH),
+        "commit_sha": ci_tags.get(ci.git.COMMIT_SHA),
+        "service": service,
+    }
+    if any(v is None for v in params.values()):
+        return None
+
+    url_format = "/ci/redirect/tests/{repo_url}/-/{service}/-/{branch}/-/{commit_sha}"
+    url = base_url + url_format.format(**{k: quote(v, safe="") for k, v in params.items()})
+    if env:
+        url += f"?env={env}".format(env=quote(env, safe=""))
+
+    return url
+
+
+def _quote_for_query(text):
+    if SAFE_FOR_QUERY.match(text):
+        return text
+
+    escaped_text = text.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped_text}"'
+
+
+def _build_test_runs_url(base_url, ci_tags):
+    ci_job_name = ci_tags.get(ci.JOB_NAME)
+    ci_pipeline_id = ci_tags.get(ci.PIPELINE_ID)
+
+    if not (ci_job_name and ci_pipeline_id):
+        return None
+
+    query = "@ci.job.name:{} @ci.pipeline.id:{}".format(_quote_for_query(ci_job_name), _quote_for_query(ci_pipeline_id))
+
+    url_format = "/ci/test-runs?query={query}&index=citest"
+    url = base_url + url_format.format(query=quote(query, safe=""))
+    return url

ddtrace/internal/ci_visibility/recorder.py

@@ -233,14 +233,14 @@ def __init__(
 
         self._git_data: GitData = get_git_data_from_tags(self._tags)
 
-        dd_env = os.getenv("_CI_DD_ENV", ddconfig.env)
+        self._dd_env = os.getenv("_CI_DD_ENV", ddconfig.env)
         dd_env_msg = ""
 
         if ddconfig._ci_visibility_agentless_enabled:
             # In agentless mode, normalize an unset env to none (this is already done by the backend in most cases, so
             # it does not override default behavior)
-            if dd_env is None:
-                dd_env = "none"
+            if self._dd_env is None:
+                self._dd_env = "none"
                 dd_env_msg = " (not set in environment)"
             if not self._api_key:
                 raise EnvironmentError(
@@ -257,13 +257,13 @@ def __init__(
                 self._dd_site,
                 ddconfig._ci_visibility_agentless_url if ddconfig._ci_visibility_agentless_url else None,
                 self._service,
-                dd_env,
+                self._dd_env,
             )
         elif evp_proxy_base_url := self._agent_evp_proxy_base_url():
             # In EVP-proxy cases, if an env is not provided, we need to get the agent's default env in order to make
             # the correct decision:
-            if dd_env is None:
-                dd_env = self._agent_get_default_env()
+            if self._dd_env is None:
+                self._dd_env = self._agent_get_default_env()
                 dd_env_msg = " (default environment provided by agent)"
             self._requests_mode = REQUESTS_MODE.EVP_PROXY_EVENTS
             requests_mode_str = "EVP Proxy"
@@ -273,7 +273,7 @@ def __init__(
                 self._configurations,
                 self.tracer._agent_url,
                 self._service,
-                dd_env,
+                self._dd_env,
                 evp_proxy_base_url=evp_proxy_base_url,
             )
         else:
@@ -293,7 +293,7 @@ def __init__(
 
         self._configure_writer(coverage_enabled=self._collect_coverage_enabled, url=self.tracer._agent_url)
 
-        log.info("Service: %s (env: %s%s)", self._service, dd_env, dd_env_msg)
+        log.info("Service: %s (env: %s%s)", self._service, self._dd_env, dd_env_msg)
         log.info("Requests mode: %s", requests_mode_str)
         log.info("Git metadata upload enabled: %s", self._should_upload_git_metadata)
         log.info("API-provided settings: coverage collection: %s", self._api_settings.coverage_enabled)
@@ -991,6 +991,16 @@ def set_library_capabilities(cls, capabilities: LibraryCapabilities) -> None:
             return
         client.set_metadata("test", capabilities.tags())
 
+    @classmethod
+    def get_ci_tags(cls):
+        instance = cls.get_instance()
+        return instance._tags
+
+    @classmethod
+    def get_dd_env(cls):
+        instance = cls.get_instance()
+        return instance._dd_env
+
     @classmethod
     def is_known_test(cls, test_id: Union[TestId, InternalTestId]) -> bool:
         instance = cls.get_instance()

hatch.toml

@@ -741,7 +741,8 @@ dependencies = [
     "requests",
     "hypothesis",
     "pytest{matrix:pytest}",
-    "pytest-cov"
+    "pytest-cov",
+    "pytest-mock"
 ]
 
 [envs.pytest_plugin_v2.env-vars]

releasenotes/notes/ci_visibility-feat-report-links-17156296dfc566a8.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    CI Visibility: This introduces report links to the pytest plugin. At the end of a test session, ddtrace shows links
+    to the Datadog Test Optimization pages with the test results for the current commit and for the current CI job
+    (provided that the CI environment variables with the current job and pipeline ID are available).