feat(ci_visibility): add github action tags for attempt to fix workflow (#14099)

## What?
- Add a tag for the head commit sha if we run inside a GitHub Action
- Whenever the HEAD commit SHA is present in the tags that come from the
CI provider, we assume that the CI provider added a commit on top of the
user's HEAD commit (e.g., GitHub Actions add a merge commit when
triggered by a pull request). In that case, we extract the metadata for
that commit specifically and add it to the tags.
- These metadata are needed for Test Optimization's attempt to fix
workflow

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: calvinbayer

ddtrace/contrib/internal/pytest/_plugin_v2.py

@@ -345,7 +345,7 @@ def pytest_sessionstart(session: pytest.Session) -> None:
             test_impact_analysis="1" if _pytest_version_supports_itr() else None,
             test_management_quarantine="1",
             test_management_disable="1",
-            test_management_attempt_to_fix="4" if _pytest_version_supports_attempt_to_fix() else None,
+            test_management_attempt_to_fix="5" if _pytest_version_supports_attempt_to_fix() else None,
         )
 
         InternalTestSession.discover(

ddtrace/ext/ci.py

@@ -105,6 +105,16 @@ def tags(env=None, cwd=None):
             break
 
     git_info = git.extract_git_metadata(cwd=cwd)
+
+    # Whenever the HEAD commit SHA is present in the tags that come from the CI provider, we assume that
+    # the CI provider added a commit on top of the user's HEAD commit (e.g., GitHub Actions add a merge
+    # commit when triggered by a pull request). In that case, we extract the metadata for that commit specifically
+    # and add it to the tags.
+    head_commit_sha = tags.get(git.COMMIT_HEAD_SHA)
+    if head_commit_sha:
+        git_head_info = git.extract_git_head_metadata(head_commit_sha=head_commit_sha, cwd=cwd)
+        git_info.update(git_head_info)
+
     try:
         git_info[WORKSPACE_PATH] = git.extract_workspace_path(cwd=cwd)
     except git.GitNotFoundError:
@@ -349,6 +359,15 @@ def extract_github_actions(env):
         github_run_id,
     )
 
+    git_commit_head_sha = None
+    if "GITHUB_EVENT_PATH" in env:
+        try:
+            with open(env["GITHUB_EVENT_PATH"]) as f:
+                github_event_data = json.load(f)
+                git_commit_head_sha = github_event_data.get("pull_request", {}).get("head", {}).get("sha")
+        except Exception as e:
+            log.error("Failed to read or parse GITHUB_EVENT_PATH: %s", e)
+
     env_vars = {
         "GITHUB_SERVER_URL": github_server_url,
         "GITHUB_REPOSITORY": github_repository,
@@ -362,6 +381,7 @@ def extract_github_actions(env):
         git.BRANCH: env.get("GITHUB_HEAD_REF") or env.get("GITHUB_REF"),
         git.COMMIT_SHA: git_commit_sha,
         git.REPOSITORY_URL: "{0}/{1}.git".format(github_server_url, github_repository),
+        git.COMMIT_HEAD_SHA: git_commit_head_sha,
         JOB_URL: "{0}/{1}/commit/{2}/checks".format(github_server_url, github_repository, git_commit_sha),
         PIPELINE_ID: github_run_id,
         PIPELINE_NAME: env.get("GITHUB_WORKFLOW"),

ddtrace/ext/git.py

@@ -33,6 +33,30 @@
 # Git Commit SHA
 COMMIT_SHA = "git.commit.sha"
 
+# Git Commit HEAD SHA
+COMMIT_HEAD_SHA = "git.commit.head.sha"
+
+# Git Commit HEAD message
+COMMIT_HEAD_MESSAGE = "git.commit.head.message"
+
+# Git Commit HEAD author date
+COMMIT_HEAD_AUTHOR_DATE = "git.commit.head.author.date"
+
+# Git Commit HEAD author email
+COMMIT_HEAD_AUTHOR_EMAIL = "git.commit.head.author.email"
+
+# Git Commit HEAD author name
+COMMIT_HEAD_AUTHOR_NAME = "git.commit.head.author.name"
+
+# Git Commit HEAD committer date
+COMMIT_HEAD_COMMITTER_DATE = "git.commit.head.committer.date"
+
+# Git Commit HEAD committer email
+COMMIT_HEAD_COMMITTER_EMAIL = "git.commit.head.committer.email"
+
+# Git Commit HEAD committer name
+COMMIT_HEAD_COMMITTER_NAME = "git.commit.head.committer.name"
+
 # Git Repository URL
 REPOSITORY_URL = "git.repository_url"
 
@@ -173,11 +197,12 @@ def _get_device_for_path(path):
     return os.stat(path).st_dev
 
 
-def _unshallow_repository_with_details(cwd=None, repo=None, refspec=None):
-    # type (Optional[str], Optional[str], Optional[str]) -> _GitSubprocessDetails
+def _unshallow_repository_with_details(
+    cwd: Optional[str] = None, repo: Optional[str] = None, refspec: Optional[str] = None, parent_only: bool = False
+) -> _GitSubprocessDetails:
     cmd = [
         "fetch",
-        '--shallow-since="1 month ago"',
+        "--deepen=1" if parent_only else '--shallow-since="1 month ago"',
         "--update-shallow",
         "--filter=blob:none",
         "--recurse-submodules=no",
@@ -190,18 +215,22 @@ def _unshallow_repository_with_details(cwd=None, repo=None, refspec=None):
     return _git_subprocess_cmd_with_details(*cmd, cwd=cwd)
 
 
-def _unshallow_repository(cwd=None, repo=None, refspec=None):
-    # type (Optional[str], Optional[str], Optional[str]) -> None
-    _unshallow_repository_with_details(cwd, repo, refspec)
+def _unshallow_repository(
+    cwd: Optional[str] = None,
+    repo: Optional[str] = None,
+    refspec: Optional[str] = None,
+    parent_only: bool = False,
+) -> None:
+    _unshallow_repository_with_details(cwd, repo, refspec, parent_only)
 
 
-def extract_user_info(cwd=None):
-    # type: (Optional[str]) -> Dict[str, Tuple[str, str, str]]
+def extract_user_info(cwd: Optional[str] = None, commit_sha: Optional[str] = None) -> Dict[str, Tuple[str, str, str]]:
     """Extract commit author info from the git repository in the current directory or one specified by ``cwd``."""
     # Note: `git show -s --format... --date...` is supported since git 2.1.4 onwards
-    stdout = _git_subprocess_cmd(
-        "show -s --format=%an|||%ae|||%ad|||%cn|||%ce|||%cd --date=format:%Y-%m-%dT%H:%M:%S%z", cwd=cwd
-    )
+    cmd = "show -s --format=%an|||%ae|||%ad|||%cn|||%ce|||%cd --date=format:%Y-%m-%dT%H:%M:%S%z"
+    if commit_sha:
+        cmd += " " + commit_sha
+    stdout = _git_subprocess_cmd(cmd=cmd, cwd=cwd)
     author_name, author_email, author_date, committer_name, committer_email, committer_date = stdout.split("|||")
     return {
         "author": (author_name, author_email, author_date),
@@ -316,6 +345,32 @@ def extract_commit_sha(cwd=None):
     return commit_sha
 
 
+def extract_git_head_metadata(head_commit_sha: str, cwd: Optional[str] = None) -> Dict[str, Optional[str]]:
+    tags: Dict[str, Optional[str]] = {}
+
+    is_shallow, *_ = _is_shallow_repository_with_details(cwd=cwd)
+    if is_shallow:
+        _unshallow_repository(cwd=cwd, repo=None, refspec=None, parent_only=True)
+
+    try:
+        users = extract_user_info(cwd=cwd, commit_sha=head_commit_sha)
+        tags[COMMIT_HEAD_AUTHOR_NAME] = users["author"][0]
+        tags[COMMIT_HEAD_AUTHOR_EMAIL] = users["author"][1]
+        tags[COMMIT_HEAD_AUTHOR_DATE] = users["author"][2]
+        tags[COMMIT_HEAD_COMMITTER_NAME] = users["committer"][0]
+        tags[COMMIT_HEAD_COMMITTER_EMAIL] = users["committer"][1]
+        tags[COMMIT_HEAD_COMMITTER_DATE] = users["committer"][2]
+        tags[COMMIT_HEAD_MESSAGE] = _git_subprocess_cmd(" ".join(("log -n 1 --format=%B", head_commit_sha)), cwd)
+    except GitNotFoundError:
+        log.error("Git executable not found, cannot extract git metadata.")
+    except ValueError as e:
+        debug_mode = log.isEnabledFor(logging.DEBUG)
+        stderr = str(e)
+        log.error("Error extracting git metadata: %s", stderr, exc_info=debug_mode)
+
+    return tags
+
+
 def extract_git_metadata(cwd=None):
     # type: (Optional[str]) -> Dict[str, Optional[str]]
     """Extract git commit metadata."""

tests/tracer/test_ci.py

@@ -2,6 +2,7 @@
 import glob
 import json
 import os
+import tempfile
 
 import mock
 import pytest
@@ -463,3 +464,73 @@ def test_build_git_packfiles_temp_dir_value_error(_temp_dir_mock, git_repo):
             pytest.fail()
     # CWD is not a temporary dir, so no deleted after using it.
     assert os.path.isdir(directory)
+
+
+def test_github_pull_request_head_sha():
+    fake_event_data = {"pull_request": {"head": {"sha": "headCommitSha"}}}
+    with tempfile.NamedTemporaryFile(mode="w+", delete=False) as event_file:
+        json.dump(fake_event_data, event_file)
+        event_file_path = event_file.name
+
+    env = {
+        "GITHUB_JOB": "test_job",
+        "GITHUB_EVENT_PATH": event_file_path,
+        "GITHUB_SERVER_URL": "https://github.com",
+        "GITHUB_REPOSITORY": "DataDog/dd-trace-py",
+        "GITHUB_REF": "refs/heads/main",
+        "GITHUB_HEAD_REF": "main",
+        "GITHUB_SHA": "mainCommitSha",
+        "GITHUB_RUN_ID": "12345",
+        "GITHUB_RUN_NUMBER": "1",
+        "GITHUB_WORKFLOW": "CI",
+        "GITHUB_WORKSPACE": "/workspace",
+    }
+
+    tags = ci.extract_github_actions(env=env)
+
+    os.remove(event_file_path)
+
+    assert tags[git.BRANCH] == "main"
+    assert tags[git.COMMIT_SHA] == "mainCommitSha"
+    assert tags[git.REPOSITORY_URL] == "https://github.com/DataDog/dd-trace-py.git"
+    assert tags[git.COMMIT_HEAD_SHA] == "headCommitSha"
+    assert tags[ci.JOB_URL] == "https://github.com/DataDog/dd-trace-py/commit/mainCommitSha/checks"
+    assert tags[ci.PIPELINE_ID] == "12345"
+    assert tags[ci.PIPELINE_NAME] == "CI"
+    assert tags[ci.PIPELINE_NUMBER] == "1"
+    assert tags[ci.PIPELINE_URL] == "https://github.com/DataDog/dd-trace-py/actions/runs/12345"
+    assert tags[ci.JOB_NAME] == "test_job"
+    assert tags[ci.PROVIDER_NAME] == "github"
+    assert tags[ci.WORKSPACE_PATH] == "/workspace"
+    assert (
+        tags[ci._CI_ENV_VARS]
+        == '{"GITHUB_SERVER_URL":"https://github.com","GITHUB_REPOSITORY":"DataDog/dd-trace-py","GITHUB_RUN_ID":"12345"}'
+    )
+
+
+def test_extract_git_head_metadata():
+    fake_user_info = {
+        "author": ("Author", "[email protected]", "date1"),
+        "committer": ("Committer", "[email protected]", "date2"),
+    }
+
+    with mock.patch(
+        "ddtrace.ext.git._is_shallow_repository_with_details", return_value=(True, 0.1, 0)
+    ) as mock_is_shallow, mock.patch("ddtrace.ext.git._unshallow_repository") as mock_unshallow_repository, mock.patch(
+        "ddtrace.ext.git.extract_user_info", return_value=fake_user_info
+    ), mock.patch(
+        "ddtrace.ext.git._git_subprocess_cmd", return_value="commit message"
+    ) as mock_git_subprocess_cmd:
+        tags = git.extract_git_head_metadata("sha123", cwd="/repo")
+
+    mock_is_shallow.assert_called_once()
+    mock_unshallow_repository.assert_called_once()
+    mock_git_subprocess_cmd.assert_called_once_with("log -n 1 --format=%B sha123", "/repo")
+
+    assert tags[git.COMMIT_HEAD_AUTHOR_NAME] == "Author"
+    assert tags[git.COMMIT_HEAD_AUTHOR_EMAIL] == "[email protected]"
+    assert tags[git.COMMIT_HEAD_AUTHOR_DATE] == "date1"
+    assert tags[git.COMMIT_HEAD_COMMITTER_NAME] == "Committer"
+    assert tags[git.COMMIT_HEAD_COMMITTER_EMAIL] == "[email protected]"
+    assert tags[git.COMMIT_HEAD_COMMITTER_DATE] == "date2"
+    assert tags[git.COMMIT_HEAD_MESSAGE] == "commit message"