fix(ci_visibility): git metadata fallback commands [backport 1.20] (#7193)

Backport 345b85a from #7178 to 1.20.

This adds some fallback commands to the git metadata gathering done by
the CI Visibility git client to handle:
* cases where there is a commit present in the branch that is not in the
remote
* cases where a branch is not tracking an upstream (eg: detached HEAD
states)

Minor extra changes to propagate log level from the parent process to
the subprocess created for the client startup operation.

No release note due to ITR still being in beta for Python/pytest.

## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
- [x] If this PR touches code that signs or publishes builds or
packages, or handles credentials of any kind, I've requested a review
from `@DataDog/security-design-and-guidance`.
- [x] This PR doesn't touch any of that.

Co-authored-by: Romain Komorn <[email protected]>

Author: github-actions[bot]

ddtrace/ext/git.py

@@ -101,7 +101,14 @@ def _set_safe_directory():
 
 
 def _extract_clone_defaultremotename(cwd=None):
-    output = _git_subprocess_cmd("config --default origin --get clone.defaultRemoteName")
+    # type: (Optional[str]) -> str
+    output = _git_subprocess_cmd("config --default origin --get clone.defaultRemoteName", cwd=cwd)
+    return output
+
+
+def _extract_upstream_sha(cwd=None):
+    # type: (Optional[str]) -> str
+    output = _git_subprocess_cmd("rev-parse @{upstream}", cwd=cwd)
     return output
 
 
@@ -111,22 +118,22 @@ def _is_shallow_repository(cwd=None):
     return output.strip() == "true"
 
 
-def _unshallow_repository(cwd=None):
-    # type (Optional[str]) -> None
-    remote_name = _extract_clone_defaultremotename(cwd)
-    head_sha = extract_commit_sha(cwd)
+def _unshallow_repository(cwd=None, repo=None, refspec=None):
+    # type (Optional[str], Optional[str], Optional[str]) -> None
 
     cmd = [
         "fetch",
+        '--shallow-since="1 month ago"',
         "--update-shallow",
         "--filter=blob:none",
         "--recurse-submodules=no",
-        '--shallow-since="1 month ago"',
-        remote_name,
-        head_sha,
     ]
+    if repo is not None:
+        cmd.append(repo)
+    if refspec is not None:
+        cmd.append(refspec)
 
-    _git_subprocess_cmd(cmd, cwd=cwd)
+    return _git_subprocess_cmd(cmd, cwd=cwd)
 
 
 def extract_user_info(cwd=None):

ddtrace/internal/ci_visibility/git_client.py

@@ -7,6 +7,8 @@
 from typing import Tuple  # noqa
 
 from ddtrace.ext import ci
+from ddtrace.ext.git import _extract_clone_defaultremotename
+from ddtrace.ext.git import _extract_upstream_sha
 from ddtrace.ext.git import _get_rev_list
 from ddtrace.ext.git import _is_shallow_repository
 from ddtrace.ext.git import _unshallow_repository
@@ -69,7 +71,7 @@ def start(self, cwd=None):
             self._worker = Process(
                 target=CIVisibilityGitClient._run_protocol,
                 args=(self._serializer, self._requests_mode, self._base_url, self._tags, self._response),
-                kwargs={"cwd": cwd},
+                kwargs={"cwd": cwd, "log_level": log.level},
             )
             self._worker.start()
 
@@ -88,8 +90,10 @@ def _run_protocol(
         _tags=None,  # Optional[Dict[str, str]]
         _response=None,  # Optional[Response]
         cwd=None,  # Optional[str]
+        log_level=0,  # int
     ):
         # type: (...) -> None
+        log.setLevel(log_level)
         if _tags is None:
             _tags = {}
         repo_url = cls._get_repository_url(tags=_tags, cwd=cwd)
@@ -98,7 +102,6 @@ def _run_protocol(
             log.debug("Shallow repository detected on git > 2.27 detected, unshallowing")
             try:
                 cls._unshallow_repository(cwd=cwd)
-                log.debug("Unshallowing done")
             except ValueError:
                 log.warning("Failed to unshallow repository, continuing to send pack data", exc_info=True)
 
@@ -163,7 +166,7 @@ def _do_request(cls, requests_mode, base_url, endpoint, payload, serializer, hea
             _headers.update(headers)
         try:
             conn = get_connection(url)
-            log.debug("Sending request: %s %s %s %s", ("POST", url, payload, _headers))
+            log.debug("Sending request: %s %s %s %s", "POST", url, payload, _headers)
             conn.request("POST", url, payload, _headers)
             resp = compat.get_connection_response(conn)
             log.debug("Response status: %s", resp.status)
@@ -209,7 +212,47 @@ def _is_shallow_repository(cls, cwd=None):
     @classmethod
     def _unshallow_repository(cls, cwd=None):
         # type () -> None
-        _unshallow_repository(cwd=cwd)
+        try:
+            remote = _extract_clone_defaultremotename()
+        except ValueError as e:
+            log.debug("Failed to get default remote: %s", e)
+            return
+
+        try:
+            CIVisibilityGitClient._unshallow_repository_to_local_head(remote, cwd=cwd)
+            return
+        except ValueError as e:
+            log.debug("Could not unshallow repository to local head: %s", e)
+
+        try:
+            CIVisibilityGitClient._unshallow_repository_to_upstream(remote, cwd=cwd)
+            return
+        except ValueError as e:
+            log.debug("Could not unshallow to upstream: %s", e)
+
+        log.debug("Unshallowing to default")
+        try:
+            _unshallow_repository(cwd=cwd, repo=remote)
+            log.debug("Unshallowing to default successful")
+            return
+        except ValueError as e:
+            log.debug("Unshallowing failed: %s", e)
+
+    @classmethod
+    def _unshallow_repository_to_local_head(cls, remote, cwd=None):
+        # type (str, Optional[str) -> None
+        head = extract_commit_sha(cwd=cwd)
+        log.debug("Unshallowing to local head %s", head)
+        _unshallow_repository(cwd=cwd, repo=remote, refspec=head)
+        log.debug("Unshallowing to local head successful")
+
+    @classmethod
+    def _unshallow_repository_to_upstream(cls, remote, cwd=None):
+        # type (str, Optional[str) -> None
+        upstream = _extract_upstream_sha(cwd=cwd)
+        log.debug("Unshallowing to upstream %s", upstream)
+        _unshallow_repository(cwd=cwd, repo=remote, refspec=upstream)
+        log.debug("Unshallowing to upstream")
 
 
 class CIVisibilityGitClientSerializerV1(object):

tests/ci_visibility/test_ci_visibility.py

@@ -954,10 +954,88 @@ def test_is_shallow_repository_false():
         mock_is_shallow_repository.assert_called_once_with(cwd="/path/to/repo")
 
 
-def test_unshallow_repository():
-    with mock.patch("ddtrace.internal.ci_visibility.git_client._unshallow_repository") as mock_unshallow_repository:
-        CIVisibilityGitClient._unshallow_repository(cwd="/path/to/repo")
-        mock_unshallow_repository.assert_called_once_with(cwd="/path/to/repo")
+def test_unshallow_repository_local_head():
+    with mock.patch(
+        "ddtrace.internal.ci_visibility.git_client._extract_clone_defaultremotename", return_value="origin"
+    ):
+        with mock.patch("ddtrace.internal.ci_visibility.git_client.extract_commit_sha", return_value="myfakesha"):
+            with mock.patch("ddtrace.ext.git._git_subprocess_cmd") as mock_git_subprocess_command:
+                CIVisibilityGitClient._unshallow_repository(cwd="/path/to/repo")
+                mock_git_subprocess_command.assert_called_once_with(
+                    [
+                        "fetch",
+                        '--shallow-since="1 month ago"',
+                        "--update-shallow",
+                        "--filter=blob:none",
+                        "--recurse-submodules=no",
+                        "origin",
+                        "myfakesha",
+                    ],
+                    cwd="/path/to/repo",
+                )
+
+
+def test_unshallow_repository_upstream():
+    with mock.patch(
+        "ddtrace.internal.ci_visibility.git_client._extract_clone_defaultremotename", return_value="origin"
+    ):
+        with mock.patch(
+            "ddtrace.internal.ci_visibility.git_client.CIVisibilityGitClient._unshallow_repository_to_local_head",
+            side_effect=ValueError,
+        ):
+            with mock.patch(
+                "ddtrace.internal.ci_visibility.git_client._extract_upstream_sha", return_value="myupstreamsha"
+            ):
+                with mock.patch("ddtrace.ext.git._git_subprocess_cmd") as mock_git_subprocess_command:
+                    CIVisibilityGitClient._unshallow_repository(cwd="/path/to/repo")
+                    mock_git_subprocess_command.assert_called_once_with(
+                        [
+                            "fetch",
+                            '--shallow-since="1 month ago"',
+                            "--update-shallow",
+                            "--filter=blob:none",
+                            "--recurse-submodules=no",
+                            "origin",
+                            "myupstreamsha",
+                        ],
+                        cwd="/path/to/repo",
+                    )
+
+
+def test_unshallow_repository_full():
+    with mock.patch(
+        "ddtrace.internal.ci_visibility.git_client._extract_clone_defaultremotename", return_value="origin"
+    ):
+        with mock.patch(
+            "ddtrace.internal.ci_visibility.git_client.CIVisibilityGitClient._unshallow_repository_to_local_head",
+            side_effect=ValueError,
+        ):
+            with mock.patch(
+                "ddtrace.internal.ci_visibility.git_client.CIVisibilityGitClient._unshallow_repository_to_upstream",
+                side_effect=ValueError,
+            ):
+                with mock.patch("ddtrace.ext.git._git_subprocess_cmd") as mock_git_subprocess_command:
+                    CIVisibilityGitClient._unshallow_repository(cwd="/path/to/repo")
+                    mock_git_subprocess_command.assert_called_once_with(
+                        [
+                            "fetch",
+                            '--shallow-since="1 month ago"',
+                            "--update-shallow",
+                            "--filter=blob:none",
+                            "--recurse-submodules=no",
+                            "origin",
+                        ],
+                        cwd="/path/to/repo",
+                    )
+
+
+def test_unshallow_respository_cant_get_remote():
+    with mock.patch(
+        "ddtrace.internal.ci_visibility.git_client._extract_clone_defaultremotename", side_effect=ValueError
+    ):
+        with mock.patch("ddtrace.ext.git._git_subprocess_cmd") as mock_git_subprocess_command:
+            CIVisibilityGitClient._unshallow_repository()
+            mock_git_subprocess_command.assert_not_called()
 
 
 def test_encoder_pack_payload():

tests/tracer/test_ci.py

@@ -275,33 +275,57 @@ def test_is_shallow_repository_false(git_repo):
         mock_git_subprocess.assert_called_once_with("rev-parse --is-shallow-repository", cwd=git_repo)
 
 
-def test_unshallow_repository(git_repo):
-    with mock.patch(
-        "ddtrace.ext.git._extract_clone_defaultremotename", return_value="myremote"
-    ) as mock_defaultremotename:
-        with mock.patch(
-            "ddtrace.ext.git.extract_commit_sha", return_value="mycommitshaaaaaaaaaaaa123"
-        ) as mock_extract_sha:
-            with mock.patch("ddtrace.ext.git._git_subprocess_cmd") as mock_git_subprocess:
-                git._unshallow_repository(cwd=git_repo)
-
-                mock_defaultremotename.assert_called_once_with(git_repo)
-                mock_extract_sha.assert_called_once_with(git_repo)
-                mock_git_subprocess.assert_called_once_with(
-                    [
-                        "fetch",
-                        "--update-shallow",
-                        "--filter=blob:none",
-                        "--recurse-submodules=no",
-                        '--shallow-since="1 month ago"',
-                        "myremote",
-                        "mycommitshaaaaaaaaaaaa123",
-                    ],
-                    cwd=git_repo,
-                )
+def test_unshallow_repository_bare(git_repo):
+    with mock.patch("ddtrace.ext.git._git_subprocess_cmd") as mock_git_subprocess:
+        git._unshallow_repository(cwd=git_repo)
+        mock_git_subprocess.assert_called_once_with(
+            [
+                "fetch",
+                '--shallow-since="1 month ago"',
+                "--update-shallow",
+                "--filter=blob:none",
+                "--recurse-submodules=no",
+            ],
+            cwd=git_repo,
+        )
+
+
+def test_unshallow_repository_bare_repo(git_repo):
+    with mock.patch("ddtrace.ext.git._git_subprocess_cmd") as mock_git_subprocess:
+        git._unshallow_repository(cwd=git_repo, repo="myremote")
+        mock_git_subprocess.assert_called_once_with(
+            [
+                "fetch",
+                '--shallow-since="1 month ago"',
+                "--update-shallow",
+                "--filter=blob:none",
+                "--recurse-submodules=no",
+                "myremote",
+            ],
+            cwd=git_repo,
+        )
+
+
+def test_unshallow_repository_bare_repo_refspec(git_repo):
+    with mock.patch("ddtrace.ext.git._git_subprocess_cmd") as mock_git_subprocess:
+        git._unshallow_repository(cwd=git_repo, repo="myremote", refspec="mycommitshaaaaaaaaaaaa123")
+        mock_git_subprocess.assert_called_once_with(
+            [
+                "fetch",
+                '--shallow-since="1 month ago"',
+                "--update-shallow",
+                "--filter=blob:none",
+                "--recurse-submodules=no",
+                "myremote",
+                "mycommitshaaaaaaaaaaaa123",
+            ],
+            cwd=git_repo,
+        )
 
 
 def test_extract_clone_defaultremotename():
     with mock.patch("ddtrace.ext.git._git_subprocess_cmd", return_value="default_remote_name") as mock_git_subprocess:
         assert git._extract_clone_defaultremotename(cwd=git_repo) == "default_remote_name"
-        mock_git_subprocess.assert_called_once_with("config --default origin --get clone.defaultRemoteName")
+        mock_git_subprocess.assert_called_once_with(
+            "config --default origin --get clone.defaultRemoteName", cwd=git_repo
+        )