ci: use mocks for GitHub calls in needs_testrun.py tests (#11692)

Avoid hitting rate limits when testing this script, especially when the
data/responses from GitHub API should be static anyways.

Avoids:
https://gitlab.ddbuild.io/DataDog/apm-reliability/dd-trace-py/-/jobs/736522083

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: brettlangdon

.riot/requirements/151a249.txt

@@ -0,0 +1,20 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/151a249.in
+#
+attrs==24.2.0
+coverage[toml]==7.6.9
+hypothesis==6.45.0
+iniconfig==2.0.0
+lxml==5.3.0
+mock==5.1.0
+opentracing==2.4.0
+packaging==24.2
+pluggy==1.5.0
+pytest==8.3.4
+pytest-cov==6.0.0
+pytest-mock==3.14.0
+ruamel-yaml==0.18.6
+sortedcontainers==2.4.0

.riot/requirements/196755b.txt

@@ -0,0 +1,20 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/4d1fa34.in
+#
+attrs==24.2.0
+coverage[toml]==7.6.9
+hypothesis==6.45.0
+iniconfig==2.0.0
+lxml==5.3.0
+mock==5.1.0
+opentracing==2.4.0
+packaging==24.2
+pluggy==1.5.0
+pytest==8.3.4
+pytest-cov==6.0.0
+pytest-mock==3.14.0
+ruamel-yaml==0.18.6
+sortedcontainers==2.4.0

.riot/requirements/4d1fa34.txt

@@ -133,8 +133,10 @@ _ = [
 detached = true
 python = "3.10"
 extra-dependencies = [
+    "lxml==5.3.0",
     "packaging==23.1",
     "ruamel.yaml==0.18.6",
+    "vcrpy==6.0.2",
 ]
 
 [envs.scripts.scripts]

.riot/requirements/b2ac981.txt

@@ -116,6 +116,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
             pys=["3"],
             pkgs={
                 "ruamel.yaml": latest,
+                "lxml": latest,
             },
         ),
         Venv(
@@ -124,6 +125,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
             pys=["3"],
             pkgs={
                 "ruamel.yaml": latest,
+                "lxml": latest,
             },
         ),
         Venv(

hatch.toml

@@ -8,14 +8,15 @@
 import logging
 import os
 from pathlib import Path
-import re
 from subprocess import check_output
 import sys
 import typing as t
 from urllib.parse import urlencode
 from urllib.request import Request
 from urllib.request import urlopen
 
+from lxml import html
+
 
 sys.path.insert(0, str(Path(__file__).parents[1]))
 
@@ -26,20 +27,34 @@
 
 LOGGER = logging.getLogger(__name__)
 
-BASE_BRANCH_PATTERN = re.compile(r':<span class="css-truncate-target">([^<]+)')
-
 
 @cache
 def get_base_branch(pr_number: int) -> str:
     """Get the base branch of a PR
 
-    >>> get_base_branch(6412)
+    >>> import vcr
+    >>> with vcr.use_cassette(
+    ...   "scripts/vcr/needs_testrun.yaml",
+    ...   filter_headers=["authorization", "user-agent"],
+    ...   record_mode="all"):
+    ...     get_base_branch(6412)
+    ...     get_base_branch(11534)
+    ...     get_base_branch(11690)
     '1.x'
+    '2.15'
+    'main'
     """
 
     pr_page_content = urlopen(f"https://github.com/DataDog/dd-trace-py/pull/{pr_number}").read().decode("utf-8")
 
-    return BASE_BRANCH_PATTERN.search(pr_page_content).group(1)
+    tree = html.fromstring(pr_page_content)
+    base_ref = tree.find_class("base-ref")
+    if base_ref:
+        ref = base_ref[0].text_content().strip()
+        # We might have `DataDog:1.x` or `DataDog:main` so we need to strip the prefix
+        _, _, ref = ref.rpartition(":")
+        return ref.strip()
+    return "main"
 
 
 @cache
@@ -116,7 +131,12 @@ def get_changed_files(pr_number: int, sha: t.Optional[str] = None) -> t.Set[str]
     or if there is a specific SHA given, use the less accurate method of
     diffing against a base commit, either the given SHA or the merge-base.
 
-    >>> sorted(get_changed_files(6388))  # doctest: +NORMALIZE_WHITESPACE
+    >>> import vcr
+    >>> with vcr.use_cassette(
+    ...   "scripts/vcr/needs_testrun.yaml",
+    ...   filter_headers=["authorization", "user-agent"],
+    ...   record_mode="all"):
+    ...     sorted(get_changed_files(6388))  # doctest: +NORMALIZE_WHITESPACE
     ['ddtrace/debugging/_expressions.py',
     'releasenotes/notes/fix-debugger-expressions-none-literal-30f3328d2e386f40.yaml',
     'tests/debugging/test_expressions.py']
@@ -141,12 +161,19 @@ def get_changed_files(pr_number: int, sha: t.Optional[str] = None) -> t.Set[str]
 def needs_testrun(suite: str, pr_number: int, sha: t.Optional[str] = None) -> bool:
     """Check if a testrun is needed for a suite and PR
 
-    >>> needs_testrun("debugger", 6485)
+    >>> import vcr
+    >>> with vcr.use_cassette(
+    ...   "scripts/vcr/needs_testrun.yaml",
+    ...   filter_headers=["authorization", "user-agent"],
+    ...   record_mode="all"):
+    ...     needs_testrun("debugger", 6485)
+    ...     needs_testrun("debugger", 6388)
+    ...     needs_testrun("foobar", 6412)
+    ...     needs_testrun("profile", 11690)
     True
-    >>> needs_testrun("debugger", 6388)
     True
-    >>> needs_testrun("foobar", 6412)
     True
+    False
     """
     if "itr:noskip" in get_latest_commit_message().lower():
         return True