Build: support cloning private repos with token

readthedocs/api/v2/serializers.py

@@ -93,6 +93,7 @@ class Meta(ProjectSerializer.Meta):
             "environment_variables",
             "max_concurrent_builds",
             "readthedocs_yaml_path",
+            "clone_token",
         )
 
 

readthedocs/doc_builder/director.py

@@ -671,6 +671,7 @@ def get_vcs_env_vars(self):
         env = self.get_rtd_env_vars()
         # Don't prompt for username, this requires Git 2.3+
         env["GIT_TERMINAL_PROMPT"] = "0"
+        env["READTHEDOCS_GIT_CLONE_TOKEN"] = self.data.project.clone_token
         return env
 
     def get_rtd_env_vars(self):

readthedocs/doc_builder/environments.py

@@ -391,6 +391,7 @@ def _escape_command(self, cmd):
         not_escape_variables = (
             "READTHEDOCS_OUTPUT",
             "READTHEDOCS_VIRTUALENV_PATH",
+            "READTHEDOCS_GIT_CLONE_TOKEN",
             "CONDA_ENVS_PATH",
             "CONDA_DEFAULT_ENV",
         )

readthedocs/oauth/services/base.py

@@ -106,6 +106,10 @@ def send_build_status(self, *, build, commit, status):
         """
         raise NotImplementedError
 
+    def attach_project(self, project):
+        # TODO: Rename this and move it to the base class
+        raise NotImplementedError
+
     def get_clone_token(self, project):
         """Get a token used for cloning the repository."""
         raise NotImplementedError

readthedocs/oauth/services/githubapp.py

@@ -483,8 +483,8 @@ def get_clone_token(self, project):
 
     def setup_webhook(self, project, integration=None):
         """When using a GitHub App, we don't need to set up a webhook."""
-        return True
+        return True, None
 
     def update_webhook(self, project, integration=None):
         """When using a GitHub App, we don't need to set up a webhook."""
-        return True
+        return True, None

readthedocs/projects/models.py

@@ -980,6 +980,7 @@ def vcs_repo(self, environment, version):
                 self,
                 version=version,
                 environment=environment,
+                use_token=bool(self.clone_token),
             )
         return repo
 
@@ -1410,6 +1411,25 @@ def get_subproject_candidates(self, user):
     def organization(self):
         return self.organizations.first()
 
+    @property
+    def clone_token(self) -> str | None:
+        """
+        Return a HTTP-based Git access token to the repository.
+
+        .. note::
+
+           - A token is only returned for projects linked to a private repository.
+           - Only repositories granted acces by a GitHub app installation will return a token.
+        """
+        service_class = self.get_git_service_class()
+        if not service_class or not self.remote_repository.private:
+            return None
+        for service in service_class.for_project(self):
+            token = service.get_clone_token(self)
+            if token:
+                return token
+        return None
+
 
 class APIProject(Project):
     """
@@ -1426,12 +1446,17 @@ class APIProject(Project):
     """
 
     features = []
+    # This is a property in the original model, in order to
+    # be able to assign it a value in the constructor, we need to re-declare it
+    # as an attribute here.
+    clone_token = None
 
     class Meta:
         proxy = True
 
     def __init__(self, *args, **kwargs):
         self.features = kwargs.pop("features", [])
+        self.clone_token = kwargs.pop("clone_token", None)
         environment_variables = kwargs.pop("environment_variables", {})
         ad_free = not kwargs.pop("show_advertising", True)
         # These fields only exist on the API return, not on the model, so we'll

readthedocs/projects/tasks/builds.py

@@ -204,6 +204,7 @@ def execute(self):
             version=self.data.version,
             environment={
                 "GIT_TERMINAL_PROMPT": "0",
+                "READTHEDOCS_GIT_CLONE_TOKEN": self.data.project.clone_token,
             },
             # Pass the api_client so that all environments have it.
             # This is needed for ``readthedocs-corporate``.

readthedocs/vcs_support/backends/git.py

@@ -1,7 +1,7 @@
 """Git-related utilities."""
 
-import re
 from typing import Iterable
+from urllib.parse import urlparse
 
 import structlog
 from django.conf import settings
@@ -27,22 +27,15 @@ class Backend(BaseVCS):
     fallback_branch = "master"  # default branch
     repo_depth = 50
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, use_token=False, **kwargs):
         super().__init__(*args, **kwargs)
-        self.token = kwargs.get("token")
+        self.use_token = use_token
         self.repo_url = self._get_clone_url()
 
     def _get_clone_url(self):
-        if "://" in self.repo_url:
-            hacked_url = self.repo_url.split("://")[1]
-            hacked_url = re.sub(".git$", "", hacked_url)
-            clone_url = "https://%s" % hacked_url
-            if self.token:
-                clone_url = "https://{}@{}".format(self.token, hacked_url)
-                return clone_url
-            # Don't edit URL because all hosts aren't the same
-            # else:
-            #     clone_url = 'git://%s' % (hacked_url)
+        if self.repo_url.startswith(("https://", "http://")) and self.use_token:
+            parsed_url = urlparse(self.repo_url)
+            return f"{parsed_url.scheme}://$READTHEDOCS_GIT_CLONE_TOKEN@{parsed_url.netloc}{parsed_url.path}"
         return self.repo_url
 
     def update(self):