Add support for installing library dependencies from repositories

Previously, library dependencies could only be installed via Library Manager.

- It is often desirable to test against non-release versions of dependencies.
- Releases take some time to be picked up by the Library Manager indexer and propagated through the CDN cache to become available for installation via LM.
- Some libraries are not available from Library Manager.

The optional source-path field allows installing libraries from a subfolder of the repository.
The optional destination-name field allows installing to a custom library name. This is useful to perfectly emulate the Library Manager installation of the library (which names the folder according to the library.properties name field).

Author: per1234

README.md

@@ -38,6 +38,14 @@ Keys:
 - `source-path` - path to install as a library. Relative paths are assumed to be relative to the root of the repository.
 - `destination-name` - folder name to install the library to. By default, the folder will be named according to the source repository or subfolder name.
 
+##### Repository
+
+Keys:
+- `source-url` - URL to clone the repository from. It must start with `git://` or end with `.git`.
+- `version` - [Git ref](https://git-scm.com/book/en/v2/Git-Internals-Git-References) of the repository to checkout. The special version name `latest` will cause the latest tag to be used. By default, the repository will be checked out to the tip of the default branch.
+- `source-path` - path to install as a library. Paths are relative to the root of the repository. The default is to install from the root of the repository.
+- `destination-name` - folder name to install the library to. By default, the folder will be named according to the source repository or subfolder name.
+
 
 ### `sketch-paths`
 

compilesketches/compilesketches.py

@@ -5,6 +5,7 @@
 import pathlib
 import re
 import shlex
+import shutil
 import subprocess
 import sys
 import tarfile
@@ -13,6 +14,7 @@
 import urllib.request
 
 import git
+import gitdb.exc
 import github
 import yaml
 import yaml.parser
@@ -89,6 +91,7 @@ class RunCommandOutput(enum.Enum):
     dependency_name_key = "name"
     dependency_version_key = "version"
     dependency_source_path_key = "source-path"
+    dependency_source_url_key = "source-url"
     dependency_destination_name_key = "destination-name"
 
     latest_release_indicator = "latest"
@@ -243,7 +246,14 @@ def sort_dependency_list(self, dependency_list):
         sorted_dependencies = self.Dependencies()
         for dependency in dependency_list:
             if dependency is not None:
-                if self.dependency_source_path_key in dependency:
+                if self.dependency_source_url_key in dependency:
+                    # Repositories are identified by the URL starting with git:// or ending in .git
+                    if (
+                        dependency[self.dependency_source_url_key].rstrip("/").endswith(".git")
+                        or dependency[self.dependency_source_url_key].startswith("git://")
+                    ):
+                        sorted_dependencies.repository.append(dependency)
+                elif self.dependency_source_path_key in dependency:
                     # Libraries with source-path and no source-url are assumed to be paths
                     sorted_dependencies.path.append(dependency)
                 else:
@@ -258,6 +268,7 @@ class Dependencies:
         def __init__(self):
             self.manager = []
             self.path = []
+            self.repository = []
 
     def install_platforms_from_board_manager(self, platform_list, additional_url_list):
         """Install platform dependencies from the Arduino Board Manager
@@ -360,6 +371,77 @@ def run_command(self, command, enable_output=RunCommandOutput.ON_FAILURE, exit_o
 
         return command_data
 
+    def get_repository_dependency_ref(self, dependency):
+        """Return the appropriate git ref value for a repository dependency
+
+        Keyword arguments:
+        dependency -- dictionary defining the repository dependency
+        """
+        if self.dependency_version_key in dependency:
+            git_ref = dependency[self.dependency_version_key]
+        else:
+            git_ref = None
+
+        return git_ref
+
+    def install_from_repository(self, url, git_ref, source_path, destination_parent_path, destination_name=None):
+        """Install by cloning a repository
+
+        Keyword arguments:
+        url -- URL to download the archive from
+        git_ref -- the Git ref (e.g., branch, tag, commit) to checkout after cloning
+        source_path -- path relative to the root of the repository to install from
+        destination_parent_path -- path under which to install
+        destination_name -- folder name to use for the installation. Set to None to use the repository name.
+                            (default None)
+        """
+        if destination_name is None:
+            if source_path.rstrip("/") == ".":
+                # Use the repository name
+                destination_name = url.rstrip("/").rsplit(sep="/", maxsplit=1)[1].rsplit(sep=".", maxsplit=1)[0]
+            else:
+                # Use the source path folder name
+                destination_name = pathlib.PurePath(source_path).name
+
+        if source_path.rstrip("/") == ".":
+            # Clone directly to the target path
+            self.clone_repository(url=url, git_ref=git_ref,
+                                  destination_path=pathlib.PurePath(destination_parent_path, destination_name))
+        else:
+            # Clone to a temporary folder
+            with tempfile.TemporaryDirectory() as clone_folder:
+                self.clone_repository(url=url, git_ref=git_ref, destination_path=clone_folder)
+                # Install by moving the source folder
+                shutil.move(src=str(pathlib.PurePath(clone_folder, source_path)),
+                            dst=str(pathlib.PurePath(destination_parent_path, destination_name)))
+
+    def clone_repository(self, url, git_ref, destination_path):
+        """Clone a Git repository to a specified location and check out the specified ref
+
+        Keyword arguments:
+        git_ref -- Git ref to check out. Set to None to leave repository checked out at the tip of the default branch.
+        destination_path -- destination for the cloned repository. This is the full path of the repository, not the
+                            parent path.
+        """
+        if git_ref is None:
+            # Shallow clone is only possible if using the tip of the branch
+            clone_arguments = {"depth": 1}
+        else:
+            clone_arguments = {}
+        cloned_repository = git.Repo.clone_from(url=url, to_path=destination_path, **clone_arguments)
+        if git_ref is not None:
+            if git_ref == self.latest_release_indicator:
+                # "latest" may be used in place of a ref to cause a checkout of the latest tag
+                try:
+                    # Check if there is a real ref named "latest", in which case it will be used
+                    cloned_repository.rev_parse(git_ref)
+                except gitdb.exc.BadName:
+                    # There is no real ref named "latest", so checkout latest (associated with most recent commit) tag
+                    git_ref = sorted(cloned_repository.tags, key=lambda tag: tag.commit.committed_date)[-1]
+
+            # checkout ref
+            cloned_repository.git.checkout(git_ref)
+
     def install_libraries(self):
         """Install Arduino libraries."""
         self.libraries_path.mkdir(parents=True, exist_ok=True)
@@ -390,6 +472,9 @@ def install_libraries(self):
         if len(library_list.path) > 0:
             self.install_libraries_from_path(library_list=library_list.path)
 
+        if len(library_list.repository) > 0:
+            self.install_libraries_from_repository(library_list=library_list.repository)
+
     def install_libraries_from_library_manager(self, library_list):
         """Install libraries using the Arduino Library Manager
 
@@ -432,6 +517,36 @@ def install_libraries_from_path(self, library_list):
             library_symlink_path = self.libraries_path.joinpath(destination_name)
             library_symlink_path.symlink_to(target=source_path, target_is_directory=True)
 
+    def install_libraries_from_repository(self, library_list):
+        """Install libraries by cloning Git repositories
+
+        Keyword arguments:
+        library_list -- list of dictionaries defining the dependencies
+        """
+        for library in library_list:
+            self.verbose_print("Installing library from repository:", library[self.dependency_source_url_key])
+
+            # Determine library folder name (important because it is a factor in dependency resolution)
+            if self.dependency_destination_name_key in library:
+                # If a folder name was specified, use it
+                destination_name = library[self.dependency_destination_name_key]
+            else:
+                # None will cause the repository name to be used by install_from_repository()
+                destination_name = None
+
+            git_ref = self.get_repository_dependency_ref(dependency=library)
+
+            if self.dependency_source_path_key in library:
+                source_path = library[self.dependency_source_path_key]
+            else:
+                source_path = "."
+
+            self.install_from_repository(url=library[self.dependency_source_url_key],
+                                         git_ref=git_ref,
+                                         source_path=source_path,
+                                         destination_parent_path=self.libraries_path,
+                                         destination_name=destination_name)
+
     def find_sketches(self):
         """Return a list of all sketches under the paths specified in the sketch paths list recursively."""
         sketch_list = []