Add support for installing library dependencies from archive downloads

- Archive downloads are often faster than cloning repositories, especially when the repository contains binaries.
- Some libraries are not available from Git repositories.

The optional source-path field allows installing libraries which are not in the archive's root folder.
The optional destination-name field allows installing libraries to a custom named library folder. This is useful for perfectly emulating the library as it will be installed by Library Manager, which uses the library.properties name field to name the library folder.

Author: per1234

README.md

@@ -46,6 +46,12 @@ Keys:
 - `source-path` - path to install as a library. Paths are relative to the root of the repository. The default is to install from the root of the repository.
 - `destination-name` - folder name to install the library to. By default, the folder will be named according to the source repository or subfolder name.
 
+##### Archive download
+
+Keys:
+- `source-url` - download URL for the archive (e.g., `https://github.com/arduino-libraries/Servo/archive/master.zip`).
+- `source-path` - path to install as a library. Paths are relative to the root folder of the archive. If the archive doesn't have a root folder, use `..` as `source-path` to install from the archive root. The default is to install from the root folder of the archive.
+- `destination-name` - folder name to install the library to. By default, the folder will be named according to the source archive or subfolder name.
 
 ### `sketch-paths`
 

compilesketches/compilesketches.py

@@ -238,6 +238,9 @@ def sort_dependency_list(self, dependency_list):
                         or dependency[self.dependency_source_url_key].startswith("git://")
                     ):
                         sorted_dependencies.repository.append(dependency)
+                    else:
+                        # All other URLs are assumed to be downloads
+                        sorted_dependencies.download.append(dependency)
                 elif self.dependency_source_path_key in dependency:
                     # Libraries with source-path and no source-url are assumed to be paths
                     sorted_dependencies.path.append(dependency)
@@ -254,6 +257,7 @@ def __init__(self):
             self.manager = []
             self.path = []
             self.repository = []
+            self.download = []
 
     def install_platforms_from_board_manager(self, platform_list, additional_url_list):
         """Install platform dependencies from the Arduino Board Manager
@@ -460,6 +464,9 @@ def install_libraries(self):
         if len(library_list.repository) > 0:
             self.install_libraries_from_repository(library_list=library_list.repository)
 
+        if len(library_list.download) > 0:
+            self.install_libraries_from_download(library_list=library_list.download)
+
     def install_libraries_from_library_manager(self, library_list):
         """Install libraries using the Arduino Library Manager
 
@@ -532,6 +539,29 @@ def install_libraries_from_repository(self, library_list):
                                          destination_parent_path=self.libraries_path,
                                          destination_name=destination_name)
 
+    def install_libraries_from_download(self, library_list):
+        """Install libraries by downloading them
+
+        Keyword arguments:
+        library_list -- list of dictionaries defining the dependencies
+        """
+        for library in library_list:
+            self.verbose_print("Installing library from download URL:", library[self.dependency_source_url_key])
+            if self.dependency_source_path_key in library:
+                source_path = library[self.dependency_source_path_key]
+            else:
+                source_path = "."
+
+            if self.dependency_destination_name_key in library:
+                destination_name = library[self.dependency_destination_name_key]
+            else:
+                destination_name = None
+
+            install_from_download(url=library[self.dependency_source_url_key],
+                                  source_path=source_path,
+                                  destination_parent_path=self.libraries_path,
+                                  destination_name=destination_name)
+
     def find_sketches(self):
         """Return a list of all sketches under the paths specified in the sketch paths list recursively."""
         sketch_list = []

compilesketches/tests/test_compilesketches.py

@@ -333,8 +333,14 @@ def test_install_platforms(mocker, fqbn_arg, expected_platform, expected_additio
          [{compilesketches.CompileSketches.dependency_source_url_key: "https://example.com/foo/bar.git/"}],
          ["repository"]),
      ([{compilesketches.CompileSketches.dependency_source_url_key: "git://example.com/foo/bar"}], ["repository"]),
+     ([{compilesketches.CompileSketches.dependency_source_url_key: "https://example.com/foo/bar"}], ["download"]),
      ([{compilesketches.CompileSketches.dependency_source_path_key: "foo/bar"}], ["path"]),
      ([{compilesketches.CompileSketches.dependency_name_key: "FooBar"}], ["manager"]),
+     ([{compilesketches.CompileSketches.dependency_source_url_key: "git://example.com/foo/bar"},
+       {compilesketches.CompileSketches.dependency_source_url_key: "https://example.com/foo/bar"},
+       {compilesketches.CompileSketches.dependency_source_path_key: "foo/bar"},
+       {compilesketches.CompileSketches.dependency_name_key: "FooBar"}],
+      ["repository", "download", "path", "manager"]),
      ([{compilesketches.CompileSketches.dependency_source_url_key: "git://example.com/foo/bar"}], ["repository"]),
      ]
 )
@@ -494,24 +500,27 @@ def test_get_repository_dependency_ref(dependency, expected_ref):
 
 
 @pytest.mark.parametrize(
-    "libraries, expected_manager, expected_path, expected_repository",
+    "libraries, expected_manager, expected_path, expected_repository, expected_download",
     [("", [], [{compilesketches.CompileSketches.dependency_source_path_key: pathlib.PurePath("/foo/GitHubWorkspace")}],
-      []),
+      [], []),
      ("foo bar", [{compilesketches.CompileSketches.dependency_name_key: "foo"},
                   {compilesketches.CompileSketches.dependency_name_key: "bar"}],
-      [{compilesketches.CompileSketches.dependency_source_path_key: pathlib.PurePath("/foo/GitHubWorkspace")}], []),
+      [{compilesketches.CompileSketches.dependency_source_path_key: pathlib.PurePath("/foo/GitHubWorkspace")}], [], []),
      ("\"foo\" \"bar\"", [{compilesketches.CompileSketches.dependency_name_key: "foo"},
                           {compilesketches.CompileSketches.dependency_name_key: "bar"}],
-      [{compilesketches.CompileSketches.dependency_source_path_key: pathlib.PurePath("/foo/GitHubWorkspace")}], []),
-     ("-", [], [], []),
+      [{compilesketches.CompileSketches.dependency_source_path_key: pathlib.PurePath("/foo/GitHubWorkspace")}], [], []),
+     ("-", [], [], [], []),
      ("- " + compilesketches.CompileSketches.dependency_name_key + ": foo",
-      [{compilesketches.CompileSketches.dependency_name_key: "foo"}], [], []),
+      [{compilesketches.CompileSketches.dependency_name_key: "foo"}], [], [], []),
      ("- " + compilesketches.CompileSketches.dependency_source_path_key + ": /foo/bar", [],
-      [{compilesketches.CompileSketches.dependency_source_path_key: "/foo/bar"}], []),
+      [{compilesketches.CompileSketches.dependency_source_path_key: "/foo/bar"}], [], []),
      ("- " + compilesketches.CompileSketches.dependency_source_url_key + ": https://example.com/foo.git", [], [],
-      [{"source-url": "https://example.com/foo.git"}])]
+      [{"source-url": "https://example.com/foo.git"}], []),
+     ("- " + compilesketches.CompileSketches.dependency_source_url_key + ": https://example.com/foo.zip", [], [], [],
+      [{"source-url": "https://example.com/foo.zip"}])]
 )
-def test_install_libraries(monkeypatch, mocker, libraries, expected_manager, expected_path, expected_repository):
+def test_install_libraries(monkeypatch, mocker, libraries, expected_manager, expected_path, expected_repository,
+                           expected_download):
     libraries_path = pathlib.Path("/foo/LibrariesPath")
 
     monkeypatch.setenv("GITHUB_WORKSPACE", "/foo/GitHubWorkspace")
@@ -523,6 +532,7 @@ def test_install_libraries(monkeypatch, mocker, libraries, expected_manager, exp
     mocker.patch("compilesketches.CompileSketches.install_libraries_from_library_manager", autospec=True)
     mocker.patch("compilesketches.CompileSketches.install_libraries_from_path", autospec=True)
     mocker.patch("compilesketches.CompileSketches.install_libraries_from_repository", autospec=True)
+    mocker.patch("compilesketches.CompileSketches.install_libraries_from_download", autospec=True)
 
     compile_sketches.install_libraries()
 
@@ -550,6 +560,13 @@ def test_install_libraries(monkeypatch, mocker, libraries, expected_manager, exp
     else:
         compile_sketches.install_libraries_from_repository.assert_not_called()
 
+    if len(expected_download) > 0:
+        compile_sketches.install_libraries_from_download.assert_called_once_with(
+            compile_sketches,
+            library_list=expected_download)
+    else:
+        compile_sketches.install_libraries_from_download.assert_not_called()
+
 
 def test_install_libraries_from_library_manager(mocker):
     run_command_output_level = unittest.mock.sentinel.run_command_output_level
@@ -658,6 +675,35 @@ def test_install_libraries_from_repository(mocker):
     compile_sketches.install_from_repository.assert_has_calls(calls=install_from_repository_calls)
 
 
+def test_install_libraries_from_download(mocker):
+    library_list = [
+        {compilesketches.CompileSketches.dependency_source_url_key: unittest.mock.sentinel.source_url1,
+         compilesketches.CompileSketches.dependency_source_path_key: unittest.mock.sentinel.source_path,
+         compilesketches.CompileSketches.dependency_destination_name_key: unittest.mock.sentinel.destination_name},
+        {compilesketches.CompileSketches.dependency_source_url_key: unittest.mock.sentinel.source_url2}
+    ]
+
+    expected_source_path_list = [unittest.mock.sentinel.source_path, "."]
+    expected_destination_name_list = [unittest.mock.sentinel.destination_name, None]
+
+    compile_sketches = get_compilesketches_object()
+
+    mocker.patch("compilesketches.install_from_download", autospec=True)
+
+    compile_sketches.install_libraries_from_download(library_list=library_list)
+
+    install_libraries_from_download_calls = []
+    for library, expected_source_path, expected_destination_name in zip(library_list, expected_source_path_list,
+                                                                        expected_destination_name_list):
+        install_libraries_from_download_calls.append(
+            unittest.mock.call(url=library[compilesketches.CompileSketches.dependency_source_url_key],
+                               source_path=expected_source_path,
+                               destination_parent_path=compilesketches.CompileSketches.libraries_path,
+                               destination_name=expected_destination_name)
+        )
+    compilesketches.install_from_download.assert_has_calls(calls=install_libraries_from_download_calls)
+
+
 def test_find_sketches(capsys, monkeypatch):
     nonexistent_sketch_path = "/foo/NonexistentSketch"