Split download code from install_arduino_cli() to general-purpose function

This code is useful for downloading and installing dependencies in addition to Arduino CLI.

Author: per1234

compilesketches/compilesketches.py

@@ -8,7 +8,6 @@
 import shutil
 import subprocess
 import sys
-import tarfile
 import tempfile
 import urllib
 import urllib.request
@@ -195,26 +194,12 @@ def install_arduino_cli(self):
         arduino_cli_archive_download_url_prefix = "https://downloads.arduino.cc/arduino-cli/"
         arduino_cli_archive_file_name = "arduino-cli_" + self.cli_version + "_Linux_64bit.tar.gz"
 
-        # Create temporary folder
-        with tempfile.TemporaryDirectory(prefix="arduino-cli-" + self.cli_version + "-") as download_folder:
-            arduino_cli_archive_file_path = pathlib.PurePath(download_folder, arduino_cli_archive_file_name)
-
-            # https://stackoverflow.com/a/38358646
-            with open(file=str(arduino_cli_archive_file_path), mode="wb") as out_file:
-                with contextlib.closing(thing=urllib.request.urlopen(
-                    url=arduino_cli_archive_download_url_prefix
-                        + arduino_cli_archive_file_name)
-                ) as file_pointer:
-                    block_size = 1024 * 8
-                    while True:
-                        block = file_pointer.read(block_size)
-                        if not block:
-                            break
-                        out_file.write(block)
-
-            # Extract arduino-cli archive
-            with tarfile.open(name=arduino_cli_archive_file_path) as tar:
-                tar.extractall(path=self.arduino_cli_installation_path)
+        install_from_download(
+            url=arduino_cli_archive_download_url_prefix + arduino_cli_archive_file_name,
+            # The Arduino CLI has no root folder, so just install the arduino-cli executable from the archive root
+            source_path="arduino-cli",
+            destination_parent_path=self.arduino_cli_installation_path
+        )
 
         # Configure the location of the Arduino CLI user directory
         os.environ["ARDUINO_DIRECTORIES_USER"] = str(self.arduino_cli_user_directory_path)
@@ -990,6 +975,83 @@ def list_to_string(list_input):
     return " ".join([str(item) for item in list_input])
 
 
+def install_from_download(url, source_path, destination_parent_path, destination_name=None):
+    """Download an archive, extract, and install.
+
+    Keyword arguments:
+    url -- URL to download the archive from
+    source_path -- path relative to the root folder of the archive to install.
+    destination_parent_path -- path under which to install
+    destination_name -- folder name to use for the installation. Set to None to take the name from source_path.
+                        (default None)
+    """
+    destination_parent_path = pathlib.Path(destination_parent_path)
+
+    # Create temporary folder for the download
+    with tempfile.TemporaryDirectory("-compilesketches-download_folder") as download_folder:
+        download_file_path = pathlib.PurePath(download_folder, url.rsplit(sep="/", maxsplit=1)[1])
+
+        # https://stackoverflow.com/a/38358646
+        with open(file=str(download_file_path), mode="wb") as out_file:
+            with contextlib.closing(thing=urllib.request.urlopen(url=url)) as file_pointer:
+                block_size = 1024 * 8
+                while True:
+                    block = file_pointer.read(block_size)
+                    if not block:
+                        break
+                    out_file.write(block)
+
+        # Create temporary folder for the extraction
+        with tempfile.TemporaryDirectory("-compilesketches-extract_folder") as extract_folder:
+            # Extract archive
+            shutil.unpack_archive(filename=str(download_file_path), extract_dir=extract_folder)
+
+            archive_root_path = get_archive_root_path(extract_folder)
+
+            absolute_source_path = pathlib.Path(archive_root_path, source_path).resolve()
+
+            if not absolute_source_path.exists():
+                print("::error::Archive source path:", source_path, "not found")
+                sys.exit(1)
+
+            if destination_name is None:
+                destination_name = absolute_source_path.name
+
+            # Create the parent path if it doesn't already exist
+            destination_parent_path.mkdir(parents=True, exist_ok=True)
+
+            # Install by moving the source folder
+            shutil.move(src=str(absolute_source_path),
+                        dst=str(destination_parent_path.joinpath(destination_name)))
+
+
+def get_archive_root_path(archive_extract_path):
+    """Return the path of the archive's root folder.
+
+    Keyword arguments:
+    archive_extract_path -- path the archive was extracted to
+    """
+    archive_root_folder_name = archive_extract_path
+    for extract_folder_content in pathlib.Path(archive_extract_path).glob("*"):
+        if extract_folder_content.is_dir():
+            # Path is a folder
+            # Ignore the __MACOSX folder
+            if extract_folder_content.name != "__MACOSX":
+                if archive_root_folder_name == archive_extract_path:
+                    # This is the first folder found
+                    archive_root_folder_name = extract_folder_content
+                else:
+                    # Multiple folders found
+                    archive_root_folder_name = archive_extract_path
+                    break
+        else:
+            # Path is a file
+            archive_root_folder_name = archive_extract_path
+            break
+
+    return archive_root_folder_name
+
+
 # Only execute the following code if the script is run directly, not imported
 if __name__ == "__main__":
     main()

compilesketches/tests/test_compilesketches.py

@@ -7,8 +7,6 @@
 import tarfile
 import tempfile
 import unittest.mock
-import urllib
-import urllib.request
 
 import git
 import github
@@ -286,33 +284,23 @@ def test_compile_sketches(mocker, compilation_success_list, expected_success, do
         )
 
 
-def test_install_arduino_cli(tmpdir, mocker):
+def test_install_arduino_cli(mocker):
     cli_version = "1.2.3"
+    arduino_cli_installation_path = unittest.mock.sentinel.arduino_cli_installation_path
     arduino_cli_user_directory_path = pathlib.PurePath("/foo/arduino_cli_user_directory_path")
-    source_file_path = test_data_path.joinpath("githubevent.json")
-    # Create temporary folder
-    arduino_cli_installation_path = pathlib.PurePath(tmpdir.mkdir("test_install_arduino_cli"))
-    output_archive_path = arduino_cli_installation_path.joinpath("foo_archive.tar.gz")
-
-    # Create an archive file
-    with tarfile.open(name=output_archive_path, mode="w:gz") as tar:
-        tar.add(name=source_file_path, arcname=source_file_path.name)
 
     compile_sketches = get_compilesketches_object(cli_version=cli_version)
     compile_sketches.arduino_cli_installation_path = arduino_cli_installation_path
     compile_sketches.arduino_cli_user_directory_path = arduino_cli_user_directory_path
 
-    # Patch urllib.request.urlopen so that the generated archive file is opened instead of the Arduino CLI download
-    mocker.patch("urllib.request.urlopen",
-                 return_value=urllib.request.urlopen(url="file:///" + str(output_archive_path)))
+    mocker.patch("compilesketches.install_from_download", autospec=True)
 
     compile_sketches.install_arduino_cli()
 
-    urllib.request.urlopen.assert_called_once_with(url="https://downloads.arduino.cc/arduino-cli/arduino-cli_"
-                                                       + cli_version + "_Linux_64bit.tar.gz")
-
-    # Verify that the installation matches the source file
-    assert filecmp.cmp(f1=source_file_path, f2=arduino_cli_installation_path.joinpath(source_file_path.name)) is True
+    compilesketches.install_from_download.assert_called_once_with(
+        url="https://downloads.arduino.cc/arduino-cli/arduino-cli_" + cli_version + "_Linux_64bit.tar.gz",
+        source_path="arduino-cli",
+        destination_parent_path=arduino_cli_installation_path)
 
     assert os.environ["ARDUINO_DIRECTORIES_USER"] == str(arduino_cli_user_directory_path)
 
@@ -1288,6 +1276,62 @@ def test_list_to_string():
     assert compilesketches.list_to_string([42, path]) == "42 " + str(path)
 
 
+@pytest.mark.parametrize("arcname, source_path, destination_name, expected_destination_name, expected_success",
+                         [("FooArcname", ".", None, "FooArcname", True),
+                          ("FooArcname", "./Sketch1", "FooDestinationName", "FooDestinationName", True),
+                          ("FooArcname", "Sketch1", None, "Sketch1", True),
+                          (".", "Sketch1", None, "Sketch1", True),
+                          ("FooArcname", "Nonexistent", None, "", False), ])
+def test_install_from_download(capsys,
+                               tmp_path,
+                               arcname,
+                               source_path,
+                               destination_name,
+                               expected_destination_name,
+                               expected_success):
+    url_source_path = test_data_path.joinpath("HasSketches")
+
+    # Create temporary folder
+    url_path = tmp_path.joinpath("url_path")
+    url_path.mkdir()
+    url_archive_path = url_path.joinpath("foo_archive.tar.gz")
+    url = url_archive_path.as_uri()
+
+    # Create an archive file
+    with tarfile.open(name=url_archive_path, mode="w:gz", format=tarfile.GNU_FORMAT) as tar:
+        tar.add(name=url_source_path, arcname=arcname)
+
+    destination_parent_path = tmp_path.joinpath("destination_parent_path")
+
+    if expected_success:
+        compilesketches.install_from_download(url=url,
+                                              source_path=source_path,
+                                              destination_parent_path=destination_parent_path,
+                                              destination_name=destination_name)
+
+        # Verify that the installation matches the source
+        assert directories_are_same(left_directory=url_source_path.joinpath(source_path),
+                                    right_directory=destination_parent_path.joinpath(expected_destination_name))
+    else:
+        with pytest.raises(expected_exception=SystemExit, match="1"):
+            compilesketches.install_from_download(url=url,
+                                                  source_path=source_path,
+                                                  destination_parent_path=destination_parent_path,
+                                                  destination_name=destination_name)
+        assert capsys.readouterr().out.strip() == ("::error::Archive source path: " + source_path + " not found")
+
+
+@pytest.mark.parametrize("archive_extract_path, expected_archive_root_path",
+                         [(test_data_path.joinpath("test_get_archive_root_folder_name", "has-root"),
+                           test_data_path.joinpath("test_get_archive_root_folder_name", "has-root", "root")),
+                          (test_data_path.joinpath("test_get_archive_root_folder_name", "has-file"),
+                           test_data_path.joinpath("test_get_archive_root_folder_name", "has-file")),
+                          (test_data_path.joinpath("test_get_archive_root_folder_name", "has-folders"),
+                           test_data_path.joinpath("test_get_archive_root_folder_name", "has-folders"))])
+def test_get_archive_root_path(archive_extract_path, expected_archive_root_path):
+    assert compilesketches.get_archive_root_path(archive_extract_path) == expected_archive_root_path
+
+
 @pytest.mark.parametrize("url, source_path, destination_name, expected_destination_name",
                          [("https://example.com/foo/FooRepositoryName.git", ".", None, "FooRepositoryName"),
                           ("https://example.com/foo/FooRepositoryName.git/", "./examples", "FooDestinationName",