Added the git clone command and all associated functionality and tests (#4)

* Added the git clone command and all associated functionality and tests

* Modified the method of retrieval to extract a git archive instead of cloning the repo.

~ Substantial modifications to `git_clone_dockerfile` (now renamed `git_extract_dockerfile`.)
~ Renamed the command to `get-archive`
~ Modified the interface accordingly

* CLI changes

+ Added a deeper description of the build commands in `init_build_parsers`
~ Made the `--archive-url command` required
~ Changed `clone` to `archive` for accuracy.

* Added a better test, a new pytest mark, and added tar as a requirement.

+ New init images now check for the `tar` command and add it if it isn't present.
+ Added a `git` mark for pytests.
+ Added a test that acquires a git archive and tests for files on it.

* `folder` -> `directory`, some documentation changes.

* Updated indentation on git archive acquisition command

---------

Co-authored-by: tyler-g-hudson <[email protected]>

Author: Tyler-g-hudson

docker_cli/_docker_git.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from textwrap import dedent
+
+from ._docker_mamba import micromamba_docker_lines
+from ._url_reader import URLReader
+
+
+def git_extract_dockerfile(
+    base: str,
+    archive_url: str,
+    url_reader: URLReader,
+    directory: str | os.PathLike[str] = Path("repo"),
+) -> str:
+    """
+    Returns a Dockerfile-formatted string with instructions to fetch a Git archive.
+
+    Parameters
+    ----------
+    base : str
+        The base image for this dockerfile.
+    archive_url : str
+        The URL of the Git archive. Must be a `tar.gz` file.
+    url_reader : URLReader
+        The URL reader program to fetch the archive with.
+    directory : path-like, optional
+        The name of the folder to store the repository in. Defaults to "repo".
+
+    Returns
+    -------
+    dockerfile : str
+        The generated Dockerfile.
+    """
+    folder_path_str = os.fspath(directory)
+
+    # Dockerfile preparation:
+    # Prepare the repository file, ensure proper ownership and permissions.
+    dockerfile = (
+        dedent(
+            f"""
+                FROM {base}
+
+                USER root
+
+                RUN mkdir -p {folder_path_str}
+                RUN chown -R $MAMBA_USER_ID:$MAMBA_USER_GID {folder_path_str}
+                RUN chmod -R 755 {folder_path_str}
+            """
+        ).strip()
+        + "\n"
+    )
+
+    # Switch user to 'MAMBA_USER'
+    dockerfile += micromamba_docker_lines() + "\n"
+
+    # Get the command to pull the git archive from the internet.
+    fetch_command = url_reader.generate_read_command(target=archive_url)
+
+    # Get the Git archive, extract it, move workdir to it, and change user back to
+    # default.
+    # The `--strip-components 1` argument to `tar` enables the archive to be unzipped
+    # without appending an additional directory in addition to the `folder_path`.
+    dockerfile += (
+        dedent(
+            f"""
+                RUN {fetch_command} | tar -xvz -C {folder_path_str} --strip-components 1
+
+                WORKDIR {directory}
+                USER $DEFAULT_USER
+            """
+        ).strip()
+        + "\n"
+    )
+
+    # Return the generated dockerfile
+    return dockerfile

docker_cli/_url_reader.py

@@ -63,6 +63,9 @@ def generate_read_command(
         retval = f"wget {target}"
         if output_file is not None:
             retval += f" -O {os.fspath(output_file)}"
+        else:
+            # Suppress program output and redirect output to command line.
+            retval += " -qO- "
         return retval
 
     @property
@@ -80,6 +83,8 @@ def generate_read_command(
         retval = f"curl --ssl {target}"
         if output_file is not None:
             retval += f" -o {os.fspath(output_file)}"
+        else:
+            retval += " -L"
         return retval
 
     @property

docker_cli/_utils.py

@@ -120,6 +120,9 @@ def image_command_check(
         url_program, url_init = _get_reader_install_lines(package_mgr=package_mgr)
         init_lines += url_init
 
+    if not image.has_command("tar"):
+        init_lines += "RUN " + package_mgr.generate_install_command(["tar"])
+
     return package_mgr, url_program, init_lines
 
 
@@ -224,8 +227,7 @@ def _package_manager_check(image: Image) -> PackageManager:
 
 def _url_reader_check(image: Image) -> Optional[URLReader]:
     """
-    Return the URL reader on a given image, and a string to install one if there is
-    none.
+    Return the URL reader on a given image, or None if there is none present.
 
     Parameters
     ----------

docker_cli/cli/build_commands.py

@@ -0,0 +1,66 @@
+import argparse
+from pathlib import Path
+from typing import List
+
+from ..commands import get_archive
+from ._utils import add_tag_argument, help_formatter
+
+
+def init_build_parsers(subparsers: argparse._SubParsersAction) -> None:
+    """
+    Augment an argument parser with build commands.
+
+    The build commands are the group of commands to be completed after the CUDA and
+    conda environments have been installed to the image, with the purpose of acquiring
+    and building the ISCE3 repository and any further repositories.
+    These commands consist of the "get-archive" command, and more are being added.
+
+    Parameters
+    -----------
+    parser : argparse.ArgumentParser
+        The parser to add setup commands to.
+    prefix : str
+        The image tag prefix.
+    """
+
+    archive_params = argparse.ArgumentParser(add_help=False)
+    archive_params.add_argument(
+        "--archive-url",
+        type=str,
+        metavar="GIT_ARCHIVE",
+        required=True,
+        help='The URL of the Git archive to be fetched. Must be a "tar.gz" file.',
+    )
+    archive_params.add_argument(
+        "--directory",
+        type=Path,
+        default=Path("/src"),
+        help="The path to place the contents of the Git archive at on the image.",
+    )
+
+    setup_parse = argparse.ArgumentParser(add_help=False)
+    setup_parse.add_argument(
+        "--base",
+        "-b",
+        type=str,
+        required=True,
+        help="The name of the base Docker image.",
+    )
+
+    archive_parser = subparsers.add_parser(
+        "get-archive",
+        parents=[setup_parse, archive_params],
+        help="Set up the GitHub repository image, in [USER]/[REPO_NAME] format.",
+        formatter_class=help_formatter,
+    )
+    add_tag_argument(parser=archive_parser, default="repo")
+
+
+def build_command_names() -> List[str]:
+    """Returns a list of all build command names."""
+    return ["get-archive"]
+
+
+def run_build(args: argparse.Namespace, command: str) -> None:
+    if command == "get-archive":
+        get_archive(**vars(args))

docker_cli/cli/cli.py

@@ -4,6 +4,7 @@
 
 from .._utils import universal_tag_prefix
 from ._utils import help_formatter
+from .build_commands import build_command_names, init_build_parsers, run_build
 from .setup_commands import init_setup_parsers, run_setup
 from .util_commands import init_util_parsers, run_util
 
@@ -25,6 +26,7 @@ def initialize_parser() -> argparse.ArgumentParser:
     subparsers = parser.add_subparsers(dest="command", required=True)
 
     init_setup_parsers(subparsers, prefix)
+    init_build_parsers(subparsers)
     init_util_parsers(subparsers, prefix)
 
     return parser
@@ -37,5 +39,7 @@ def main(args: Sequence[str] = sys.argv[1:]):
     del args_parsed.command
     if command == "setup":
         run_setup(args_parsed)
+    elif command in build_command_names():
+        run_build(args_parsed, command)
     else:
         run_util(args_parsed, command)

docker_cli/commands.py

@@ -1,11 +1,69 @@
+from __future__ import annotations
+
 import os
 from shlex import split
 from subprocess import DEVNULL, PIPE, run
-from typing import Iterable, List, Union
+from typing import Iterable, List
 
+from ._docker_git import git_extract_dockerfile
 from ._docker_mamba import mamba_lockfile_command
 from ._image import Image
-from ._utils import is_conda_pkg_name, universal_tag_prefix
+from ._url_reader import URLReader
+from ._utils import (
+    image_command_check,
+    is_conda_pkg_name,
+    temp_image,
+    universal_tag_prefix,
+)
+
+
+def get_archive(
+    tag: str,
+    base: str,
+    archive_url: str,
+    directory: os.PathLike[str],
+    url_reader: URLReader | None = None,
+):
+    """
+    Builds a docker image containing the requested Git archive.
+
+    .. note:
+        With this image, the workdir is moved to `directory`.
+
+    Parameters
+    ----------
+    tag : str
+        The image tag.
+    base : str
+        The base image tag.
+    archive_url : str
+        The URL of the Git archive to add to the image. Must be a `tar.gz` file.
+    directory : path-like
+        The path to the folder that the archive will be held at within the image.
+    url_reader : URLReader | None, optional
+        If given, will use the given URL reader to acquire the Git archive. If None,
+        will check the base image and use whichever one it can find. Defaults to None.
+
+    Returns
+    -------
+    Image
+        The generated image.
+    """
+    if url_reader is None:
+        with temp_image(base) as temp_img:
+            _, url_reader, _ = image_command_check(temp_img)
+
+    prefix = universal_tag_prefix()
+    img_tag = tag if tag.startswith(prefix) else f"{prefix}-{tag}"
+
+    dockerfile = git_extract_dockerfile(
+        base=base,
+        directory=directory,
+        archive_url=archive_url,
+        url_reader=url_reader,
+    )
+
+    return Image.build(tag=img_tag, dockerfile_string=dockerfile, no_cache=True)
 
 
 def dropin(tag: str) -> None:
@@ -80,7 +138,7 @@ def remove(
 
 
 def make_lockfile(
-    tag: str, file: Union[str, os.PathLike[str]], env_name: str = "base"
+    tag: str, file: os.PathLike[str] | str, env_name: str = "base"
 ) -> None:
     """
     Makes a lockfile from an image.
@@ -91,8 +149,8 @@ def make_lockfile(
     Parameters
     ----------
     tag : str
-        The tag or ID of the image.
-    file : Union[str, os.PathLike[str]]
+        The tag of the image.
+    file : os.PathLike[str] | str
         The file to be output to.
     env_name: str
         The name of the environment. Defaults to "base".

pytest.ini

@@ -6,3 +6,4 @@ markers =
     dockerfiles: mark a test as a dockerfile test
     mamba: mark a test as a mamba test
     cuda: mark a test as a CUDA test
+    git: mark a test as a Git test

test/test_docker_git.py

@@ -0,0 +1,55 @@
+from pathlib import Path
+from subprocess import PIPE
+from typing import Any, Tuple
+
+from pytest import mark
+
+from docker_cli._docker_git import git_extract_dockerfile
+from docker_cli._image import Image
+from docker_cli._url_reader import URLReader, get_supported_url_readers, get_url_reader
+
+from .utils import generate_tag, rough_dockerfile_validity_check
+
+
+@mark.dockerfiles
+@mark.git
+def test_git_dockerfile():
+    """Performs a rough validity check of the Git dockerfile"""
+    for supported_reader in get_supported_url_readers():
+        url_reader = get_url_reader(supported_reader)
+        dockerfile = git_extract_dockerfile(
+            base="base",
+            archive_url="www.url.com/a.tar.gz",
+            directory="/",
+            url_reader=url_reader,
+        )
+        rough_dockerfile_validity_check(dockerfile=dockerfile)
+
+
+@mark.images
+@mark.git
+def test_docker_git(
+    init_tag: str,
+    init_image: Image,  # type: ignore
+    base_properties: Tuple[Any, URLReader],
+):
+    # This is a basic github archive from octocat's test-repo1 archive.
+    # If this test fails, one thing to do is check that this archive exists.
+    archive = "https://github.com/octocat/test-repo1/archive/refs/tags/1.0.tar.gz"
+    _, url_reader = base_properties
+    img_tag = generate_tag("git-archive")
+
+    dockerfile = git_extract_dockerfile(
+        base=init_tag,
+        archive_url=archive,
+        directory=Path("/src/"),
+        url_reader=url_reader,
+    )
+
+    image = Image.build(tag=img_tag, dockerfile_string=dockerfile, no_cache=True)
+
+    # Test that three files, which should be present at the archive, are present and
+    # in the right location.
+    image.run("test -f /src/2015-04-12-test-post-last-year.md")
+    image.run("test -f /src/2016-02-24-first-post.md")
+    image.run("test -f /src/2016-02-26-sample-post-jekyll.md")