diff --git a/test/fixtures_isce3.py b/test/fixtures_isce3.py index dc38c49..05dd061 100644 --- a/test/fixtures_isce3.py +++ b/test/fixtures_isce3.py @@ -123,7 +123,7 @@ def isce3_git_repo_image( dockerfile = git_extract_dockerfile( base=isce3_env_dev_image_tag, archive_url=archive, - directory=Path("/src/"), + dst_path=Path("/src/"), url_reader=url_reader, ) diff --git a/test/test_docker_git.py b/test/test_docker_git.py index 4715c7b..1a66003 100644 --- a/test/test_docker_git.py +++ b/test/test_docker_git.py @@ -20,7 +20,7 @@ def test_git_dockerfile(): dockerfile = git_extract_dockerfile( base="base", archive_url="www.url.com/a.tar.gz", - directory="/", + dst_path="/", url_reader=url_reader, ) rough_dockerfile_validity_check(dockerfile=dockerfile) @@ -44,7 +44,7 @@ def test_docker_git( dockerfile = git_extract_dockerfile( base=init_tag, archive_url=archive, - directory=Path("/src/"), + dst_path=Path("/src/"), url_reader=url_reader, ) diff --git a/wigwam/_docker_cuda.py b/wigwam/_docker_cuda.py index e7d4e21..d5f5b87 100644 --- a/wigwam/_docker_cuda.py +++ b/wigwam/_docker_cuda.py @@ -101,6 +101,7 @@ def generate_runtime_dockerfile( ).strip() + "\n\n" + install_lines + + "\n\n" + "USER $DEFAULT_USER" ) diff --git a/wigwam/_docker_git.py b/wigwam/_docker_git.py index 7a8282c..3950d4c 100644 --- a/wigwam/_docker_git.py +++ b/wigwam/_docker_git.py @@ -12,7 +12,7 @@ def git_extract_dockerfile( base: str, archive_url: str, url_reader: URLReader, - directory: str | os.PathLike[str] = Path("repo"), + dst_path: str | os.PathLike[str] = Path("repo"), ) -> str: """ Returns a Dockerfile-formatted string with instructions to fetch a Git archive. @@ -25,15 +25,15 @@ def git_extract_dockerfile( The URL of the Git archive. Must be a `tar.gz` file. url_reader : URLReader The URL reader program to fetch the archive with. - directory : path-like, optional - The name of the folder to store the repository in. Defaults to "repo". + dst_path : path-like, optional + The prefix of the directory to store the repository in. Defaults to "repo". Returns ------- dockerfile : str The generated Dockerfile. """ - folder_path_str = os.fspath(directory) + folder_path_str = os.fspath(dst_path) # Dockerfile preparation: # Prepare the repository file, ensure proper ownership and permissions. @@ -67,7 +67,7 @@ def git_extract_dockerfile( f""" RUN {fetch_command} | tar -xvz -C {folder_path_str} --strip-components 1 - WORKDIR {directory} + WORKDIR {dst_path} USER $DEFAULT_USER """ ).strip() diff --git a/wigwam/cli/build_commands.py b/wigwam/cli/build_commands.py index 4d18974..9946636 100644 --- a/wigwam/cli/build_commands.py +++ b/wigwam/cli/build_commands.py @@ -3,6 +3,7 @@ from typing import List from ..commands import ( + build_all, cmake_install, compile_cmake, configure_cmake, @@ -10,6 +11,7 @@ get_archive, make_distrib, ) +from ..defaults import universal_tag_prefix from ._utils import add_tag_argument, help_formatter @@ -26,6 +28,7 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: cmake-config, cmake-compile, cmake-install, + build-all, and more are being added. Parameters @@ -45,7 +48,7 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: help='The URL of the Git archive to be fetched. Must be a "tar.gz" file.', ) archive_params.add_argument( - "--directory", + "--dst-path", type=Path, default=Path("/src"), help="The path to place the contents of the Git archive at on the image.", @@ -85,15 +88,15 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: help="Run Docker build with no cache if used.", ) - archive_parser = subparsers.add_parser( + archive_parser: argparse.ArgumentParser = subparsers.add_parser( "get-archive", parents=[setup_params, archive_params, no_cache_params], - help="Set up the GitHub repository image, in [USER]/[REPO_NAME] format.", + help="Set up the GitHub repository image.", formatter_class=help_formatter, ) add_tag_argument(parser=archive_parser, default="repo") - copy_dir_parser = subparsers.add_parser( + copy_dir_parser: argparse.ArgumentParser = subparsers.add_parser( "copydir", parents=[setup_params, no_cache_params], help="Insert the contents of a directory at the given path.", @@ -101,14 +104,14 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: ) add_tag_argument(parser=copy_dir_parser, default="dir-copy") copy_dir_parser.add_argument( - "--directory", + "--src-path", "-d", type=Path, required=True, help="The directory to be copied to the image.", ) copy_dir_parser.add_argument( - "--target-path", + "--dst-path", "-p", type=Path, default=None, @@ -116,7 +119,7 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: "the base name of the path given by the directory argument will be used.", ) - config_parser = subparsers.add_parser( + config_parser: argparse.ArgumentParser = subparsers.add_parser( "cmake-config", parents=[setup_params, config_params, no_cache_params], help="Creates an image with a configured compiler.", @@ -124,7 +127,7 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: ) add_tag_argument(parser=config_parser, default="configured") - compile_parser = subparsers.add_parser( + compile_parser: argparse.ArgumentParser = subparsers.add_parser( "cmake-compile", parents=[setup_params, no_cache_params], help="Creates an image with the project built.", @@ -132,7 +135,7 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: ) add_tag_argument(parser=compile_parser, default="compiled") - install_parser = subparsers.add_parser( + install_parser: argparse.ArgumentParser = subparsers.add_parser( "cmake-install", parents=[setup_params, no_cache_params], help="Creates an image with the project installed.", @@ -140,7 +143,7 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: ) add_tag_argument(parser=install_parser, default="installed") - distrib_parser = subparsers.add_parser( + distrib_parser: argparse.ArgumentParser = subparsers.add_parser( "make-distrib", parents=[no_cache_params], help="Creates a distributable image.", @@ -151,23 +154,72 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None: "-t", default="isce3", type=str, - help="The complete tag of the Docker image to be created. " 'Default: "isce3"', + help="The complete tag of the Docker image to be created.", ) distrib_parser.add_argument( "--base", "-b", default="setup-mamba-runtime", type=str, - help="The complete tag of the Docker image to be created. " - 'Default: "setup-mamba-runtime"', + help="The complete tag of the Docker image to be created.", ) distrib_parser.add_argument( "--source-tag", "-s", default="build-installed", type=str, - help="The tag or ID of the source image which has the project installed. " - ' Defaults to "build-installed".', + help="The tag or ID of the source image which has the project installed.", + ) + + parser_build_all: argparse.ArgumentParser = subparsers.add_parser( + "build-all", + parents=[config_params, no_cache_params], + help="Performs the complete compilation process, from initial GitHub checkout " + "to installation.", + formatter_class=help_formatter, + ) + parser_build_all.add_argument( + "--base", + "-b", + type=str, + default="setup-mamba-dev", + help="The name of the parent Docker image.", + ) + parser_build_all.add_argument( + "--tag", + "-t", + default="build", + type=str, + help="The sub-prefix of the Docker images to be created. Generated images will " + f'have tags fitting "{universal_tag_prefix()}-[TAG]-*".', + ) + parser_build_all.add_argument( + "--dst-path", + type=Path, + default=Path("/src"), + help="The path to place the contents of the Git archive or copied directory " + "into on the image.", + ) + # This group ensures that only one of --archive-url or --src-path is used, since + # this command only builds either the contents of a source directory or the contents + # of a Git archive. + build_all_mutex_group = parser_build_all.add_mutually_exclusive_group(required=True) + build_all_mutex_group.add_argument( + "--src-path", + "-p", + metavar="FILEPATH", + type=str, + default=None, + help="The path to the source prefix on the host to be copied to the image. " + "Cannot be used with --archive-url.", + ) + build_all_mutex_group.add_argument( + "--archive-url", + type=str, + metavar="GIT_ARCHIVE", + default=None, + help='The URL of the Git archive to be fetched. Must be a "tar.gz" file. ' + "Cannot be used with --src-path.", ) @@ -180,6 +232,7 @@ def build_command_names() -> List[str]: "cmake-compile", "cmake-install", "make-distrib", + "build-all", ] @@ -196,3 +249,5 @@ def run_build(args: argparse.Namespace, command: str) -> None: cmake_install(**vars(args)) elif command == "make-distrib": make_distrib(**vars(args)) + elif command == "build-all": + build_all(**vars(args)) diff --git a/wigwam/cli/cli.py b/wigwam/cli/cli.py index 9999910..b9c4de2 100644 --- a/wigwam/cli/cli.py +++ b/wigwam/cli/cli.py @@ -23,7 +23,9 @@ def initialize_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog=__package__, formatter_class=help_formatter) # Add arguments - subparsers = parser.add_subparsers(dest="command", required=True) + subparsers: argparse._SubParsersAction = parser.add_subparsers( + dest="command", required=True + ) init_setup_parsers(subparsers, prefix) init_build_parsers(subparsers) diff --git a/wigwam/cli/setup_commands.py b/wigwam/cli/setup_commands.py index 5fbfd60..9a92dd9 100644 --- a/wigwam/cli/setup_commands.py +++ b/wigwam/cli/setup_commands.py @@ -60,15 +60,15 @@ def init_setup_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> N metavar="REPO_NAME", ) - setup_parser = subparsers.add_parser( + setup_parser: argparse.ArgumentParser = subparsers.add_parser( "setup", help="Docker image setup commands.", formatter_class=help_formatter ) - setup_subparsers = setup_parser.add_subparsers( + setup_subparsers: argparse._SubParsersAction = setup_parser.add_subparsers( dest="setup_subcommand", required=True ) - setup_all_parser = setup_subparsers.add_parser( + setup_all_parser: argparse.ArgumentParser = setup_subparsers.add_parser( "all", parents=[cuda_run_parse, no_cache_parse], help="Set up the full Docker image stack.", @@ -111,7 +111,7 @@ def init_setup_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> N help="If used, output informational messages upon completion.", ) - setup_init_parser = setup_subparsers.add_parser( + setup_init_parser: argparse.ArgumentParser = setup_subparsers.add_parser( "init", parents=[no_cache_parse], help="Set up the configuration image.", @@ -126,7 +126,7 @@ def init_setup_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> N ) add_tag_argument(parser=setup_init_parser, default="init") - setup_cuda_parser = setup_subparsers.add_parser( + setup_cuda_parser: argparse.ArgumentParser = setup_subparsers.add_parser( "cuda", help="Set up a CUDA image. Designate dev or runtime.", formatter_class=help_formatter, @@ -159,7 +159,7 @@ def init_setup_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> N ) add_tag_argument(parser=setup_cuda_dev_parser, default="cuda-dev") - setup_conda_parser = setup_subparsers.add_parser( + setup_conda_parser: argparse.ArgumentParser = setup_subparsers.add_parser( "conda", help="Set up a conda environment image. Designate dev or runtime.", formatter_class=help_formatter, @@ -168,7 +168,7 @@ def init_setup_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> N conda_subparsers = setup_conda_parser.add_subparsers( dest="conda_subcommand", required=True ) - setup_conda_runtime_parser = conda_subparsers.add_parser( + setup_conda_runtime_parser: argparse.ArgumentParser = conda_subparsers.add_parser( "runtime", parents=[setup_parse, no_cache_parse], help="Set up the runtime conda environment image", @@ -183,7 +183,7 @@ def init_setup_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> N ) add_tag_argument(parser=setup_conda_runtime_parser, default="conda-runtime") - setup_conda_dev_parser = conda_subparsers.add_parser( + setup_conda_dev_parser: argparse.ArgumentParser = conda_subparsers.add_parser( "dev", parents=[setup_parse, no_cache_parse], help="Set up the dev conda environment image", diff --git a/wigwam/cli/util_commands.py b/wigwam/cli/util_commands.py index 9b30a11..80b4321 100644 --- a/wigwam/cli/util_commands.py +++ b/wigwam/cli/util_commands.py @@ -18,7 +18,7 @@ def init_util_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> No The image tag prefix. """ - test_parser = subparsers.add_parser( + test_parser: argparse.ArgumentParser = subparsers.add_parser( "test", help="Run unit tests on an image.", formatter_class=help_formatter ) test_parser.add_argument( @@ -38,7 +38,7 @@ def init_util_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> No "--quiet-fail", action="store_true", help="Less verbose output on test failure." ) - dropin_parser = subparsers.add_parser( + dropin_parser: argparse.ArgumentParser = subparsers.add_parser( "dropin", help="Start a drop-in session.", formatter_class=help_formatter ) dropin_parser.add_argument( @@ -50,8 +50,16 @@ def init_util_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> No help="Run as the default user on the image. If not used, will run as the " "current user on the host machine.", ) + dropin_parser.add_argument( + "--no-prefix", + action="store_false", + dest="use_prefix", + default=True, + help="Run as the default user on the image. If not used, will run as the " + "current user on the host machine.", + ) - remove_parser = subparsers.add_parser( + remove_parser: argparse.ArgumentParser = subparsers.add_parser( "remove", help=f"Remove all Docker images beginning with {prefix}-[IMAGE_TAG] for each " "image tag provided.", @@ -82,7 +90,7 @@ def init_util_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> No "if not already prefixed.", ) - lockfile_parser = subparsers.add_parser( + lockfile_parser: argparse.ArgumentParser = subparsers.add_parser( "lockfile", help="Produce a lockfile for the image.", formatter_class=help_formatter, diff --git a/wigwam/commands.py b/wigwam/commands.py index 5a9861f..b7d3c6c 100644 --- a/wigwam/commands.py +++ b/wigwam/commands.py @@ -33,7 +33,7 @@ def get_archive( tag: str, base: str, archive_url: str, - directory: os.PathLike[str], + dst_path: str | os.PathLike[str], url_reader: URLReader | None = None, no_cache: bool = False, ): @@ -51,8 +51,8 @@ def get_archive( The base image tag. archive_url : str The URL of the Git archive to add to the image. Must be a `tar.gz` file. - directory : path-like - The path to the folder that the archive will be held at within the image. + dst_path : path-like + The prefix of the directory that the archive will be copied to on the image. url_reader : URLReader | None, optional If given, will use the given URL reader to acquire the Git archive. If None, will check the base image and use whichever one it can find. Defaults to None. @@ -73,7 +73,7 @@ def get_archive( dockerfile = git_extract_dockerfile( base=base_tag, - directory=directory, + dst_path=dst_path, archive_url=archive_url, url_reader=url_reader, ) @@ -84,8 +84,8 @@ def get_archive( def copy_dir( tag: str, base: str, - directory: str | os.PathLike[str], - target_path: str | os.PathLike[str] | None = None, + src_path: str | os.PathLike[str], + dst_path: str | os.PathLike[str] | None = None, no_cache: bool = False, ): """ @@ -104,13 +104,13 @@ def copy_dir( The image tag. base : str The base image tag. - directory : path-like - The directory to be copied. - target_path : path-like or None - The directory to copy to, on the image, or None. If given, the contents of the - source directory will be copied to the given path. If None, the target path will - default to the base name of the path given by the `directory` argument. - Defaults to None. + src_path : path-like + The prefix of the directory on the host machine to be copied. + dst_path : path-like or None + The prefix of the directory to copy to on the image, or None. If given, the + contents of the source directory will be copied to the given path. If None, the + target path will default to the base name of the path given by the `directory` + argument. Defaults to None. no_cache : bool, optional Run Docker build with no cache if True. Defaults to False. @@ -122,14 +122,14 @@ def copy_dir( img_tag = prefix_image_tag(tag) - dir_str = os.fspath(directory) + dir_str = os.fspath(src_path) # The absolute path of the given directory will be the build context. # This is necessary because otherwise docker may be unable to find the directory if # the build context is at the current working directory. path_absolute = os.path.abspath(dir_str) - if target_path is None: + if dst_path is None: # No argument was passed to target_path, so the lowest-level directory of the # input path will be the name of the directory in the image. if os.path.isdir(dir_str): @@ -137,7 +137,7 @@ def copy_dir( else: raise ValueError(f"{dir_str} is not a valid directory on this machine.") else: - target_dir = os.fspath(target_path) + target_dir = os.fspath(dst_path) # Generate the dockerfile. The source directory will be "." since the build context # will be at the source path when the image is built. @@ -303,6 +303,115 @@ def make_distrib(tag: str, base: str, source_tag: str, no_cache: bool = False) - return Image.build(tag=tag, dockerfile_string=dockerfile, no_cache=no_cache) +def build_all( + tag: str, + base: str, + src_path: str | os.PathLike | None, + archive_url: str | None, + dst_path: str | os.PathLike, + build_type: str, + no_cuda: bool, + no_cache: bool = False, +) -> dict[str, Image]: + """ + Fully compiles and installs a CMake project. + + Parameters + ---------- + tag : str + The image tag prefix. + base : str + The base image tag. + src_path : path-like or None + The path to the source prefix on the host to copy to an image. + archive_url : str or None + The URL of the Git archive to install on an image. No archive will be installed + if `copy_dir` is given. + dst_path : str + The path to place the contents of the Git archive or copied directory to. + build_type : str + The CMake build type. See + `here `_ + for possible values. + no_cuda : bool + If True, build without CUDA. + no_cache : bool, optional + Run Docker build with no cache if True. Defaults to False. + + Returns + ------- + dict[str, Image] + A dict of images produced by this process. + """ + + prefixed_tag: str = prefix_image_tag(tag) + prefixed_base_tag: str = prefix_image_tag(base) + + images: dict[str, Image] = {} + + initial_tag: str = "" + if src_path is not None: + src_path = os.fspath(src_path) + path_absolute = os.path.abspath(src_path) + if os.path.isdir(src_path): + top_dir = os.path.basename(path_absolute) + else: + raise ValueError(f"src_path must be a directory. Given value: {src_path}") + + insert_tag = f"{prefixed_tag}-file-{top_dir}" + insert_image = copy_dir( + base=prefixed_base_tag, + tag=insert_tag, + src_path=src_path, + dst_path=dst_path, + no_cache=no_cache, + ) + images[insert_tag] = insert_image + initial_tag = insert_tag + else: + git_repo_tag = f"{prefixed_tag}-git-repo" + if archive_url is None: + raise ValueError("Either archive_url or src_path must be passed.") + + git_repo_image = get_archive( + base=prefixed_base_tag, + tag=git_repo_tag, + archive_url=archive_url, + dst_path=dst_path, + no_cache=no_cache, + ) + images[git_repo_tag] = git_repo_image + initial_tag = git_repo_tag + + configure_tag = f"{prefixed_tag}-configured" + configure_image = configure_cmake( + tag=configure_tag, + base=initial_tag, + build_type=build_type, + no_cuda=no_cuda, + no_cache=no_cache, + ) + images[configure_tag] = configure_image + + build_tag = f"{prefixed_tag}-built" + build_image = compile_cmake( + tag=build_tag, + base=configure_tag, + no_cache=no_cache, + ) + images[build_tag] = build_image + + install_tag = f"{prefixed_tag}-installed" + install_image = cmake_install( + tag=install_tag, + base=build_tag, + no_cache=no_cache, + ) + images[install_tag] = install_image + + return images + + def test( tag: str, output_xml: os.PathLike[str] | str, @@ -366,7 +475,7 @@ def test( image.run(command=command, host_user=True, bind_mounts=[bind_mount]) -def dropin(tag: str, default_user: bool = False) -> None: +def dropin(tag: str, default_user: bool = False, use_prefix: bool = True) -> None: """ Initiates a drop-in session on an image. @@ -378,7 +487,8 @@ def dropin(tag: str, default_user: bool = False) -> None: If True, run as the default user in the image. Else, run as the current user on the host machine. Defaults to False. """ - tag = prefix_image_tag(tag) + if use_prefix: + tag = prefix_image_tag(tag) image: Image = Image(tag) image.drop_in(host_user=not default_user)