Added distributable docker image generation (#10)

* Added distributable docker image generation

+ And a unit test!

* fixed CLI error on test command

* Doc changes, bugfixes

* removed root user from the distributable dockerfile

---------

Co-authored-by: tyler-g-hudson <[email protected]>

Author: Tyler-g-hudson

test/fixtures_isce3.py

@@ -13,7 +13,10 @@
     cmake_config_dockerfile,
     cmake_install_dockerfile,
 )
+from wigwam._docker_distrib import distrib_dockerfile
 from wigwam._docker_git import git_extract_dockerfile
+from wigwam._utils import get_libdir
+from wigwam.defaults import install_prefix
 from wigwam.setup_commands import setup_all
 
 from .utils import determine_scope, generate_tag, remove_docker_image
@@ -87,6 +90,20 @@ def isce3_env_dev_image_tag(
     raise ValueError("No development environment image tag found.")
 
 
+@fixture(scope=determine_scope)
+def isce3_env_runtime_image_tag(
+    isce3_build_tag: str,
+    isce3_setup_images: dict[str, Image],
+) -> str:
+    """Return the tag of the ISCE3 development environment image."""
+    pattern = re.compile(rf".*{isce3_build_tag}.*mamba-runtime")
+    for tag in isce3_setup_images:
+        if pattern.match(tag):
+            return tag
+
+    raise ValueError("No development environment image tag found.")
+
+
 @fixture(scope=determine_scope)
 def isce3_git_repo_tag() -> str:
     """Return a tag for the ISCE3 repository image."""
@@ -191,3 +208,36 @@ def isce3_cmake_install_image(
     )
 
     remove_docker_image(isce3_cmake_install_tag)
+
+
+@fixture(scope=determine_scope)
+def isce3_distributable_tag() -> str:
+    """Return a tag for the ISCE3 CMake install image."""
+    return generate_tag("isce3-distributable")
+
+
+@fixture(scope=determine_scope)
+def isce3_distributable_image(
+    isce3_distributable_tag: str,
+    isce3_cmake_install_tag: str,
+    isce3_env_runtime_image_tag: str,
+    isce3_cmake_install_image: Image,  # type: ignore
+) -> Iterator[Image]:
+    """Return the ISCE3 distributable image."""
+    libdir = get_libdir(isce3_cmake_install_tag)
+
+    dockerfile = distrib_dockerfile(
+        base=isce3_env_runtime_image_tag,
+        source_tag=isce3_cmake_install_tag,
+        source_path=install_prefix(),
+        distrib_path=install_prefix(),
+        libdir=libdir,
+    )
+
+    yield Image.build(
+        tag=isce3_distributable_tag,
+        dockerfile_string=dockerfile,
+        no_cache=False,
+    )
+
+    remove_docker_image(isce3_distributable_tag)

test/test_docker_cmake.py

@@ -308,3 +308,18 @@ def test_test_command(
                     quiet_fail=False,
                 )
                 assert output_xml.is_file()
+
+        @mark.isce3
+        @mark.slow
+        def test_isce3_distributable_image(
+            self,
+            isce3_distributable_image: Image,
+        ):
+            """
+            Test the ISCE3 Distributable image.
+
+            NOTE: This test runs very slowly because it requires the building of all
+            ISCE3 base images.
+            """
+            isce3_distributable_image.run(command='python -c "import isce3"')
+            isce3_distributable_image.run(command='python -c "import nisar"')

wigwam/_docker_distrib.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import os
+from textwrap import dedent
+
+
+def distrib_dockerfile(
+    base: str,
+    source_tag: str,
+    source_path: os.PathLike[str] | str,
+    distrib_path: os.PathLike[str] | str,
+    libdir: str,
+) -> str:
+    """
+    Returns a Dockerfile for a distributable build.
+
+    The distributable image includes the installed software and all of its runtime
+    dependencies, but excludes build-time dependencies. The working directory on the
+    generated image will also be moved to the location given at `distrib_path`.
+
+    Parameters
+    ----------
+    base : str
+        The base image tag.
+    source_tag : str
+        The tag of the image on which the project is installed.
+    source_path : os.PathLike[str] or str
+        The installation path of the project on the source image.
+    distrib_path : os.PathLike[str] or str
+        The desired installation path on the distributable image.
+    libdir : str
+        The base name of the directory where object libraries are stored (e.g. "lib" or
+        "lib64"). This should match the value of `LIBDIR` in CMake's `GNUInstallDirs`
+        module.
+        See https://cmake.org/cmake/help/latest/module/GNUInstallDirs.html.
+
+    Returns
+    -------
+    dockerfile: str
+        The generated Dockerfile.
+    """
+    dockerfile: str = dedent(
+        f"""
+            FROM {source_tag} as source
+
+            FROM {base}
+
+            COPY --from=source {source_path} {distrib_path}
+
+            ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:{distrib_path}/{libdir}
+            ENV PYTHONPATH $PYTHONPATH:{distrib_path}/packages
+
+            ENV ISCE3_PREFIX={distrib_path}
+            WORKDIR $ISCE3_PREFIX
+        """
+    ).strip()
+
+    return dockerfile

wigwam/_utils.py

@@ -203,6 +203,24 @@ def test_image(image: Image, expression: str) -> bool:
             raise
 
 
+def get_libdir(base_tag: str) -> str:
+    """
+    Determine if the given image uses `lib64` or `lib` as its `LIBDIR`.
+
+    This function assumes that the base_tag has something at either
+    `$INSTALL_PREFIX/lib64` or `$INSTALL_PREFIX/lib`.
+    """
+    with temp_image(base_tag) as temp_img:
+        for libdir in ["lib64", "lib"]:
+            if test_image(temp_img, f'"$INSTALL_PREFIX/{libdir}"'):
+                return libdir
+        else:
+            raise ValueError(
+                "could not find a directory named $INSTALL_PREFIX/lib64"
+                " or $INSTALL_PREFIX/lib in the specified image"
+            )
+
+
 def _package_manager_check(image: Image) -> PackageManager:
     """
     Returns the package manager present on an image.

wigwam/cli/build_commands.py

@@ -8,6 +8,7 @@
     configure_cmake,
     copy_dir,
     get_archive,
+    make_distrib,
 )
 from ._utils import add_tag_argument, help_formatter
 
@@ -139,10 +140,47 @@ def init_build_parsers(subparsers: argparse._SubParsersAction) -> None:
     )
     add_tag_argument(parser=install_parser, default="installed")
 
+    distrib_parser = subparsers.add_parser(
+        "make-distrib",
+        parents=[no_cache_params],
+        help="Creates a distributable image.",
+        formatter_class=help_formatter,
+    )
+    distrib_parser.add_argument(
+        "--tag",
+        "-t",
+        default="isce3",
+        type=str,
+        help="The complete tag of the Docker image to be created. " 'Default: "isce3"',
+    )
+    distrib_parser.add_argument(
+        "--base",
+        "-b",
+        default="setup-mamba-runtime",
+        type=str,
+        help="The complete tag of the Docker image to be created. "
+        'Default: "setup-mamba-runtime"',
+    )
+    distrib_parser.add_argument(
+        "--source-tag",
+        "-s",
+        default="build-installed",
+        type=str,
+        help="The tag or ID of the source image which has the project installed. "
+        ' Defaults to "build-installed".',
+    )
+
 
 def build_command_names() -> List[str]:
     """Returns a list of all build command names."""
-    return ["get-archive", "copydir", "cmake-config", "cmake-compile", "cmake-install"]
+    return [
+        "get-archive",
+        "copydir",
+        "cmake-config",
+        "cmake-compile",
+        "cmake-install",
+        "make-distrib",
+    ]
 
 
 def run_build(args: argparse.Namespace, command: str) -> None:
@@ -156,3 +194,5 @@ def run_build(args: argparse.Namespace, command: str) -> None:
         compile_cmake(**vars(args))
     elif command == "cmake-install":
         cmake_install(**vars(args))
+    elif command == "make-distrib":
+        make_distrib(**vars(args))

wigwam/cli/util_commands.py

@@ -44,7 +44,7 @@ def init_util_parsers(subparsers: argparse._SubParsersAction, prefix: str) -> No
     dropin_parser.add_argument(
         "tag", metavar="IMAGE_TAG", type=str, help="The tag or ID of the desired image."
     )
-    test_parser.add_argument(
+    dropin_parser.add_argument(
         "--default-user",
         action="store_true",
         help="Run as the default user on the image. If not used, will run as the "

wigwam/commands.py

@@ -13,13 +13,20 @@
     cmake_config_dockerfile,
     cmake_install_dockerfile,
 )
+from ._docker_distrib import distrib_dockerfile
 from ._docker_git import git_extract_dockerfile
 from ._docker_insert import insert_dir_dockerfile
 from ._docker_mamba import mamba_lockfile_command
 from ._image import Image
 from ._url_reader import URLReader
-from ._utils import image_command_check, is_conda_pkg_name, prefix_image_tag, temp_image
-from .defaults import build_prefix
+from ._utils import (
+    get_libdir,
+    image_command_check,
+    is_conda_pkg_name,
+    prefix_image_tag,
+    temp_image,
+)
+from .defaults import build_prefix, install_prefix
 
 
 def get_archive(
@@ -57,13 +64,13 @@ def get_archive(
     Image
         The generated image.
     """
-    if url_reader is None:
-        with temp_image(base) as temp_img:
-            _, url_reader, _ = image_command_check(temp_img)
-
     img_tag = prefix_image_tag(tag)
     base_tag = prefix_image_tag(base)
 
+    if url_reader is None:
+        with temp_image(base_tag) as temp_img:
+            _, url_reader, _ = image_command_check(temp_img)
+
     dockerfile = git_extract_dockerfile(
         base=base_tag,
         directory=directory,
@@ -256,6 +263,46 @@ def cmake_install(tag: str, base: str, no_cache: bool = False) -> Image:
     )
 
 
+def make_distrib(tag: str, base: str, source_tag: str, no_cache: bool = False) -> Image:
+    """
+    Produces a distributable image.
+
+    Parameters
+    ----------
+    tag : str
+        The image tag.
+    base : str
+        The base image tag.
+    source_tag : str
+        The tag of the source image from which to acquire the install directory.
+    no_cache : bool, optional
+        Run Docker build with no cache if True. Defaults to False.
+
+    Returns
+    -------
+    Image
+        The generated image.
+    """
+
+    prefixed_base_tag: str = prefix_image_tag(base)
+    prefixed_source_tag: str = prefix_image_tag(source_tag)
+
+    # Unlike with the CMake Install function, `libdir` can be checked directly on this
+    # image because it has something at $INSTALL_PREFIX. Check the base tag for lib64 or
+    # lib.
+    libdir: str = get_libdir(prefixed_base_tag)
+
+    dockerfile = distrib_dockerfile(
+        base=prefixed_base_tag,
+        source_tag=prefixed_source_tag,
+        source_path=install_prefix(),
+        distrib_path=install_prefix(),
+        libdir=libdir,
+    )
+
+    return Image.build(tag=tag, dockerfile_string=dockerfile, no_cache=no_cache)
+
+
 def test(
     tag: str,
     output_xml: os.PathLike[str] | str,