Merge pull request #37 from epics-containers/dev

Add support for local docker deployment

Author: gilesknap

environment.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# This is an example environment.sh for local docker deployments.
+# It is used for testing epics-contianers-cli. It sets up a number of
+# environment varaibles used to direct 'ec' commands to the beamline you
+# are working on. Typically each beamline repository will have its own.
+
+# check we are sourced
+if [ "$0" = "$BASH_SOURCE" ]; then
+    echo "ERROR: Please source this script"
+    exit 1
+fi
+
+echo "Loading IOC environment for BL45P ..."
+
+# a mapping between genenric IOC repo roots and the related container registry
+export EC_REGISTRY_MAPPING='github.com=ghcr.io gitlab.diamond.ac.uk=gcr.io/diamond-privreg/controls/ioc'
+
+# the git organisation used for beamline repositories
+export EC_GIT_ORG=https://github.com/epics-containers
+
+# the git rempos used for the current beamline
+export EC_DOMAIN_REPO=git@github.com:epics-containers/bl45p.git
+
+# to use kubernetes specify the namespace - otherwise docker local deployments are used
+# export EC_K8S_NAMESPACE=bl45p
+
+# enforce a specific container cli (docker or podman)
+# export EC_CONTAINER_CLI=podman
+
+# enable debug output in all 'ec' commands
+# export EC_DEBUG=1
+
+# declare your centralised log server Web UI
+# export EC_LOG_URL='https://graylog2.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{ioc_name}*'
+
+# if you are using kubernetes then this script must enable access to the cluster.
+# An example for how this is done at DLS is below.
+
+# # the following configures kubernetes inside DLS.
+# if module --version &> /dev/null; then
+#     if module avail pollux > /dev/null; then
+#         module unload pollux > /dev/null
+#         module load pollux > /dev/null
+#     fi
+# fi
+
+# check if epics-containers-cli (ec command) is installed and install if not
+if ! ec --version &> /dev/null; then
+    # must be in a venv and this is the reliable check
+    if python3 -c 'import sys; sys.exit(0 if sys.base_prefix==sys.prefix else 1)'; then
+        echo "ERROR: Please activate a virtualenv and re-run"
+        return
+    elif ! ec --version &> /dev/null; then
+        pip install epics-containers-cli
+    fi
+fi
+
+# enable shell completion for ec commands
+source <(ec --show-completion ${SHELL})

pyproject.toml

@@ -38,6 +38,7 @@ dev = [
     "sphinx-design",
     "tox-direct",
     "types-mock",
+    "types-requests",
 ]
 
 [project.scripts]

src/epics_containers_cli/__main__.py

@@ -1,3 +1,4 @@
+import os
 from typing import Optional
 
 import typer
@@ -9,13 +10,7 @@
 from .k8s.k8s_cli import cluster
 from .k8s.kubectl import fmt_deploys, fmt_pods, fmt_pods_wide
 from .logging import init_logging
-from .shell import (
-    EC_DOMAIN_REPO,
-    EC_GIT_ORG,
-    EC_K8S_NAMESPACE,
-    check_domain,
-    run_command,
-)
+from .shell import run_command
 
 __all__ = ["main"]
 
@@ -55,19 +50,19 @@ def main(
         help="Log the version of ec and exit",
     ),
     org: str = typer.Option(
-        EC_GIT_ORG,
+        "",
         "-o",
         "--org",
         help="git remote organisation of domain repos",
     ),
     repo: str = typer.Option(
-        EC_DOMAIN_REPO,
+        "",
         "-r",
         "--repo",
         help="beamline or accelerator domain repository of ioc instances",
     ),
     namespace: str = typer.Option(
-        EC_K8S_NAMESPACE, "-n", "--namespace", help="kubernetes namespace to use"
+        "", "-n", "--namespace", help="kubernetes namespace to use"
     ),
     log_level: str = typer.Option(
         "WARN", help="Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)"
@@ -81,6 +76,9 @@ def main(
     init_logging(log_level.upper(), debug)
 
     # create a context dictionary to pass to all sub commands
+    org = org or os.environ.get("EC_GIT_ORG", "")
+    repo = repo or os.environ.get("EC_DOMAIN_REPO", "")
+    namespace = namespace or os.environ.get("EC_K8S_NAMESPACE", "")
     ctx.ensure_object(Context)
     context = Context(namespace, repo, org)
     ctx.obj = context
@@ -98,7 +96,6 @@ def ps(
 ):
     """List the IOCs running in the current domain"""
     domain = ctx.obj.namespace
-    check_domain(domain)
 
     if all:
         run_command(f"kubectl -n {domain} get deploy -l is_ioc==True -o {fmt_deploys}")

src/epics_containers_cli/dev/dev_commands.py

@@ -6,15 +6,18 @@
 
 import typer
 
+from epics_containers_cli.git import get_git_name, get_image_name
+from epics_containers_cli.logging import log
+from epics_containers_cli.shell import EC_CONTAINER_CLI, run_command
+from epics_containers_cli.utils import get_instance_image_name
+
 from ..globals import (
     CONFIG_FOLDER,
     IOC_CONFIG_FOLDER,
     IOC_START,
     Architecture,
     Targets,
 )
-from ..logging import log
-from ..shell import EC_CONTAINER_CLI, get_git_name, get_image_name, run_command
 
 dev = typer.Typer()
 
@@ -185,25 +188,11 @@ def launch(
             f" execute={execute} target={target} args={args}"
         )
 
-        mounts = []
+        mounts = [f"-v {ioc_instance}/{CONFIG_FOLDER}:{IOC_CONFIG_FOLDER}"]
 
-        ioc_instance = ioc_instance.resolve()
-        values = ioc_instance / "values.yaml"
-        if not values.exists():
-            log.error(f"values.yaml not found in {ioc_instance}")
-            raise typer.Exit(1)
-        mounts.append(f"-v {ioc_instance}/{CONFIG_FOLDER}:{IOC_CONFIG_FOLDER}")
+        image_name = image or get_instance_image_name(ioc_instance, tag)
 
-        values_text = values.read_text()
-        matches = re.findall(r"image: (.*):(.*)", values_text)
-        if len(matches) == 1:
-            tag = tag or matches[0][1]
-            image = matches[0][0] + f":{tag}"
-        else:
-            log.error(f"image tag definition not found in {values}")
-            raise typer.Exit(1)
-
-        self._do_launch(ioc_name, target, image, execute, args, mounts)
+        self._do_launch(ioc_name, target, image_name, execute, args, mounts)
 
     def debug_last(self, generic_ioc: Path, mount_repos: bool):
         """

src/epics_containers_cli/git.py

@@ -0,0 +1,113 @@
+"""
+Utility functions for working with git
+"""
+
+import os
+import re
+from pathlib import Path
+from typing import Tuple
+
+import typer
+
+from epics_containers_cli.globals import Architecture
+from epics_containers_cli.logging import log
+from epics_containers_cli.shell import (
+    EC_REGISTRY_MAPPING,
+    check_beamline_repo,
+    run_command,
+)
+
+
+def get_image_name(
+    repo: str, arch: Architecture = Architecture.linux, target: str = "developer"
+) -> str:
+    registry = repo2registry(repo).lower().removesuffix(".git")
+
+    image = f"{registry}-{arch}-{target}"
+    log.info("repo = %s image  = %s", repo, image)
+    return image
+
+
+def get_git_name(folder: Path = Path(".")) -> Tuple[str, Path]:
+    """
+    work out the git repo name and top level folder for a local clone
+    """
+    os.chdir(folder)
+    path = str(run_command("git rev-parse --show-toplevel", interactive=False))
+    git_root = Path(path.strip())
+
+    remotes = str(run_command("git remote -v", interactive=False))
+    log.debug(f"remotes = {remotes}")
+
+    matches = re.findall(r"((?:(?:git@)|(?:http[s]+:\/\/)).*) (?:.fetch.)", remotes)
+
+    if len(matches) > 0:
+        repo_name = str(matches[0])
+    else:
+        log.error(f"folder {folder.absolute()} cannot parse repo name {remotes}")
+        raise typer.Exit(1)
+
+    log.debug(f"repo_name = {repo_name}, git_root = {git_root}")
+    return repo_name, git_root
+
+
+# work out what the registry name is for a given repo remote e.g.
+def repo2registry(repo_name: str) -> str:
+    """convert a repo name to the related a container registry name"""
+
+    log.debug("extracting fields from repo name %s", repo_name)
+
+    match_git = re.match(r"git@([^:]*):(.*)\/(.*)(?:.git)", repo_name)
+    match_http = re.match(r"https:\/\/([^\/]*)\/([^\/]*)\/([^\/]*)", repo_name)
+    for match in [match_git, match_http]:
+        if match is not None:
+            source_reg, org, repo = match.groups()
+            break
+    else:
+        log.error(f"repo {repo_name} is not a valid git remote")
+        raise typer.Exit(1)
+
+    log.debug("source_reg = %s org = %s repo = %s", source_reg, org, repo)
+
+    if not EC_REGISTRY_MAPPING:
+        log.error("environment variable EC_REGISTRY_MAPPING not set")
+        raise typer.Exit(1)
+
+    for mapping in EC_REGISTRY_MAPPING.split():
+        if mapping.split("=")[0] == source_reg:
+            registry = mapping.split("=")[1]
+            registry = f"{registry}/{org}/{repo}"
+            break
+    else:
+        log.error(f"repo {repo_name} does not match any registry mapping")
+        log.error("please update the environment variable EC_REGISTRY_MAPPING")
+        raise typer.Exit(1)
+
+    return registry
+
+
+def versions(beamline_repo: str, ioc_name: str, folder: Path):
+    """
+    determine the versions of an IOC instance by discovering the tags in the
+    beamline repo at which changes to the instance were made since the last
+    tag
+    """
+    check_beamline_repo(beamline_repo)
+    typer.echo(f"Available instance versions for {ioc_name}:")
+
+    run_command(f"git clone {beamline_repo} {folder}", interactive=False)
+
+    ioc_name = Path(ioc_name).name
+    os.chdir(folder)
+    result = str(run_command("git tag", interactive=False))
+    log.debug(f"checking these tags for changes in the instance: {result}")
+
+    tags = result.split("\n")
+    for tag in tags:
+        if tag == "":
+            continue
+        cmd = f"git diff --name-only {tag} {tag}^"
+        result = str(run_command(cmd, interactive=False))
+
+        if ioc_name in result:
+            typer.echo(f"  {tag}")

src/epics_containers_cli/ioc/__init__.py

@@ -1,4 +1,3 @@
-import os
 import shutil
 from datetime import datetime
 from pathlib import Path
@@ -11,6 +10,7 @@
 from epics_containers_cli.globals import BEAMLINE_CHART_FOLDER, CONFIG_FOLDER
 from epics_containers_cli.logging import log
 from epics_containers_cli.shell import run_command
+from epics_containers_cli.utils import check_ioc_instance_path
 
 
 class Helm:
@@ -31,7 +31,7 @@ def __init__(
         Create a helm chart from a local or a remote repo
         """
         self.ioc_name = ioc_name
-        self.repo = repo
+        self.beamline_repo = repo
         self.namespace = domain
         self.args = args
         self.version = version or datetime.strftime(
@@ -59,14 +59,7 @@ def deploy_local(self, ioc_path: Path, yes: bool = False):
         Deploy a local IOC helm chart directly to the cluster with dated beta version
         """
 
-        ioc_path = ioc_path.absolute()
-        ioc_name = ioc_path.name.lower()
-        if (
-            not (ioc_path / "values.yaml").exists()
-            or not (ioc_path / CONFIG_FOLDER).is_dir()
-        ):
-            log.error("ERROR: IOC instance requires values.yaml and config")
-            raise typer.Exit(1)
+        ioc_name, ioc_path = check_ioc_instance_path(ioc_path)
 
         if not yes and not self.template:
             typer.echo(
@@ -91,7 +84,7 @@ def deploy(self):
             raise typer.Exit("ERROR: version is required")
 
         run_command(
-            f"git clone {self.repo} {self.tmp} --depth=1 "
+            f"git clone {self.beamline_repo} {self.tmp} --depth=1 "
             f"--single-branch --branch={self.version}",
             interactive=False,
         )
@@ -139,24 +132,3 @@ def _install(
 
         output = run_command(cmd, interactive=False)
         typer.echo(output)
-
-    def versions(self):
-        typer.echo(f"Available instance versions for {self.ioc_name}:")
-
-        run_command(f"git clone {self.repo} {self.tmp}", interactive=False)
-
-        ioc_name = Path(self.ioc_name).name
-        cmd = "git tag"
-        os.chdir(self.tmp)
-        result = run_command(cmd, interactive=False)
-        log.debug(f"checking these tags for changes in the instance: {result}")
-
-        tags = result.split("\n")
-        for tag in tags:
-            if tag == "":
-                continue
-            cmd = f"git diff --name-only {tag} {tag}^"
-            result = run_command(cmd, interactive=False)
-
-            if ioc_name in result:
-                typer.echo(f"  {tag}")