Merge pull request #49 from epics-containers/dev

Add ioc validate command for beamline CI

Author: gilesknap

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-added-large-files
       - id: check-yaml

pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 description = "One line description of your module"
-dependencies = ["typer[all]", "ruamel.yaml", "jinja2"]
+dependencies = ["typer[all]", "requests", "ruamel.yaml", "jinja2"]
 
 # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
@@ -27,7 +27,7 @@ dev = [
     "mock",
     "mypy",
     "pipdeptree",
-    "pre-commit",
+    "pre-commit==3.5.0",
     "pydata-sphinx-theme>=0.12",
     "pytest",
     "pytest-cov",

src/epics_containers_cli/dev/dev_cli.py

@@ -213,6 +213,10 @@ def build(
     cache_from: Optional[str] = typer.Option(None, help="buildx cache from folder"),
     push: bool = typer.Option(False, help="buildx push to registry"),
     rebuild: bool = typer.Option(True, help="rebuild the image even if it exists"),
+    target: Optional[str] = typer.Option(
+        None, help="target to build (default: developer and runtime)"
+    ),
+    suffix: Optional[str] = typer.Option(None, help="suffix for image"),
 ):
     """
     Build a generic IOC container locally from a container project.
@@ -229,4 +233,6 @@ def build(
         cache_to=cache_to,
         push=push,
         rebuild=rebuild,
+        target=target,
+        suffix=suffix,
     )

src/epics_containers_cli/dev/dev_commands.py

@@ -190,15 +190,21 @@ def build(
         cache_to: Optional[str],
         push: bool,
         rebuild: bool,
+        target: Optional[str],
+        suffix: Optional[str],
     ):
         """
         build a local image from a Dockerfile
         """
         repo, _ = get_git_name(generic_ioc)
         args = f"--platform {platform} {'--no-cache' if not cache else ''}"
 
-        for target in Targets:
-            image = get_image_name(repo, arch, target)
+        if target is None:
+            targets = [Targets.developer.value, Targets.runtime.value]
+        else:
+            targets = [target]
+        for target in targets:
+            image = get_image_name(repo, arch, target, suffix)
             image_name = f"{image}:{tag}"
 
             if not rebuild:

src/epics_containers_cli/docker.py

@@ -27,12 +27,13 @@ class Docker:
     which CLI is being used and whether buildx is available.
     """
 
-    def __init__(self, devcontainer: bool = False):
+    def __init__(self, devcontainer: bool = False, check: bool = True):
         self.devcontainer = devcontainer
         self.docker: str = "podman"
         self.is_docker: bool = False
         self.is_buildx: bool = False
-        self._check_docker()
+        if check:
+            self._check_docker()
 
     def _check_docker(self):
         """

src/epics_containers_cli/git.py

@@ -5,7 +5,7 @@
 import os
 import re
 from pathlib import Path
-from typing import Tuple
+from typing import Optional, Tuple
 
 import typer
 
@@ -19,11 +19,17 @@
 
 
 def get_image_name(
-    repo: str, arch: Architecture = Architecture.linux, target: str = "developer"
+    repo: str,
+    arch: Architecture = Architecture.linux,
+    target: str = "developer",
+    suffix: Optional[str] = None,
 ) -> str:
+    if suffix is None:
+        suffix = "-{arch}-{target}"
     registry = repo2registry(repo).lower().removesuffix(".git")
+    img_suffix = suffix.format(repo=repo, arch=arch, target=target, registry=registry)
 
-    image = f"{registry}-{arch}-{target}"
+    image = f"{registry}{img_suffix}"
     log.info("repo = %s image  = %s", repo, image)
     return image
 
@@ -55,6 +61,24 @@ def get_git_name(folder: Path = Path(".")) -> Tuple[str, Path]:
 def repo2registry(repo_name: str) -> str:
     """convert a repo name to the related a container registry name"""
 
+    # First try matching using the regex mappings environment variable
+    registry = ""
+
+    for mapping in glob_vars.EC_REGISTRY_MAPPING_REGEX.split("\n"):
+        if mapping == "":
+            continue
+        regex, replacement = mapping.split(" ")
+        log.debug("regex = %s replacement = %s", regex, replacement)
+        match = re.match(regex, repo_name)
+        if match is not None:
+            registry = match.expand(replacement)
+            break
+
+    if registry:
+        return registry
+
+    # Now try matching using the simple mappings environment variable.
+    # Here automatically add the organization name to the image root URL
     log.debug("extracting fields from repo name %s", repo_name)
 
     match_git = re.match(r"git@([^:]*):(.*)\/(.*)(?:.git)", repo_name)
@@ -69,10 +93,6 @@ def repo2registry(repo_name: str) -> str:
 
     log.debug("source_reg = %s org = %s repo = %s", source_reg, org, repo)
 
-    if not glob_vars.EC_REGISTRY_MAPPING:
-        log.error("environment variable EC_REGISTRY_MAPPING not set")
-        raise typer.Exit(1)
-
     for mapping in glob_vars.EC_REGISTRY_MAPPING.split():
         if mapping.split("=")[0] == source_reg:
             registry = mapping.split("=")[1]

src/epics_containers_cli/globals.py

@@ -33,16 +33,39 @@ def __str__(self):
 # common stings used in paths
 BEAMLINE_CHART_FOLDER = "beamline-chart"
 CONFIG_FOLDER = "config"
+CONFIG_FILE = "ioc.yaml"
 IOC_CONFIG_FOLDER = "/epics/ioc/config/"
 IOC_START = "/epics/ioc/start.sh"
 IOC_NAME = "test-ioc"
 # these should be set to 0 or 1 in the environment - blank is treated as false
-EC_DEBUG = bool(os.environ.get("EC_DEBUG"))
-EC_VERBOSE = bool(os.environ.get("EC_VERBOSE"))
+EC_DEBUG = bool(os.environ.get("EC_DEBUG", 0))
+EC_VERBOSE = bool(os.environ.get("EC_VERBOSE", 0))
 
+"""
+each mapping is a string of the form <source registry>=<container registry>
+the container registry is used to build the image name as
+<container registry>/<source organisation>/<repo name (without .git)>:<tag>
+mappings are separated by space or line break.
+
+For a much more flexible way to define mappings see EC_REGISTRY_MAPPING_REGEX
+"""
 EC_REGISTRY_MAPPING = os.environ.get(
     "EC_REGISTRY_MAPPING",
-    "github.com=ghcr.io gitlab.diamond.ac.uk=gcr.io/diamond-privreg/controls/ioc",
+    "github.com=ghcr.io",
 )
+
+"""
+each mapping is a regex to match the repo name and a replacement string
+to generate the container registry name. Mappings are separated by line
+break and regex and replacement are separated by space
+"""
+EC_REGISTRY_MAPPING_REGEX = os.environ.get(
+    "EC_REGISTRY_MAPPING",
+    r"""
+.*github.com:(.*)\/(.*)\.git ghcr.io/\1/\2
+.*gitlab.diamond.ac.uk.*\/(.*).git gcr.io/diamond-privreg/controls/prod/ioc/\1
+""",
+)
+
 EC_CONTAINER_CLI = os.environ.get("EC_CONTAINER_CLI")  # default to auto choice
 EC_LOG_URL = os.environ.get("EC_LOG_URL", None)

src/epics_containers_cli/ioc/ioc_cli.py

@@ -184,3 +184,24 @@ def stop(
         IocLocalCommands(ctx.obj, ioc_name).stop()
     else:
         IocK8sCommands(ctx.obj, ioc_name).stop()
+
+
+@ioc.command()
+def validate(
+    ctx: typer.Context,
+    ioc_instance: Path = typer.Argument(
+        ...,
+        help="folder of local ioc definition",
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+    ),
+):
+    """
+    Verify a local IOC definition folder is valid
+
+    Checks that values.yaml points at a valid image
+    Checks that ioc.yaml has the matching schema header and that it passes
+      scheme validation
+    """
+    IocLocalCommands(ctx.obj).validate_instance(ioc_instance)

src/epics_containers_cli/ioc/local_commands.py

@@ -8,29 +8,41 @@
 'ec deploy <ioc_name> <ioc_version> and then managing the network with a
 tool like Portainer is a decent workflow.
 """
-
+import re
 import shutil
 from datetime import datetime
 from pathlib import Path
 from tempfile import mkdtemp
 from typing import Optional
 
+import requests
 import typer
 
 import epics_containers_cli.globals as glob_vars
 from epics_containers_cli.docker import Docker
-from epics_containers_cli.globals import CONFIG_FOLDER, IOC_CONFIG_FOLDER, Context
+from epics_containers_cli.globals import (
+    CONFIG_FILE,
+    CONFIG_FOLDER,
+    IOC_CONFIG_FOLDER,
+    Context,
+)
 from epics_containers_cli.logging import log
 from epics_containers_cli.shell import check_beamline_repo, run_command
-from epics_containers_cli.utils import check_ioc_instance_path, get_instance_image_name
+from epics_containers_cli.utils import (
+    check_ioc_instance_path,
+    generic_ioc_from_image,
+    get_instance_image_name,
+)
 
 
 class IocLocalCommands:
     """
     A class for implementing the ioc command namespace for local docker/podman
     """
 
-    def __init__(self, ctx: Optional[Context], ioc_name: str = ""):
+    def __init__(
+        self, ctx: Optional[Context], ioc_name: str = "", with_docker: bool = True
+    ):
         self.beamline_repo: str = ""
         if ctx is not None:
             self.beamline_repo = ctx.beamline_repo
@@ -39,7 +51,7 @@ def __init__(self, ctx: Optional[Context], ioc_name: str = ""):
 
         self.tmp = Path(mkdtemp())
         self.ioc_folder = self.tmp / "iocs" / ioc_name
-        self.docker = Docker()
+        self.docker = Docker(check=with_docker)
 
     def __del__(self):
         # keep the tmp folder if debug is enabled for inspection_del
@@ -160,3 +172,56 @@ def ps(self, all: bool, wide: bool):
 
         for row in rows:
             print("{0: <20.20} {1: <20.20} {2: <23.23} {3}".format(*row))
+
+    def validate_instance(self, ioc_instance: Path):
+        check_ioc_instance_path(ioc_instance)
+
+        typer.echo(f"Validating {ioc_instance}")
+
+        ioc_config_file = ioc_instance / CONFIG_FOLDER / CONFIG_FILE
+        image = get_instance_image_name(ioc_instance)
+        image_name, image_tag = image.split(":")
+
+        tmp = Path(mkdtemp())
+        schema_file = tmp / "schema.json"
+
+        # not all IOCs have a config file so no config validation for them
+        if ioc_config_file.exists():
+            config = ioc_config_file.read_text()
+            matches = re.findall(r"#.* \$schema=(.*)", config)
+            if not matches:
+                raise RuntimeError("No schema modeline found in {ioc_config_file}")
+
+            schema_url = matches[0]
+
+            with requests.get(schema_url, allow_redirects=True) as r:
+                schema_file.write_text(r.content.decode())
+
+            if not run_command("yajsv -v", interactive=False, error_OK=True):
+                typer.echo(
+                    "yajsv, used for schema validation of ioc.yaml, is not installed. "
+                    "Please install from https://github.com/neilpa/yajsv"
+                )
+                raise typer.Exit(1)
+
+            run_command(f"yajsv -s {schema_file} {ioc_config_file}", interactive=False)
+
+            # check that the image name and the schema are from the same generic IOC
+            if image_tag not in schema_url:
+                log.error(f"image version {image_tag} and {schema_url} do not match")
+                raise typer.Exit(1)
+
+            # make sure that generic IOC name matches the schema
+            generic_ioc = generic_ioc_from_image(image_name)
+            if generic_ioc not in schema_url:
+                log.error(
+                    f"ioc.yaml schema {schema_url} does not match generic IOC {generic_ioc}"
+                )
+                raise typer.Exit(1)
+
+        # verify that the values.yaml file points to a container image that exists
+        run_command(f"{self.docker.docker} image inspect {image}", interactive=False)
+
+        shutil.rmtree(tmp, ignore_errors=True)
+
+        typer.echo(f"{ioc_instance} validated successfully")

src/epics_containers_cli/utils.py

@@ -32,7 +32,7 @@ def get_instance_image_name(ioc_instance: Path, tag: Optional[str] = None) -> st
     return image
 
 
-def check_ioc_instance_path(ioc_path: Path, yes: bool = False):
+def check_ioc_instance_path(ioc_path: Path):
     """
     verify that the ioc instance path is valid
     """
@@ -52,3 +52,15 @@ def check_ioc_instance_path(ioc_path: Path, yes: bool = False):
         raise typer.Exit(1)
 
     return ioc_name, ioc_path
+
+
+def generic_ioc_from_image(image_name: str) -> str:
+    """
+    return the generic IOC name from an image name
+    """
+    match = re.findall(r".*\/(.*)-.*-(?:runtime|developer)", image_name)
+    if not match:
+        log.error(f"cannot extract generic IOC name from {image_name}")
+        raise typer.Exit(1)
+
+    return match[0]