♻️🎨 ooil config init to create first .osparc config layout (#5913)

Author: pcrespov

packages/service-integration/requirements/_base.in

@@ -6,7 +6,9 @@
 --requirement ../../../packages/models-library/requirements/_base.in
 
 click
+cookiecutter
 docker # pytest-plugin
+jinja2_time
 jsonschema # pytest-plugin
 pytest # pytest-plugin
 pyyaml

packages/service-integration/requirements/_base.txt

@@ -1,20 +1,30 @@
 arrow==1.3.0
-    # via -r requirements/../../../packages/models-library/requirements/_base.in
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   cookiecutter
+    #   jinja2-time
 attrs==23.2.0
     # via
     #   jsonschema
     #   referencing
+binaryornot==0.4.4
+    # via cookiecutter
 certifi==2024.2.2
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   requests
+chardet==5.2.0
+    # via binaryornot
 charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via
     #   -r requirements/_base.in
+    #   cookiecutter
     #   typer
+cookiecutter==2.6.0
+    # via -r requirements/_base.in
 dnspython==2.6.1
     # via email-validator
 docker==7.1.0
@@ -29,6 +39,14 @@ idna==3.7
     #   requests
 iniconfig==2.0.0
     # via pytest
+jinja2==3.1.4
+    # via
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   cookiecutter
+    #   jinja2-time
+jinja2-time==0.2.0
+    # via -r requirements/_base.in
 jsonschema==4.22.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
@@ -37,6 +55,8 @@ jsonschema-specifications==2023.12.1
     # via jsonschema
 markdown-it-py==3.0.0
     # via rich
+markupsafe==2.1.5
+    # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
 orjson==3.10.3
@@ -59,19 +79,26 @@ pytest==8.2.0
     # via -r requirements/_base.in
 python-dateutil==2.9.0.post0
     # via arrow
+python-slugify==8.0.4
+    # via cookiecutter
 pyyaml==6.0.1
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/_base.in
+    #   cookiecutter
 referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
 requests==2.32.2
-    # via docker
+    # via
+    #   cookiecutter
+    #   docker
 rich==13.7.1
-    # via typer
+    # via
+    #   cookiecutter
+    #   typer
 rpds-py==0.18.0
     # via
     #   jsonschema
@@ -80,6 +107,8 @@ shellingham==1.5.4
     # via typer
 six==1.16.0
     # via python-dateutil
+text-unidecode==1.3
+    # via python-slugify
 tomli==2.0.1
     # via pytest
 typer==0.12.3

packages/service-integration/scripts/ooil.bash

@@ -6,7 +6,7 @@ set -o nounset
 set -o pipefail
 IFS=$'\n\t'
 
-IMAGE_NAME="${DOCKER_REGISTRY:-itisfoundation}/service-integration:${OOIL_IMAGE_TAG:-master-github-latest}"
+IMAGE_NAME="${DOCKER_REGISTRY:-local}/service-integration:${OOIL_IMAGE_TAG:-production}"
 WORKDIR="$(pwd)"
 
 #
@@ -20,6 +20,7 @@ WORKDIR="$(pwd)"
 run() {
   docker run \
     --rm \
+    --tty \
     --volume="/etc/group:/etc/group:ro" \
     --volume="/etc/passwd:/etc/passwd:ro" \
     --user="$(id --user "$USER")":"$(id --group "$USER")" \

packages/service-integration/src/service_integration/cli.py

@@ -0,0 +1,72 @@
+# Allows entrypoint via python -m as well
+
+from typing import Annotated
+
+import rich
+import typer
+
+from .._meta import __version__
+from ..settings import AppSettings
+from . import _compose_spec, _metadata, _run_creator, _test
+from ._config import config_app
+
+app = typer.Typer()
+
+
+def _version_callback(value: bool):  # noqa: FBT002
+    if value:
+        rich.print(__version__)
+        raise typer.Exit
+
+
+@app.callback()
+def main(
+    ctx: typer.Context,
+    registry_name: Annotated[
+        str,
+        typer.Option(
+            "--REGISTRY_NAME",
+            help="image registry name. Full url or prefix used as prefix in an image name",
+        ),
+    ] = None,
+    compose_version: Annotated[
+        str,
+        typer.Option(
+            "--COMPOSE_VERSION",
+            help="version used for docker compose specification",
+        ),
+    ] = None,
+    version: Annotated[  # noqa: FBT002
+        bool,
+        typer.Option(
+            "--version",
+            callback=_version_callback,
+            is_eager=True,
+        ),
+    ] = False,
+):
+    """o2s2parc service Integration Library (OOIL in short)"""
+    assert isinstance(version, bool | None)  # nosec
+
+    overrides = {}
+    if registry_name:
+        overrides["REGISTRY_NAME"] = registry_name
+
+    if compose_version:
+        overrides["COMPOSE_VERSION"] = compose_version
+
+    # save states
+    ctx.settings = AppSettings.parse_obj(overrides)
+
+
+#
+# REGISTER commands and/or sub-apps
+#
+
+app.command("compose")(_compose_spec.create_compose)
+app.add_typer(config_app, name="config", help="Manage osparc config files")
+app.command("test")(_test.run_tests)
+# legacy
+app.command("bump-version")(_metadata.bump_version)
+app.command("get-version")(_metadata.get_version)
+app.command("run-creator")(_run_creator.run_creator)

packages/service-integration/src/service_integration/cli/__init__.py

@@ -1,14 +1,16 @@
 import subprocess
-from datetime import datetime
 from pathlib import Path
+from typing import Annotated
 
+import arrow
 import rich
 import typer
 import yaml
 from models_library.utils.labels_annotations import to_labels
 from rich.console import Console
 
 from ..compose_spec_model import ComposeSpecification
+from ..errors import UndefinedOciImageSpecError
 from ..oci_image_spec import LS_LABEL_PREFIX, OCI_LABEL_PREFIX
 from ..osparc_config import (
     OSPARC_CONFIG_DIRNAME,
@@ -61,10 +63,10 @@ def create_docker_compose_image_spec(
 
     config_basedir = meta_config_path.parent
 
-    # required
+    # REQUIRED
     meta_cfg = MetadataConfig.from_yaml(meta_config_path)
 
-    # required
+    # REQUIRED
     if docker_compose_overwrite_path:
         docker_compose_overwrite_cfg = DockerComposeOverwriteConfig.from_yaml(
             docker_compose_overwrite_path
@@ -74,11 +76,10 @@ def create_docker_compose_image_spec(
             service_name=meta_cfg.service_name()
         )
 
-    # optional
+    # OPTIONAL
     runtime_cfg = None
     if service_config_path:
         try:
-            # TODO: should include default?
             runtime_cfg = RuntimeConfig.from_yaml(service_config_path)
         except FileNotFoundError:
             rich.print("No runtime config found (optional), using default.")
@@ -90,13 +91,11 @@ def create_docker_compose_image_spec(
             (config_basedir / f"{OCI_LABEL_PREFIX}.yml").read_text()
         )
         if not oci_spec:
-            msg = "Undefined OCI image spec"
-            raise ValueError(msg)
+            raise UndefinedOciImageSpecError
 
         oci_labels = to_labels(oci_spec, prefix_key=OCI_LABEL_PREFIX)
         extra_labels.update(oci_labels)
-    except (FileNotFoundError, ValueError):
-
+    except (FileNotFoundError, UndefinedOciImageSpecError):
         try:
             # if not OCI, try label-schema
             ls_spec = yaml.safe_load(
@@ -109,9 +108,11 @@ def create_docker_compose_image_spec(
                 "No explicit config for OCI/label-schema found (optional), skipping OCI annotations."
             )
     # add required labels
-    extra_labels[f"{LS_LABEL_PREFIX}.build-date"] = datetime.utcnow().strftime(
-        "%Y-%m-%dT%H:%M:%SZ"
-    )
+
+    # SEE https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+    # Format the datetime object as a string following RFC-3339
+    rfc3339_format = arrow.now().format("YYYY-MM-DDTHH:mm:ssZ")
+    extra_labels[f"{LS_LABEL_PREFIX}.build-date"] = rfc3339_format
     extra_labels[f"{LS_LABEL_PREFIX}.schema-version"] = "1.0"
 
     extra_labels[f"{LS_LABEL_PREFIX}.vcs-ref"] = _run_git_or_empty_string(
@@ -130,25 +131,28 @@ def create_docker_compose_image_spec(
     )
 
 
-def main(
+def create_compose(
     ctx: typer.Context,
-    config_path: Path = typer.Option(
-        OSPARC_CONFIG_DIRNAME,
-        "-m",
-        "--metadata",
-        help="osparc config file or folder. "
-        "If the latter, it will scan for configs using the glob pattern 'config_path/**/metadata.yml' ",
-    ),
-    to_spec_file: Path = typer.Option(
-        Path("docker-compose.yml"),
-        "-f",
-        "--to-spec-file",
-        help="Output docker-compose image spec",
-    ),
+    config_path: Annotated[
+        Path,
+        typer.Option(
+            "-m",
+            "--metadata",
+            help="osparc config file or folder. "
+            "If the latter, it will scan for configs using the glob pattern 'config_path/**/metadata.yml' ",
+        ),
+    ] = Path(OSPARC_CONFIG_DIRNAME),
+    to_spec_file: Annotated[
+        Path,
+        typer.Option(
+            "-f",
+            "--to-spec-file",
+            help="Output docker-compose image spec",
+        ),
+    ] = Path("docker-compose.yml"),
 ):
-    """create docker image/runtime compose-specs from an osparc config"""
+    """Creates the docker image/runtime compose-spec file from an .osparc config"""
 
-    # TODO: all these MUST be replaced by osparc_config.ConfigFilesStructure
     if not config_path.exists():
         msg = "Invalid path to metadata file or folder"
         raise typer.BadParameter(msg)
@@ -168,10 +172,10 @@ def main(
         config_name = meta_config.parent.name
         configs_kwargs_map[config_name] = {}
 
-        # load meta [required]
+        # load meta REQUIRED
         configs_kwargs_map[config_name]["meta_config_path"] = meta_config
 
-        # others [optional]
+        # others OPTIONAL
         for file_name, arg_name in (
             ("docker-compose.overwrite.yml", "docker_compose_overwrite_path"),
             ("runtime.yml", "service_config_path"),
@@ -194,7 +198,6 @@ def main(
             settings, **configs_kwargs_map[config_name]
         ).dict(exclude_unset=True)
 
-        # FIXME: shaky! why first decides ??
         if n == 0:
             compose_spec_dict = nth_compose_spec
         else: