✨ Enhancement/3022/allow setting services scheduler specs (⚠️ devops) (ITISFoundation#3034)

Author: sanderegg

.env-devel

@@ -15,6 +15,8 @@ BF_API_KEY=none
 BF_API_SECRET=none
 
 CATALOG_DEV_FEATURES_ENABLED=0
+CATALOG_SERVICES_DEFAULT_RESOURCE='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}'
+CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}'
 
 DASK_SCHEDULER_HOST=dask-scheduler
 DASK_SCHEDULER_PORT=8786

.pylintrc

@@ -9,6 +9,10 @@ extension-pkg-whitelist=pydantic,orjson,ujson
 # paths.
 ignore=CVS,migration
 
+# Add files or directories matching the regex patterns to the ignore-list.
+# The regex matches against paths and can be in Posix or Windows format.
+ignore-paths=^.*/generated_models/.*$
+
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.
 ignore-patterns=
@@ -18,7 +22,7 @@ ignore-patterns=
 #init-hook=
 
 # Use multiple processes to speed up Pylint.
-jobs=4
+jobs=0
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.

Makefile

@@ -364,7 +364,7 @@ push-version: tag-version
 	python3 -m venv $@
 	## upgrading tools to latest version in $(shell python3 --version)
 	$@/bin/pip3 --quiet install --upgrade \
-		pip \
+		pip~=22.0.0 \
 		wheel \
 		setuptools
 	@$@/bin/pip3 list --verbose

packages/models-library/Makefile

@@ -31,3 +31,19 @@ project-jsonschema.ignore.json: ## creates project-v0.0.1.json for DEV purposes
 .PHONY: service-jsonschema.ignore.json
 node-meta-jsonschema.ignore.json: ## creates node-meta-v0.0.1.json for DEV purposes
 	python3 -c "from models_library.services import ServiceDockerData as cls; print(cls.schema_json(indent=2))" > $@
+
+DOCKER_API_VERSION ?= 1.41
+.PHONY: docker_rest_api.py
+docker_rest_api.py: ## auto-generates pydantic models for Docker REST API models
+	# auto-generates $@ from $<
+	@$(SCRIPTS_DIR)/openapi-pydantic-models-generator.bash \
+			--url https://docs.docker.com/engine/api/v$(DOCKER_API_VERSION).yaml \
+			--output $@
+
+	# formats
+	@black $@
+	# copy output to src/models_library/generated_models...
+	@mkdir --parents src/models_library/generated_models
+	@mv $@ src/models_library/generated_models/$@
+	@touch src/models_library/generated_models/__init__.py
+	# done

packages/models-library/src/models_library/aiodocker_api.py

@@ -0,0 +1,41 @@
+from typing import Optional
+
+from pydantic import Field, validator
+
+from .generated_models.docker_rest_api import ContainerSpec, ServiceSpec, TaskSpec
+from .utils.converters import to_snake_case
+
+
+class AioDockerContainerSpec(ContainerSpec):
+    Env: Optional[dict[str, str]] = Field(
+        None,
+        description="aiodocker expects here a dictionary and re-convert it back internally`.\n",
+    )
+
+    @validator("Env", pre=True)
+    @classmethod
+    def convert_list_to_dict(cls, v):
+        if v is not None and isinstance(v, list):
+            converted_dict = {}
+            for env in v:
+                splitted_env = f"{env}".split("=", maxsplit=1)
+                converted_dict[splitted_env[0]] = (
+                    splitted_env[1] if len(splitted_env) > 1 else None
+                )
+            return converted_dict
+        return v
+
+
+class AioDockerTaskSpec(TaskSpec):
+    ContainerSpec: Optional[AioDockerContainerSpec] = Field(
+        None,
+    )
+
+
+class AioDockerServiceSpec(ServiceSpec):
+
+    TaskTemplate: Optional[AioDockerTaskSpec] = None
+
+    class Config(ServiceSpec.Config):
+        alias_generator = to_snake_case
+        allow_population_by_field_name = True