Custom release_tags support

Make all packages ignore unknown tags
Introduce specific meta types for docker
Add module_versions into args and docker meta
Make docker accept arbitrary release tag

Author: Peter-Sh

src/redis_release/bht/behaviours_docker.py

@@ -1,22 +1,19 @@
+from pydoc import Doc
 from typing import Optional
 
 from py_trees.common import Status
 
-from redis_release.bht.behaviours import LoggingAction, ReleaseAction
-from redis_release.bht.state import PackageMeta, ReleaseMeta, Workflow
-from redis_release.models import RedisVersion, ReleaseType
+from ..models import RedisVersion, ReleaseType
+from .behaviours import LoggingAction, ReleaseAction
+from .state import DockerMeta, ReleaseMeta, Workflow
 
 
 class DockerWorkflowInputs(ReleaseAction):
-    """
-    Docker uses only release_tag input which is set automatically in TriggerWorkflow
-    """
-
     def __init__(
         self,
         name: str,
         workflow: Workflow,
-        package_meta: PackageMeta,
+        package_meta: DockerMeta,
         release_meta: ReleaseMeta,
         log_prefix: str = "",
     ) -> None:
@@ -26,6 +23,9 @@ def __init__(
         super().__init__(name=name, log_prefix=log_prefix)
 
     def update(self) -> Status:
+        if self.package_meta.module_versions is not None:
+            for module, version in self.package_meta.module_versions.items():
+                self.workflow.inputs[f"{module.value}_version"] = version
         return Status.SUCCESS
 
 
@@ -35,7 +35,7 @@ class DetectReleaseTypeDocker(LoggingAction):
     def __init__(
         self,
         name: str,
-        package_meta: PackageMeta,
+        package_meta: DockerMeta,
         release_meta: ReleaseMeta,
         log_prefix: str = "",
     ) -> None:
@@ -52,7 +52,14 @@ def initialise(self) -> None:
             return
         if self.release_meta.tag == "unstable":
             return
-        self.release_version = RedisVersion.parse(self.release_meta.tag)
+        try:
+            self.release_version = RedisVersion.parse(self.release_meta.tag)
+        except ValueError as e:
+            if self.release_meta.tag != "":
+                self.logger.info(
+                    f"Failed to parse release tag: {e}, assuming custom release with tag {self.release_meta.tag}"
+                )
+            return
 
     def update(self) -> Status:
         result: Status = Status.FAILURE
@@ -70,8 +77,9 @@ def update(self) -> Status:
                 f"Detected release type for docker: {self.package_meta.release_type}"
             )
         else:
-            self.feedback_message = "Failed to detect release type"
-            result = Status.FAILURE
+            self.package_meta.release_type = ReleaseType.INTERNAL
+            self.feedback_message = "Set release type to internal for custom build"
+            result = Status.SUCCESS
 
         if self.log_once(
             "release_type_detected", self.package_meta.ephemeral.log_once_flags
@@ -92,7 +100,7 @@ class NeedToReleaseDocker(LoggingAction):
     def __init__(
         self,
         name: str,
-        package_meta: PackageMeta,
+        package_meta: DockerMeta,
         release_meta: ReleaseMeta,
         log_prefix: str = "",
     ) -> None:
@@ -123,9 +131,6 @@ def update(self) -> Status:
         if self.release_meta.tag is None:
             self.feedback_message = "Release tag is not set"
             result = Status.FAILURE
-        if self.release_meta.tag == "unstable":
-            self.feedback_message = "Skip unstable release for docker"
-            result = Status.FAILURE
 
         if self.release_version is not None:
             if self.release_version.major < 8:
@@ -138,6 +143,9 @@ def update(self) -> Status:
                     f"Need to release docker version {str(self.release_version)}"
                 )
                 result = Status.SUCCESS
+        else:
+            self.feedback_message = "Custom build, need to release"
+            result = Status.SUCCESS
 
         if self.log_once("need_to_release", self.package_meta.ephemeral.log_once_flags):
             color_open = "" if result == Status.SUCCESS else "[yellow]"

src/redis_release/bht/behaviours_homebrew.py

@@ -82,6 +82,12 @@ def update(self) -> Status:
                     self.logger.info(self.feedback_message)
                 return Status.SUCCESS
 
+            if self.release_version is None and self.release_meta.tag != "":
+                self.logger.info(
+                    f"Release version is not set, skipping probably custom release {self.release_meta.tag}"
+                )
+                return Status.SUCCESS
+
             assert self.release_version is not None
             if self.package_meta.release_type is None:
                 if self.release_version.is_internal:
@@ -147,6 +153,12 @@ def initialise(self) -> None:
             self.package_meta.remote_version = "unstable"
             return
 
+        if self.release_meta.tag != "":
+            self.package_meta.ephemeral.is_version_acceptable = False
+            # we need to set remote version to not None as it is a sign of successful classify step
+            self.package_meta.remote_version = "custom"
+            return
+
         self.feedback_message = ""
         # Validate homebrew_channel is set
         if self.package_meta.homebrew_channel is None:

src/redis_release/bht/behaviours_snap.py

@@ -86,6 +86,11 @@ def update(self) -> Status:
                 self.package_meta.release_type = ReleaseType.PUBLIC
                 self.package_meta.snap_risk_level = SnapRiskLevel.EDGE
             else:
+                if self.release_version is None and self.release_meta.tag != "":
+                    self.logger.info(
+                        f"Release version is not set, skipping probably custom release: {self.release_meta.tag}"
+                    )
+                    return Status.SUCCESS
                 assert self.release_version is not None
                 if self.package_meta.release_type is None:
                     if self.release_version.is_internal:
@@ -144,6 +149,18 @@ def initialise(self) -> None:
         if self.package_meta.ephemeral.is_version_acceptable is not None:
             return
 
+        # TODO: don't like this expresion, should be better way to handle custom releases skipping
+        if (
+            self.release_meta.tag is not None
+            and self.release_meta.tag != ""
+            and self.release_meta.tag != "unstable"
+            and self.package_meta.snap_risk_level is None
+        ):
+            self.package_meta.ephemeral.is_version_acceptable = False
+            # we need to set remote version to not None as it is a sign of successful classify step
+            self.package_meta.remote_version = "custom"
+            return
+
         self.feedback_message = ""
         # Validate snap_risk_level is set
         if self.package_meta.snap_risk_level is None:

src/redis_release/bht/state.py

@@ -1,3 +1,60 @@
+"""State of release - central data model for the release process.
+
+Used by the behavior tree to keep track of the release progress, plays
+blackboard role for behavior tree.
+
+Ephemeral and persistent fields
+--------------------------------
+
+The main purpose of ephemeral fields is to prevent retry loops and to allow
+extensive status reporting.
+
+Each workflow step has a pair of fields indicating the step status:
+One ephemeral field is set when the step is attempted. It may have four states:
+- `None` (default): Step has not been attempted
+- `common.Status.RUNNING`: Step is currently running
+- `common.Status.FAILURE`: Step has been attempted and failed
+- `common.Status.SUCCESS`: Step has been attempted and succeeded
+
+Ephemeral fields are reset on each run. Their values are persisted but only until
+next run is started.
+So they indicate either current (if run is in progress) or last run state.
+
+The other field indicates the step result, it may either have some value or be empty.
+This field is persisted across runs.
+
+For example for trigger step we have `trigger_workflow` ephemeral
+and `triggered_at` result fields.
+
+Optional message field may be used to provide additional information about the step.
+For example wait_for_completion_message may contain information about timeout.
+
+Given combination of ephemeral and result fields we can determine step status.
+Each step may be in one of the following states:
+    Not started
+    Failed
+    Succeeded or OK
+    Incorrect (this shouln't happen)
+
+The following decision table show how step status is determined for trigger step.
+In general this is applicable to all steps.
+
+tigger_workflow -> | None (default) |     Running    |   Failure   |  Success   |
+triggered_at:      |                |                |             |            |
+   None            |   Not started  |   In progress  |    Failed   |  Incorrect |
+  Has value        |       OK       |    Incorrect   |  Incorrect  |     OK     |
+
+The result field (triggered_at in this case) should not be set while step is
+running, if step was not started or if it's failed.
+And it should be set if trigger_workflow is successful.
+It may be set if trigger_workflow is None, which is the case when release
+process was restarted and all ephemeral fields are reset, but the particular
+step was successful in previous run.
+
+Correct values are not eforced it's up to the implementation to correctly
+set the fields.
+"""
+
 import json
 import logging
 from datetime import datetime
@@ -8,72 +65,25 @@
 from py_trees.common import Status
 from pydantic import BaseModel, Field
 
-from redis_release.models import (
+from ..config import Config, PackageConfig
+from ..models import (
     HomebrewChannel,
     PackageType,
+    RedisModule,
     ReleaseType,
     SnapRiskLevel,
     WorkflowConclusion,
     WorkflowStatus,
     WorkflowType,
 )
 
-from ..config import Config, PackageConfig
-
 logger = logging.getLogger(__name__)
 
-SUPPORTED_STATE_VERSION = 3
+SUPPORTED_STATE_VERSION = 4
 
 
 class WorkflowEphemeral(BaseModel):
-    """Ephemeral workflow state. Reset on each run.
-
-    The main purpose of ephemeral fields is to prevent retry loops and to allow extensive status reporting.
-
-    Each workflow step has a pair of fields indicating the step status:
-    One ephemeral field is set when the step is attempted. It may have four states:
-    - `None` (default): Step has not been attempted
-    - `common.Status.RUNNING`: Step is currently running
-    - `common.Status.FAILURE`: Step has been attempted and failed
-    - `common.Status.SUCCESS`: Step has been attempted and succeeded
-
-    Ephemeral fields are reset on each run. Their values are persisted but only until
-    next run is started.
-    So they indicate either current (if run is in progress) or last run state.
-
-    The other field indicates the step result, it may either have some value or be empty.
-
-    For example for trigger step we have `trigger_workflow` ephemeral
-    and `triggered_at` result fields.
-
-    Optional message field may be used to provide additional information about the step.
-    For example wait_for_completion_message may contain information about timeout.
-
-    Given combination of ephemeral and result fields we can determine step status.
-    Each step may be in one of the following states:
-        Not started
-        Failed
-        Succeeded or OK
-        Incorrect (this shouln't happen)
-
-    The following decision table show how step status is determined for trigger step.
-    In general this is applicable to all steps.
-
-    tigger_workflow -> | None (default) |     Running    |   Failure   |  Success   |
-    triggered_at:      |                |                |             |            |
-       None            |   Not started  |   In progress  |    Failed   |  Incorrect |
-      Has value        |       OK       |    Incorrect   |  Incorrect  |     OK     |
-
-    The result field (triggered_at in this case) should not be set while step is
-    running, if step was not started or if it's failed.
-    And it should be set if trigger_workflow is successful.
-    It may be set if trigger_workflow is None, which is the case when release
-    process was restarted and all ephemeral fields are reset, but the particular
-    step was successful in previous run.
-
-    Correct values are not eforced it's up to the implementation to correctly
-    set the fields.
-    """
+    """Ephemeral workflow state. Reset on each run."""
 
     identify_workflow: Optional[common.Status] = None
     trigger_workflow: Optional[common.Status] = None
@@ -138,6 +148,10 @@ class SnapMetaEphemeral(PackageMetaEphemeral):
     pass
 
 
+class DockerMetaEphemeral(PackageMetaEphemeral):
+    pass
+
+
 class PackageMeta(BaseModel):
     """Metadata for a package (base/generic type)."""
 
@@ -176,6 +190,14 @@ class SnapMeta(PackageMeta):
     ephemeral: SnapMetaEphemeral = Field(default_factory=SnapMetaEphemeral)  # type: ignore[assignment]
 
 
+class DockerMeta(PackageMeta):
+    """Metadata for Docker package."""
+
+    serialization_hint: Literal["docker"] = "docker"  # type: ignore[assignment]
+    module_versions: Optional[Dict[RedisModule, str]] = None
+    ephemeral: DockerMetaEphemeral = Field(default_factory=DockerMetaEphemeral)  # type: ignore[assignment]
+
+
 class Package(BaseModel):
     """State for a package in the release.
 
@@ -186,7 +208,7 @@ class Package(BaseModel):
     - serialization_hint="snap" -> SnapMeta
     """
 
-    meta: Union[HomebrewMeta, SnapMeta, PackageMeta] = Field(
+    meta: Union[HomebrewMeta, SnapMeta, PackageMeta, DockerMeta] = Field(
         default_factory=PackageMeta, discriminator="serialization_hint"
     )
     build: Workflow = Field(default_factory=Workflow)
@@ -227,7 +249,7 @@ def _create_package_meta_from_config(
             package_config: Package configuration
 
         Returns:
-            PackageMeta subclass instance (HomebrewMeta, SnapMeta, or PackageMeta)
+            PackageMeta subclass instance (HomebrewMeta, SnapMeta, DockerMeta or PackageMeta)
 
         Raises:
             ValueError: If package_type is None
@@ -246,6 +268,13 @@ def _create_package_meta_from_config(
                 package_type=package_config.package_type,
                 publish_internal_release=package_config.publish_internal_release,
             )
+        elif package_config.package_type == PackageType.DOCKER:
+            return DockerMeta(
+                repo=package_config.repo,
+                ref=package_config.ref,
+                package_type=package_config.package_type,
+                publish_internal_release=package_config.publish_internal_release,
+            )
         elif package_config.package_type is not None:
             return PackageMeta(
                 repo=package_config.repo,