Merge branch 'develop' into ple-global-activities

Author: pa-lem

backend/infrahub/api/schema.py

@@ -25,7 +25,7 @@
     SchemaDiff,
     SchemaUpdateValidationResult,
 )
-from infrahub.core.schema import GenericSchema, MainSchemaTypes, NodeSchema, ProfileSchema, SchemaRoot
+from infrahub.core.schema import GenericSchema, MainSchemaTypes, NodeSchema, ProfileSchema, SchemaRoot, TemplateSchema
 from infrahub.core.schema.constants import SchemaNamespace  # noqa: TC001
 from infrahub.core.validators.models.validate_migration import (
     SchemaValidateMigrationData,
@@ -87,11 +87,17 @@ class APIProfileSchema(ProfileSchema, APISchemaMixin):
     hash: str
 
 
+class APITemplateSchema(TemplateSchema, APISchemaMixin):
+    api_kind: str | None = Field(default=None, alias="kind", validate_default=True)
+    hash: str
+
+
 class SchemaReadAPI(BaseModel):
     main: str = Field(description="Main hash for the entire schema")
     nodes: list[APINodeSchema] = Field(default_factory=list)
     generics: list[APIGenericSchema] = Field(default_factory=list)
     profiles: list[APIProfileSchema] = Field(default_factory=list)
+    templates: list[APITemplateSchema] = Field(default_factory=list)
     namespaces: list[SchemaNamespace] = Field(default_factory=list)
 
 
@@ -191,6 +197,11 @@ async def get_schema(
             for value in all_schemas
             if isinstance(value, ProfileSchema) and value.namespace != "Internal"
         ],
+        templates=[
+            APITemplateSchema.from_schema(value)
+            for value in all_schemas
+            if isinstance(value, TemplateSchema) and value.namespace != "Internal"
+        ],
         namespaces=schema_branch.get_namespaces(),
     )
 
@@ -207,15 +218,16 @@ async def get_schema_summary(
 @router.get("/{schema_kind}")
 async def get_schema_by_kind(
     schema_kind: str, branch: Branch = Depends(get_branch_dep), _: AccountSession = Depends(get_current_user)
-) -> APIProfileSchema | APINodeSchema | APIGenericSchema:
+) -> APIProfileSchema | APINodeSchema | APIGenericSchema | APITemplateSchema:
     log.debug("schema_kind_request", branch=branch.name)
 
     schema = registry.schema.get(name=schema_kind, branch=branch, duplicate=False)
 
-    api_schema: dict[str, type[APIProfileSchema | APINodeSchema | APIGenericSchema]] = {
+    api_schema: dict[str, type[APIProfileSchema | APINodeSchema | APIGenericSchema | APITemplateSchema]] = {
         "profile": APIProfileSchema,
         "node": APINodeSchema,
         "generic": APIGenericSchema,
+        "template": APITemplateSchema,
     }
     key = ""
 
@@ -225,6 +237,8 @@ async def get_schema_by_kind(
         key = "node"
     if isinstance(schema, GenericSchema):
         key = "generic"
+    if isinstance(schema, TemplateSchema):
+        key = "template"
 
     return api_schema[key].from_schema(schema=schema)
 

backend/infrahub/artifacts/__init__.py

@@ -1,11 +1,9 @@
 from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
-from infrahub.message_bus import InfrahubMessage
 
-
-class CheckArtifactCreate(InfrahubMessage):
+class CheckArtifactCreate(BaseModel):
     """Runs a check to verify the creation of an artifact."""
 
     artifact_name: str = Field(..., description="Name of the artifact")

backend/infrahub/message_bus/messages/check_artifact_create.py renamed to backend/infrahub/artifacts/models.py

@@ -2,44 +2,38 @@
 
 from prefect import flow
 
+from infrahub.artifacts.models import CheckArtifactCreate
 from infrahub.core.constants import InfrahubKind, ValidatorConclusion
 from infrahub.core.timestamp import Timestamp
-from infrahub.git.repository import InfrahubReadOnlyRepository, InfrahubRepository
-from infrahub.log import get_logger
-from infrahub.message_bus import messages
+from infrahub.git import InfrahubReadOnlyRepository, InfrahubRepository
 from infrahub.services import InfrahubServices
 from infrahub.tasks.artifact import define_artifact
-from infrahub.tasks.check import set_check_status
 from infrahub.workflows.utils import add_tags
 
-log = get_logger()
-
 
 @flow(name="git-repository-check-artifact-create", flow_run_name="Check artifact creation")
-async def create(message: messages.CheckArtifactCreate, service: InfrahubServices) -> None:
-    await add_tags(branches=[message.branch_name], nodes=[message.target_id])
-    validator = await service.client.get(
-        kind=InfrahubKind.ARTIFACTVALIDATOR, id=message.validator_id, include=["checks"]
-    )
+async def create(model: CheckArtifactCreate, service: InfrahubServices) -> ValidatorConclusion:
+    await add_tags(branches=[model.branch_name], nodes=[model.target_id])
+    validator = await service.client.get(kind=InfrahubKind.ARTIFACTVALIDATOR, id=model.validator_id, include=["checks"])
 
+    repo: InfrahubReadOnlyRepository | InfrahubRepository
     if InfrahubKind.READONLYREPOSITORY:
         repo = await InfrahubReadOnlyRepository.init(
-            id=message.repository_id,
-            name=message.repository_name,
+            id=model.repository_id,
+            name=model.repository_name,
             client=service.client,
             service=service,
         )
     else:
         repo = await InfrahubRepository.init(
-            id=message.repository_id,
-            name=message.repository_name,
+            id=model.repository_id,
+            name=model.repository_name,
             client=service.client,
             service=service,
         )
 
-    artifact = await define_artifact(message=message, service=service)
+    artifact = await define_artifact(model=model, service=service)
 
-    conclusion = ValidatorConclusion.SUCCESS.value
     severity = "info"
     artifact_result: dict[str, Union[str, bool, None]] = {
         "changed": None,
@@ -50,22 +44,23 @@ async def create(message: messages.CheckArtifactCreate, service: InfrahubService
     check_message = "Failed to render artifact"
 
     try:
-        result = await repo.render_artifact(artifact=artifact, message=message)
+        result = await repo.render_artifact(artifact=artifact, message=model)
         artifact_result["changed"] = result.changed
         artifact_result["checksum"] = result.checksum
         artifact_result["artifact_id"] = result.artifact_id
         artifact_result["storage_id"] = result.storage_id
         check_message = "Artifact rendered successfully"
+        conclusion = ValidatorConclusion.SUCCESS
 
     except Exception as exc:
-        conclusion = ValidatorConclusion.FAILURE.value
         artifact.status.value = "Error"
+        await artifact.save()
         severity = "critical"
+        conclusion = ValidatorConclusion.FAILURE
         check_message += f": {str(exc)}"
-        await artifact.save()
 
     check = None
-    check_name = f"{message.artifact_name}: {message.target_name}"
+    check_name = f"{model.artifact_name}: {model.target_name}"
     existing_check = await service.client.filters(
         kind=InfrahubKind.ARTIFACTCHECK, validator__ids=validator.id, name__value=check_name
     )
@@ -74,7 +69,7 @@ async def create(message: messages.CheckArtifactCreate, service: InfrahubService
 
     if check:
         check.created_at.value = Timestamp().to_string()
-        check.conclusion.value = conclusion
+        check.conclusion.value = conclusion.value
         check.severity.value = severity
         check.changed.value = artifact_result["changed"]
         check.checksum.value = artifact_result["checksum"]
@@ -86,12 +81,12 @@ async def create(message: messages.CheckArtifactCreate, service: InfrahubService
             kind=InfrahubKind.ARTIFACTCHECK,
             data={
                 "name": check_name,
-                "origin": message.repository_id,
+                "origin": model.repository_id,
                 "kind": "ArtifactDefinition",
-                "validator": message.validator_id,
+                "validator": model.validator_id,
                 "created_at": Timestamp().to_string(),
                 "message": check_message,
-                "conclusion": conclusion,
+                "conclusion": conclusion.value,
                 "severity": severity,
                 "changed": artifact_result["changed"],
                 "checksum": artifact_result["checksum"],
@@ -101,4 +96,4 @@ async def create(message: messages.CheckArtifactCreate, service: InfrahubService
         )
         await check.save()
 
-    await set_check_status(message=message, conclusion=conclusion, service=service)
+    return conclusion

backend/infrahub/message_bus/operations/check/artifact.py renamed to backend/infrahub/artifacts/tasks.py

@@ -102,10 +102,10 @@ class RelationshipCardinalityOneChangelog(BaseModel):
     properties: dict[str, PropertyChangelog] = Field(
         default_factory=dict, description="Changes to properties of this relationship if any were made"
     )
-    _parent: ChangelogNodeParent | None = PrivateAttr(default=None)
+    _parent: ChangelogRelatedNode | None = PrivateAttr(default=None)
 
     @property
-    def parent(self) -> ChangelogNodeParent | None:
+    def parent(self) -> ChangelogRelatedNode | None:
         return self._parent
 
     @computed_field
@@ -127,7 +127,7 @@ def add_property(self, name: str, value_current: bool | str | None, value_previo
         self.properties[name] = PropertyChangelog(name=name, value=value_current, value_previous=value_previous)
 
     def set_parent(self, parent_id: str, parent_kind: str) -> None:
-        self._parent = ChangelogNodeParent(node_id=parent_id, node_kind=parent_kind)
+        self._parent = ChangelogRelatedNode(node_id=parent_id, node_kind=parent_kind)
 
     def set_parent_from_relationship(self, relationship: Relationship) -> None:
         if relationship.schema.kind == RelationshipKind.PARENT:
@@ -136,9 +136,9 @@ def set_parent_from_relationship(self, relationship: Relationship) -> None:
                 and self.peer_id
                 and self.peer_kind
             ):
-                self._parent = ChangelogNodeParent(node_id=self.peer_id, node_kind=self.peer_kind)
+                self._parent = ChangelogRelatedNode(node_id=self.peer_id, node_kind=self.peer_kind)
             elif self.peer_id_previous and self.peer_kind_previous:
-                self._parent = ChangelogNodeParent(node_id=self.peer_id_previous, node_kind=self.peer_kind_previous)
+                self._parent = ChangelogRelatedNode(node_id=self.peer_id_previous, node_kind=self.peer_kind_previous)
 
     @property
     def is_empty(self) -> bool:
@@ -200,7 +200,7 @@ def is_empty(self) -> bool:
         return not self.peers
 
 
-class ChangelogNodeParent(BaseModel):
+class ChangelogRelatedNode(BaseModel):
     node_id: str
     node_kind: str
 
@@ -217,36 +217,40 @@ class NodeChangelog(BaseModel):
         default_factory=dict
     )
 
-    _parent: ChangelogNodeParent | None = PrivateAttr(default=None)
+    _parent: ChangelogRelatedNode | None = PrivateAttr(default=None)
 
     @property
-    def parent(self) -> ChangelogNodeParent | None:
+    def parent(self) -> ChangelogRelatedNode | None:
         return self._parent
 
     @property
     def updated_fields(self) -> list[str]:
         """Return a list of update fields i.e. attributes and relationships"""
         return list(self.relationships.keys()) + list(self.attributes.keys())
 
+    @property
+    def has_changes(self) -> bool:
+        return len(self.updated_fields) > 0
+
     @property
     def root_node_id(self) -> str:
         """Return the top level node_id"""
         if self.parent:
             return self.parent.node_id
         return self.node_id
 
-    def add_parent(self, parent: ChangelogNodeParent) -> None:
+    def add_parent(self, parent: ChangelogRelatedNode) -> None:
         self._parent = parent
 
     def add_parent_from_relationship(self, parent: Relationship) -> None:
-        self._parent = ChangelogNodeParent(node_id=parent.get_peer_id(), node_kind=parent.get_peer_kind())
+        self._parent = ChangelogRelatedNode(node_id=parent.get_peer_id(), node_kind=parent.get_peer_kind())
 
     def create_relationship(self, relationship: Relationship) -> None:
         if relationship.schema.cardinality == RelationshipCardinality.ONE:
             peer_id = relationship.get_peer_id()
             peer_kind = relationship.get_peer_kind()
             if relationship.schema.kind == RelationshipKind.PARENT:
-                self._parent = ChangelogNodeParent(node_id=peer_id, node_kind=peer_kind)
+                self._parent = ChangelogRelatedNode(node_id=peer_id, node_kind=peer_kind)
             changelog_relationship = RelationshipCardinalityOneChangelog(
                 name=relationship.schema.name,
                 peer_id=peer_id,
@@ -321,6 +325,27 @@ def create_attribute(self, attribute: BaseAttribute) -> None:
         changelog_attribute.add_property(name="is_visible", value_current=attribute.is_visible, value_previous=None)
         self.attributes[changelog_attribute.name] = changelog_attribute
 
+    def get_related_nodes(self) -> list[ChangelogRelatedNode]:
+        related_nodes: dict[str, ChangelogRelatedNode] = {}
+        for relationship in self.relationships.values():
+            if isinstance(relationship, RelationshipCardinalityOneChangelog):
+                if relationship.peer_id and relationship.peer_kind:
+                    related_nodes[relationship.peer_id] = ChangelogRelatedNode(
+                        node_id=relationship.peer_id, node_kind=relationship.peer_kind
+                    )
+                if relationship.peer_id_previous and relationship.peer_kind_previous:
+                    related_nodes[relationship.peer_id_previous] = ChangelogRelatedNode(
+                        node_id=relationship.peer_id_previous, node_kind=relationship.peer_kind_previous
+                    )
+            elif isinstance(relationship, RelationshipCardinalityManyChangelog):
+                for peer in relationship.peers:
+                    related_nodes[peer.peer_id] = ChangelogRelatedNode(node_id=peer.peer_id, node_kind=peer.peer_kind)
+
+        if self.parent:
+            related_nodes[self.parent.node_id] = self.parent
+
+        return list(related_nodes.values())
+
 
 class ChangelogRelationshipMapper:
     def __init__(self, schema: RelationshipSchema) -> None:

backend/infrahub/core/changelog/models.py

@@ -225,6 +225,7 @@ class RelationshipKind(InfrahubStringEnum):
     GROUP = "Group"
     HIERARCHY = "Hierarchy"
     PROFILE = "Profile"
+    TEMPLATE = "Template"
 
 
 class RelationshipStatus(InfrahubStringEnum):
@@ -301,6 +302,7 @@ class AttributeDBNodeType(InfrahubStringEnum):
     "Lineage",
     "Schema",
     "Profile",
+    "Template",
 ]
 
 NODE_NAME_REGEX = r"^[A-Z][a-zA-Z0-9]+$"
@@ -315,3 +317,6 @@ class AttributeDBNodeType(InfrahubStringEnum):
 NAMESPACE_REGEX = r"^[A-Z][a-z0-9]+$"
 NODE_KIND_REGEX = r"^[A-Z][a-zA-Z0-9]+$"
 DEFAULT_REL_IDENTIFIER_LENGTH = 128
+
+OBJECT_TEMPLATE_RELATIONSHIP_NAME = "object_template"
+OBJECT_TEMPLATE_NAME_ATTR = "template_name"

backend/infrahub/core/constants/__init__.py

@@ -42,6 +42,7 @@
 LINEAGEOWNER = "LineageOwner"
 LINEAGESOURCE = "LineageSource"
 OBJECTPERMISSION = "CoreObjectPermission"
+OBJECTTEMPLATE = "CoreObjectTemplate"
 OBJECTTHREAD = "CoreObjectThread"
 PASSWORDCREDENTIAL = "CorePasswordCredential"
 PROFILE = "CoreProfile"

backend/infrahub/core/constants/infrahubkind.py

@@ -1,7 +1,6 @@
 from copy import deepcopy
 from dataclasses import dataclass, field, replace
 from typing import Iterable
-from uuid import uuid4
 
 from infrahub.core.constants import NULL_VALUE, DiffAction, RelationshipCardinality
 from infrahub.core.constants.database import DatabaseEdgeType
@@ -418,10 +417,16 @@ async def combine(self, earlier_diffs: EnrichedDiffs, later_diffs: EnrichedDiffs
             filtered_node_pairs = self._filter_nodes_to_keep(earlier_diff=earlier, later_diff=later)
             combined_nodes = self._combine_nodes(node_pairs=filtered_node_pairs)
             self._link_child_nodes(nodes=combined_nodes)
+            if earlier.exists_on_database:
+                diff_uuid = earlier.uuid
+                partner_uuid = earlier.partner_uuid
+            else:
+                diff_uuid = later.uuid
+                partner_uuid = later.partner_uuid
             combined_diffs.append(
                 EnrichedDiffRoot(
-                    uuid=str(uuid4()),
-                    partner_uuid=later.partner_uuid,
+                    uuid=diff_uuid,
+                    partner_uuid=partner_uuid,
                     base_branch_name=later.base_branch_name,
                     diff_branch_name=later.diff_branch_name,
                     from_time=earlier.from_time,