Merge branch 'develop' of github.com:opsmill/infrahub into ple-global-activities

Author: pa-lem

CHANGELOG.md

@@ -11,6 +11,28 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang
 
 <!-- towncrier release notes start -->
 
+## [Infrahub - v1.1.7](https://github.com/opsmill/infrahub/tree/infrahub-v1.1.7) - 2025-02-18
+
+### Added
+
+- Data diffs are loaded in sequential batches for faster performance with large changes.
+- The diff tree and diff list can now be scrolled independently.
+
+### Changed
+
+- Modified node mutation events to not send metadata properties as part of the mutation payload. The reason is that the property lookup was time consuming. This information will return again in Infrahub 1.2 with a completely updated format. ([#5664](https://github.com/opsmill/infrahub/issues/5664))
+
+### Fixed
+
+- Fix nodes remaining in the database after a create mutation fails when using pools. ([#4303](https://github.com/opsmill/infrahub/issues/4303))
+- Modify the query for the current tasks, ensuring the correct determination of the merge button state. ([#5565](https://github.com/opsmill/infrahub/issues/5565))
+- Fix Docker `task-manager-db` PostgreSQL health check test by adding database and user parameters. ([#5739](https://github.com/opsmill/infrahub/issues/5739))
+- Fixed issue causing a gap in menu sidebar when text is too long.
+- Prevent avatar from being cut off in menu sidebar.
+- Enforce permission checks when using relationship add or delete mutation.
+- Enhance the data integrity checks UI to enable navigation from the check to the diff view.
+- Improved performance when updating an existing diff.
+
 ## [Infrahub - v1.1.6](https://github.com/opsmill/infrahub/tree/infrahub-v1.1.6) - 2025-01-30
 
 ### Artifact improvements

backend/infrahub/api/schema.py

@@ -25,7 +25,7 @@
     SchemaDiff,
     SchemaUpdateValidationResult,
 )
-from infrahub.core.schema import GenericSchema, MainSchemaTypes, NodeSchema, ProfileSchema, SchemaRoot
+from infrahub.core.schema import GenericSchema, MainSchemaTypes, NodeSchema, ProfileSchema, SchemaRoot, TemplateSchema
 from infrahub.core.schema.constants import SchemaNamespace  # noqa: TC001
 from infrahub.core.validators.models.validate_migration import (
     SchemaValidateMigrationData,
@@ -87,11 +87,17 @@ class APIProfileSchema(ProfileSchema, APISchemaMixin):
     hash: str
 
 
+class APITemplateSchema(TemplateSchema, APISchemaMixin):
+    api_kind: str | None = Field(default=None, alias="kind", validate_default=True)
+    hash: str
+
+
 class SchemaReadAPI(BaseModel):
     main: str = Field(description="Main hash for the entire schema")
     nodes: list[APINodeSchema] = Field(default_factory=list)
     generics: list[APIGenericSchema] = Field(default_factory=list)
     profiles: list[APIProfileSchema] = Field(default_factory=list)
+    templates: list[APITemplateSchema] = Field(default_factory=list)
     namespaces: list[SchemaNamespace] = Field(default_factory=list)
 
 
@@ -191,6 +197,11 @@ async def get_schema(
             for value in all_schemas
             if isinstance(value, ProfileSchema) and value.namespace != "Internal"
         ],
+        templates=[
+            APITemplateSchema.from_schema(value)
+            for value in all_schemas
+            if isinstance(value, TemplateSchema) and value.namespace != "Internal"
+        ],
         namespaces=schema_branch.get_namespaces(),
     )
 
@@ -207,15 +218,16 @@ async def get_schema_summary(
 @router.get("/{schema_kind}")
 async def get_schema_by_kind(
     schema_kind: str, branch: Branch = Depends(get_branch_dep), _: AccountSession = Depends(get_current_user)
-) -> APIProfileSchema | APINodeSchema | APIGenericSchema:
+) -> APIProfileSchema | APINodeSchema | APIGenericSchema | APITemplateSchema:
     log.debug("schema_kind_request", branch=branch.name)
 
     schema = registry.schema.get(name=schema_kind, branch=branch, duplicate=False)
 
-    api_schema: dict[str, type[APIProfileSchema | APINodeSchema | APIGenericSchema]] = {
+    api_schema: dict[str, type[APIProfileSchema | APINodeSchema | APIGenericSchema | APITemplateSchema]] = {
         "profile": APIProfileSchema,
         "node": APINodeSchema,
         "generic": APIGenericSchema,
+        "template": APITemplateSchema,
     }
     key = ""
 
@@ -225,6 +237,8 @@ async def get_schema_by_kind(
         key = "node"
     if isinstance(schema, GenericSchema):
         key = "generic"
+    if isinstance(schema, TemplateSchema):
+        key = "template"
 
     return api_schema[key].from_schema(schema=schema)
 

backend/infrahub/artifacts/__init__.py

@@ -1,11 +1,9 @@
 from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
-from infrahub.message_bus import InfrahubMessage
 
-
-class CheckArtifactCreate(InfrahubMessage):
+class CheckArtifactCreate(BaseModel):
     """Runs a check to verify the creation of an artifact."""
 
     artifact_name: str = Field(..., description="Name of the artifact")

backend/infrahub/message_bus/messages/check_artifact_create.py renamed to backend/infrahub/artifacts/models.py

@@ -2,44 +2,38 @@
 
 from prefect import flow
 
+from infrahub.artifacts.models import CheckArtifactCreate
 from infrahub.core.constants import InfrahubKind, ValidatorConclusion
 from infrahub.core.timestamp import Timestamp
-from infrahub.git.repository import InfrahubReadOnlyRepository, InfrahubRepository
-from infrahub.log import get_logger
-from infrahub.message_bus import messages
+from infrahub.git import InfrahubReadOnlyRepository, InfrahubRepository
 from infrahub.services import InfrahubServices
 from infrahub.tasks.artifact import define_artifact
-from infrahub.tasks.check import set_check_status
 from infrahub.workflows.utils import add_tags
 
-log = get_logger()
-
 
 @flow(name="git-repository-check-artifact-create", flow_run_name="Check artifact creation")
-async def create(message: messages.CheckArtifactCreate, service: InfrahubServices) -> None:
-    await add_tags(branches=[message.branch_name], nodes=[message.target_id])
-    validator = await service.client.get(
-        kind=InfrahubKind.ARTIFACTVALIDATOR, id=message.validator_id, include=["checks"]
-    )
+async def create(model: CheckArtifactCreate, service: InfrahubServices) -> ValidatorConclusion:
+    await add_tags(branches=[model.branch_name], nodes=[model.target_id])
+    validator = await service.client.get(kind=InfrahubKind.ARTIFACTVALIDATOR, id=model.validator_id, include=["checks"])
 
+    repo: InfrahubReadOnlyRepository | InfrahubRepository
     if InfrahubKind.READONLYREPOSITORY:
         repo = await InfrahubReadOnlyRepository.init(
-            id=message.repository_id,
-            name=message.repository_name,
+            id=model.repository_id,
+            name=model.repository_name,
             client=service.client,
             service=service,
         )
     else:
         repo = await InfrahubRepository.init(
-            id=message.repository_id,
-            name=message.repository_name,
+            id=model.repository_id,
+            name=model.repository_name,
             client=service.client,
             service=service,
         )
 
-    artifact = await define_artifact(message=message, service=service)
+    artifact = await define_artifact(model=model, service=service)
 
-    conclusion = ValidatorConclusion.SUCCESS.value
     severity = "info"
     artifact_result: dict[str, Union[str, bool, None]] = {
         "changed": None,
@@ -50,22 +44,23 @@ async def create(message: messages.CheckArtifactCreate, service: InfrahubService
     check_message = "Failed to render artifact"
 
     try:
-        result = await repo.render_artifact(artifact=artifact, message=message)
+        result = await repo.render_artifact(artifact=artifact, message=model)
         artifact_result["changed"] = result.changed
         artifact_result["checksum"] = result.checksum
         artifact_result["artifact_id"] = result.artifact_id
         artifact_result["storage_id"] = result.storage_id
         check_message = "Artifact rendered successfully"
+        conclusion = ValidatorConclusion.SUCCESS
 
     except Exception as exc:
-        conclusion = ValidatorConclusion.FAILURE.value
         artifact.status.value = "Error"
+        await artifact.save()
         severity = "critical"
+        conclusion = ValidatorConclusion.FAILURE
         check_message += f": {str(exc)}"
-        await artifact.save()
 
     check = None
-    check_name = f"{message.artifact_name}: {message.target_name}"
+    check_name = f"{model.artifact_name}: {model.target_name}"
     existing_check = await service.client.filters(
         kind=InfrahubKind.ARTIFACTCHECK, validator__ids=validator.id, name__value=check_name
     )
@@ -74,7 +69,7 @@ async def create(message: messages.CheckArtifactCreate, service: InfrahubService
 
     if check:
         check.created_at.value = Timestamp().to_string()
-        check.conclusion.value = conclusion
+        check.conclusion.value = conclusion.value
         check.severity.value = severity
         check.changed.value = artifact_result["changed"]
         check.checksum.value = artifact_result["checksum"]
@@ -86,12 +81,12 @@ async def create(message: messages.CheckArtifactCreate, service: InfrahubService
             kind=InfrahubKind.ARTIFACTCHECK,
             data={
                 "name": check_name,
-                "origin": message.repository_id,
+                "origin": model.repository_id,
                 "kind": "ArtifactDefinition",
-                "validator": message.validator_id,
+                "validator": model.validator_id,
                 "created_at": Timestamp().to_string(),
                 "message": check_message,
-                "conclusion": conclusion,
+                "conclusion": conclusion.value,
                 "severity": severity,
                 "changed": artifact_result["changed"],
                 "checksum": artifact_result["checksum"],
@@ -101,4 +96,4 @@ async def create(message: messages.CheckArtifactCreate, service: InfrahubService
         )
         await check.save()
 
-    await set_check_status(message=message, conclusion=conclusion, service=service)
+    return conclusion

backend/infrahub/message_bus/operations/check/artifact.py renamed to backend/infrahub/artifacts/tasks.py

@@ -102,10 +102,10 @@ class RelationshipCardinalityOneChangelog(BaseModel):
     properties: dict[str, PropertyChangelog] = Field(
         default_factory=dict, description="Changes to properties of this relationship if any were made"
     )
-    _parent: ChangelogNodeParent | None = PrivateAttr(default=None)
+    _parent: ChangelogRelatedNode | None = PrivateAttr(default=None)
 
     @property
-    def parent(self) -> ChangelogNodeParent | None:
+    def parent(self) -> ChangelogRelatedNode | None:
         return self._parent
 
     @computed_field
@@ -127,7 +127,7 @@ def add_property(self, name: str, value_current: bool | str | None, value_previo
         self.properties[name] = PropertyChangelog(name=name, value=value_current, value_previous=value_previous)
 
     def set_parent(self, parent_id: str, parent_kind: str) -> None:
-        self._parent = ChangelogNodeParent(node_id=parent_id, node_kind=parent_kind)
+        self._parent = ChangelogRelatedNode(node_id=parent_id, node_kind=parent_kind)
 
     def set_parent_from_relationship(self, relationship: Relationship) -> None:
         if relationship.schema.kind == RelationshipKind.PARENT:
@@ -136,9 +136,9 @@ def set_parent_from_relationship(self, relationship: Relationship) -> None:
                 and self.peer_id
                 and self.peer_kind
             ):
-                self._parent = ChangelogNodeParent(node_id=self.peer_id, node_kind=self.peer_kind)
+                self._parent = ChangelogRelatedNode(node_id=self.peer_id, node_kind=self.peer_kind)
             elif self.peer_id_previous and self.peer_kind_previous:
-                self._parent = ChangelogNodeParent(node_id=self.peer_id_previous, node_kind=self.peer_kind_previous)
+                self._parent = ChangelogRelatedNode(node_id=self.peer_id_previous, node_kind=self.peer_kind_previous)
 
     @property
     def is_empty(self) -> bool:
@@ -200,7 +200,7 @@ def is_empty(self) -> bool:
         return not self.peers
 
 
-class ChangelogNodeParent(BaseModel):
+class ChangelogRelatedNode(BaseModel):
     node_id: str
     node_kind: str
 
@@ -217,36 +217,40 @@ class NodeChangelog(BaseModel):
         default_factory=dict
     )
 
-    _parent: ChangelogNodeParent | None = PrivateAttr(default=None)
+    _parent: ChangelogRelatedNode | None = PrivateAttr(default=None)
 
     @property
-    def parent(self) -> ChangelogNodeParent | None:
+    def parent(self) -> ChangelogRelatedNode | None:
         return self._parent
 
     @property
     def updated_fields(self) -> list[str]:
         """Return a list of update fields i.e. attributes and relationships"""
         return list(self.relationships.keys()) + list(self.attributes.keys())
 
+    @property
+    def has_changes(self) -> bool:
+        return len(self.updated_fields) > 0
+
     @property
     def root_node_id(self) -> str:
         """Return the top level node_id"""
         if self.parent:
             return self.parent.node_id
         return self.node_id
 
-    def add_parent(self, parent: ChangelogNodeParent) -> None:
+    def add_parent(self, parent: ChangelogRelatedNode) -> None:
         self._parent = parent
 
     def add_parent_from_relationship(self, parent: Relationship) -> None:
-        self._parent = ChangelogNodeParent(node_id=parent.get_peer_id(), node_kind=parent.get_peer_kind())
+        self._parent = ChangelogRelatedNode(node_id=parent.get_peer_id(), node_kind=parent.get_peer_kind())
 
     def create_relationship(self, relationship: Relationship) -> None:
         if relationship.schema.cardinality == RelationshipCardinality.ONE:
             peer_id = relationship.get_peer_id()
             peer_kind = relationship.get_peer_kind()
             if relationship.schema.kind == RelationshipKind.PARENT:
-                self._parent = ChangelogNodeParent(node_id=peer_id, node_kind=peer_kind)
+                self._parent = ChangelogRelatedNode(node_id=peer_id, node_kind=peer_kind)
             changelog_relationship = RelationshipCardinalityOneChangelog(
                 name=relationship.schema.name,
                 peer_id=peer_id,
@@ -321,6 +325,27 @@ def create_attribute(self, attribute: BaseAttribute) -> None:
         changelog_attribute.add_property(name="is_visible", value_current=attribute.is_visible, value_previous=None)
         self.attributes[changelog_attribute.name] = changelog_attribute
 
+    def get_related_nodes(self) -> list[ChangelogRelatedNode]:
+        related_nodes: dict[str, ChangelogRelatedNode] = {}
+        for relationship in self.relationships.values():
+            if isinstance(relationship, RelationshipCardinalityOneChangelog):
+                if relationship.peer_id and relationship.peer_kind:
+                    related_nodes[relationship.peer_id] = ChangelogRelatedNode(
+                        node_id=relationship.peer_id, node_kind=relationship.peer_kind
+                    )
+                if relationship.peer_id_previous and relationship.peer_kind_previous:
+                    related_nodes[relationship.peer_id_previous] = ChangelogRelatedNode(
+                        node_id=relationship.peer_id_previous, node_kind=relationship.peer_kind_previous
+                    )
+            elif isinstance(relationship, RelationshipCardinalityManyChangelog):
+                for peer in relationship.peers:
+                    related_nodes[peer.peer_id] = ChangelogRelatedNode(node_id=peer.peer_id, node_kind=peer.peer_kind)
+
+        if self.parent:
+            related_nodes[self.parent.node_id] = self.parent
+
+        return list(related_nodes.values())
+
 
 class ChangelogRelationshipMapper:
     def __init__(self, schema: RelationshipSchema) -> None:

backend/infrahub/core/changelog/models.py

@@ -225,6 +225,7 @@ class RelationshipKind(InfrahubStringEnum):
     GROUP = "Group"
     HIERARCHY = "Hierarchy"
     PROFILE = "Profile"
+    TEMPLATE = "Template"
 
 
 class RelationshipStatus(InfrahubStringEnum):
@@ -301,6 +302,7 @@ class AttributeDBNodeType(InfrahubStringEnum):
     "Lineage",
     "Schema",
     "Profile",
+    "Template",
 ]
 
 NODE_NAME_REGEX = r"^[A-Z][a-zA-Z0-9]+$"
@@ -315,3 +317,6 @@ class AttributeDBNodeType(InfrahubStringEnum):
 NAMESPACE_REGEX = r"^[A-Z][a-z0-9]+$"
 NODE_KIND_REGEX = r"^[A-Z][a-zA-Z0-9]+$"
 DEFAULT_REL_IDENTIFIER_LENGTH = 128
+
+OBJECT_TEMPLATE_RELATIONSHIP_NAME = "object_template"
+OBJECT_TEMPLATE_NAME_ATTR = "template_name"

backend/infrahub/core/constants/__init__.py

@@ -42,6 +42,7 @@
 LINEAGEOWNER = "LineageOwner"
 LINEAGESOURCE = "LineageSource"
 OBJECTPERMISSION = "CoreObjectPermission"
+OBJECTTEMPLATE = "CoreObjectTemplate"
 OBJECTTHREAD = "CoreObjectThread"
 PASSWORDCREDENTIAL = "CorePasswordCredential"
 PROFILE = "CoreProfile"

backend/infrahub/core/constants/infrahubkind.py

@@ -1,7 +1,6 @@
 from copy import deepcopy
 from dataclasses import dataclass, field, replace
 from typing import Iterable
-from uuid import uuid4
 
 from infrahub.core.constants import NULL_VALUE, DiffAction, RelationshipCardinality
 from infrahub.core.constants.database import DatabaseEdgeType
@@ -418,10 +417,16 @@ async def combine(self, earlier_diffs: EnrichedDiffs, later_diffs: EnrichedDiffs
             filtered_node_pairs = self._filter_nodes_to_keep(earlier_diff=earlier, later_diff=later)
             combined_nodes = self._combine_nodes(node_pairs=filtered_node_pairs)
             self._link_child_nodes(nodes=combined_nodes)
+            if earlier.exists_on_database:
+                diff_uuid = earlier.uuid
+                partner_uuid = earlier.partner_uuid
+            else:
+                diff_uuid = later.uuid
+                partner_uuid = later.partner_uuid
             combined_diffs.append(
                 EnrichedDiffRoot(
-                    uuid=str(uuid4()),
-                    partner_uuid=later.partner_uuid,
+                    uuid=diff_uuid,
+                    partner_uuid=partner_uuid,
                     base_branch_name=later.base_branch_name,
                     diff_branch_name=later.diff_branch_name,
                     from_time=earlier.from_time,