Merge pull request #7450 from opsmill/stable-to-release-1.5

Stable to release 1.5

Author: ajtmccarty

backend/infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 42
+GRAPH_VERSION = 43

backend/infrahub/core/migrations/graph/__init__.py

@@ -41,9 +41,10 @@
 from .m037_index_attr_vals import Migration037
 from .m038_redo_0000_prefix_fix import Migration038
 from .m039_ipam_reconcile import Migration039
-from .m040_profile_attrs_in_db import Migration040
-from .m041_create_hfid_display_label_in_db import Migration041
-from .m042_backfill_hfid_display_label_in_db import Migration042
+from .m040_duplicated_attributes import Migration040
+from .m041_profile_attrs_in_db import Migration041
+from .m042_create_hfid_display_label_in_db import Migration042
+from .m043_backfill_hfid_display_label_in_db import Migration043
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -93,6 +94,7 @@
     Migration040,
     Migration041,
     Migration042,
+    Migration043,
 ]
 
 

backend/infrahub/core/migrations/graph/m040_duplicated_attributes.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Sequence
+
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+
+from ..shared import GraphMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+
+class DeleteDuplicatedAttributesQuery(Query):
+    name: str = "delete_duplicated_attributes"
+    type: QueryType = QueryType.WRITE
+    insert_return: bool = False
+    insert_limit: bool = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+// -------------
+// get all the Nodes linked to multiple Attributes with the same name to drastically reduce the search space
+// -------------
+MATCH (n:Node)-[:HAS_ATTRIBUTE]->(attr:Attribute)
+WITH DISTINCT n, attr
+WITH n, attr.name AS attr_name, count(*) AS num_attrs
+WHERE num_attrs > 1
+// -------------
+// for each Node-attr_name pair, get the possible duplicate Attributes
+// -------------
+MATCH (n)-[:HAS_ATTRIBUTE]->(dup_attr:Attribute {name: attr_name})
+WITH DISTINCT n, dup_attr
+// -------------
+// get the branch(es) for each possible duplicate Attribute
+// -------------
+CALL (n, dup_attr) {
+    MATCH (n)-[r:HAS_ATTRIBUTE {status: "active"}]->(dup_attr)
+    WHERE r.to IS NULL
+    AND NOT exists((n)-[:HAS_ATTRIBUTE {status: "deleted", branch: r.branch}]->(dup_attr))
+    RETURN r.branch AS branch
+}
+// -------------
+// get the latest update time for each duplicate Attribute on each branch
+// -------------
+CALL (dup_attr, branch) {
+    MATCH (dup_attr)-[r {branch: branch}]-()
+    RETURN max(r.from) AS latest_update
+}
+// -------------
+// order the duplicate Attributes by latest update time
+// -------------
+WITH n, dup_attr, branch, latest_update
+ORDER BY n, branch, dup_attr.name, latest_update DESC
+// -------------
+// for any Node-dup_attr_name pairs with multiple duplicate Attributes, keep the Attribute with the latest update
+// on this branch and delete all the other edges on this branch for this Attribute
+// -------------
+WITH n, branch, dup_attr.name AS dup_attr_name, collect(dup_attr) AS dup_attrs_reverse_chronological
+WHERE size(dup_attrs_reverse_chronological) > 1
+WITH branch, tail(dup_attrs_reverse_chronological) AS dup_attrs_to_delete
+UNWIND dup_attrs_to_delete AS dup_attr_to_delete
+MATCH (dup_attr_to_delete)-[r {branch: branch}]-()
+DELETE r
+// -------------
+// delete any orphaned Attributes
+// -------------
+WITH DISTINCT dup_attr_to_delete
+WHERE NOT exists((dup_attr_to_delete)--())
+DELETE dup_attr_to_delete
+        """
+        self.add_to_query(query)
+
+
+class Migration040(GraphMigration):
+    name: str = "040_duplicated_attributes"
+    queries: Sequence[type[Query]] = [DeleteDuplicatedAttributesQuery]
+    minimum_version: int = 39
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()

backend/infrahub/core/migrations/graph/m040_profile_attrs_in_db.py renamed to backend/infrahub/core/migrations/graph/m041_profile_attrs_in_db.py

@@ -84,7 +84,7 @@ def get_node_ids_by_branch(self) -> dict[str, set[str]]:
         return nodes_by_branch
 
 
-class Migration040(ArbitraryMigration):
+class Migration041(ArbitraryMigration):
     """
     Save profile attribute values on each node using the profile in the database
     For any profile that has updates on a given branch (including default branch)
@@ -93,8 +93,8 @@ class Migration040(ArbitraryMigration):
     - run NodeProfilesApplier.apply_profiles on the node on that branch
     """
 
-    name: str = "040_profile_attrs_in_db"
-    minimum_version: int = 39
+    name: str = "041_profile_attrs_in_db"
+    minimum_version: int = 40
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

backend/infrahub/core/migrations/graph/m041_create_hfid_display_label_in_db.py renamed to backend/infrahub/core/migrations/graph/m042_create_hfid_display_label_in_db.py

@@ -17,9 +17,9 @@
     from infrahub.database import InfrahubDatabase
 
 
-class Migration041(InternalSchemaMigration):
-    name: str = "041_create_hfid_display_label_in_db"
-    minimum_version: int = 40
+class Migration042(InternalSchemaMigration):
+    name: str = "042_create_hfid_display_label_in_db"
+    minimum_version: int = 41
 
     @classmethod
     def init(cls, **kwargs: Any) -> Self:

backend/infrahub/core/migrations/graph/m042_backfill_hfid_display_label_in_db.py renamed to backend/infrahub/core/migrations/graph/m043_backfill_hfid_display_label_in_db.py

@@ -18,13 +18,13 @@
     from infrahub.database import InfrahubDatabase
 
 
-class Migration042(ArbitraryMigration):
+class Migration043(ArbitraryMigration):
     """
     Backfill `human_friendly_id` and `display_label` attributes for nodes with schemas that define them.
     """
 
-    name: str = "042_backfill_hfid_display_label_in_db"
-    minimum_version: int = 41
+    name: str = "043_backfill_hfid_display_label_in_db"
+    minimum_version: int = 42
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

backend/infrahub/core/migrations/schema/node_attribute_add.py

@@ -42,6 +42,16 @@ class NodeAttributeAddMigration(AttributeSchemaMigration):
     name: str = "node.attribute.add"
     queries: Sequence[type[AttributeMigrationQuery]] = [NodeAttributeAddMigrationQuery01]  # type: ignore[assignment]
 
+    async def execute(
+        self,
+        db: InfrahubDatabase,
+        branch: Branch,
+        at: Timestamp | str | None = None,
+    ) -> MigrationResult:
+        if self.new_attribute_schema.inherited is True:
+            return MigrationResult()
+        return await super().execute(db=db, branch=branch, at=at)
+
     async def execute_post_queries(
         self,
         db: InfrahubDatabase,

backend/infrahub/core/query/node.py

@@ -246,11 +246,15 @@ async def query_init(self, db: InfrahubDatabase, **kwargs) -> None:  # noqa: ARG
         ipnetwork_prop_list = [f"{key}: {value}" for key, value in ipnetwork_prop.items()]
 
         attrs_nonindexed_query = """
-        WITH distinct n
+        WITH DISTINCT n
         UNWIND $attrs AS attr
         // Try to find a matching vertex
-        OPTIONAL MATCH (existing_av:AttributeValue {value: attr.content.value, is_default: attr.content.is_default})
-        WHERE NOT existing_av:AttributeValueIndexed
+        CALL (attr) {
+            OPTIONAL MATCH (existing_av:AttributeValue {value: attr.content.value, is_default: attr.content.is_default})
+            WHERE NOT existing_av:AttributeValueIndexed
+            RETURN existing_av
+            LIMIT 1
+        }
         CALL (attr, existing_av) {
             // If none found, create a new one
             WITH existing_av

backend/tests/helpers/db_validation.py

@@ -130,3 +130,38 @@ async def verify_no_duplicate_paths(db: InfrahubDatabase) -> None:
         raise ValueError(
             f"{num_paths} paths ({branch=},{edge_type=},{from_time=}) between nodes '{node_id1}' and '{node_id2}'"
         )
+
+
+async def validate_no_duplicate_attributes(db: InfrahubDatabase, branch: Branch) -> list[str]:
+    """
+    Validate that no Nodes have duplicated attribute or relationship names
+    """
+    branch_filter, branch_params = branch.get_query_filter_path()
+
+    query = """
+// -------------
+// get all the active Attributes this branch and count them up
+// -------------
+MATCH (n:Node)-[:HAS_ATTRIBUTE]->(field:Attribute)
+WITH DISTINCT n, field
+CALL (n, field) {
+MATCH (n)-[r:HAS_ATTRIBUTE]->(field)
+WHERE %(branch_filter)s
+RETURN r
+ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
+LIMIT 1
+}
+WITH n, field, r
+WHERE r.status = "active" AND r.to IS NULL
+WITH n.uuid AS node_id, field.name AS field_name, count(*) AS num_fields
+WHERE num_fields > 1
+RETURN node_id, field_name, num_fields
+    """ % {"branch_filter": branch_filter}
+    results = await db.execute_query(query=query, params=branch_params)
+    errors = []
+    for result in results:
+        node_id = result.get("node_id")
+        field_name = result.get("field_name")
+        num_fields = result.get("num_fields")
+        errors.append(f"Node '{node_id}' has {num_fields} duplicated attributes with {field_name=}")
+    return errors