Merge release-1.5 into develop

.devcontainer/onCreateCommand.sh

@@ -8,4 +8,4 @@ poetry install --no-interaction --no-ansi
 
 git submodule update --init
 
-invoke demo.pull
+poetry run invoke demo.pull

.devcontainer/postCreateCommand.sh

@@ -2,4 +2,4 @@
 
 git pull
 git submodule update
-invoke demo.start --wait
+poetry run invoke demo.start --wait

.devcontainer/updateContentCommand.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 
 export WEB_CONCURRENCY=2
-invoke demo.start
+poetry run invoke demo.start
 sleep 120
 docker logs infrahub-server-1
-invoke demo.load-infra-schema
+poetry run invoke demo.load-infra-schema
 docker logs infrahub-server-1
 sleep 90
 docker logs infrahub-server-1
-invoke demo.load-infra-data
-invoke demo.stop
+poetry run invoke demo.load-infra-data
+poetry run invoke demo.stop

backend/infrahub/computed_attribute/tasks.py

@@ -2,6 +2,7 @@
 
 from typing import TYPE_CHECKING
 
+from infrahub_sdk.exceptions import URLNotFoundError
 from infrahub_sdk.protocols import CoreTransformPython
 from infrahub_sdk.template import Jinja2Template
 from prefect import flow
@@ -229,7 +230,13 @@ async def process_jinja2(
 
         for id_filter in computed_macro.node_filters:
             query = attribute_graphql.render_graphql_query(query_filter=id_filter, filter_id=object_id)
-            response = await client.execute_graphql(query=query, branch_name=branch_name)
+            try:
+                response = await client.execute_graphql(query=query, branch_name=branch_name)
+            except URLNotFoundError:
+                log.warning(
+                    f"Process computed attributes for {computed_attribute_kind}.{computed_attribute_name} failed for branch {branch_name} (not found)"
+                )
+                return
             output = attribute_graphql.parse_response(response=response)
             found.extend(output)
 

backend/infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 42
+GRAPH_VERSION = 43

backend/infrahub/core/migrations/__init__.py

@@ -1,5 +1,6 @@
 from .schema.attribute_kind_update import AttributeKindUpdateMigration
 from .schema.attribute_name_update import AttributeNameUpdateMigration
+from .schema.attribute_supports_profile import AttributeSupportsProfileUpdateMigration
 from .schema.node_attribute_add import NodeAttributeAddMigration
 from .schema.node_attribute_remove import NodeAttributeRemoveMigration
 from .schema.node_kind_update import NodeKindUpdateMigration
@@ -19,6 +20,8 @@
     "attribute.name.update": AttributeNameUpdateMigration,
     "attribute.branch.update": None,
     "attribute.kind.update": AttributeKindUpdateMigration,
+    "attribute.optional.update": AttributeSupportsProfileUpdateMigration,
+    "attribute.read_only.update": AttributeSupportsProfileUpdateMigration,
     "relationship.branch.update": None,
     "relationship.direction.update": None,
     "relationship.identifier.update": PlaceholderDummyMigration,

backend/infrahub/core/migrations/graph/__init__.py

@@ -41,9 +41,10 @@
 from .m037_index_attr_vals import Migration037
 from .m038_redo_0000_prefix_fix import Migration038
 from .m039_ipam_reconcile import Migration039
-from .m040_profile_attrs_in_db import Migration040
-from .m041_create_hfid_display_label_in_db import Migration041
-from .m042_backfill_hfid_display_label_in_db import Migration042
+from .m040_duplicated_attributes import Migration040
+from .m041_profile_attrs_in_db import Migration041
+from .m042_create_hfid_display_label_in_db import Migration042
+from .m043_backfill_hfid_display_label_in_db import Migration043
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -93,6 +94,7 @@
     Migration040,
     Migration041,
     Migration042,
+    Migration043,
 ]
 
 

backend/infrahub/core/migrations/graph/m013_convert_git_password_credential.py

@@ -286,7 +286,7 @@ def __init__(self, **kwargs: Any):
         kwargs.pop("branch", None)
 
         super().__init__(
-            node_kind="CoreGenericRepository",
+            node_kinds=["CoreGenericRepository"],
             attribute_name="internal_status",
             attribute_kind="Dropdown",
             branch_support=BranchSupportType.LOCAL.value,

backend/infrahub/core/migrations/graph/m040_duplicated_attributes.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Sequence
+
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+
+from ..shared import GraphMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+
+class DeleteDuplicatedAttributesQuery(Query):
+    name: str = "delete_duplicated_attributes"
+    type: QueryType = QueryType.WRITE
+    insert_return: bool = False
+    insert_limit: bool = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+// -------------
+// get all the Nodes linked to multiple Attributes with the same name to drastically reduce the search space
+// -------------
+MATCH (n:Node)-[:HAS_ATTRIBUTE]->(attr:Attribute)
+WITH DISTINCT n, attr
+WITH n, attr.name AS attr_name, count(*) AS num_attrs
+WHERE num_attrs > 1
+// -------------
+// for each Node-attr_name pair, get the possible duplicate Attributes
+// -------------
+MATCH (n)-[:HAS_ATTRIBUTE]->(dup_attr:Attribute {name: attr_name})
+WITH DISTINCT n, dup_attr
+// -------------
+// get the branch(es) for each possible duplicate Attribute
+// -------------
+CALL (n, dup_attr) {
+    MATCH (n)-[r:HAS_ATTRIBUTE {status: "active"}]->(dup_attr)
+    WHERE r.to IS NULL
+    AND NOT exists((n)-[:HAS_ATTRIBUTE {status: "deleted", branch: r.branch}]->(dup_attr))
+    RETURN r.branch AS branch
+}
+// -------------
+// get the latest update time for each duplicate Attribute on each branch
+// -------------
+CALL (dup_attr, branch) {
+    MATCH (dup_attr)-[r {branch: branch}]-()
+    RETURN max(r.from) AS latest_update
+}
+// -------------
+// order the duplicate Attributes by latest update time
+// -------------
+WITH n, dup_attr, branch, latest_update
+ORDER BY n, branch, dup_attr.name, latest_update DESC
+// -------------
+// for any Node-dup_attr_name pairs with multiple duplicate Attributes, keep the Attribute with the latest update
+// on this branch and delete all the other edges on this branch for this Attribute
+// -------------
+WITH n, branch, dup_attr.name AS dup_attr_name, collect(dup_attr) AS dup_attrs_reverse_chronological
+WHERE size(dup_attrs_reverse_chronological) > 1
+WITH branch, tail(dup_attrs_reverse_chronological) AS dup_attrs_to_delete
+UNWIND dup_attrs_to_delete AS dup_attr_to_delete
+MATCH (dup_attr_to_delete)-[r {branch: branch}]-()
+DELETE r
+// -------------
+// delete any orphaned Attributes
+// -------------
+WITH DISTINCT dup_attr_to_delete
+WHERE NOT exists((dup_attr_to_delete)--())
+DELETE dup_attr_to_delete
+        """
+        self.add_to_query(query)
+
+
+class Migration040(GraphMigration):
+    name: str = "040_duplicated_attributes"
+    queries: Sequence[type[Query]] = [DeleteDuplicatedAttributesQuery]
+    minimum_version: int = 39
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()

backend/infrahub/core/migrations/graph/m040_profile_attrs_in_db.py → backend/infrahub/core/migrations/graph/m041_profile_attrs_in_db.py

@@ -84,7 +84,7 @@ def get_node_ids_by_branch(self) -> dict[str, set[str]]:
         return nodes_by_branch
 
 
-class Migration040(ArbitraryMigration):
+class Migration041(ArbitraryMigration):
     """
     Save profile attribute values on each node using the profile in the database
     For any profile that has updates on a given branch (including default branch)
@@ -93,8 +93,8 @@ class Migration040(ArbitraryMigration):
     - run NodeProfilesApplier.apply_profiles on the node on that branch
     """
 
-    name: str = "040_profile_attrs_in_db"
-    minimum_version: int = 39
+    name: str = "041_profile_attrs_in_db"
+    minimum_version: int = 40
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

backend/infrahub/core/migrations/graph/m041_create_hfid_display_label_in_db.py → backend/infrahub/core/migrations/graph/m042_create_hfid_display_label_in_db.py

@@ -17,9 +17,9 @@
     from infrahub.database import InfrahubDatabase
 
 
-class Migration041(InternalSchemaMigration):
-    name: str = "041_create_hfid_display_label_in_db"
-    minimum_version: int = 40
+class Migration042(InternalSchemaMigration):
+    name: str = "042_create_hfid_display_label_in_db"
+    minimum_version: int = 41
 
     @classmethod
     def init(cls, **kwargs: Any) -> Self:

backend/infrahub/core/migrations/graph/m042_backfill_hfid_display_label_in_db.py → backend/infrahub/core/migrations/graph/m043_backfill_hfid_display_label_in_db.py

@@ -18,13 +18,13 @@
     from infrahub.database import InfrahubDatabase
 
 
-class Migration042(ArbitraryMigration):
+class Migration043(ArbitraryMigration):
     """
     Backfill `human_friendly_id` and `display_label` attributes for nodes with schemas that define them.
     """
 
-    name: str = "042_backfill_hfid_display_label_in_db"
-    minimum_version: int = 41
+    name: str = "043_backfill_hfid_display_label_in_db"
+    minimum_version: int = 42
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

backend/infrahub/core/migrations/query/__init__.py

@@ -8,7 +8,12 @@
     from ..shared import AttributeSchemaMigration, SchemaMigration
 
 
-class MigrationQuery(Query):
+class MigrationBaseQuery(Query):
+    def get_nbr_migrations_executed(self) -> int:
+        return self.num_of_results
+
+
+class MigrationQuery(MigrationBaseQuery):
     type: QueryType = QueryType.WRITE
 
     def __init__(
@@ -19,11 +24,8 @@ def __init__(
         self.migration = migration
         super().__init__(**kwargs)
 
-    def get_nbr_migrations_executed(self) -> int:
-        return self.num_of_results
-
 
-class AttributeMigrationQuery(Query):
+class AttributeMigrationQuery(MigrationBaseQuery):
     type: QueryType = QueryType.WRITE
 
     def __init__(
@@ -33,6 +35,3 @@ def __init__(
     ):
         self.migration = migration
         super().__init__(**kwargs)
-
-    def get_nbr_migrations_executed(self) -> int:
-        return self.num_of_results

backend/infrahub/core/migrations/query/attribute_add.py

@@ -17,14 +17,14 @@ class AttributeAddQuery(Query):
 
     def __init__(
         self,
-        node_kind: str,
+        node_kinds: list[str],
         attribute_name: str,
         attribute_kind: str,
         branch_support: str,
         default_value: Any | None = None,
         **kwargs: Any,
     ) -> None:
-        self.node_kind = node_kind
+        self.node_kinds = node_kinds
         self.attribute_name = attribute_name
         self.attribute_kind = attribute_kind
         self.branch_support = branch_support
@@ -36,7 +36,7 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> No
         branch_filter, branch_params = self.branch.get_query_filter_path(at=self.at.to_string())
         self.params.update(branch_params)
 
-        self.params["node_kind"] = self.node_kind
+        self.params["node_kinds"] = self.node_kinds
         self.params["attr_name"] = self.attribute_name
         self.params["branch_support"] = self.branch_support
         self.params["current_time"] = self.at.to_string()
@@ -79,12 +79,13 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> No
             LIMIT 1
             """ % {"attr_value_label": attr_value_label}
 
+        node_kinds_str = "|".join(self.node_kinds)
         query = """
         %(match_query)s
         MERGE (is_protected_value:Boolean { value: $is_protected_default })
         MERGE (is_visible_value:Boolean { value: $is_visible_default })
         WITH av, is_protected_value, is_visible_value
-        MATCH p = (n:%(node_kind)s)
+        MATCH (n:%(node_kinds_str)s)
         CALL (n) {
             MATCH (:Root)<-[r:IS_PART_OF]-(n)
             WHERE %(branch_filter)s
@@ -110,7 +111,7 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> No
         """ % {
             "match_query": match_query,
             "branch_filter": branch_filter,
-            "node_kind": self.node_kind,
+            "node_kinds_str": node_kinds_str,
             "uuid_generation": db.render_uuid_generation(node_label="a", node_attr="uuid"),
         }