IFC-1348 Make templates for generics generics (#5938)

Author: gmazoyer

backend/infrahub/core/migrations/graph/m018_uniqueness_nulls.py

@@ -50,7 +50,7 @@ async def execute(self, db: InfrahubDatabase) -> MigrationResult:
         schema_branch = await manager.load_schema_from_db(db=db, branch=default_branch)
         manager.set_schema_branch(name=default_branch.name, schema=schema_branch)
 
-        for schema_kind in schema_branch.node_names + schema_branch.generic_names:
+        for schema_kind in schema_branch.node_names + schema_branch.generic_names_without_templates:
             schema = schema_branch.get(name=schema_kind, duplicate=False)
             if not isinstance(schema, NodeSchema | GenericSchema):
                 continue

backend/infrahub/core/schema/manager.py

@@ -251,7 +251,7 @@ async def load_schema_to_db(
 
         branch = await registry.get_branch(branch=branch, db=db)
 
-        for item_kind in schema.node_names + schema.generic_names:
+        for item_kind in schema.node_names + schema.generic_names_without_templates:
             if limit and item_kind not in limit:
                 continue
             item = schema.get(name=item_kind, duplicate=False)

backend/infrahub/core/schema/schema_branch.py

@@ -107,6 +107,10 @@ def node_names(self) -> list[str]:
     def generic_names(self) -> list[str]:
         return list(self.generics.keys())
 
+    @property
+    def generic_names_without_templates(self) -> list[str]:
+        return [g for g in self.generic_names if not g.startswith("Template")]
+
     @property
     def profile_names(self) -> list[str]:
         return list(self.profiles.keys())
@@ -115,10 +119,10 @@ def profile_names(self) -> list[str]:
     def template_names(self) -> list[str]:
         return list(self.templates.keys())
 
-    def get_all_kind_id_map(self, exclude_profiles: bool = False) -> dict[str, str]:
+    def get_all_kind_id_map(self, nodes_and_generics_only: bool = False) -> dict[str, str]:
         kind_id_map = {}
-        if exclude_profiles:
-            names = self.node_names + self.generic_names
+        if nodes_and_generics_only:
+            names = self.node_names + self.generic_names_without_templates
         else:
             names = self.all_names
         for name in names:
@@ -182,8 +186,8 @@ def from_dict_schema_object(cls, data: dict) -> Self:
 
     def diff(self, other: SchemaBranch) -> SchemaDiff:
         # Identify the nodes or generics that have been added or removed
-        local_kind_id_map = self.get_all_kind_id_map(exclude_profiles=True)
-        other_kind_id_map = other.get_all_kind_id_map(exclude_profiles=True)
+        local_kind_id_map = self.get_all_kind_id_map(nodes_and_generics_only=True)
+        other_kind_id_map = other.get_all_kind_id_map(nodes_and_generics_only=True)
         clean_local_ids = [id for id in local_kind_id_map.values() if id is not None]
         clean_other_ids = [id for id in other_kind_id_map.values() if id is not None]
         shared_ids = intersection(list1=clean_local_ids, list2=clean_other_ids)
@@ -687,7 +691,7 @@ def validate_schema_path(
         return schema_attribute_path
 
     def sync_uniqueness_constraints_and_unique_attributes(self) -> None:
-        for name in self.generic_names + self.node_names:
+        for name in self.generic_names_without_templates + self.node_names:
             node_schema = self.get(name=name, duplicate=False)
 
             if not node_schema.unique_attributes and not node_schema.uniqueness_constraints:
@@ -802,7 +806,7 @@ def validate_default_filters(self) -> None:
             )
 
     def validate_default_values(self) -> None:
-        for name in self.generic_names + self.node_names:
+        for name in self.generic_names_without_templates + self.node_names:
             node_schema = self.get(name=name, duplicate=False)
             for node_attr in node_schema.local_attributes:
                 if node_attr.default_value is None:
@@ -822,7 +826,7 @@ def validate_default_values(self) -> None:
                     ) from exc
 
     def validate_human_friendly_id(self) -> None:
-        for name in self.generic_names + self.node_names:
+        for name in self.generic_names_without_templates + self.node_names:
             node_schema = self.get(name=name, duplicate=False)
             hf_attr_names = set()
 
@@ -843,7 +847,7 @@ def validate_human_friendly_id(self) -> None:
 
     def validate_required_relationships(self) -> None:
         reverse_dependency_map: dict[str, set[str]] = {}
-        for name in self.node_names + self.generic_names:
+        for name in self.node_names + self.generic_names_without_templates:
             node_schema = self.get(name=name, duplicate=False)
             for relationship_schema in node_schema.relationships:
                 if relationship_schema.optional:
@@ -861,7 +865,7 @@ def validate_required_relationships(self) -> None:
     def validate_parent_component(self) -> None:
         # {parent_kind: {component_kind_1, component_kind_2, ...}}
         dependency_map: dict[str, set[str]] = defaultdict(set)
-        for name in self.generic_names + self.node_names:
+        for name in self.generic_names_without_templates + self.node_names:
             node_schema = self.get(name=name, duplicate=False)
 
             parent_relationships: list[RelationshipSchema] = []
@@ -1147,7 +1151,7 @@ def process_relationships(self) -> None:
                 self.set(name=schema_to_update.kind, schema=schema_to_update)
 
     def process_human_friendly_id(self) -> None:
-        for name in self.generic_names + self.node_names:
+        for name in self.generic_names_without_templates + self.node_names:
             node = self.get(name=name, duplicate=False)
 
             # If human_friendly_id IS NOT defined
@@ -1634,7 +1638,7 @@ def manage_profile_schemas(self) -> None:
             self.set(name=core_profile_schema.kind, schema=core_profile_schema)
 
         profile_schema_kinds = set()
-        for node_name in self.node_names + self.generic_names:
+        for node_name in self.node_names + self.generic_names_without_templates:
             node = self.get(name=node_name, duplicate=False)
             if (
                 node.namespace in RESTRICTED_NAMESPACES
@@ -1865,37 +1869,64 @@ def add_relationships_to_template(self, node: NodeSchema) -> None:
                 )
             )
 
-    def generate_object_template_from_node(self, node: NodeSchema) -> TemplateSchema:
+    def generate_object_template_from_node(
+        self, node: NodeSchema | GenericSchema, need_templates: set[NodeSchema | GenericSchema]
+    ) -> TemplateSchema | GenericSchema:
         core_template_schema = self.get(name=InfrahubKind.OBJECTTEMPLATE, duplicate=False)
         core_name_attr = core_template_schema.get_attribute(name=OBJECT_TEMPLATE_NAME_ATTR)
         template_name_attr = AttributeSchema(
             **core_name_attr.model_dump(exclude=["id", "inherited"]),
         )
         template_name_attr.branch = node.branch
 
-        template = TemplateSchema(
-            name=node.kind,
-            namespace="Template",
-            label=f"Object template {node.label}",
-            description=f"Object template for {node.kind}",
-            branch=node.branch,
-            include_in_menu=False,
-            display_labels=["template_name__value"],
-            inherit_from=[InfrahubKind.LINEAGESOURCE, InfrahubKind.OBJECTTEMPLATE, InfrahubKind.NODE],
-            human_friendly_id=["template_name__value"],
-            default_filter="template_name__value",
-            attributes=[template_name_attr],
-            relationships=[
-                RelationshipSchema(
-                    name="related_nodes",
-                    identifier="node__objecttemplate",
-                    peer=node.kind,
-                    kind=RelationshipKind.TEMPLATE,
-                    cardinality=RelationshipCardinality.MANY,
-                    branch=BranchSupportType.AWARE,
-                )
-            ],
-        )
+        template: TemplateSchema | GenericSchema
+        need_template_kinds = [n.kind for n in need_templates]
+
+        if isinstance(node, GenericSchema):
+            # When needing a template for a generic, we generate an empty shell mostly to make sure that schemas (including the GraphQL one) will
+            # look right. We don't really care about applying inheritance of fields as it was already processed and actual templates will have the
+            # correct attributes and relationships
+            template = GenericSchema(
+                name=node.kind,
+                namespace="Template",
+                label=f"Generic object template {node.label}",
+                description=f"Generic object template for generic {node.kind}",
+                generate_profile=False,
+                branch=node.branch,
+                include_in_menu=False,
+            )
+
+            for used in node.used_by:
+                if used in need_template_kinds:
+                    template.used_by.append(self._get_object_template_kind(node_kind=used))
+        else:
+            template = TemplateSchema(
+                name=node.kind,
+                namespace="Template",
+                label=f"Object template {node.label}",
+                description=f"Object template for {node.kind}",
+                branch=node.branch,
+                include_in_menu=False,
+                display_labels=["template_name__value"],
+                inherit_from=[InfrahubKind.LINEAGESOURCE, InfrahubKind.OBJECTTEMPLATE, InfrahubKind.NODE],
+                human_friendly_id=["template_name__value"],
+                default_filter="template_name__value",
+                attributes=[template_name_attr],
+                relationships=[
+                    RelationshipSchema(
+                        name="related_nodes",
+                        identifier="node__objecttemplate",
+                        peer=node.kind,
+                        kind=RelationshipKind.TEMPLATE,
+                        cardinality=RelationshipCardinality.MANY,
+                        branch=BranchSupportType.AWARE,
+                    )
+                ],
+            )
+
+            for inherited in node.inherit_from:
+                if inherited in need_template_kinds:
+                    template.inherit_from.append(self._get_object_template_kind(node_kind=inherited))
 
         for node_attr in node.attributes:
             if node_attr.unique:
@@ -1918,15 +1949,22 @@ def identify_required_object_templates(
         identified.add(node_schema)
 
         for relationship in node_schema.relationships:
-            if relationship.peer in [
-                InfrahubKind.GENERICGROUP,
-                InfrahubKind.PROFILE,
-            ] or relationship.kind not in [RelationshipKind.COMPONENT, RelationshipKind.PARENT]:
+            if relationship.peer in [InfrahubKind.GENERICGROUP, InfrahubKind.PROFILE] or relationship.kind not in [
+                RelationshipKind.COMPONENT,
+                RelationshipKind.PARENT,
+            ]:
                 continue
 
             peer_schema = self.get(name=relationship.peer, duplicate=False)
             if not isinstance(peer_schema, NodeSchema | GenericSchema) or peer_schema in identified:
                 continue
+            # In a context of a generic, we won't be able to create objects out of it, so any kind of nodes implementing the generic is a valid
+            # option, we therefore need to have a template for each of those nodes
+            if isinstance(peer_schema, GenericSchema) and peer_schema.used_by:
+                for used_by in peer_schema.used_by:
+                    identified |= self.identify_required_object_templates(
+                        node_schema=self.get(name=used_by, duplicate=False), identified=identified
+                    )
 
             identified |= self.identify_required_object_templates(node_schema=peer_schema, identified=identified)
 
@@ -1936,7 +1974,7 @@ def manage_object_template_schemas(self) -> None:
         need_templates: set[NodeSchema | GenericSchema] = set()
         template_schema_kinds: set[str] = set()
 
-        for node_name in self.node_names + self.generic_names:
+        for node_name in self.node_names + self.generic_names_without_templates:
             node = self.get(name=node_name, duplicate=False)
 
             # Delete old object templates if schemas were removed
@@ -1955,7 +1993,7 @@ def manage_object_template_schemas(self) -> None:
 
         # Generate templates with their attributes
         for node in need_templates:
-            template = self.generate_object_template_from_node(node=node)
+            template = self.generate_object_template_from_node(node=node, need_templates=need_templates)
             self.set(name=template.kind, schema=template)
             template_schema_kinds.add(template.kind)
 

backend/infrahub/graphql/mutations/main.py

@@ -288,8 +288,13 @@ async def _handle_template_relationships(
             if not template_relationship_peers:
                 continue
 
-            obj_peer_schema = relationship.get_peer_schema(db=db, branch=branch)
             for template_relationship_peer in template_relationship_peers.values():
+                # We retrieve peer schema for each peer in case we are processing a relationship which is based on a generic
+                obj_peer_schema = registry.schema.get_node_schema(
+                    name=template_relationship_peer.get_schema().kind.removeprefix("Template"),
+                    branch=branch,
+                    duplicate=False,
+                )
                 obj_peer_data = await cls._extract_peer_data(
                     db=db,
                     template_peer=template_relationship_peer,

backend/tests/constants/kind.py

@@ -5,11 +5,14 @@
 COUNTRY = "TestingCountry"
 DEVICE = "TestingDevice"
 INTERFACE = "TestingInterface"
+INTERFACE_HOLDER = "TestingInterfaceHolder"
 LOCATION = "TestingLocation"
 MANUFACTURER = "TestingManufacturer"
 PERSON = "TestingPerson"
+PHYSICAL_INTERFACE = "TestingPhysicalInterface"
 SFP = "TestingSfp"
 SITE = "TestingSite"
 THING = "TestingThing"
 TICKET = "TestingTicket"
 TSHIRT = "TestingTShirt"
+VIRTUAL_INTERFACE = "TestingVirtualInterface"

backend/tests/helpers/schema/__init__.py

@@ -9,7 +9,7 @@
 from .car import CAR
 from .child import CHILD
 from .color import COLOR
-from .device import DEVICE, INTERFACE, SFP
+from .device import DEVICE, INTERFACE, INTERFACE_HOLDER, PHYSICAL_INTERFACE, SFP, VIRTUAL_INTERFACE
 from .location import CONTINENT, COUNTRY, LOCATION, SITE
 from .manufacturer import MANUFACTURER
 from .person import PERSON
@@ -23,7 +23,9 @@
 
 
 CAR_SCHEMA = SchemaRoot(nodes=[CAR, MANUFACTURER, PERSON])
-DEVICE_SCHEMA = SchemaRoot(nodes=[DEVICE, INTERFACE, SFP])
+DEVICE_SCHEMA = SchemaRoot(
+    generics=[INTERFACE, INTERFACE_HOLDER], nodes=[DEVICE, PHYSICAL_INTERFACE, VIRTUAL_INTERFACE, SFP]
+)
 LOCATION_SCHEMA = SchemaRoot(generics=[LOCATION], nodes=[CONTINENT, COUNTRY, SITE])
 
 
@@ -52,14 +54,17 @@ async def load_schema(
     "DEVICE",
     "DEVICE_SCHEMA",
     "INTERFACE",
+    "INTERFACE_HOLDER",
     "LOCATION",
     "LOCATION_SCHEMA",
     "MANUFACTURER",
     "PERSON",
+    "PHYSICAL_INTERFACE",
     "SFP",
     "SITE",
     "THING",
     "TICKET",
     "TSHIRT",
+    "VIRTUAL_INTERFACE",
     "WIDGET",
 ]