Allow to load data from git repo (#6527)

Author: LucasG0

backend/infrahub/core/constants/__init__.py

@@ -146,6 +146,11 @@ class AllowOverrideType(InfrahubStringEnum):
     ANY = "any"
 
 
+class RepositoryObjects(InfrahubStringEnum):
+    OBJECT = "object"
+    MENU = "menu"
+
+
 class ContentType(InfrahubStringEnum):
     APPLICATION_JSON = "application/json"
     APPLICATION_YAML = "application/yaml"

backend/infrahub/core/constants/infrahubkind.py

@@ -58,6 +58,7 @@
 PROPOSEDCHANGE = "CoreProposedChange"
 REFRESHTOKEN = "InternalRefreshToken"
 REPOSITORY = "CoreRepository"
+REPOSITORYGROUP = "CoreRepositoryGroup"
 RESOURCEPOOL = "CoreResourcePool"
 GENERICREPOSITORY = "CoreGenericRepository"
 READONLYREPOSITORY = "CoreReadOnlyRepository"

backend/infrahub/core/protocols.py

@@ -497,6 +497,11 @@ class CoreRepository(LineageOwner, LineageSource, CoreGenericRepository, CoreTas
     commit: StringOptional
 
 
+class CoreRepositoryGroup(CoreGroup):
+    content: Dropdown
+    repository: RelationshipManager
+
+
 class CoreRepositoryValidator(CoreValidator):
     repository: RelationshipManager
 

backend/infrahub/core/schema/definitions/core/__init__.py

@@ -28,7 +28,13 @@
 from .core import core_node, core_task_target
 from .generator import core_generator_definition, core_generator_instance
 from .graphql_query import core_graphql_query
-from .group import core_generator_group, core_graphql_query_group, core_group, core_standard_group
+from .group import (
+    core_generator_group,
+    core_graphql_query_group,
+    core_group,
+    core_repository_group,
+    core_standard_group,
+)
 from .ipam import builtin_ip_address, builtin_ip_prefix, builtin_ipam, core_ipam_namespace
 from .lineage import lineage_owner, lineage_source
 from .menu import generic_menu_item, menu_item
@@ -116,6 +122,7 @@
         core_standard_group,
         core_generator_group,
         core_graphql_query_group,
+        core_repository_group,
         builtin_tag,
         core_account,
         core_account_token,

backend/infrahub/core/schema/definitions/core/group.py

@@ -1,9 +1,11 @@
 from infrahub.core.constants import (
     BranchSupportType,
     InfrahubKind,
+    RepositoryObjects,
 )
 from infrahub.core.constants import RelationshipCardinality as Cardinality
 from infrahub.core.constants import RelationshipKind as RelKind
+from infrahub.core.schema.dropdown import DropdownChoice
 
 from ...attribute_schema import AttributeSchema as Attr
 from ...generic_schema import GenericSchema
@@ -80,6 +82,7 @@
     generate_profile=False,
 )
 
+
 core_graphql_query_group = NodeSchema(
     name="GraphQLQueryGroup",
     namespace="Core",
@@ -106,3 +109,45 @@
         ),
     ],
 )
+
+
+core_repository_group = NodeSchema(
+    name="RepositoryGroup",
+    namespace="Core",
+    description="Group of nodes associated with a given repository.",
+    include_in_menu=False,
+    icon="mdi:account-group",
+    label="Repository Group",
+    default_filter="name__value",
+    order_by=["name__value"],
+    display_labels=["name__value"],
+    branch=BranchSupportType.LOCAL,
+    inherit_from=[InfrahubKind.GENERICGROUP],
+    generate_profile=False,
+    attributes=[
+        Attr(
+            name="content",
+            kind="Dropdown",
+            description="Type of data to load, can be either `object` or `menu`",
+            choices=[
+                DropdownChoice(
+                    name=RepositoryObjects.OBJECT.value,
+                    label="Objects",
+                ),
+                DropdownChoice(
+                    name=RepositoryObjects.MENU.value,
+                    label="Menus",
+                ),
+            ],
+            optional=False,
+        ),
+    ],
+    relationships=[
+        Rel(
+            name="repository",
+            peer=InfrahubKind.GENERICREPOSITORY,
+            optional=False,
+            cardinality=Cardinality.ONE,
+        ),
+    ],
+)

backend/infrahub/git/base.py

@@ -162,6 +162,11 @@ class InfrahubRepositoryBase(BaseModel, ABC):
     infrahub_branch_name: str | None = Field(None, description="Infrahub branch on which to sync the remote repository")
     model_config = ConfigDict(arbitrary_types_allowed=True, ignored_types=(Flow, Task))
 
+    def get_client(self) -> InfrahubClient:
+        if self.client is None:
+            raise ValueError("Client is not set")
+        return self.client
+
     @property
     def sdk(self) -> InfrahubClient:
         if self.client:
@@ -445,9 +450,6 @@ def get_worktrees(self) -> list[Worktree]:
 
         return [Worktree.init(response) for response in responses]
 
-    def get_client(self) -> InfrahubClient:
-        return self.sdk
-
     def get_location(self) -> str:
         if self.location:
             return self.location

backend/infrahub/git/integrator.py

@@ -29,18 +29,20 @@
     InfrahubPythonTransformConfig,
     InfrahubRepositoryConfig,
 )
+from infrahub_sdk.spec.menu import MenuFile
+from infrahub_sdk.spec.object import ObjectFile
 from infrahub_sdk.template import Jinja2Template
 from infrahub_sdk.template.exceptions import JinjaTemplateError
 from infrahub_sdk.utils import compare_lists
-from infrahub_sdk.yaml import SchemaFile
+from infrahub_sdk.yaml import InfrahubFile, SchemaFile
 from prefect import flow, task
 from prefect.cache_policies import NONE
 from prefect.logging import get_run_logger
 from pydantic import BaseModel, Field
 from pydantic import ValidationError as PydanticValidationError
 from typing_extensions import Self
 
-from infrahub.core.constants import ArtifactStatus, ContentType, InfrahubKind, RepositorySyncStatus
+from infrahub.core.constants import ArtifactStatus, ContentType, InfrahubKind, RepositoryObjects, RepositorySyncStatus
 from infrahub.core.registry import registry
 from infrahub.events.artifact_action import ArtifactCreatedEvent, ArtifactUpdatedEvent
 from infrahub.events.models import EventMeta
@@ -54,6 +56,7 @@
     import types
 
     from infrahub_sdk.checks import InfrahubCheck
+    from infrahub_sdk.ctl.utils import YamlFileVar
     from infrahub_sdk.schema.repository import InfrahubRepositoryArtifactDefinitionConfig
     from infrahub_sdk.transforms import InfrahubTransform
 
@@ -159,7 +162,7 @@ async def init(cls, service: InfrahubServices, commit: str | None = None, **kwar
     async def ensure_location_is_defined(self) -> None:
         if self.location:
             return
-        client = self.get_client()
+        client = self.sdk
         repo = await client.get(
             kind=CoreGenericRepository, name__value=self.name, exclude=["tags", "credential"], raise_when_missing=True
         )
@@ -179,6 +182,7 @@ async def import_objects_from_files(
 
         config_file = await self.get_repository_config(branch_name=infrahub_branch_name, commit=commit)  # type: ignore[misc]
         sync_status = RepositorySyncStatus.IN_SYNC if config_file else RepositorySyncStatus.ERROR_IMPORT
+
         error: Exception | None = None
 
         try:
@@ -189,6 +193,17 @@ async def import_objects_from_files(
                     branch_name=infrahub_branch_name, commit=commit, config_file=config_file
                 )  # type: ignore[misc]
 
+                await self.import_objects(
+                    branch_name=infrahub_branch_name,
+                    commit=commit,
+                    config_file=config_file,
+                )  # type: ignore[misc]
+                await self.import_objects(
+                    branch_name=infrahub_branch_name,
+                    commit=commit,
+                    config_file=config_file,
+                )  # type: ignore[misc]
+
                 await self.import_all_python_files(  # type: ignore[call-overload]
                     branch_name=infrahub_branch_name, commit=commit, config_file=config_file
                 )  # type: ignore[misc]
@@ -815,6 +830,80 @@ async def import_python_transforms(
             log.info(f"TransformPython {transform_name!r} not found locally, deleting")
             await transform_definition_in_graph[transform_name].delete()
 
+    async def _load_yamlfile_from_disk(self, paths: list[Path], file_type: type[YamlFileVar]) -> list[YamlFileVar]:
+        data_files = file_type.load_from_disk(paths=paths)
+
+        for data_file in data_files:
+            if not data_file.valid or not data_file.content:
+                raise ValueError(f"{data_file.error_message} ({data_file.location})")
+
+        return data_files
+
+    async def _load_objects(
+        self,
+        paths: list[Path],
+        branch: str,
+        file_type: type[InfrahubFile],
+    ) -> None:
+        """Load one or multiple objects files into Infrahub."""
+
+        log = get_run_logger()
+        files = await self._load_yamlfile_from_disk(paths=paths, file_type=file_type)
+
+        for file in files:
+            await file.validate_format(client=self.sdk, branch=branch)
+            schema = await self.sdk.schema.get(kind=file.spec.kind, branch=branch)
+            if not schema.human_friendly_id and not schema.default_filter:
+                raise ValueError(
+                    f"Schemas of objects or menus defined within {file.location} "
+                    "should have a `human_friendly_id` defined to avoid creating duplicated objects."
+                )
+
+        for file in files:
+            log.info(f"Loading objects defined in {file.location}")
+            await file.process(client=self.sdk, branch=branch)
+
+    async def _import_file_paths(
+        self, branch_name: str, commit: str, files_pathes: list[Path], object_type: RepositoryObjects
+    ) -> None:
+        branch_wt = self.get_worktree(identifier=commit or branch_name)
+        file_pathes = [branch_wt.directory / file_path for file_path in files_pathes]
+
+        # We currently assume there can't be concurrent imports, but if so, we might need to clone the client before tracking here.
+        async with self.sdk.start_tracking(
+            identifier=f"group-repo-{object_type.value}-{self.id}",
+            delete_unused_nodes=True,
+            branch=branch_name,
+            group_type="CoreRepositoryGroup",
+            group_params={"content": object_type.value, "repository": str(self.id)},
+        ):
+            file_type = repo_object_type_to_file_type(object_type)
+            await self._load_objects(
+                paths=file_pathes,
+                branch=branch_name,
+                file_type=file_type,
+            )
+
+    @task(name="import-objects", task_run_name="Import Objects", cache_policy=NONE)  # type: ignore[arg-type]
+    async def import_objects(
+        self,
+        branch_name: str,
+        commit: str,
+        config_file: InfrahubRepositoryConfig,
+    ) -> None:
+        await self._import_file_paths(
+            branch_name=branch_name,
+            commit=commit,
+            files_pathes=config_file.objects,
+            object_type=RepositoryObjects.OBJECT,
+        )
+        await self._import_file_paths(
+            branch_name=branch_name,
+            commit=commit,
+            files_pathes=config_file.menus,
+            object_type=RepositoryObjects.MENU,
+        )
+
     @task(name="check-definition-get", task_run_name="Get Check Definition", cache_policy=NONE)  # type: ignore[arg-type]
     async def get_check_definition(
         self,
@@ -1342,3 +1431,13 @@ async def render_artifact(
 
         await self.service.event.send(event=event)
         return ArtifactGenerateResult(changed=True, checksum=checksum, storage_id=storage_id, artifact_id=artifact.id)
+
+
+def repo_object_type_to_file_type(repo_object: RepositoryObjects) -> type[InfrahubFile]:
+    match repo_object:
+        case RepositoryObjects.OBJECT:
+            return ObjectFile
+        case RepositoryObjects.MENU:
+            return MenuFile
+        case _:
+            raise ValueError(f"Unknown repository object type: {repo_object}")

backend/tests/fixtures/repos/car-dealership/initial__main/.infrahub.yml

@@ -98,3 +98,11 @@ queries:
     file_path: "generators/cartags.gql"
   - name: person_with_cars
     file_path: "templates/person_with_cars.gql"
+
+objects:
+  - "objects/persons.yml"
+  - "objects/manufacturers.yml"
+
+menus:
+  - "menus/person_base.yml"
+  - "menus/manufacturer_base.yml"

backend/tests/fixtures/repos/car-dealership/initial__main/menus/manufacturer_base.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: infrahub.app/v1
+kind: Menu
+spec:
+  data:
+    - namespace: Testing
+      name: Manufacturer
+      label: Manufacturer
+      kind: TestingManufacturer
+      children:
+        data:
+          - namespace: Testing
+            name: Car
+            label: Car
+            kind: TestingCar

backend/tests/fixtures/repos/car-dealership/initial__main/menus/person_base.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: infrahub.app/v1
+kind: Menu
+spec:
+  data:
+    - namespace: Testing
+      name: Person
+      label: Person
+      kind: TestingPerson
+      children:
+        data:
+          - namespace: Testing
+            name: Manufacturer
+            label: Manufacturer
+            kind: TestingManufacturer