Add group trackingé

Author: LucasG0

backend/infrahub/core/constants/__init__.py

@@ -146,6 +146,11 @@ class AllowOverrideType(InfrahubStringEnum):
     ANY = "any"
 
 
+class RepositoryObjects(InfrahubStringEnum):
+    OBJECT = "object"
+    MENU = "menu"
+
+
 class ContentType(InfrahubStringEnum):
     APPLICATION_JSON = "application/json"
     APPLICATION_YAML = "application/yaml"

backend/infrahub/core/protocols.py

@@ -493,6 +493,11 @@ class CoreRepository(LineageOwner, LineageSource, CoreGenericRepository, CoreTas
     commit: StringOptional
 
 
+class CoreRepositoryGroup(CoreGroup):
+    content: Dropdown
+    repository: RelationshipManager
+
+
 class CoreRepositoryValidator(CoreValidator):
     repository: RelationshipManager
 

backend/infrahub/core/schema/definitions/core/__init__.py

@@ -28,7 +28,13 @@
 from .core import core_node, core_task_target
 from .generator import core_generator_definition, core_generator_instance
 from .graphql_query import core_graphql_query
-from .group import core_generator_group, core_graphql_query_group, core_group, core_standard_group
+from .group import (
+    core_generator_group,
+    core_graphql_query_group,
+    core_group,
+    core_repository_group,
+    core_standard_group,
+)
 from .ipam import builtin_ip_address, builtin_ip_prefix, builtin_ipam, core_ipam_namespace
 from .lineage import lineage_owner, lineage_source
 from .menu import generic_menu_item, menu_item
@@ -109,6 +115,7 @@
         core_standard_group,
         core_generator_group,
         core_graphql_query_group,
+        core_repository_group,
         builtin_tag,
         core_account,
         core_account_token,

backend/infrahub/core/schema/definitions/core/group.py

@@ -1,9 +1,11 @@
 from infrahub.core.constants import (
     BranchSupportType,
     InfrahubKind,
+    RepositoryObjects,
 )
 from infrahub.core.constants import RelationshipCardinality as Cardinality
 from infrahub.core.constants import RelationshipKind as RelKind
+from infrahub.core.schema.dropdown import DropdownChoice
 
 from ...attribute_schema import AttributeSchema as Attr
 from ...generic_schema import GenericSchema
@@ -80,6 +82,7 @@
     generate_profile=False,
 )
 
+
 core_graphql_query_group = NodeSchema(
     name="GraphQLQueryGroup",
     namespace="Core",
@@ -106,3 +109,45 @@
         ),
     ],
 )
+
+
+core_repository_group = NodeSchema(
+    name="RepositoryGroup",
+    namespace="Core",
+    description="Group of nodes associated with a given repository.",
+    include_in_menu=False,
+    icon="mdi:account-group",
+    label="Repository Group",
+    default_filter="name__value",
+    order_by=["name__value"],
+    display_labels=["name__value"],
+    branch=BranchSupportType.LOCAL,
+    inherit_from=[InfrahubKind.GENERICGROUP],
+    generate_profile=False,
+    attributes=[
+        Attr(
+            name="content",
+            kind="Dropdown",
+            description="Type of data to load, can be either `object` or `menu`",
+            choices=[
+                DropdownChoice(
+                    name=RepositoryObjects.OBJECT.value,
+                    label="Objects",
+                ),
+                DropdownChoice(
+                    name=RepositoryObjects.MENU.value,
+                    label="Menus",
+                ),
+            ],
+            optional=False,
+        ),
+    ],
+    relationships=[
+        Rel(
+            name="repository",
+            peer=InfrahubKind.GENERICREPOSITORY,
+            optional=False,
+            cardinality=Cardinality.ONE,
+        ),
+    ],
+)

backend/infrahub/git/base.py

@@ -162,6 +162,11 @@ class InfrahubRepositoryBase(BaseModel, ABC):
     infrahub_branch_name: str | None = Field(None, description="Infrahub branch on which to sync the remote repository")
     model_config = ConfigDict(arbitrary_types_allowed=True, ignored_types=(Flow, Task))
 
+    def get_client(self) -> InfrahubClient:
+        if self.client is None:
+            raise ValueError("Client is not set")
+        return self.client
+
     @property
     def sdk(self) -> InfrahubClient:
         if self.client:
@@ -445,9 +450,6 @@ def get_worktrees(self) -> list[Worktree]:
 
         return [Worktree.init(response) for response in responses]
 
-    def get_client(self) -> InfrahubClient:
-        return self.sdk
-
     def get_location(self) -> str:
         if self.location:
             return self.location

backend/infrahub/git/integrator.py

@@ -29,19 +29,20 @@
     InfrahubPythonTransformConfig,
     InfrahubRepositoryConfig,
 )
+from infrahub_sdk.spec.menu import MenuFile
 from infrahub_sdk.spec.object import ObjectFile
 from infrahub_sdk.template import Jinja2Template
 from infrahub_sdk.template.exceptions import JinjaTemplateError
 from infrahub_sdk.utils import compare_lists
-from infrahub_sdk.yaml import SchemaFile
+from infrahub_sdk.yaml import InfrahubFile, SchemaFile
 from prefect import flow, task
 from prefect.cache_policies import NONE
 from prefect.logging import get_run_logger
 from pydantic import BaseModel, Field
 from pydantic import ValidationError as PydanticValidationError
 from typing_extensions import Self
 
-from infrahub.core.constants import ArtifactStatus, ContentType, InfrahubKind, RepositorySyncStatus
+from infrahub.core.constants import ArtifactStatus, ContentType, InfrahubKind, RepositoryObjects, RepositorySyncStatus
 from infrahub.core.registry import registry
 from infrahub.events.artifact_action import ArtifactCreatedEvent, ArtifactUpdatedEvent
 from infrahub.events.models import EventMeta
@@ -161,7 +162,7 @@ async def init(cls, service: InfrahubServices, commit: str | None = None, **kwar
     async def ensure_location_is_defined(self) -> None:
         if self.location:
             return
-        client = self.get_client()
+        client = self.sdk
         repo = await client.get(
             kind=CoreGenericRepository, name__value=self.name, exclude=["tags", "credential"], raise_when_missing=True
         )
@@ -181,6 +182,9 @@ async def import_objects_from_files(
 
         config_file = await self.get_repository_config(branch_name=infrahub_branch_name, commit=commit)  # type: ignore[misc]
         sync_status = RepositorySyncStatus.IN_SYNC if config_file else RepositorySyncStatus.ERROR_IMPORT
+        if sync_status == RepositorySyncStatus.ERROR_IMPORT:
+            raise ValueError("Unable to import the repository here")
+
         error: Exception | None = None
 
         try:
@@ -191,6 +195,19 @@ async def import_objects_from_files(
                     branch_name=infrahub_branch_name, commit=commit, config_file=config_file
                 )  # type: ignore[misc]
 
+                await self.import_objects(
+                    branch_name=infrahub_branch_name,
+                    commit=commit,
+                    files_pathes=config_file.objects,
+                    object_type=RepositoryObjects.OBJECT,
+                )  # type: ignore[misc]
+                await self.import_objects(
+                    branch_name=infrahub_branch_name,
+                    commit=commit,
+                    files_pathes=config_file.menus,
+                    object_type=RepositoryObjects.MENU,
+                )  # type: ignore[misc]
+
                 await self.import_all_python_files(  # type: ignore[call-overload]
                     branch_name=infrahub_branch_name, commit=commit, config_file=config_file
                 )  # type: ignore[misc]
@@ -200,11 +217,15 @@ async def import_objects_from_files(
                 await self.import_artifact_definitions(
                     branch_name=infrahub_branch_name, commit=commit, config_file=config_file
                 )  # type: ignore[misc]
-                await self.import_objects(branch_name=infrahub_branch_name, commit=commit, config_file=config_file)  # type: ignore[misc]
+
+                # TODO: import menu, check def load() in menu.^y + add a deidcated group for menu, work as for objects
+                # "CoreRepositoryMenuGroup"
 
         except Exception as exc:
             sync_status = RepositorySyncStatus.ERROR_IMPORT
             error = exc
+            log = get_run_logger()
+            log.error(f"Error importing the repository {self.name} : {exc}")
 
         await self._update_sync_status(branch_name=infrahub_branch_name, status=sync_status)
 
@@ -831,23 +852,65 @@ async def _load_objects(
         self,
         paths: list[Path],
         branch: str,
+        file_type: type[InfrahubFile],
     ) -> None:
         """Load one or multiple objects files into Infrahub."""
 
-        files = await self._load_yamlfile_from_disk(paths=paths, file_type=ObjectFile)
+        log = get_run_logger()
+        files = await self._load_yamlfile_from_disk(paths=paths, file_type=file_type)
+        log.info(f"Loaded files {files=}")
+
         for file in files:
-            await file.validate_format(client=self.client, branch=branch)
+            await file.validate_format(client=self.get_client(), branch=branch)
+            schema = await self.get_client().schema.get(kind=file.spec.kind, branch=branch)
+            if not schema.human_friendly_id and not schema.default_filter:
+                raise ValueError(
+                    f"Schemas of objects or menus defined within {file.location} "
+                    "should have a `human_friendly_id` defined to avoid creating duplicated objects."
+                )
+
         for file in files:
-            await file.process(client=self.client, branch=branch)
+            # TODO: client.log that is then used should log into infrahub.tasks
+            log.info(f"Loading objects defined in {file.location}")
+            await file.process(client=self.get_client(), branch=branch)
 
     @task(name="import-objects", task_run_name="Import Objects", cache_policy=NONE)  # type: ignore[arg-type]
-    async def import_objects(self, branch_name: str, commit: str, config_file: InfrahubRepositoryConfig) -> None:
+    async def import_objects(
+        self, branch_name: str, commit: str, files_pathes: list[Path], object_type: RepositoryObjects
+    ) -> None:
+        log = get_run_logger()
+        log.info(f"Importing {object_type.value} from {files_pathes=}")
+
         branch_wt = self.get_worktree(identifier=commit or branch_name)
 
-        file_pathes = [branch_wt.directory / obj.file_path for obj in config_file.objects]
-        await self._load_objects(paths=file_pathes, branch=branch_name)
+        file_pathes = [branch_wt.directory / file_path for file_path in files_pathes]
+
+        log.info("Before getting the repository object")
+        # do not clone for now, assume only one import at a time
+        if self.is_read_only:
+            sdk_repo_obj = await self.get_client().get(
+                kind=InfrahubKind.READONLYREPOSITORY, id=str(self.id), raise_when_missing=True
+            )
+        else:
+            sdk_repo_obj = await self.get_client().get(
+                kind=InfrahubKind.REPOSITORY, id=str(self.id), raise_when_missing=True
+            )
+
+        log.info(f"After getting the repository object {sdk_repo_obj.id}")
 
-        # TODO attach to a group?
+        async with self.get_client().start_tracking(
+            identifier=f"group-repo-{object_type.value}-{self.id}",
+            delete_unused_nodes=True,
+            group_type="CoreRepositoryGroup",
+            group_params={"content": object_type.value, "repository": sdk_repo_obj},
+        ):
+            log.info("Started to track")
+            file_type = ObjectFile if object_type == RepositoryObjects.OBJECT else MenuFile
+            await self._load_objects(
+                paths=file_pathes,
+                branch=branch_name,
+                file_type=file_type,
+            )
 
     @task(name="check-definition-get", task_run_name="Get Check Definition", cache_policy=NONE)  # type: ignore[arg-type]
     async def get_check_definition(

backend/tests/fixtures/repos/car-dealership/initial__main/.infrahub.yml

@@ -100,7 +100,9 @@ queries:
     file_path: "templates/person_with_cars.gql"
 
 objects:
-  - name: "persons"
-    file_path: "objects/persons.yml"
-  - name: "manufacturers"
-    file_path: "objects/manufacturers.yml"
+  - "objects/persons.yml"
+  - "objects/manufacturers.yml"
+
+menus:
+  - "menus/person_base.yml"
+  - "menus/manufacturer_base.yml"

backend/tests/fixtures/repos/car-dealership/initial__main/menus/manufacturer_base.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: infrahub.app/v1
+kind: Menu
+spec:
+  data:
+    - namespace: Testing
+      name: Manufacturer
+      label: Manufacturer
+      kind: TestingManufacturer
+      children:
+        data:
+          - namespace: Testing
+            name: Car
+            label: Car
+            kind: TestingCar

backend/tests/fixtures/repos/car-dealership/initial__main/menus/person_base.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: infrahub.app/v1
+kind: Menu
+spec:
+  data:
+    - namespace: Testing
+      name: Person
+      label: Person
+      kind: TestingPerson
+      children:
+        data:
+          - namespace: Testing
+            name: Manufacturer
+            label: Manufacturer
+            kind: TestingManufacturer

backend/tests/integration/git/test_repository.py

@@ -6,7 +6,7 @@
 import pytest
 from git import GitCommandError
 
-from infrahub.core.constants import InfrahubKind, RepositoryOperationalStatus
+from infrahub.core.constants import InfrahubKind, RepositoryObjects, RepositoryOperationalStatus
 from infrahub.core.manager import NodeManager
 from infrahub.core.node import Node
 from infrahub.exceptions import RepositoryError
@@ -83,6 +83,43 @@ async def test_create_repository(
         assert manufacturer_mercedes.name.value == "Mercedes"
         assert list((await manufacturer_mercedes.customers.get_peers(db=db)).values())[0].name.value == "Ethan Carter"
 
+        repository_group = await NodeManager.get_one_by_default_filter(
+            db=db,
+            id=f"group-repo-{RepositoryObjects.OBJECT.value}-{repository.id}",
+            kind="CoreRepositoryGroup",
+            raise_on_error=True,
+            prefetch_relationships=True,
+        )
+        assert repository_group.content.value == RepositoryObjects.OBJECT.value
+        members = (await repository_group.members.get_peers(db=db)).values()
+        assert len(members) == 4
+        assert manufacturer_mercedes.id in {m.id for m in members}
+        assert person_ethan.id in {m.id for m in members}
+
+        # TODO Retrieve menus
+
+        repository_group_menus = await NodeManager.get_one_by_default_filter(
+            db=db,
+            id=f"group-repo-{RepositoryObjects.MENU.value}-{repository.id}",
+            kind="CoreRepositoryGroup",
+            raise_on_error=True,
+            prefetch_relationships=True,
+        )
+
+        assert repository_group_menus.content.value == RepositoryObjects.MENU.value
+
+        _ = await NodeManager.get_one_by_hfid(
+            db=db,
+            hfid=["Testing", "Manufacturer"],
+            kind="CoreMenu",
+            raise_on_error=True,
+            prefetch_relationships=True,
+        )
+
+        _ = await NodeManager.get_one_by_hfid(
+            db=db, hfid=["Testing", "Person"], kind="CoreMenu", raise_on_error=True, prefetch_relationships=True
+        )
+
     # TODO add a test with invalid yml file OR invalid order of objects in the yml file, and make sure the repository ends
     # up in error import state