Allow to load data from git repo

Author: LucasG0

backend/infrahub/git/integrator.py

@@ -29,6 +29,7 @@
     InfrahubPythonTransformConfig,
     InfrahubRepositoryConfig,
 )
+from infrahub_sdk.spec.object import ObjectFile
 from infrahub_sdk.template import Jinja2Template
 from infrahub_sdk.template.exceptions import JinjaTemplateError
 from infrahub_sdk.utils import compare_lists
@@ -54,6 +55,7 @@
     import types
 
     from infrahub_sdk.checks import InfrahubCheck
+    from infrahub_sdk.ctl.utils import YamlFileVar
     from infrahub_sdk.schema.repository import InfrahubRepositoryArtifactDefinitionConfig
     from infrahub_sdk.transforms import InfrahubTransform
 
@@ -198,6 +200,7 @@ async def import_objects_from_files(
                 await self.import_artifact_definitions(
                     branch_name=infrahub_branch_name, commit=commit, config_file=config_file
                 )  # type: ignore[misc]
+                await self.import_objects(branch_name=infrahub_branch_name, commit=commit, config_file=config_file)  # type: ignore[misc]
 
         except Exception as exc:
             sync_status = RepositorySyncStatus.ERROR_IMPORT
@@ -815,6 +818,37 @@ async def import_python_transforms(
             log.info(f"TransformPython {transform_name!r} not found locally, deleting")
             await transform_definition_in_graph[transform_name].delete()
 
+    async def _load_yamlfile_from_disk(self, paths: list[Path], file_type: type[YamlFileVar]) -> list[YamlFileVar]:
+        data_files = file_type.load_from_disk(paths=paths)
+
+        for data_file in data_files:
+            if not data_file.valid or not data_file.content:
+                raise ValueError(f"{data_file.error_message} ({data_file.location})")
+
+        return data_files
+
+    async def _load_objects(
+        self,
+        paths: list[Path],
+        branch: str,
+    ) -> None:
+        """Load one or multiple objects files into Infrahub."""
+
+        files = await self._load_yamlfile_from_disk(paths=paths, file_type=ObjectFile)
+        for file in files:
+            await file.validate_format(client=self.client, branch=branch)
+        for file in files:
+            await file.process(client=self.client, branch=branch)
+
+    @task(name="import-objects", task_run_name="Import Objects", cache_policy=NONE)  # type: ignore[arg-type]
+    async def import_objects(self, branch_name: str, commit: str, config_file: InfrahubRepositoryConfig) -> None:
+        branch_wt = self.get_worktree(identifier=commit or branch_name)
+
+        file_pathes = [branch_wt.directory / obj.file_path for obj in config_file.objects]
+        await self._load_objects(paths=file_pathes, branch=branch_name)
+
+        # TODO attach to a group?
+
     @task(name="check-definition-get", task_run_name="Get Check Definition", cache_policy=NONE)  # type: ignore[arg-type]
     async def get_check_definition(
         self,

backend/tests/fixtures/repos/car-dealership/initial__main/.infrahub.yml

@@ -98,3 +98,9 @@ queries:
     file_path: "generators/cartags.gql"
   - name: person_with_cars
     file_path: "templates/person_with_cars.gql"
+
+objects:
+  - name: "persons"
+    file_path: "objects/persons.yml"
+  - name: "manufacturers"
+    file_path: "objects/manufacturers.yml"

backend/tests/fixtures/repos/car-dealership/initial__main/objects/manufacturers.yml

@@ -0,0 +1,12 @@
+---
+apiVersion: infrahub.app/v1
+kind: Object
+spec:
+  kind: TestingManufacturer
+  data:
+    - name: Mercedes
+      customers:
+        - "Ethan Carter"
+    - name: Ford
+      customers:
+        - "Olivia Bennett"

backend/tests/fixtures/repos/car-dealership/initial__main/objects/persons.yml

@@ -0,0 +1,10 @@
+---
+apiVersion: infrahub.app/v1
+kind: Object
+spec:
+  kind: TestingPerson
+  data:
+    - name: Ethan Carter
+      height: 180
+    - name: Olivia Bennett
+      height: 170

backend/tests/integration/git/test_repository.py

@@ -71,6 +71,21 @@ async def test_create_repository(
         assert repository.operational_status.value == "online"
         assert check_definition.file_path.value == "checks/car_overview.py"
 
+        person_ethan = await NodeManager.get_one_by_default_filter(
+            db=db, id="Ethan Carter", kind="TestingPerson", raise_on_error=True
+        )
+        assert person_ethan.name.value == "Ethan Carter"
+        assert person_ethan.height.value == 180
+
+        manufacturer_mercedes = await NodeManager.get_one_by_default_filter(
+            db=db, id="Mercedes", kind="TestingManufacturer", raise_on_error=True, prefetch_relationships=True
+        )
+        assert manufacturer_mercedes.name.value == "Mercedes"
+        assert list((await manufacturer_mercedes.customers.get_peers(db=db)).values())[0].name.value == "Ethan Carter"
+
+    # TODO add a test with invalid yml file OR invalid order of objects in the yml file, and make sure the repository ends
+    # up in error import state
+
     @pytest.mark.parametrize(
         "stderr,expected_operational_status",
         [