Add support for multiple yaml documents in a file

Author: dgarros

.yamllint.yml

@@ -5,6 +5,7 @@ ignore: |
   /.venv
   /examples
   tests/unit/sdk/test_data/schema_encoding_error.yml
+  tests/unit/sdk/test_data/multiple_files_valid_not_valid.yml
 
 rules:
   new-lines: disable

infrahub_sdk/yaml.py

@@ -7,6 +7,7 @@
 import yaml
 from pydantic import BaseModel, Field
 from typing_extensions import Self
+from yaml.parser import ParserError
 
 from .exceptions import FileNotValidError
 from .utils import find_files, read_file
@@ -31,6 +32,8 @@ class InfrahubFileData(BaseModel):
 class LocalFile(BaseModel):
     identifier: str | None = None
     location: Path
+    multiple_documents: bool = False
+    document_position: int | None = None
     content: dict | None = None
     valid: bool = True
     error_message: str | None = None
@@ -57,20 +60,52 @@ def load_content(self) -> None:
     def validate_content(self) -> None:
         pass
 
+    @classmethod
+    def load_file_from_disk(cls, path: Path) -> list[Self]:
+        yaml_files: list[Self] = []
+
+        try:
+            file_content = read_file(path)
+
+            has_multiple_document = bool(file_content.count("---") > 1)
+
+            if has_multiple_document:
+                for content in yaml.safe_load_all(file_content):
+                    yaml_files.append(cls(location=path, multiple_documents=has_multiple_document, content=content))
+            else:
+                yaml_files.append(
+                    cls(location=path, multiple_documents=has_multiple_document, content=yaml.safe_load(file_content))
+                )
+        except FileNotValidError as exc:
+            yaml_files.append(
+                cls(location=path, multiple_documents=has_multiple_document, error_message=exc.message, valid=False)
+            )
+        except (yaml.YAMLError, ParserError):
+            yaml_files.append(
+                cls(
+                    location=path,
+                    multiple_documents=has_multiple_document,
+                    error_message="Invalid YAML/JSON file",
+                    valid=False,
+                )
+            )
+
+        if has_multiple_document:
+            for idx, file in enumerate(yaml_files):
+                file.document_position = idx + 1
+
+        return yaml_files
+
     @classmethod
     def load_from_disk(cls, paths: list[Path]) -> list[Self]:
         yaml_files: list[Self] = []
         for file_path in paths:
             if file_path.is_file():
-                yaml_file = cls(location=file_path)
-                yaml_file.load_content()
-                yaml_files.append(yaml_file)
+                yaml_files.extend(cls.load_file_from_disk(path=file_path))
             elif file_path.is_dir():
                 files = find_files(extension=["yaml", "yml", "json"], directory=file_path)
                 for item in files:
-                    yaml_file = cls(location=item)
-                    yaml_file.load_content()
-                    yaml_files.append(yaml_file)
+                    yaml_files.extend(cls.load_file_from_disk(path=item))
             else:
                 raise FileNotValidError(name=str(file_path), message=f"{file_path} does not exist!")
 

tests/unit/sdk/test_data/multiple_files_valid.yml

@@ -0,0 +1,10 @@
+---
+version: "1.0"
+nodes:
+  - name: Node
+    namespace: Testing
+---
+version: "1.0"
+nodes:
+  - name: Node2
+    namespace: Testing

tests/unit/sdk/test_data/multiple_files_valid_not_valid.yml

@@ -0,0 +1,11 @@
+---
+version: "1.0"
+nodes:
+  - name: Node
+    namespace: Testing
+---
+version: "1.0"
+nodes:
+  - name: Node2
+    namespace: Testing
+    - name: Node3

tests/unit/sdk/test_data/single_file.yml

@@ -0,0 +1,5 @@
+---
+version: "1.0"
+nodes:
+  - name: Node
+    namespace: Testing

tests/unit/sdk/test_yaml.py

@@ -19,3 +19,23 @@ def test_read_incorrect_encoding() -> None:
     yaml_file.load_content()
     assert not yaml_file.valid
     assert yaml_file.error_message == f"Unable to read {file} with utf-8 encoding"
+
+
+def test_read_multiple_files() -> None:
+    file = here / "test_data/multiple_files_valid.yml"
+    yaml_files = YamlFile.load_file_from_disk(path=file)
+    assert len(yaml_files) == 2
+    assert yaml_files[0].document_position == 1
+    assert yaml_files[0].valid is True
+    assert yaml_files[1].document_position == 2
+    assert yaml_files[1].valid is True
+
+
+def test_read_multiple_files_invalid() -> None:
+    file = here / "test_data/multiple_files_valid_not_valid.yml"
+    yaml_files = YamlFile.load_file_from_disk(path=file)
+    assert len(yaml_files) == 2
+    assert yaml_files[0].document_position == 1
+    assert yaml_files[0].valid is True
+    assert yaml_files[1].document_position == 2
+    assert yaml_files[1].valid is False