Merge pull request #458 from opsmill/stable

Merge stable into develop

Author: ogenstad

CHANGELOG.md

@@ -11,6 +11,28 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang
 
 <!-- towncrier release notes start -->
 
+## [1.13.5](https://github.com/opsmill/infrahub-sdk-python/tree/v1.13.5) - 2025-07-23
+
+### Fixed
+
+- Respect ordering when loading files from a directory
+
+## [1.13.4](https://github.com/opsmill/infrahub-sdk-python/tree/v1.13.4) - 2025-07-22
+
+### Fixed
+
+- Fix processing of relationshhip during nodes retrieval using the Sync Client, when prefecthing related_nodes. ([#461](https://github.com/opsmill/infrahub-sdk-python/issues/461))
+- Fix schema loading to ignore non-YAML files in folders. ([#462](https://github.com/opsmill/infrahub-sdk-python/issues/462))
+- Fix ignored node variable in filters(). ([#469](https://github.com/opsmill/infrahub-sdk-python/issues/469))
+- Fix use of parallel with filters for Infrahub Client Sync.
+- Avoid sending empty list to infrahub if no valids schemas are found.
+
+## [1.13.3](https://github.com/opsmill/infrahub-sdk-python/tree/v1.13.3) - 2025-06-30
+
+### Fixed
+
+- Update InfrahubNode creation to include __typename, display_label, and kind from a RelatedNode ([#455](https://github.com/opsmill/infrahub-sdk-python/issues/455))
+
 ## [1.13.2](https://github.com/opsmill/infrahub-sdk-python/tree/v1.13.2) - 2025-06-27
 
 ### Fixed

changelog/+batch.fixed.md

@@ -0,0 +1 @@
+Create a new batch while fetching relationships instead of using the reusing the same one.

changelog/+branch-in-count.fixed.md

@@ -0,0 +1 @@
+Update internal calls to `count` to include the branch parameter so that the query is performed on the correct branch

infrahub_sdk/client.py

@@ -784,7 +784,6 @@ async def filters(
         if at:
             at = Timestamp(at)
 
-        node = InfrahubNode(client=self, schema=schema, branch=branch)
         filters = kwargs
         pagination_size = self.pagination_size
 
@@ -825,12 +824,12 @@ async def process_batch() -> tuple[list[InfrahubNode], list[InfrahubNode]]:
             nodes = []
             related_nodes = []
             batch_process = await self.create_batch()
-            count = await self.count(kind=schema.kind, partial_match=partial_match, **filters)
+            count = await self.count(kind=schema.kind, branch=branch, partial_match=partial_match, **filters)
             total_pages = (count + pagination_size - 1) // pagination_size
 
             for page_number in range(1, total_pages + 1):
                 page_offset = (page_number - 1) * pagination_size
-                batch_process.add(task=process_page, node=node, page_offset=page_offset, page_number=page_number)
+                batch_process.add(task=process_page, page_offset=page_offset, page_number=page_number)
 
             async for _, response in batch_process.execute():
                 nodes.extend(response[1]["nodes"])
@@ -847,7 +846,7 @@ async def process_non_batch() -> tuple[list[InfrahubNode], list[InfrahubNode]]:
 
             while has_remaining_items:
                 page_offset = (page_number - 1) * pagination_size
-                response, process_result = await process_page(page_offset, page_number)
+                response, process_result = await process_page(page_offset=page_offset, page_number=page_number)
 
                 nodes.extend(process_result["nodes"])
                 related_nodes.extend(process_result["related_nodes"])
@@ -1946,9 +1945,9 @@ def filters(
         """
         branch = branch or self.default_branch
         schema = self.schema.get(kind=kind, branch=branch)
-        node = InfrahubNodeSync(client=self, schema=schema, branch=branch)
         if at:
             at = Timestamp(at)
+
         filters = kwargs
         pagination_size = self.pagination_size
 
@@ -1990,12 +1989,12 @@ def process_batch() -> tuple[list[InfrahubNodeSync], list[InfrahubNodeSync]]:
             related_nodes = []
             batch_process = self.create_batch()
 
-            count = self.count(kind=schema.kind, partial_match=partial_match, **filters)
+            count = self.count(kind=schema.kind, branch=branch, partial_match=partial_match, **filters)
             total_pages = (count + pagination_size - 1) // pagination_size
 
             for page_number in range(1, total_pages + 1):
                 page_offset = (page_number - 1) * pagination_size
-                batch_process.add(task=process_page, node=node, page_offset=page_offset, page_number=page_number)
+                batch_process.add(task=process_page, page_offset=page_offset, page_number=page_number)
 
             for _, response in batch_process.execute():
                 nodes.extend(response[1]["nodes"])
@@ -2012,7 +2011,7 @@ def process_non_batch() -> tuple[list[InfrahubNodeSync], list[InfrahubNodeSync]]
 
             while has_remaining_items:
                 page_offset = (page_number - 1) * pagination_size
-                response, process_result = process_page(page_offset, page_number)
+                response, process_result = process_page(page_offset=page_offset, page_number=page_number)
 
                 nodes.extend(process_result["nodes"])
                 related_nodes.extend(process_result["related_nodes"])

infrahub_sdk/ctl/utils.py

@@ -187,6 +187,9 @@ def load_yamlfile_from_disk_and_exit(
     has_error = False
     try:
         data_files = file_type.load_from_disk(paths=paths)
+        if not data_files:
+            console.print("[red]No valid files found to load.")
+            raise typer.Exit(1)
     except FileNotValidError as exc:
         console.print(f"[red]{exc.message}")
         raise typer.Exit(1) from exc

infrahub_sdk/node/node.py

@@ -402,10 +402,10 @@ def generate_query_data_init(
         if order:
             data["@filters"]["order"] = order
 
-        if offset:
+        if offset is not None:
             data["@filters"]["offset"] = offset
 
-        if limit:
+        if limit is not None:
             data["@filters"]["limit"] = limit
 
         if include and exclude:
@@ -507,11 +507,17 @@ def _init_relationships(self, data: dict | RelatedNode | None = None) -> None:
 
             if rel_schema.cardinality == "one":
                 if isinstance(rel_data, RelatedNode):
-                    peer_id_data: dict[str, Any] = {}
-                    if rel_data.id:
-                        peer_id_data["id"] = rel_data.id
-                    if rel_data.hfid:
-                        peer_id_data["hfid"] = rel_data.hfid
+                    peer_id_data: dict[str, Any] = {
+                        key: value
+                        for key, value in (
+                            ("id", rel_data.id),
+                            ("hfid", rel_data.hfid),
+                            ("__typename", rel_data.typename),
+                            ("kind", rel_data.kind),
+                            ("display_label", rel_data.display_label),
+                        )
+                        if value is not None
+                    }
                     if peer_id_data:
                         rel_data = peer_id_data
                     else:
@@ -1090,11 +1096,17 @@ def _init_relationships(self, data: dict | None = None) -> None:
 
             if rel_schema.cardinality == "one":
                 if isinstance(rel_data, RelatedNodeSync):
-                    peer_id_data: dict[str, Any] = {}
-                    if rel_data.id:
-                        peer_id_data["id"] = rel_data.id
-                    if rel_data.hfid:
-                        peer_id_data["hfid"] = rel_data.hfid
+                    peer_id_data: dict[str, Any] = {
+                        key: value
+                        for key, value in (
+                            ("id", rel_data.id),
+                            ("hfid", rel_data.hfid),
+                            ("__typename", rel_data.typename),
+                            ("kind", rel_data.kind),
+                            ("display_label", rel_data.display_label),
+                        )
+                        if value is not None
+                    }
                     if peer_id_data:
                         rel_data = peer_id_data
                     else:
@@ -1481,15 +1493,15 @@ def _process_relationships(
         for rel_name in self._relationships:
             rel = getattr(self, rel_name)
             if rel and isinstance(rel, RelatedNodeSync):
-                relation = node_data["node"].get(rel_name)
-                if relation.get("node", None):
+                relation = node_data["node"].get(rel_name, None)
+                if relation and relation.get("node", None):
                     related_node = InfrahubNodeSync.from_graphql(
                         client=self._client, branch=branch, data=relation, timeout=timeout
                     )
                     related_nodes.append(related_node)
             elif rel and isinstance(rel, RelationshipManagerSync):
-                peers = node_data["node"].get(rel_name)
-                if peers:
+                peers = node_data["node"].get(rel_name, None)
+                if peers and peers["edges"]:
                     for peer in peers["edges"]:
                         related_node = InfrahubNodeSync.from_graphql(
                             client=self._client, branch=branch, data=peer, timeout=timeout

infrahub_sdk/node/related_node.py

@@ -39,6 +39,7 @@ def __init__(self, branch: str, schema: RelationshipSchemaAPI, data: Any | dict,
         self._hfid: list[str] | None = None
         self._display_label: str | None = None
         self._typename: str | None = None
+        self._kind: str | None = None
 
         if isinstance(data, (CoreNodeBase)):
             self._peer = data
@@ -118,6 +119,12 @@ def typename(self) -> str | None:
             return self._peer.typename
         return self._typename
 
+    @property
+    def kind(self) -> str | None:
+        if self._peer:
+            return self._peer.get_kind()
+        return self._kind
+
     def _generate_input_data(self, allocate_from_pool: bool = False) -> dict[str, Any]:
         data: dict[str, Any] = {}
 

infrahub_sdk/node/relationship.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
+from collections import defaultdict
 from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any
 
+from ..batch import InfrahubBatch
 from ..exceptions import (
+    Error,
     UninitializedError,
 )
+from ..types import Order
 from .constants import PROPERTIES_FLAG, PROPERTIES_OBJECT
 from .related_node import RelatedNode, RelatedNodeSync
 
@@ -156,8 +160,26 @@ async def fetch(self) -> None:
             self.peers = rm.peers
             self.initialized = True
 
+        ids_per_kind_map = defaultdict(list)
         for peer in self.peers:
-            await peer.fetch()  # type: ignore[misc]
+            if not peer.id or not peer.typename:
+                raise Error("Unable to fetch the peer, id and/or typename are not defined")
+            ids_per_kind_map[peer.typename].append(peer.id)
+
+        batch = InfrahubBatch(max_concurrent_execution=self.client.max_concurrent_execution)
+        for kind, ids in ids_per_kind_map.items():
+            batch.add(
+                task=self.client.filters,
+                kind=kind,
+                ids=ids,
+                populate_store=True,
+                branch=self.branch,
+                parallel=True,
+                order=Order(disable=True),
+            )
+
+        async for _ in batch.execute():
+            pass
 
     def add(self, data: str | RelatedNode | dict) -> None:
         """Add a new peer to this relationship."""
@@ -261,8 +283,27 @@ def fetch(self) -> None:
             self.peers = rm.peers
             self.initialized = True
 
+        ids_per_kind_map = defaultdict(list)
         for peer in self.peers:
-            peer.fetch()
+            if not peer.id or not peer.typename:
+                raise Error("Unable to fetch the peer, id and/or typename are not defined")
+            ids_per_kind_map[peer.typename].append(peer.id)
+
+        # Unlike Async, no need to create a new batch from scratch because we are not using a semaphore
+        batch = self.client.create_batch()
+        for kind, ids in ids_per_kind_map.items():
+            batch.add(
+                task=self.client.filters,
+                kind=kind,
+                ids=ids,
+                populate_store=True,
+                branch=self.branch,
+                parallel=True,
+                order=Order(disable=True),
+            )
+
+        for _ in batch.execute():
+            pass
 
     def add(self, data: str | RelatedNodeSync | dict) -> None:
         """Add a new peer to this relationship."""

infrahub_sdk/yaml.py

@@ -120,16 +120,22 @@ def load_file_from_disk(cls, path: Path) -> list[Self]:
     @classmethod
     def load_from_disk(cls, paths: list[Path]) -> list[Self]:
         yaml_files: list[Self] = []
+        file_extensions = {".yaml", ".yml", ".json"}  # FIXME: .json is not a YAML file, should be removed
+
         for file_path in paths:
-            if file_path.is_file() and file_path.suffix in [".yaml", ".yml", ".json"]:
-                yaml_files.extend(cls.load_file_from_disk(path=file_path))
+            if not file_path.exists():
+                # Check if the provided path exists, relevant for the first call coming from the user
+                raise FileNotValidError(name=str(file_path), message=f"{file_path} does not exist!")
+            if file_path.is_file():
+                if file_path.suffix in file_extensions:
+                    yaml_files.extend(cls.load_file_from_disk(path=file_path))
+                # else: silently skip files with unrelevant extensions (e.g. .md, .py...)
             elif file_path.is_dir():
+                # Introduce recursion to handle sub-folders
                 sub_paths = [Path(sub_file_path) for sub_file_path in file_path.glob("*")]
-                sub_files = cls.load_from_disk(paths=sub_paths)
-                sorted_sub_files = sorted(sub_files, key=lambda x: x.location)
-                yaml_files.extend(sorted_sub_files)
-            else:
-                raise FileNotValidError(name=str(file_path), message=f"{file_path} does not exist!")
+                sub_paths = sorted(sub_paths, key=lambda p: p.name)
+                yaml_files.extend(cls.load_from_disk(paths=sub_paths))
+            # else: skip non-file, non-dir (e.g., symlink...)
 
         return yaml_files
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "infrahub-sdk"
-version = "1.13.2"
+version = "1.13.5"
 description = "Python Client to interact with Infrahub"
 authors = ["OpsMill <[email protected]>"]
 readme = "README.md"