Merge pull request #136 from opsmill/develop

Merge develop into stable

Author: wvandeun

changelog/+479a6128.removed.md

@@ -0,0 +1 @@
+Removed previously deprecated InfrahubTransform.init() method

changelog/+89d1d0b7.deprecated.md

@@ -0,0 +1 @@
+Marked InfrahubCheck.init() as deprecated and scheduled to be removed in Infrahub SDK 2.0.0

changelog/27.fixed.md

@@ -0,0 +1 @@
+Fix generated GraphQL query when having a relationship to a pool node

changelog/81.fixed.md

@@ -0,0 +1 @@
+CTL: Fix support for relative imports for transforms and generators

infrahub_sdk/_importer.py

@@ -2,20 +2,26 @@
 
 import importlib
 import sys
+from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
 from .exceptions import ModuleImportError
 
 if TYPE_CHECKING:
-    from pathlib import Path
     from types import ModuleType
 
+module_mtime_cache: dict[str, float] = {}
+
 
 def import_module(
     module_path: Path, import_root: Optional[str] = None, relative_path: Optional[str] = None
 ) -> ModuleType:
     import_root = import_root or str(module_path.parent)
 
+    file_on_disk = module_path
+    if import_root and relative_path:
+        file_on_disk = Path(import_root, relative_path, module_path.name)
+
     if import_root not in sys.path:
         sys.path.append(import_root)
 
@@ -25,7 +31,21 @@ def import_module(
         module_name = relative_path.replace("/", ".") + f".{module_name}"
 
     try:
-        module = importlib.import_module(module_name)
+        if module_name in sys.modules:
+            module = sys.modules[module_name]
+            current_mtime = file_on_disk.stat().st_mtime
+
+            if module_name in module_mtime_cache:
+                last_mtime = module_mtime_cache[module_name]
+                if current_mtime == last_mtime:
+                    return module
+
+            module_mtime_cache[module_name] = current_mtime
+            module = importlib.reload(module)
+        else:
+            module = importlib.import_module(module_name)
+            module_mtime_cache[module_name] = file_on_disk.stat().st_mtime
+
     except ModuleNotFoundError as exc:
         raise ModuleImportError(message=f"{exc!s} ({module_path})") from exc
     except SyntaxError as exc:

infrahub_sdk/checks.py

@@ -3,23 +3,26 @@
 import asyncio
 import importlib
 import os
+import warnings
 from abc import abstractmethod
 from typing import TYPE_CHECKING, Any, Optional
 
 import ujson
 from git.repo import Repo
 from pydantic import BaseModel, Field
 
-from . import InfrahubClient
 from .exceptions import InfrahubCheckNotFoundError, UninitializedError
 
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from . import InfrahubClient
     from .schema import InfrahubCheckDefinitionConfig
 
 INFRAHUB_CHECK_VARIABLE_TO_IMPORT = "INFRAHUB_CHECKS"
 
+_client_class = "InfrahubClient"
+
 
 class InfrahubCheckInitializer(BaseModel):
     """Information about the originator of the check."""
@@ -81,11 +84,17 @@ def client(self, value: InfrahubClient) -> None:
     @classmethod
     async def init(cls, client: Optional[InfrahubClient] = None, *args: Any, **kwargs: Any) -> InfrahubCheck:
         """Async init method, If an existing InfrahubClient client hasn't been provided, one will be created automatically."""
-
-        instance = cls(*args, **kwargs)
-        instance.client = client or InfrahubClient()
-
-        return instance
+        warnings.warn(
+            "InfrahubCheck.init has been deprecated and will be removed in the version in Infrahub SDK 2.0.0",
+            DeprecationWarning,
+            stacklevel=1,
+        )
+        if not client:
+            client_module = importlib.import_module("infrahub_sdk.client")
+            client_class = getattr(client_module, _client_class)
+            client = client_class()
+        kwargs["client"] = client
+        return cls(*args, **kwargs)
 
     @property
     def errors(self) -> list[dict[str, Any]]:

infrahub_sdk/ctl/check.py

@@ -1,11 +1,9 @@
-import importlib
 import logging
 import sys
 from asyncio import run as aiorun
 from dataclasses import dataclass
 from pathlib import Path
-from types import ModuleType
-from typing import Optional
+from typing import Optional, Type
 
 import typer
 from rich.console import Console
@@ -18,6 +16,7 @@
 from ..ctl.exceptions import QueryNotFoundError
 from ..ctl.repository import get_repository_config
 from ..ctl.utils import catch_exception, execute_graphql_query
+from ..exceptions import ModuleImportError
 from ..schema import InfrahubCheckDefinitionConfig, InfrahubRepositoryConfig
 
 app = typer.Typer()
@@ -27,12 +26,9 @@
 @dataclass
 class CheckModule:
     name: str
-    module: ModuleType
+    check_class: Type[InfrahubCheck]
     definition: InfrahubCheckDefinitionConfig
 
-    def get_check(self) -> InfrahubCheck:
-        return getattr(self.module, self.definition.class_name)
-
 
 @app.callback()
 def callback() -> None:
@@ -67,11 +63,7 @@ def run(
 
     check_definitions = repository_config.check_definitions
     if name:
-        check_definitions = [check for check in repository_config.check_definitions if check.name == name]  # pylint: disable=not-an-iterable
-        if not check_definitions:
-            console.print(f"[red]Unable to find requested transform: {name}")
-            list_checks(repository_config=repository_config)
-            return
+        check_definitions = [repository_config.get_check_definition(name=name)]
 
     check_modules = get_modules(check_definitions=check_definitions)
     aiorun(
@@ -99,8 +91,8 @@ async def run_check(
     output = "stdout" if format_json else None
     log = logging.getLogger("infrahub")
     passed = True
-    check_class = check_module.get_check()
-    check = await check_class.init(client=client, params=params, output=output, root_directory=path, branch=branch)
+    check_class = check_module.check_class
+    check = check_class(client=client, params=params, output=output, root_directory=path, branch=branch)
     param_log = f" - {params}" if params else ""
     try:
         data = execute_graphql_query(
@@ -231,25 +223,19 @@ async def run_checks(
 
 
 def get_modules(check_definitions: list[InfrahubCheckDefinitionConfig]) -> list[CheckModule]:
-    log = logging.getLogger("infrahub")
     modules = []
     for check_definition in check_definitions:
-        directory_name = str(check_definition.file_path.parent)
         module_name = check_definition.file_path.stem
 
-        if directory_name not in sys.path:
-            sys.path.append(directory_name)
+        relative_path = str(check_definition.file_path.parent) if check_definition.file_path.parent != Path() else None
 
         try:
-            module = importlib.import_module(module_name)
-        except ModuleNotFoundError:
-            log.error(f"Unable to load {check_definition.file_path}")
-            continue
-
-        if check_definition.class_name not in dir(module):
-            log.error(f"{check_definition.class_name} class not found in {check_definition.file_path}")
-            continue
-        modules.append(CheckModule(name=module_name, module=module, definition=check_definition))
+            check_class = check_definition.load_class(import_root=str(Path.cwd()), relative_path=relative_path)
+        except ModuleImportError as exc:
+            console.print(f"[red]{exc.message}")
+            raise typer.Exit(1) from exc
+
+        modules.append(CheckModule(name=module_name, check_class=check_class, definition=check_definition))
 
     return modules
 

infrahub_sdk/ctl/cli_commands.py

@@ -36,14 +36,13 @@
     parse_cli_vars,
 )
 from ..ctl.validate import app as validate_app
-from ..exceptions import GraphQLError, InfrahubTransformNotFoundError
+from ..exceptions import GraphQLError, ModuleImportError
 from ..jinja2 import identify_faulty_jinja_code
 from ..schema import (
     InfrahubRepositoryConfig,
     MainSchemaTypes,
     SchemaRoot,
 )
-from ..transforms import get_transform_class_instance
 from ..utils import get_branch, write_to_file
 from ..yaml import SchemaFile
 from .exporter import dump
@@ -322,32 +321,22 @@ def transform(
         list_transforms(config=repository_config)
         return
 
-    # Load transform config
-    try:
-        matched = [transform for transform in repository_config.python_transforms if transform.name == transform_name]  # pylint: disable=not-an-iterable
-        if not matched:
-            raise ValueError(f"{transform_name} does not exist")
-    except ValueError as exc:
-        console.print(f"[red]Unable to find requested transform: {transform_name}")
-        list_transforms(config=repository_config)
-        raise typer.Exit(1) from exc
-
-    transform_config = matched[0]
+    transform_config = repository_config.get_python_transform(name=transform_name)
 
     # Get client
     client = initialize_client(branch=branch)
 
     # Get python transform class instance
+
+    relative_path = str(transform_config.file_path.parent) if transform_config.file_path.parent != Path() else None
+
     try:
-        transform = get_transform_class_instance(
-            transform_config=transform_config,
-            branch=branch,
-            client=client,
-        )
-    except InfrahubTransformNotFoundError as exc:
-        console.print(f"Unable to load {transform_name} from python_transforms")
+        transform_class = transform_config.load_class(import_root=str(Path.cwd()), relative_path=relative_path)
+    except ModuleImportError as exc:
+        console.print(f"[red]{exc.message}")
         raise typer.Exit(1) from exc
 
+    transform = transform_class(client=client, branch=branch)
     # Get data
     query_str = repository_config.get_query(name=transform.query).load_query()
     data = asyncio.run(

infrahub_sdk/ctl/generator.py

@@ -1,12 +1,14 @@
 from pathlib import Path
 from typing import Optional
 
+import typer
 from rich.console import Console
 
 from ..ctl import config
 from ..ctl.client import initialize_client
 from ..ctl.repository import get_repository_config
 from ..ctl.utils import execute_graphql_query, parse_cli_vars
+from ..exceptions import ModuleImportError
 from ..node import InfrahubNode
 from ..schema import InfrahubRepositoryConfig
 
@@ -18,24 +20,25 @@ async def run(
     list_available: bool,
     branch: Optional[str] = None,
     variables: Optional[list[str]] = None,
-):  # pylint: disable=unused-argument
+) -> None:  # pylint: disable=unused-argument
     repository_config = get_repository_config(Path(config.INFRAHUB_REPO_CONFIG_FILE))
 
     if list_available:
         list_generators(repository_config=repository_config)
         return
 
-    matched = [generator for generator in repository_config.generator_definitions if generator.name == generator_name]  # pylint: disable=not-an-iterable
+    generator_config = repository_config.get_generator_definition(name=generator_name)
 
     console = Console()
 
-    if not matched:
-        console.print(f"[red]Unable to find requested generator: {generator_name}")
-        list_generators(repository_config=repository_config)
-        return
+    relative_path = str(generator_config.file_path.parent) if generator_config.file_path.parent != Path() else None
+
+    try:
+        generator_class = generator_config.load_class(import_root=str(Path.cwd()), relative_path=relative_path)
+    except ModuleImportError as exc:
+        console.print(f"[red]{exc.message}")
+        raise typer.Exit(1) from exc
 
-    generator_config = matched[0]
-    generator_class = generator_config.load_class()
     variables_dict = parse_cli_vars(variables)
 
     param_key = list(generator_config.parameters.keys())
@@ -69,6 +72,13 @@ async def run(
             kind="CoreGroup", branch=branch, include=["members"], name__value=generator_config.targets
         )
         await targets.members.fetch()
+
+        if not targets.members.peers:
+            console.print(
+                f"[red]No members found within '{generator_config.targets}', not running generator '{generator_name}'"
+            )
+            return
+
         for member in targets.members.peers:
             check_parameter = {}
             if identifier:

infrahub_sdk/ctl/utils.py

@@ -20,6 +20,7 @@
     Error,
     GraphQLError,
     NodeNotFoundError,
+    ResourceNotDefinedError,
     SchemaNotFoundError,
     ServerNotReachableError,
     ServerNotResponsiveError,
@@ -59,7 +60,7 @@ def handle_exception(exc: Exception, console: Console, exit_code: int) -> NoRetu
     if isinstance(exc, GraphQLError):
         print_graphql_errors(console=console, errors=exc.errors)
         raise typer.Exit(code=exit_code)
-    if isinstance(exc, (SchemaNotFoundError, NodeNotFoundError)):
+    if isinstance(exc, (SchemaNotFoundError, NodeNotFoundError, ResourceNotDefinedError)):
         console.print(f"[red]Error: {exc!s}")
         raise typer.Exit(code=exit_code)