Merge pull request #4819 from opsmill/dga-20241101-develop-1.1

Merge develop-1.1 into develop

Author: dgarros

.github/workflows/ci.yml

@@ -776,11 +776,11 @@ jobs:
 
       - name: Display task worker 1 logs
         if: always()
-        run: docker logs "${INFRAHUB_BUILD_NAME}-infrahub-git-1"
+        run: docker logs "${INFRAHUB_BUILD_NAME}-task-worker-1"
 
       - name: Display task worker 2 logs
         if: always()
-        run: docker logs "${INFRAHUB_BUILD_NAME}-infrahub-git-2"
+        run: docker logs "${INFRAHUB_BUILD_NAME}-task-worker-2"
 
       - name: Display task manager logs
         if: always()

backend/infrahub/core/utils.py

@@ -184,10 +184,10 @@ def convert_ip_to_binary_str(
     obj: Union[ipaddress.IPv6Network, ipaddress.IPv4Network, ipaddress.IPv4Interface, ipaddress.IPv6Interface],
 ) -> str:
     if isinstance(obj, (ipaddress.IPv6Network, ipaddress.IPv4Network)):
-        prefix_bin = bin(int(obj.network_address))[2:]
+        prefix_bin = f"{int(obj.network_address):b}"
         return prefix_bin.zfill(obj.max_prefixlen)
 
-    ip_bin = bin(int(obj))[2:]
+    ip_bin = f"{int(obj):b}"
     return ip_bin.zfill(obj.max_prefixlen)
 
 

backend/infrahub/generators/tasks.py

@@ -7,10 +7,12 @@
 from prefect import flow, task
 
 from infrahub import lock
-from infrahub.core.constants import GeneratorInstanceStatus
+from infrahub.core.constants import GeneratorInstanceStatus, InfrahubKind
 from infrahub.generators.models import RequestGeneratorRun
 from infrahub.git.base import extract_repo_file_information
 from infrahub.git.repository import get_initialized_repo
+from infrahub.message_bus import messages
+from infrahub.message_bus.types import ProposedChangeGeneratorDefinition
 from infrahub.services import InfrahubServices, services
 
 
@@ -103,3 +105,34 @@ async def _define_instance(model: RequestGeneratorRun, service: InfrahubServices
                 )
                 await instance.save()
     return instance
+
+
+@flow(name="generator_definition_run")
+async def run_generator_definition(branch: str) -> None:
+    service = services.service
+    generators = await service.client.filters(
+        kind=InfrahubKind.GENERATORDEFINITION, prefetch_relationships=True, populate_store=True, branch=branch
+    )
+
+    generator_definitions = [
+        ProposedChangeGeneratorDefinition(
+            definition_id=generator.id,
+            definition_name=generator.name.value,
+            class_name=generator.class_name.value,
+            file_path=generator.file_path.value,
+            query_name=generator.query.peer.name.value,
+            query_models=generator.query.peer.models.value,
+            repository_id=generator.repository.peer.id,
+            parameters=generator.parameters.value,
+            group_id=generator.targets.peer.id,
+            convert_query_response=generator.convert_query_response.value,
+        )
+        for generator in generators
+    ]
+
+    events = [
+        messages.RequestGeneratorDefinitionRun(branch=branch, generator_definition=generator_definition)
+        for generator_definition in generator_definitions
+    ]
+    for event in events:
+        await service.send(message=event)

backend/infrahub/git/base.py

@@ -316,10 +316,10 @@ async def create_locally(
             log.warning(f"Found an existing file at {self.directory_root}, deleted it", repository=self.name)
 
         # Initialize directory structure
-        os.makedirs(self.directory_root)
-        os.makedirs(self.directory_branches)
-        os.makedirs(self.directory_commits)
-        os.makedirs(self.directory_temp)
+        Path(self.directory_root).mkdir(parents=True)
+        Path(self.directory_branches).mkdir(parents=True)
+        Path(self.directory_commits).mkdir(parents=True)
+        Path(self.directory_temp).mkdir(parents=True)
 
         try:
             repo = Repo.clone_from(self.location, self.directory_default)

backend/infrahub/git/directory.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 from infrahub import config
 from infrahub.log import get_logger
@@ -25,7 +26,8 @@ def initialize_repositories_directory() -> bool:
     """
     repos_dir = get_repositories_directory()
     if not os.path.isdir(repos_dir):
-        os.makedirs(repos_dir)
+        Path(repos_dir).mkdir(parents=True)
+
         log.debug(f"Initialized the repositories_directory at {repos_dir}")
         return True
 

backend/infrahub/message_bus/messages/__init__.py

@@ -41,7 +41,6 @@
 from .schema_migration_path import SchemaMigrationPath, SchemaMigrationPathResponse
 from .schema_validator_path import SchemaValidatorPath, SchemaValidatorPathResponse
 from .send_echo_request import SendEchoRequest, SendEchoRequestResponse
-from .trigger_generatordefinition_run import TriggerGeneratorDefinitionRun
 from .trigger_proposed_change_cancel import TriggerProposedChangeCancel
 from .trigger_webhook_actions import TriggerWebhookActions
 
@@ -87,7 +86,6 @@
     "request.repository.checks": RequestRepositoryChecks,
     "request.repository.user_checks": RequestRepositoryUserChecks,
     "send.echo.request": SendEchoRequest,
-    "trigger.generator_definition.run": TriggerGeneratorDefinitionRun,
     "trigger.proposed_change.cancel": TriggerProposedChangeCancel,
     "trigger.webhook.actions": TriggerWebhookActions,
 }

backend/infrahub/message_bus/operations/__init__.py

@@ -61,7 +61,6 @@
     "send.echo.request": send.echo.request,
     "schema.migration.path": schema.migration.path,
     "schema.validator.path": schema.validator.path,
-    "trigger.generator_definition.run": trigger.generator_definition.run,
     "trigger.proposed_change.cancel": trigger.proposed_change.cancel,
     "trigger.webhook.actions": trigger.webhook.actions,
 }

backend/infrahub/message_bus/operations/event/branch.py

@@ -15,6 +15,7 @@
     REQUEST_DIFF_REFRESH,
     REQUEST_DIFF_UPDATE,
     TRIGGER_ARTIFACT_DEFINITION_GENERATE,
+    TRIGGER_GENERATOR_DEFINITION_RUN,
 )
 
 log = get_logger()
@@ -56,7 +57,6 @@ async def merge(message: messages.EventBranchMerge, service: InfrahubServices) -
 
     events: List[InfrahubMessage] = [
         messages.RefreshRegistryBranches(),
-        messages.TriggerGeneratorDefinitionRun(branch=message.target_branch),
     ]
     component_registry = get_component_registry()
     default_branch = registry.get_branch_from_registry()
@@ -69,6 +69,11 @@ async def merge(message: messages.EventBranchMerge, service: InfrahubServices) -
         parameters={"branch": message.target_branch},
     )
 
+    await service.workflow.submit_workflow(
+        workflow=TRIGGER_GENERATOR_DEFINITION_RUN,
+        parameters={"branch": message.target_branch},
+    )
+
     for diff_root in branch_diff_roots:
         if (
             diff_root.base_branch_name != diff_root.diff_branch_name

backend/infrahub/message_bus/operations/trigger/__init__.py

@@ -1,3 +1,3 @@
-from . import generator_definition, proposed_change, webhook
+from . import proposed_change, webhook
 
-__all__ = ["generator_definition", "proposed_change", "webhook"]
+__all__ = ["proposed_change", "webhook"]