Merge pull request #4814 from opsmill/lgu-migrate-git-repo-pull-read-only

Migrate GitRepositoryPullReadOnly to prefect

Author: dgarros

backend/infrahub/git/models.py

@@ -33,3 +33,14 @@ class RequestArtifactGenerate(BaseModel):
     query: str = Field(..., description="The name of the query to use when collecting data")
     timeout: int = Field(..., description="Timeout for requests used to generate this artifact")
     variables: dict = Field(..., description="Input variables when generating the artifact")
+
+
+class GitRepositoryPullReadOnly(BaseModel):
+    """Update a read-only repository to the latest commit for its ref"""
+
+    location: str = Field(..., description="The external URL of the repository")
+    repository_id: str = Field(..., description="The unique ID of the Repository")
+    repository_name: str = Field(..., description="The name of the repository")
+    ref: Optional[str] = Field(None, description="Ref to track on the external repository")
+    commit: Optional[str] = Field(None, description="Specific commit to pull")
+    infrahub_branch_name: str = Field(..., description="Infrahub branch on which to sync the remote repository")

backend/infrahub/git/tasks.py

@@ -12,8 +12,8 @@
 from ..tasks.artifact import define_artifact
 from ..workflows.catalogue import REQUEST_ARTIFACT_DEFINITION_GENERATE, REQUEST_ARTIFACT_GENERATE
 from ..workflows.utils import add_branch_tag
-from .models import RequestArtifactDefinitionGenerate, RequestArtifactGenerate
-from .repository import InfrahubRepository, get_initialized_repo
+from .models import GitRepositoryPullReadOnly, RequestArtifactDefinitionGenerate, RequestArtifactGenerate
+from .repository import InfrahubReadOnlyRepository, InfrahubRepository, get_initialized_repo
 
 log = get_logger()
 
@@ -234,3 +234,44 @@ async def generate_request_artifact_definition(model: RequestArtifactDefinitionG
         await service.workflow.submit_workflow(
             workflow=REQUEST_ARTIFACT_GENERATE, parameters={"model": request_artifact_generate_model}
         )
+
+
+@flow(name="git-repository-pull-read-only")
+async def pull_read_only(model: GitRepositoryPullReadOnly) -> None:
+    service = services.service
+    if not model.ref and not model.commit:
+        log.warning(
+            "No commit or ref in GitRepositoryPullReadOnly message",
+            name=model.repository_name,
+            repository_id=model.repository_id,
+        )
+        return
+    async with service.git_report(related_node=model.repository_id, title="Pulling read-only repository") as git_report:
+        async with lock.registry.get(name=model.repository_name, namespace="repository"):
+            init_failed = False
+            try:
+                repo = await InfrahubReadOnlyRepository.init(
+                    id=model.repository_id,
+                    name=model.repository_name,
+                    location=model.location,
+                    client=service.client,
+                    ref=model.ref,
+                    infrahub_branch_name=model.infrahub_branch_name,
+                    task_report=git_report,
+                )
+            except RepositoryError:
+                init_failed = True
+
+            if init_failed:
+                repo = await InfrahubReadOnlyRepository.new(
+                    id=model.repository_id,
+                    name=model.repository_name,
+                    location=model.location,
+                    client=service.client,
+                    ref=model.ref,
+                    infrahub_branch_name=model.infrahub_branch_name,
+                    task_report=git_report,
+                )
+
+            await repo.import_objects_from_files(infrahub_branch_name=model.infrahub_branch_name, commit=model.commit)
+            await repo.sync_from_remote(commit=model.commit)

backend/infrahub/graphql/mutations/repository.py

@@ -10,10 +10,12 @@
 from infrahub.core.protocols import CoreGenericRepository, CoreReadOnlyRepository, CoreRepository
 from infrahub.core.schema import NodeSchema
 from infrahub.exceptions import ValidationError
+from infrahub.git.models import GitRepositoryPullReadOnly
 from infrahub.graphql.types.common import IdentifierInput
 from infrahub.log import get_logger
 from infrahub.message_bus import messages
 from infrahub.message_bus.messages.git_repository_connectivity import GitRepositoryConnectivityResponse
+from infrahub.workflows.catalogue import GIT_REPOSITORIES_PULL_READ_ONLY
 
 from .main import InfrahubMutationMixin, InfrahubMutationOptions
 
@@ -166,7 +168,7 @@ async def mutate_update(
             ref=data.ref.value if data.ref else None,
         )
 
-        message = messages.GitRepositoryPullReadOnly(
+        model = GitRepositoryPullReadOnly(
             repository_id=obj.id,
             repository_name=obj.name.value,
             location=obj.location.value,
@@ -175,7 +177,9 @@ async def mutate_update(
             infrahub_branch_name=branch.name,
         )
         if context.service:
-            await context.service.send(message=message)
+            await context.service.workflow.submit_workflow(
+                workflow=GIT_REPOSITORIES_PULL_READ_ONLY, parameters={"model": model}
+            )
         return obj, result
 
 

backend/infrahub/message_bus/messages/__init__.py

@@ -20,7 +20,6 @@
 from .git_repository_importobjects import GitRepositoryImportObjects
 from .git_repository_merge import GitRepositoryMerge
 from .git_repository_read_only_add import GitRepositoryAddReadOnly
-from .git_repository_read_only_pull import GitRepositoryPullReadOnly
 from .proposed_change.request_proposedchange_dataintegrity import RequestProposedChangeDataIntegrity
 from .proposed_change.request_proposedchange_refreshartifacts import RequestProposedChangeRefreshArtifacts
 from .proposed_change.request_proposedchange_repositorychecks import RequestProposedChangeRepositoryChecks
@@ -65,7 +64,6 @@
     "git.repository.merge": GitRepositoryMerge,
     "git.repository.add_read_only": GitRepositoryAddReadOnly,
     "git.repository.import_objects": GitRepositoryImportObjects,
-    "git.repository.pull_read_only": GitRepositoryPullReadOnly,
     "schema.migration.path": SchemaMigrationPath,
     "schema.validator.path": SchemaValidatorPath,
     "refresh.registry.branches": RefreshRegistryBranches,

backend/infrahub/message_bus/messages/git_repository_read_only_pull.py

@@ -39,7 +39,6 @@
     "git.repository.add_read_only": git.repository.add_read_only,
     "git.repository.connectivity": git.repository.connectivity,
     "git.repository.import_objects": git.repository.import_objects,
-    "git.repository.pull_read_only": git.repository.pull_read_only,
     "git.repository.merge": git.repository.merge,
     "refresh.registry.branches": refresh.registry.branches,
     "refresh.registry.rebased_branch": refresh.registry.rebased_branch,

backend/infrahub/message_bus/operations/__init__.py

@@ -104,57 +104,6 @@ async def import_objects(message: messages.GitRepositoryImportObjects, service:
         await repo.import_objects_from_files(infrahub_branch_name=message.infrahub_branch_name, commit=message.commit)
 
 
-@flow(name="git-repository-pull-read-only")
-async def pull_read_only(message: messages.GitRepositoryPullReadOnly, service: InfrahubServices) -> None:
-    if not message.ref and not message.commit:
-        log.warning(
-            "No commit or ref in GitRepositoryPullReadOnly message",
-            name=message.repository_name,
-            repository_id=message.repository_id,
-        )
-        return
-    log.info(
-        "Pulling read-only repository",
-        repository=message.repository_name,
-        location=message.location,
-        ref=message.ref,
-        commit=message.commit,
-    )
-    async with service.git_report(
-        related_node=message.repository_id, title="Pulling read-only repository"
-    ) as git_report:
-        async with lock.registry.get(name=message.repository_name, namespace="repository"):
-            init_failed = False
-            try:
-                repo = await InfrahubReadOnlyRepository.init(
-                    id=message.repository_id,
-                    name=message.repository_name,
-                    location=message.location,
-                    client=service.client,
-                    ref=message.ref,
-                    infrahub_branch_name=message.infrahub_branch_name,
-                    task_report=git_report,
-                )
-            except RepositoryError:
-                init_failed = True
-
-            if init_failed:
-                repo = await InfrahubReadOnlyRepository.new(
-                    id=message.repository_id,
-                    name=message.repository_name,
-                    location=message.location,
-                    client=service.client,
-                    ref=message.ref,
-                    infrahub_branch_name=message.infrahub_branch_name,
-                    task_report=git_report,
-                )
-
-            await repo.import_objects_from_files(
-                infrahub_branch_name=message.infrahub_branch_name, commit=message.commit
-            )
-            await repo.sync_from_remote(commit=message.commit)
-
-
 @flow(name="git-repository-merge")
 async def merge(message: messages.GitRepositoryMerge, service: InfrahubServices) -> None:
     log.info(

backend/infrahub/message_bus/operations/git/repository.py

@@ -129,6 +129,16 @@
     branch_support=BranchSupportType.AWARE,
     tags=[WorkflowTag.DATABASE_CHANGE],
 )
+
+GIT_REPOSITORIES_PULL_READ_ONLY = WorkflowDefinition(
+    name="git-repository-pull-read-only",
+    type=WorkflowType.INTERNAL,
+    module="infrahub.git.tasks",
+    function="pull_read_only",
+    branch_support=BranchSupportType.AWARE,
+    tags=[WorkflowTag.DATABASE_CHANGE],
+)
+
 BRANCH_REBASE = WorkflowDefinition(
     name="branch-rebase",
     type=WorkflowType.INTERNAL,
@@ -167,5 +177,6 @@
     REQUEST_GENERATOR_RUN,
     REQUEST_DIFF_UPDATE,
     REQUEST_DIFF_REFRESH,
+    GIT_REPOSITORIES_PULL_READ_ONLY,
     TRIGGER_GENERATOR_DEFINITION_RUN,
 ]

backend/infrahub/workflows/catalogue.py

@@ -10,11 +10,14 @@
 from infrahub.core.constants import InfrahubKind, RepositoryInternalStatus
 from infrahub.exceptions import RepositoryError
 from infrahub.git import InfrahubRepository
+from infrahub.git.models import GitRepositoryPullReadOnly
 from infrahub.git.repository import InfrahubReadOnlyRepository
+from infrahub.git.tasks import pull_read_only
 from infrahub.lock import InfrahubLockRegistry
 from infrahub.message_bus import Meta, messages
 from infrahub.message_bus.operations import git
-from infrahub.services import InfrahubServices
+from infrahub.services import InfrahubServices, services
+from infrahub.services.adapters.workflow.local import WorkflowLocalExecution
 from tests.helpers.test_client import dummy_async_request
 
 # pylint: disable=redefined-outer-name
@@ -230,16 +233,18 @@ class TestPullReadOnly:
     def setup_method(self):
         self.client = AsyncMock(spec=InfrahubClient)
         self.git_report = AsyncContextManagerMock()
-        self.services = InfrahubServices(client=self.client)
-        self.services.git_report = self.git_report
+        self.original_services = services.service
+        services.service = InfrahubServices(client=self.client, workflow=WorkflowLocalExecution())
+        services.service.git_report = self.git_report
+
         self.commit = str(UUIDT())
         self.infrahub_branch_name = "read-only-branch"
         self.repo_id = str(UUIDT())
         self.location = "/some/directory/over/here"
         self.repo_name = "dont-update-this-dude"
         self.ref = "stable-branch"
 
-        self.message = messages.GitRepositoryPullReadOnly(
+        self.model = GitRepositoryPullReadOnly(
             location=self.location,
             repository_id=self.repo_id,
             repository_name=self.repo_name,
@@ -248,31 +253,30 @@ def setup_method(self):
             infrahub_branch_name=self.infrahub_branch_name,
         )
 
-        lock_patcher = patch("infrahub.message_bus.operations.git.repository.lock")
+        lock_patcher = patch("infrahub.git.tasks.lock")
         self.mock_infra_lock = lock_patcher.start()
-        self.mock_infra_lock.registry = AsyncMock(spec=InfrahubLockRegistry)
-        repo_class_patcher = patch(
-            "infrahub.message_bus.operations.git.repository.InfrahubReadOnlyRepository", spec=InfrahubReadOnlyRepository
-        )
+        self.mock_infra_lock.registry = AsyncMock(spec=InfrahubLockRegistry)  # TODO fix mock?
+        repo_class_patcher = patch("infrahub.git.tasks.InfrahubReadOnlyRepository", spec=InfrahubReadOnlyRepository)
         self.mock_repo_class = repo_class_patcher.start()
         self.mock_repo = AsyncMock(spec=InfrahubReadOnlyRepository)
         self.mock_repo_class.new.return_value = self.mock_repo
         self.mock_repo_class.init.return_value = self.mock_repo
 
     def teardown_method(self):
         patch.stopall()
+        services.service = self.original_services
 
     async def test_improper_message(self):
-        self.message.ref = None
-        self.message.commit = None
+        self.model.ref = None
+        self.model.commit = None
 
-        await git.repository.pull_read_only(message=self.message, service=self.services)
+        await pull_read_only(model=self.model)
 
         self.mock_repo_class.new.assert_not_awaited()
         self.mock_repo_class.init.assert_not_awaited()
 
     async def test_existing_repository(self):
-        await git.repository.pull_read_only(message=self.message, service=self.services)
+        await pull_read_only(model=self.model)
 
         self.mock_infra_lock.registry.get(name=self.repo_name, namespace="repository")
         self.mock_repo_class.init.assert_awaited_once_with(
@@ -292,7 +296,7 @@ async def test_existing_repository(self):
     async def test_new_repository(self):
         self.mock_repo_class.init.side_effect = RepositoryError(self.repo_name, "it is broken")
 
-        await git.repository.pull_read_only(message=self.message, service=self.services)
+        await pull_read_only(model=self.model)
 
         self.mock_infra_lock.registry.get(name=self.repo_name, namespace="repository")
         self.mock_repo_class.init.assert_awaited_once_with(

backend/tests/unit/git/test_git_rpc.py

@@ -448,25 +448,6 @@ For more detailed explanations on how to use these events within Infrahub, see t
 | **commit** | Specific commit to pull | string | None |
 | **infrahub_branch_name** | Infrahub branch on which to sync the remote repository | string | None |
 <!-- vale on -->
-<!-- vale off -->
-#### Event git.repository.pull_read_only
-<!-- vale on -->
-
-**Description**: Update a read-only repository to the latest commit for its ref
-
-**Priority**: 3
-
-<!-- vale off -->
-| Key | Description | Type | Default Value |
-|-----|-------------|------|---------------|
-| **meta** | Meta properties for the message | N/A | None |
-| **location** | The external URL of the repository | string | None |
-| **repository_id** | The unique ID of the Repository | string | None |
-| **repository_name** | The name of the repository | string | None |
-| **ref** | Ref to track on the external repository | N/A | None |
-| **commit** | Specific commit to pull | N/A | None |
-| **infrahub_branch_name** | Infrahub branch on which to sync the remote repository | string | None |
-<!-- vale on -->
 
 
 <!-- vale off -->
@@ -1356,26 +1337,6 @@ For more detailed explanations on how to use these events within Infrahub, see t
 | **commit** | Specific commit to pull | string | None |
 | **infrahub_branch_name** | Infrahub branch on which to sync the remote repository | string | None |
 <!-- vale on -->
-<!-- vale off -->
-#### Event git.repository.pull_read_only
-<!-- vale on -->
-
-**Description**: Update a read-only repository to the latest commit for its ref
-
-**Priority**: 3
-
-
-<!-- vale off -->
-| Key | Description | Type | Default Value |
-|-----|-------------|------|---------------|
-| **meta** | Meta properties for the message | N/A | None |
-| **location** | The external URL of the repository | string | None |
-| **repository_id** | The unique ID of the Repository | string | None |
-| **repository_name** | The name of the repository | string | None |
-| **ref** | Ref to track on the external repository | N/A | None |
-| **commit** | Specific commit to pull | N/A | None |
-| **infrahub_branch_name** | Infrahub branch on which to sync the remote repository | string | None |
-<!-- vale on -->
 
 
 <!-- vale off -->