feat(cli): add tasks flush commands

Signed-off-by: Fatih Acar <[email protected]>

Author: fatih-acar

backend/infrahub/cli/tasks.py

@@ -3,9 +3,11 @@
 import typer
 from infrahub_sdk.async_typer import AsyncTyper
 from prefect.client.orchestration import get_client
+from prefect.client.schemas.objects import StateType
 
 from infrahub import config
 from infrahub.services.adapters.workflow.worker import WorkflowWorkerExecution
+from infrahub.task_manager.task import PrefectTask
 from infrahub.tasks.dummy import DUMMY_FLOW, DummyInput
 from infrahub.workflows.initialization import setup_task_manager
 from infrahub.workflows.models import WorkerPoolDefinition
@@ -50,3 +52,47 @@ async def execute(
             workflow=DUMMY_FLOW, parameters={"data": DummyInput(firstname="John", lastname="Doe")}
         )  # type: ignore[var-annotated]
         print(result)
+
+
+flush_app = AsyncTyper()
+
+app.add_typer(flush_app, name="flush")
+
+
+@flush_app.command()
+async def flow_runs(
+    ctx: typer.Context,  # noqa: ARG001
+    config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
+    days_to_keep: int = 30,
+    batch_size: int = 100,
+) -> None:
+    """Flush old task runs"""
+    logging.getLogger("infrahub").setLevel(logging.WARNING)
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    logging.getLogger("prefect").setLevel(logging.ERROR)
+
+    config.load_and_exit(config_file_name=config_file)
+
+    await PrefectTask.delete_flow_runs(
+        days_to_keep=days_to_keep,
+        batch_size=batch_size,
+    )
+
+
+@flush_app.command()
+async def stale_runs(
+    ctx: typer.Context,  # noqa: ARG001
+    config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
+    days_to_keep: int = 2,
+    batch_size: int = 100,
+) -> None:
+    """Flush stale task runs"""
+    logging.getLogger("infrahub").setLevel(logging.WARNING)
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    logging.getLogger("prefect").setLevel(logging.ERROR)
+
+    config.load_and_exit(config_file_name=config_file)
+
+    await PrefectTask.delete_flow_runs(
+        states=[StateType.RUNNING], delete=False, days_to_keep=days_to_keep, batch_size=batch_size
+    )

backend/infrahub/task_manager/task.py

@@ -1,7 +1,10 @@
+import asyncio
 import uuid
+from datetime import datetime, timedelta, timezone
 from typing import Any
 from uuid import UUID
 
+from prefect import State
 from prefect.client.orchestration import PrefectClient, get_client
 from prefect.client.schemas.filters import (
     ArtifactFilter,
@@ -12,6 +15,7 @@
     FlowRunFilter,
     FlowRunFilterId,
     FlowRunFilterName,
+    FlowRunFilterStartTime,
     FlowRunFilterState,
     FlowRunFilterStateType,
     FlowRunFilterTags,
@@ -311,3 +315,72 @@ async def query(
                     )
 
         return {"count": count or 0, "edges": nodes}
+
+    @classmethod
+    async def delete_flow_runs(
+        cls,
+        states: list[StateType] = [StateType.COMPLETED, StateType.FAILED, StateType.CANCELLED],  # noqa: B006
+        delete: bool = True,
+        days_to_keep: int = 2,
+        batch_size: int = 100,
+    ) -> None:
+        """Delete flow runs in the specified states and older than specified days."""
+
+        logger = get_logger()
+
+        async with get_client(sync_client=False) as client:
+            cutoff = datetime.now(timezone.utc) - timedelta(days=days_to_keep)
+
+            flow_run_filter = FlowRunFilter(
+                start_time=FlowRunFilterStartTime(before_=cutoff),  # type: ignore[arg-type]
+                state=FlowRunFilterState(type=FlowRunFilterStateType(any_=states)),
+            )
+
+            # Get flow runs to delete
+            flow_runs = await client.read_flow_runs(flow_run_filter=flow_run_filter, limit=batch_size)
+
+            deleted_total = 0
+
+            while True:
+                batch_deleted = 0
+                failed_deletes = []
+
+                # Delete each flow run through the API
+                for flow_run in flow_runs:
+                    try:
+                        if delete:
+                            await client.delete_flow_run(flow_run_id=flow_run.id)
+                        else:
+                            await client.set_flow_run_state(
+                                flow_run_id=flow_run.id,
+                                state=State(type=StateType.CRASHED),
+                                force=True,
+                            )
+                        deleted_total += 1
+                        batch_deleted += 1
+                    except Exception as e:
+                        logger.warning(f"Failed to delete flow run {flow_run.id}: {e}")
+                        failed_deletes.append(flow_run.id)
+
+                    # Rate limiting
+                    if batch_deleted % 10 == 0:
+                        await asyncio.sleep(0.5)
+
+                logger.info(f"Delete {batch_deleted}/{len(flow_runs)} flow runs (total: {deleted_total})")
+
+                # Get next batch
+                previous_flow_run_ids = [fr.id for fr in flow_runs]
+                flow_runs = await client.read_flow_runs(flow_run_filter=flow_run_filter, limit=batch_size)
+
+                if not flow_runs:
+                    logger.info("No more flow runs to delete")
+                    break
+
+                if previous_flow_run_ids == [fr.id for fr in flow_runs]:
+                    logger.info("Found same flow runs to delete, aborting")
+                    break
+
+                # Delay between batches to avoid overwhelming the API
+                await asyncio.sleep(1.0)
+
+            logger.info(f"Retention complete. Total deleted tasks: {deleted_total}")