WIP: Run ETL as K8s Jobs from backoffice

apps/vedana/README.md

@@ -1,3 +1,36 @@
 # Vedana
 
-see README.md in the repository root
+## Setup
+1. `uv sync`
+2. Для запуска скриптов - `uv run python app/update_graph_db.py` и т.д.
+
+## Backoffice
+...
+
+## Kubernetes Job Execution
+### Prerequisites
+- kind (Kubernetes in Docker) installed
+
+### Setup
+
+```bash
+# Build Vedana image for cluster
+docker-compose -f apps/vedana/docker-compose.yml build
+
+# Create local K8s cluster
+cd apps/vedana
+apps/vedana/scripts/setup-kind.sh
+
+docker-compose -f apps/vedana/docker-compose.yml -f up --build -d vedana-backoffice
+```
+
+### Test ETL Job Execution
+
+1. Open backoffice UI
+2. Navigate to ETL page
+3. Select steps
+4. Select Execution mode "k8s"
+5. Click "Run Selected"
+6. Watch logs stream from Job in real-time or open logs from Jobs card
+
+See also README.md in the repository root.

apps/vedana/docker-compose.yml

@@ -1,14 +1,19 @@
 x-app-common: &app-common
+  image: vedana/vedana:dev
   build:
     context: ../../
     dockerfile: apps/vedana/Dockerfile
   volumes:
     - ./data:/app/vedana/data
+    - ${HOME}/.kube/config-kind-vedana-local-docker:/root/.kube/config:ro
+    - /var/run/docker.sock:/var/run/docker.sock:ro
   env_file:
     - .env
   depends_on:
     - memgraph
     - db
+  extra_hosts:
+    - "host.docker.internal:host-gateway"
 
 services:
   db-migrate:

apps/vedana/scripts/external-services.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: memgraph
+spec:
+  type: ExternalName
+  externalName: host.docker.internal
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: db
+spec:
+  type: ExternalName
+  externalName: host.docker.internal

apps/vedana/scripts/kind-config.yaml

@@ -0,0 +1,15 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerAddress: 127.0.0.1
+  apiServerPort: 0
+kubeadmConfigPatches:
+- |
+  kind: ClusterConfiguration
+  apiServer:
+    certSANs:
+    - "host.docker.internal"
+    - "localhost"
+    - "127.0.0.1"
+nodes:
+- role: control-plane

apps/vedana/scripts/setup-kind.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+set -e
+
+CLUSTER_NAME="${KIND_CLUSTER_NAME:-vedana-local}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KIND_CONFIG="${SCRIPT_DIR}/kind-config.yaml"
+EXTERNAL_SERVICES="${SCRIPT_DIR}/external-services.yaml"
+
+
+echo "▶ Setting up kind cluster: ${CLUSTER_NAME}"
+
+# Ensure kind exists
+command -v kind >/dev/null || {
+  echo "kind is not installed"
+  exit 1
+}
+
+# Delete existing cluster if present
+if kind get clusters | grep -q "^${CLUSTER_NAME}$"; then
+  echo "Deleting existing cluster..."
+  kind delete cluster --name "${CLUSTER_NAME}"
+fi
+
+# Create cluster (no special networking needed)
+kind create cluster --name "${CLUSTER_NAME}" --config "${KIND_CONFIG}"
+
+# Wait for readiness
+kubectl wait --for=condition=Ready nodes --all --timeout=120s \
+  --context "kind-${CLUSTER_NAME}"
+
+echo "✔ kind cluster ready"
+
+echo "Configuring external services..."
+kubectl apply -f "${EXTERNAL_SERVICES}"
+echo "✔ done"
+
+# Create host kubeconfig
+HOST_KUBECONFIG="${HOME}/.kube/config-kind-${CLUSTER_NAME}"
+DOCKER_KUBECONFIG="${HOME}/.kube/config-kind-${CLUSTER_NAME}-docker"
+
+mkdir -p "${HOME}/.kube"
+
+# Ensure files, not directories
+rm -rf "${HOST_KUBECONFIG}" "${DOCKER_KUBECONFIG}"
+
+kind get kubeconfig --name "${CLUSTER_NAME}" > "${HOST_KUBECONFIG}"
+
+echo "Loading local vedana image into cluster..."
+kind load docker-image vedana/vedana:dev --name vedana-local
+echo "✔ vedana image loaded"
+
+cp "${HOST_KUBECONFIG}" "${DOCKER_KUBECONFIG}"
+
+sed -i.bak \
+  -e "s|https://127.0.0.1:|https://host.docker.internal:|g" \
+  -e "s|https://localhost:|https://host.docker.internal:|g" \
+  "${DOCKER_KUBECONFIG}"
+
+rm -f "${DOCKER_KUBECONFIG}.bak"
+
+
+echo "✔ Docker kubeconfig written to ${DOCKER_KUBECONFIG}"
+
+echo ""
+echo "NEXT STEPS:"
+echo "  export KUBECONFIG=${HOST_KUBECONFIG}"
+echo "  kubectl get nodes"
+echo ""
+echo "Docker containers should mount:"
+echo "  ${DOCKER_KUBECONFIG} -> /root/.kube/config"

libs/vedana-backoffice/pyproject.toml

@@ -12,6 +12,7 @@ authors = [
 dependencies = [
     "reflex>=0.8.26,<0.9.0",
     "orjson>=3.11.3",
+    "kubernetes>=34.1.0",
     "vedana-core",
     "vedana-etl",
 ]

libs/vedana-backoffice/src/vedana_backoffice/components/etl_graph.py

@@ -3,6 +3,43 @@
 from vedana_backoffice.states.etl import EtlState
 
 
+def _execution_status_badge(node: dict) -> rx.Component:
+    """Display execution status badge for a step."""
+    return rx.cond(
+        node.get("is_running", False),
+        rx.hstack(
+            rx.spinner(size="1"),
+            rx.badge(
+                rx.hstack(
+                    rx.text("Running", size="1"),
+                    rx.text(node.get("exec_progress_str", ""), size="1"),
+                    spacing="1",
+                ),
+                color_scheme="blue",
+                variant="soft",
+            ),
+            spacing="1",
+            align="center",
+        ),
+        rx.cond(
+            node.get("is_pending", False),
+            rx.badge("Queued", color_scheme="amber", variant="soft", size="1"),
+            rx.cond(
+                node.get("is_exec_failed", False),
+                rx.tooltip(
+                    rx.badge("Failed", color_scheme="red", variant="soft", size="1"),
+                    content=node.get("exec_error", "Unknown error"),
+                ),
+                rx.cond(
+                    node.get("is_completed", False),
+                    rx.badge("Done", color_scheme="green", variant="soft", size="1"),
+                    rx.box()
+                ),
+            ),
+        ),
+    )
+
+
 def _node_card(node: dict) -> rx.Component:
     is_table = node.get("node_type") == "table"  # step for transform, table for table
     return rx.card(
@@ -55,7 +92,16 @@ def _node_card(node: dict) -> rx.Component:
                             rx.text(node.get("last_run", "—"), size="1", color="gray"),
                             content="last run time (that produced changes)",
                         ),
+                        _execution_status_badge(node),
                         rx.hstack(
+                            rx.cond(
+                                node.get("has_pending", False) & ~node.get("is_running", False),
+                                rx.tooltip(
+                                    rx.badge("Stale", color_scheme="orange", variant="outline", size="1"),
+                                    content=node.get("pending_str", "Has pending records"),
+                                ),
+                                rx.box(),  # No badge for idle state
+                            ),
                             rx.tooltip(
                                 rx.text(node.get("rows_processed", 0), size="1", color="gray"),
                                 content="rows processed in last run",
@@ -78,7 +124,7 @@ def _node_card(node: dict) -> rx.Component:
                         width="100%",
                         justify="between",
                     ),
-                    rx.box(),
+                    _execution_status_badge(node),
                 ),
             ),
             spacing="2",

libs/vedana-backoffice/src/vedana_backoffice/k8s/config.py

@@ -0,0 +1,20 @@
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class K8sConfig(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="K8S",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    enable_jobs: bool = True
+    job_cpu_limit: str = "2"
+    job_memory_limit: str = "4Gi"
+    job_cpu_request: str = "500m"
+    job_memory_request: str = "2Gi"
+
+
+k8s_config = K8sConfig()  # type: ignore
+