Merge pull request #111 from reconurge/feat/node-refactor

feat: big codebase refactor + new node display features

Author: dextmorgn

Makefile

@@ -1,126 +1,140 @@
 PROJECT_ROOT := $(shell pwd)
 
-.PHONY: install run stop stop-dev stop-prod infra api frontend celery clean dev prod check-env build-dev build-prod open-browser test
-ENV_DIRS := . flowsint-api flowsint-core flowsint-app
-
-open-browser:
-	@echo "⏳ Waiting for frontend to be ready..."
-	@bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 1; done'
-	@echo "🌐 Opening browser..."
-	@open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ Flowsint ready at http://localhost:5173"
+export DOCKER_BUILDKIT=0
+export COMPOSE_PARALLEL_LIMIT=1
 
-dev:
-	@echo "🐙 Starting Flowsint in DEVELOPMENT mode..."
-	$(MAKE) check-env
-	docker compose -f docker-compose.dev.yml up --build -d
-	$(MAKE) open-browser
-	docker compose -f docker-compose.dev.yml logs -f
-
-prod:
-	@echo "🐙 Starting Flowsint in PRODUCTION mode..."
-	$(MAKE) check-env
-	docker compose -f docker-compose.prod.yml up --build -d
-	$(MAKE) open-browser
+COMPOSE_DEV  := docker compose -f docker-compose.dev.yml
+COMPOSE_PROD := docker compose -f docker-compose.prod.yml
 
-build-dev:
-	@echo "🔨 Building development images..."
-	docker compose -f docker-compose.dev.yml build
+.PHONY: \
+	dev prod \
+	build-dev build-prod \
+	up-dev up-prod down \
+	infra-dev infra-prod infra-stop-dev infra-stop-prod \
+	migrate-dev migrate-prod \
+	api frontend celery \
+	test install clean check-env open-browser
 
-build-prod:
-	@echo "🔨 Building production images..."
-	docker compose -f docker-compose.prod.yml build
+ENV_DIRS := . flowsint-api flowsint-core flowsint-app
 
 check-env:
-	@echo "🔎 Checking .env files..."
+	@echo "Checking .env files..."
 	@for dir in $(ENV_DIRS); do \
 		env_file="$$dir/.env"; \
 		env_example="$(PROJECT_ROOT)/.env.example"; \
-		if [ -f "$$env_file" ]; then \
-			echo "✅ Using existing .env in $$dir"; \
-		else \
-			echo "⚠️ .env missing in $$dir, copying from .env.example"; \
+		if [ ! -f "$$env_file" ]; then \
 			cp "$$env_example" "$$env_file"; \
+			echo "Created $$env_file"; \
 		fi; \
 	done
 
-test:
-	@echo "🔎 Running tests..."
-	cd $(PROJECT_ROOT)/flowsint-types && poetry run pytest
-	cd $(PROJECT_ROOT)/flowsint-core && poetry run pytest
-	cd $(PROJECT_ROOT)/flowsint-enrichers && poetry run pytest
+open-browser:
+	@echo "Waiting for frontend..."
+	@bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 1; done'
+	@open http://localhost:5173 2>/dev/null || \
+	 xdg-open http://localhost:5173 2>/dev/null || \
+	 echo "Frontend ready at http://localhost:5173"
 
-install:
-	@echo "🐙 Installing Flowsint project modules..."
-	@if ! command -v poetry >/dev/null 2>&1; then \
-		echo "⚠️ Poetry is not installed. Please install it:"; \
-		echo "pipx install poetry"; \
-		echo "or"; \
-		echo "curl -sSL https://install.python-poetry.org | python3 -"; \
-		exit 1; \
+build-dev:
+	@echo "Building DEV images..."
+	$(COMPOSE_DEV) build
+
+build-prod:
+	@echo "Building PROD images..."
+	$(COMPOSE_PROD) build
+
+infra-dev:
+	@echo "Starting DEV infra (postgres / redis / neo4j)..."
+	$(COMPOSE_DEV) up -d postgres redis neo4j
+
+infra-prod:
+	@echo "Starting PROD infra (postgres / redis / neo4j)..."
+	$(COMPOSE_PROD) up -d postgres redis neo4j
+
+infra-stop-dev:
+	@echo "Stopping DEV infra..."
+	$(COMPOSE_DEV) stop postgres redis neo4j
+
+infra-stop-prod:
+	@echo "Stopping PROD infra..."
+	$(COMPOSE_PROD) stop postgres redis neo4j
+
+migrate-dev:
+	@echo "Running DEV migrations..."
+	@if ! $(COMPOSE_DEV) ps -q neo4j | grep -q .; then \
+		echo "Neo4j not running → starting DEV infra"; \
+		$(COMPOSE_DEV) up -d --wait neo4j; \
 	fi
-	poetry config virtualenvs.in-project true --local
-	docker compose up -d postgres redis neo4j
-	poetry install
-	cd $(PROJECT_ROOT)/flowsint-core && poetry install
-	cd $(PROJECT_ROOT)/flowsint-enrichers && poetry install
-	cd $(PROJECT_ROOT)/flowsint-api && poetry install && poetry run alembic upgrade head
-	@echo "✅ All modules installed successfully!"
+	yarn migrate
+
+migrate-prod:
+	@echo "⚠️  Running PROD migrations"
+	@echo "This will ALTER production data."
+	@read -p "Type 'prod' to continue: " confirm; \
+	if [ "$$confirm" != "prod" ]; then \
+		echo "Aborted."; exit 1; \
+	fi
+	yarn migrate
 
-infra:
-	docker compose up -d
+dev:
+	@echo "Starting DEV environment..."
+	$(MAKE) check-env
+	$(MAKE) infra-dev
+	$(MAKE) migrate-dev
+	$(MAKE) build-dev
+	$(MAKE) up-dev
+	$(MAKE) open-browser
+	$(COMPOSE_DEV) logs -f
 
-api:
-	cd $(PROJECT_ROOT)/flowsint-api && poetry run uvicorn app.main:app --host 0.0.0.0 --port 5001 --reload
+prod:
+	@echo "Starting PROD environment..."
+	$(MAKE) check-env
+	$(MAKE) build-prod
+	$(MAKE) up-prod
 
-frontend:
-	@echo "🐙 Starting frontend and opening browser..."
-	@docker compose up -d flowsint-app
-	@bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 1; done; open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ Frontend ready at http://localhost:5173"'
+up-dev:
+	$(COMPOSE_DEV) up -d --no-build
 
-frontend_prod:
-	cd $(PROJECT_ROOT)/flowsint-app && npm run build
+up-prod:
+	$(COMPOSE_PROD) up -d --no-build
 
-celery:
-	cd $(PROJECT_ROOT)/flowsint-core && poetry run celery -A flowsint_core.core.celery worker --loglevel=info --pool=threads --concurrency=10
+down:
+	-$(COMPOSE_DEV) down
+	-$(COMPOSE_PROD) down
 
-run:
-	@echo "🐙 Starting all services..."
-	docker compose up -d
-	@echo "⏳ Waiting for frontend to be ready..."
-	@bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 1; done'
-	@echo "🌐 Opening browser..."
-	@open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ All services ready! Flowsint at http://localhost:5173"
-	$(MAKE) -j2 api celery
+api:
+	cd $(PROJECT_ROOT)/flowsint-api && \
+	poetry run uvicorn app.main:app --host 0.0.0.0 --port 5001 --reload
 
-stop:
-	@echo "🛑 Stopping all services..."
-	-docker compose -f docker-compose.dev.yml down
-	-docker compose -f docker-compose.prod.yml down
-	-docker compose down
+frontend:
+	$(COMPOSE_DEV) up -d flowsint-app
+	$(MAKE) open-browser
+
+celery:
+	cd $(PROJECT_ROOT)/flowsint-core && \
+	poetry run celery -A flowsint_core.core.celery \
+	worker --loglevel=info --pool=threads --concurrency=10
 
-stop-dev:
-	@echo "🛑 Stopping development services..."
-	docker compose -f docker-compose.dev.yml down
+test:
+	cd flowsint-types && poetry run pytest
+	cd flowsint-core && poetry run pytest
+	cd flowsint-enrichers && poetry run pytest
 
-stop-prod:
-	@echo "🛑 Stopping production services..."
-	docker compose -f docker-compose.prod.yml down
+install:
+	poetry config virtualenvs.in-project true --local
+	$(MAKE) infra-dev
+	poetry install
+	cd flowsint-core && poetry install
+	cd flowsint-enrichers && poetry install
+	cd flowsint-api && poetry install && poetry run alembic upgrade head
 
 clean:
-	@echo "⚠️  WARNING: This will remove ALL Docker containers, images, volumes, and virtual environments."
-	@echo "⚠️  ALL DATA in databases and volumes will be permanently deleted!"
-	@echo ""
-	@read -p "Are you sure you want to continue? [y/N]: " confirm; \
-	if [ "$$confirm" != "y" ] && [ "$$confirm" != "Y" ]; then \
-		echo "❌ Cleanup cancelled."; \
-		exit 1; \
-	fi
-	@echo "🧹 Removing containers, images, volumes and venvs..."
-	-docker compose -f docker-compose.dev.yml down -v --rmi all --remove-orphans
-	-docker compose -f docker-compose.prod.yml down -v --rmi all --remove-orphans
-	-docker compose down -v --rmi all --remove-orphans
-	rm -rf $(PROJECT_ROOT)/flowsint-app/node_modules
-	rm -rf $(PROJECT_ROOT)/flowsint-core/.venv
-	rm -rf $(PROJECT_ROOT)/flowsint-enrichers/.venv
-	rm -rf $(PROJECT_ROOT)/flowsint-api/.venv
-	@echo "✅ Cleanup complete!"
+	@echo "This will remove ALL Docker data. Continue? [y/N]"
+	@read confirm; \
+	if [ "$$confirm" != "y" ]; then exit 1; fi
+	-$(COMPOSE_DEV) down -v --rmi all --remove-orphans
+	-$(COMPOSE_PROD) down -v --rmi all --remove-orphans
+	rm -rf flowsint-app/node_modules
+	rm -rf flowsint-core/.venv
+	rm -rf flowsint-enrichers/.venv
+	rm -rf flowsint-api/.venv

docker-compose.dev.yml

@@ -58,10 +58,10 @@ services:
     networks:
       - flowsint_network
     healthcheck:
-      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"]
-      interval: 10s
+      test: cypher-shell -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} "RETURN 1"
+      interval: 5s
       timeout: 5s
-      retries: 5
+      retries: 10
 
   # FastAPI Backend
   api:

docker-compose.prod.yml

@@ -59,10 +59,10 @@ services:
     networks:
       - flowsint_network
     healthcheck:
-      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"]
-      interval: 10s
+      test: cypher-shell -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} "RETURN 1"
+      interval: 5s
       timeout: 5s
-      retries: 5
+      retries: 10
 
   # FastAPI Backend (Production)
   api:

docker-compose.yml

@@ -44,6 +44,11 @@ services:
       - neo4j_logs:/logs
       - neo4j_import:/var/lib/neo4j/import
       - neo4j_plugins:/plugins
+    healthcheck:
+      test: cypher-shell -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} "RETURN 1"
+      interval: 5s
+      timeout: 5s
+      retries: 10
     restart: unless-stopped
     networks:
       - flowsint_network

flowsint-api/app/api/routes/enrichers.py

@@ -1,24 +1,25 @@
-from fastapi import APIRouter, HTTPException, Depends, Query
-from typing import List, Any, Optional
-from pydantic import BaseModel
-from flowsint_enrichers import ENRICHER_REGISTRY, load_all_enrichers
+from typing import Any, List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
 from flowsint_core.core.celery import celery
-from flowsint_core.core.types import Node, Edge, FlowBranch
+from flowsint_core.core.graph import create_graph_service, GraphEdge, GraphNode
 from flowsint_core.core.models import CustomType, Profile
-from flowsint_core.core.graph_repository import GraphRepository
-from flowsint_types import clean_neo4j_node_data
-from app.api.deps import get_current_user
 from flowsint_core.core.postgre_db import get_db
-from sqlalchemy.orm import Session
+from flowsint_core.core.types import FlowBranch
+from flowsint_enrichers import ENRICHER_REGISTRY, load_all_enrichers
+from pydantic import BaseModel
 from sqlalchemy import func
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user
 
 # Auto-discover and register all enrichers
 load_all_enrichers()
 
 
 class FlowComputationRequest(BaseModel):
-    nodes: List[Node]
-    edges: List[Edge]
+    nodes: List[GraphNode]
+    edges: List[GraphEdge]
     inputType: Optional[str] = None
 
 
@@ -74,21 +75,20 @@ async def launch_enricher(
 ):
     try:
         # Retrieve nodes from Neo4J by their element IDs
-        graph_repo = GraphRepository()
-        nodes_data = graph_repo.get_nodes_by_ids(payload.node_ids, payload.sketch_id)
-
-        if not nodes_data:
-            raise HTTPException(status_code=404, detail="No nodes found with provided IDs")
-
-        # Clean Neo4J-specific fields from node data
-        # The enricher's preprocess() will handle Pydantic validation
-        cleaned_nodes = [clean_neo4j_node_data(node_data) for node_data in nodes_data]
-
+        graph_service = create_graph_service(sketch_id=payload.sketch_id)
+        entities = graph_service.get_nodes_by_ids_for_task(payload.node_ids)
+
+        # send deserialized nodes
+        entities = [entity.model_dump(mode="json", serialize_as_any=True) for entity in entities]
+        if not entities:
+            raise HTTPException(
+                status_code=404, detail="No entities found with provided IDs"
+            )
         task = celery.send_task(
             "run_enricher",
             args=[
                 enricher_name,
-                cleaned_nodes,
+                entities,
                 payload.sketch_id,
                 str(current_user.id),
             ],
@@ -98,4 +98,7 @@ async def launch_enricher(
     except HTTPException:
         raise
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Error launching enricher: {str(e)}")
+        print(e)
+        raise HTTPException(
+            status_code=500, detail=f"Error launching enricher: {str(e)}"
+        )