fixes

denv3rr · denv3rr · commit 07a51af91093 · 2026-01-06T20:25:35.000-06:00
diff --git a/modules/client_store.py b/modules/client_store.py
@@ -59,7 +59,7 @@ def _normalize_holdings_map(raw: Any) -> Dict[str, Any]:
     return holdings
 
 
-def _normalize_account_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
+def _normalize_account_payload(payload: Dict[str, Any]) -> Dict[str, Any]:      
     account = Account.from_dict(payload)
     normalized = account.to_dict()
     extra = _split_extra(
@@ -85,7 +85,25 @@ def _normalize_account_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
     return normalized
 
 
-def _normalize_client_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
+def _account_fingerprint(account: models.Account) -> str:
+    payload: Dict[str, Any] = {
+        "name": account.name or "",
+        "account_type": account.account_type or "",
+        "ownership_type": account.ownership_type or "",
+        "custodian": account.custodian or "",
+        "tags": sorted(list(account.tags or [])),
+        "tax_settings": account.tax_settings or {},
+        "holdings_map": account.holdings_map or {},
+        "lots": account.lots or {},
+        "manual_holdings": account.manual_holdings or [],
+        "extra": account.extra or {},
+        "current_value": float(account.current_value or 0.0),
+        "active_interval": account.active_interval or "1M",
+    }
+    return json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+
+
+def _normalize_client_payload(payload: Dict[str, Any]) -> Dict[str, Any]:       
     client = Client.from_dict(payload)
     normalized = client.to_dict()
     extra = _split_extra(
@@ -324,6 +342,73 @@ def update_account(self, client_ref: Any, account_ref: Any, payload: Dict[str, A
             db.refresh(account)
             return self._account_to_dict(account)
 
+    def find_duplicate_accounts(self) -> Dict[str, Any]:
+        self.ensure_schema()
+        with _session_scope(self._db) as db:
+            duplicates: List[Dict[str, Any]] = []
+            total_duplicates = 0
+            client_count = 0
+            for client in db.query(models.Client).all():
+                groups: Dict[str, List[models.Account]] = {}
+                for account in client.accounts:
+                    key = _account_fingerprint(account)
+                    groups.setdefault(key, []).append(account)
+                has_duplicates = False
+                for accounts in groups.values():
+                    if len(accounts) <= 1:
+                        continue
+                    has_duplicates = True
+                    accounts_sorted = sorted(accounts, key=lambda item: item.id or 0)
+                    keeper = accounts_sorted[0]
+                    dupes = accounts_sorted[1:]
+                    duplicates.append(
+                        {
+                            "client_id": client.client_uid or str(client.id),
+                            "client_name": client.name or "",
+                            "account_name": keeper.name or "",
+                            "account_type": keeper.account_type or "",
+                            "keep_account_id": keeper.account_uid or str(keeper.id),
+                            "duplicate_ids": [
+                                dup.account_uid or str(dup.id) for dup in dupes
+                            ],
+                            "duplicate_count": len(dupes),
+                        }
+                    )
+                    total_duplicates += len(dupes)
+                if has_duplicates:
+                    client_count += 1
+            return {
+                "count": total_duplicates,
+                "clients": client_count,
+                "details": duplicates,
+            }
+
+    def remove_duplicate_accounts(self) -> Dict[str, Any]:
+        self.ensure_schema()
+        with _session_scope(self._db) as db:
+            removed = 0
+            client_count = 0
+            for client in db.query(models.Client).all():
+                groups: Dict[str, List[models.Account]] = {}
+                for account in client.accounts:
+                    key = _account_fingerprint(account)
+                    groups.setdefault(key, []).append(account)
+                removed_for_client = False
+                for accounts in groups.values():
+                    if len(accounts) <= 1:
+                        continue
+                    accounts_sorted = sorted(accounts, key=lambda item: item.id or 0)
+                    dupes = accounts_sorted[1:]
+                    for account in dupes:
+                        db.delete(account)
+                        removed += 1
+                        removed_for_client = True
+                if removed_for_client:
+                    client_count += 1
+            if removed:
+                db.commit()
+            return {"removed": removed, "clients": client_count}
+
     def _sync_accounts(
         self,
         db: Session,
diff --git a/tests/test_client_store_duplicates.py b/tests/test_client_store_duplicates.py
@@ -0,0 +1,75 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+import core.database as database
+import core.db_management as db_management
+from core import models
+import modules.client_store as client_store
+
+
+def _setup_temp_db(tmp_path, monkeypatch):
+    db_path = tmp_path / "clients.db"
+    engine = create_engine(
+        f"sqlite:///{db_path}", connect_args={"check_same_thread": False}
+    )
+    session_local = sessionmaker(
+        autocommit=False, autoflush=False, bind=engine
+    )
+    monkeypatch.setattr(database, "engine", engine)
+    monkeypatch.setattr(database, "SessionLocal", session_local)
+    monkeypatch.setattr(db_management, "engine", engine)
+    monkeypatch.setattr(client_store, "SessionLocal", session_local)
+    return session_local
+
+
+def _seed_duplicates(session):
+    client = models.Client(client_uid="c1", name="Dup Client")
+    session.add(client)
+    session.flush()
+    account_payload = dict(
+        name="Primary",
+        account_type="Taxable",
+        ownership_type="Individual",
+        custodian="Fidelity",
+        tags=["Core"],
+        tax_settings={"jurisdiction": "US"},
+        holdings_map={"AAPL": 1.0},
+        lots={"AAPL": [{"qty": 1.0, "basis": 100.0, "timestamp": "2024-01-01T00:00:00"}]},
+        manual_holdings=[],
+        extra={"source": "seed"},
+        current_value=100.0,
+        active_interval="1M",
+        client_id=client.id,
+    )
+    session.add(models.Account(account_uid="a1", **account_payload))
+    session.add(models.Account(account_uid="a2", **account_payload))
+    session.commit()
+
+
+def test_detect_duplicate_accounts(tmp_path, monkeypatch):
+    session_local = _setup_temp_db(tmp_path, monkeypatch)
+    db_management.create_db_and_tables()
+    session = session_local()
+    try:
+        _seed_duplicates(session)
+        store = client_store.DbClientStore(session)
+        result = store.find_duplicate_accounts()
+        assert result["count"] == 1
+        assert result["clients"] == 1
+        assert result["details"][0]["duplicate_count"] == 1
+    finally:
+        session.close()
+
+
+def test_cleanup_duplicate_accounts(tmp_path, monkeypatch):
+    session_local = _setup_temp_db(tmp_path, monkeypatch)
+    db_management.create_db_and_tables()
+    session = session_local()
+    try:
+        _seed_duplicates(session)
+        store = client_store.DbClientStore(session)
+        result = store.remove_duplicate_accounts()
+        assert result["removed"] == 1
+        assert session.query(models.Account).count() == 1
+    finally:
+        session.close()
diff --git a/tests/test_web_api_clients.py b/tests/test_web_api_clients.py
@@ -5,6 +5,7 @@
 from sqlalchemy.orm import sessionmaker
 
 from core.database import Base
+from core import models
 from web_api.app import app
 from web_api.routes.clients import get_db
 
@@ -110,3 +111,44 @@ def test_account_metadata_persists(client):
     assert "AAPL" in account["holdings"]
     assert account["lots"]["AAPL"][0]["basis"] == 100.0
     assert account["manual_holdings"][0]["total_value"] == 1234.5
+
+
+def test_duplicate_account_cleanup(client, session):
+    dup_client = models.Client(client_uid="dup1", name="Dup Client")
+    session.add(dup_client)
+    session.flush()
+    account_payload = dict(
+        name="Primary",
+        account_type="Taxable",
+        ownership_type="Individual",
+        custodian="Fidelity",
+        tags=["Core"],
+        tax_settings={"jurisdiction": "US"},
+        holdings_map={"AAPL": 1.0},
+        lots={"AAPL": [{"qty": 1.0, "basis": 100.0, "timestamp": "2024-01-01T00:00:00"}]},
+        manual_holdings=[],
+        extra={"source": "seed"},
+        current_value=100.0,
+        active_interval="1M",
+        client_id=dup_client.id,
+    )
+    session.add(models.Account(account_uid="dup-a1", **account_payload))
+    session.add(models.Account(account_uid="dup-a2", **account_payload))
+    session.commit()
+
+    response = client.get("/api/clients/duplicates")
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["count"] == 1
+
+    response = client.post(
+        "/api/clients/duplicates/cleanup",
+        json={"confirm": True},
+    )
+    assert response.status_code == 200
+    cleaned = response.json()
+    assert cleaned["removed"] == 1
+
+    response = client.get("/api/clients/duplicates")
+    assert response.status_code == 200
+    assert response.json()["count"] == 0
diff --git a/web/src/pages/Dashboard.tsx b/web/src/pages/Dashboard.tsx
@@ -105,6 +105,7 @@ export default function Dashboard() {
   const [riskOpen, setRiskOpen] = useState(true);
   const [mapOpen, setMapOpen] = useState(true);
   const [feedOpen, setFeedOpen] = useState(true);
+  const accentColor = "#48f1a6";
   const { paused } = useTrackerPause();
   const {
     data: trackerStream,
@@ -261,7 +262,7 @@ export default function Dashboard() {
           source: "tracker-points",
           paint: {
             "circle-radius": 6,
-            "circle-color": "var(--green-500)",
+            "circle-color": accentColor,
             "circle-opacity": 0.12
           }
         });
@@ -276,7 +277,7 @@ export default function Dashboard() {
               ["get", "kind"],
               "ship",
               "#8892a0",
-              "var(--green-500)"
+              accentColor
             ],
             "circle-opacity": 0.8
           }
@@ -412,7 +413,7 @@ export default function Dashboard() {
       (point) => Number.isFinite(point.lat) && Number.isFinite(point.lon)
     );
     points.slice(0, 200).forEach((point) => {
-      const color = point.kind === "ship" ? "#8892a0" : "var(--green-500)";
+      const color = point.kind === "ship" ? "#8892a0" : accentColor;
       L.circleMarker([point.lat, point.lon], {
         radius: 4,
         color,
diff --git a/web/src/pages/System.tsx b/web/src/pages/System.tsx
@@ -4,6 +4,7 @@ import { ErrorBanner } from "../components/ui/ErrorBanner";
 import { KpiCard } from "../components/ui/KpiCard";
 import { SectionHeader } from "../components/ui/SectionHeader";
 import {
+  apiPost,
   clearApiKey,
   getApiKey,
   setApiKey as setApiKeyLocal,
@@ -42,6 +43,7 @@ type DiagnosticsPayload = {
   trackers?: { warning_count?: number; count?: number };
   intel?: { news_cache?: { status?: string; items?: number; age_hours?: number | null } };
   clients?: { clients?: number; accounts?: number; holdings?: number; lots?: number };
+  duplicates?: { accounts?: { count?: number; clients?: number } };
   reports?: { items?: number; status?: string };
 };
 
@@ -66,6 +68,9 @@ export default function System() {
       : null
   ].filter(Boolean) as string[];
 
+  const duplicateCount = data?.duplicates?.accounts?.count ?? 0;
+  const duplicateClients = data?.duplicates?.accounts?.clients ?? 0;
+
   const onNormalize = () => {
     // Implement normalization logic here
   };
@@ -74,6 +79,20 @@ export default function System() {
     // Implement clear cache logic here
   };
 
+  const onCleanupDuplicates = async () => {
+    if (!duplicateCount) return;
+    const confirmed = window.confirm(
+      `Remove ${duplicateCount} duplicate account${duplicateCount === 1 ? '' : 's'} across ${duplicateClients} client${duplicateClients === 1 ? '' : 's'}? Originals will be preserved.`
+    );
+    if (!confirmed) return;
+    try {
+      await apiPost('/api/clients/duplicates/cleanup', { confirm: true });
+      refresh();
+    } catch (err) {
+      alert(err instanceof Error ? err.message : 'Duplicate cleanup failed.');
+    }
+  };
+
   const onSetApiKey = () => {
     const key = prompt("Enter API key:");
     if (key) {
@@ -106,6 +125,22 @@ export default function System() {
       <div className="mt-4">
         <ErrorBanner messages={errorMessages} onRetry={refresh} />
       </div>
+      {duplicateCount > 0 ? (
+        <div className="mt-4 rounded-xl border border-amber-400/30 bg-amber-400/10 p-4 text-sm text-amber-200">
+          <div className="flex flex-col gap-3 md:flex-row md:items-center md:justify-between">
+            <p>
+              Duplicate accounts detected: {duplicateCount} across{' '}
+              {duplicateClients} client{duplicateClients === 1 ? '' : 's'}.
+            </p>
+            <button
+              onClick={onCleanupDuplicates}
+              className="rounded-lg border border-amber-400/50 px-3 py-1 text-xs text-amber-100 hover:border-amber-300"
+            >
+              Remove duplicates
+            </button>
+          </div>
+        </div>
+      ) : null}
       <div className="mt-6 grid grid-cols-1 lg:grid-cols-4 gap-4">
         <KpiCard label="Clients" value={`${data?.clients?.clients ?? 0}`} tone="text-green-300" />
         <KpiCard label="Accounts" value={`${data?.clients?.accounts ?? 0}`} tone="text-slate-100" />
diff --git a/web/src/pages/Trackers.tsx b/web/src/pages/Trackers.tsx
@@ -146,6 +146,7 @@ export default function Trackers() {
   const [riskOpen, setRiskOpen] = useState(true);
   const [mapOpen, setMapOpen] = useState(true);
   const [feedOpen, setFeedOpen] = useState(true);
+  const accentColor = "#48f1a6";
 
   useEffect(() => {
     if (!paused) {
@@ -429,7 +430,7 @@ export default function Trackers() {
           source: "tracker-points",
           paint: {
             "circle-radius": 6,
-            "circle-color": "var(--green-500)",
+            "circle-color": accentColor,
             "circle-opacity": 0.12
           }
         });
@@ -444,7 +445,7 @@ export default function Trackers() {
               ["get", "kind"],
               "ship",
               "#8892a0",
-              "var(--green-500)"
+              accentColor
             ],
             "circle-opacity": 0.8
           }
@@ -476,7 +477,7 @@ export default function Trackers() {
           type: "line",
           source: "history-line",
           paint: {
-            "line-color": "var(--green-500)",
+            "line-color": accentColor,
             "line-width": 2,
             "line-opacity": 0.8
           }
diff --git a/web_api/diagnostics.py b/web_api/diagnostics.py
@@ -6,6 +6,7 @@
 from typing import Dict, List, Optional
 
 from modules.client_mgr.data_handler import DataHandler
+from modules.client_store import DbClientStore
 from modules.market_data.trackers import GlobalTrackers
 from utils.system import SystemHost
 
@@ -81,6 +82,11 @@ def client_counts() -> Dict[str, int]:
     }
 
 
+def duplicate_account_summary() -> Dict[str, object]:
+    store = DbClientStore()
+    return store.find_duplicate_accounts()
+
+
 def news_cache_info(max_age_hours: int = 6) -> Dict[str, Optional[object]]:
     path = os.path.join("data", "intel_news.json")
     if not os.path.exists(path):
diff --git a/web_api/routes/clients.py b/web_api/routes/clients.py
diff --git a/web_api/routes/tools.py b/web_api/routes/tools.py
