Stability and lifecycle improvements (#59)

Author: eeiaao

demo/include/demo/simple_components.h

@@ -195,6 +195,11 @@ class SimplePositionManager : public IPositionManager
     update(order, order.quantity);
   }
 
+  void onOrderPendingCancel(const Order& order) override
+  {
+    FLOX_LOG("[position] order pending cancel: id=" << order.id);
+  }
+
   void onOrderCanceled(const Order& order) override
   {
     FLOX_LOG("[position] order canceled: id=" << order.id);

docs/components/book/nlevel_order_book.md

@@ -39,6 +39,32 @@ class NLevelOrderBook : public IOrderBook {
 5. **No Dynamic Allocation**
    Uses `std::array` of fixed size; fully cache-friendly and allocation-free after construction.
 
+## Market State Helpers
+
+```cpp
+[[nodiscard]] bool isCrossed() const noexcept;
+[[nodiscard]] std::optional<Price> spread() const noexcept;
+[[nodiscard]] std::optional<Price> mid() const noexcept;
+```
+
+| Method      | Description                                                      |
+| ----------- | ---------------------------------------------------------------- |
+| `isCrossed` | Returns `true` if best bid >= best ask (crossed/locked market).  |
+| `spread`    | Returns ask - bid spread, or `nullopt` if either side is empty.  |
+| `mid`       | Returns midpoint price, or `nullopt` if either side is empty.    |
+
+These methods are useful for detecting market anomalies and calculating fair value:
+
+```cpp
+if (book.isCrossed()) {
+  // Handle crossed book condition
+}
+
+if (auto mid = book.mid()) {
+  // Use midpoint for fair value calculations
+}
+```
+
 ## Notes
 
 * Extremely fast and deterministic — suitable for backtests and production.

docs/components/connector/exchange_connector.md

@@ -1,23 +1,34 @@
 # IExchangeConnector
 
-`IExchangeConnector` is the abstract interface for real-time market data adapters. It provides lifecycle control and typed callback delivery for `BookUpdateEvent` and `TradeEvent`.
+`IExchangeConnector` is the abstract interface for real-time market data adapters. It provides lifecycle control, typed callback delivery for market data events, and error notifications.
 
 ```cpp
-class IExchangeConnector : public ISubsystem
+class IExchangeConnector : public ISubsystem, public IDrainable
 {
 public:
-  using BookUpdateCallback = std::move_only_function<void(const BookUpdateEvent&)>;
-  using TradeCallback = std::move_only_function<void(const TradeEvent&)>;
+  using BookUpdateCallback = MoveOnlyFunction<void(const BookUpdateEvent&)>;
+  using TradeCallback = MoveOnlyFunction<void(const TradeEvent&)>;
+  using DisconnectCallback = MoveOnlyFunction<void(std::string_view reason)>;
+  using SequenceGapCallback = MoveOnlyFunction<void(uint64_t expected, uint64_t received)>;
+  using StaleDataCallback = MoveOnlyFunction<void(SymbolId symbol, uint64_t lastUpdateMs)>;
 
   virtual ~IExchangeConnector() = default;
 
+  bool drain(uint32_t timeoutMs) override { return true; }
+
   virtual std::string exchangeId() const = 0;
 
   virtual void setCallbacks(BookUpdateCallback onBookUpdate, TradeCallback onTrade);
+  virtual void setErrorCallbacks(DisconnectCallback onDisconnect,
+                                 SequenceGapCallback onSequenceGap,
+                                 StaleDataCallback onStaleData);
 
 protected:
   void emitBookUpdate(const BookUpdateEvent& bu);
   void emitTrade(const TradeEvent& t);
+  void emitDisconnect(std::string_view reason);
+  void emitSequenceGap(uint64_t expected, uint64_t received);
+  void emitStaleData(SymbolId symbol, uint64_t lastUpdateMs);
 };
 ```
 
@@ -27,16 +38,26 @@ protected:
 
 ## Responsibilities
 
-| Aspect        | Details                                                                 |
-| ------------- | ----------------------------------------------------------------------- |
-| Lifecycle     | Inherits `start()` and `stop()` from `ISubsystem`.                      |
-| Identity      | `exchangeId()` provides a stable identifier for the connector instance. |
-| Callbacks     | `setCallbacks()` binds downstream handlers for book and trade events.   |
-| Event Routing | `emitBookUpdate()` and `emitTrade()` dispatch data to subscribers.      |
+| Aspect          | Details                                                                 |
+| --------------- | ----------------------------------------------------------------------- |
+| Lifecycle       | Inherits `start()` and `stop()` from `ISubsystem`.                      |
+| Draining        | Implements `IDrainable` for graceful shutdown with pending operations.  |
+| Identity        | `exchangeId()` provides a stable identifier for the connector instance. |
+| Data Callbacks  | `setCallbacks()` binds handlers for book and trade events.              |
+| Error Callbacks | `setErrorCallbacks()` binds handlers for connection/data errors.        |
+| Event Routing   | `emit*()` methods dispatch data and errors to subscribers.              |
+
+## Error Callbacks
+
+| Callback       | When Called                                                    |
+| -------------- | -------------------------------------------------------------- |
+| onDisconnect   | Connection lost or closed unexpectedly.                        |
+| onSequenceGap  | Sequence number gap detected (expected vs received).           |
+| onStaleData    | No updates received for a symbol beyond threshold.             |
 
 ## Notes
 
-* Inherits from `ISubsystem`, enabling unified lifecycle management via `start()` and `stop()`.
-* Callbacks use `std::move_only_function` to avoid `std::function` overhead and enable capturing closures with ownership.
+* Inherits from `ISubsystem` and `IDrainable`, enabling unified lifecycle and graceful shutdown.
+* Callbacks use `MoveOnlyFunction` to avoid `std::function` overhead and enable capturing closures with ownership.
 * Implementations must call `emit*()` manually from internal processing (e.g. websocket handler).
 * The class is intentionally non-copyable and non-thread-safe — connectors are expected to run in isolated threads.

docs/components/engine/abstract_subsystem.md

@@ -30,3 +30,31 @@ public:
 * Used by core modules like `CandleAggregator`, `Strategy`, `ExecutionTracker`, `SymbolRegistry`, etc.
 * Lifecycle is typically orchestrated by the engine or test harness.
 * No assumptions about threading — start/stop are always externally coordinated.
+
+---
+
+# IDrainable
+
+`IDrainable` is a separate interface for components with pending async work that must complete before shutdown.
+
+```cpp
+class IDrainable
+{
+public:
+  virtual ~IDrainable() = default;
+  virtual bool drain(uint32_t timeoutMs) = 0;
+};
+```
+
+## Purpose
+
+* Wait for in-flight operations (e.g., pending orders, network requests) to complete.
+
+## When to Implement
+
+Only implement `IDrainable` for components with actual async work:
+* Exchange connectors (in-flight order confirmations)
+* Order executors (pending order submissions)
+* Network transports (pending requests)
+
+Do NOT implement for stateless or synchronous components like strategies, aggregators, or registries.

docs/components/engine/engine_config.md

@@ -9,6 +9,7 @@ struct EngineConfig
   KillSwitchConfig killSwitchConfig;
   std::string logLevel = "info";
   std::string logFile;
+  uint32_t drainTimeoutMs = 5000;
 };
 ```
 
@@ -24,6 +25,7 @@ struct EngineConfig
 | killSwitchConfig | Limits for order size, frequency, and loss (see `KillSwitchConfig`). |
 | logLevel         | Runtime log verbosity (`info`, `debug`, `trace`, etc.).              |
 | logFile          | Optional path to write logs to disk.                                 |
+| drainTimeoutMs   | Timeout for draining subsystems during shutdown (default: 5000ms).   |
 
 
 ## Substructures

docs/components/execution/events/order_event.md

@@ -10,6 +10,7 @@ struct OrderEvent {
   Order order{};
   Order newOrder{};
   Quantity fillQty{0};
+  std::string rejectReason;
   uint64_t tickSequence = 0;
 
   void dispatchTo(IOrderExecutionListener& listener) const;
@@ -28,6 +29,7 @@ struct OrderEvent {
 | order        | The primary order involved in the event.                    |
 | newOrder     | Used only for `REPLACED` events.                            |
 | fillQty      | Quantity filled (used only in `PARTIALLY_FILLED`).          |
+| rejectReason | Reason string (used only in `REJECTED` events).             |
 | tickSequence | Event ordering marker for sequencing and backtesting.       |
 
 ## Dispatch Logic
@@ -38,16 +40,17 @@ void dispatchTo(IOrderExecutionListener& listener) const;
 
 Routes the event to the appropriate method:
 
-| Type               | Dispatched Method                        |
-| ------------------ | ---------------------------------------- |
-| `SUBMITTED`        | `onOrderSubmitted(order)`                |
-| `ACCEPTED`         | `onOrderAccepted(order)`                 |
-| `PARTIALLY_FILLED` | `onOrderPartiallyFilled(order, fillQty)` |
-| `FILLED`           | `onOrderFilled(order)`                   |
-| `CANCELED`         | `onOrderCanceled(order)`                 |
-| `EXPIRED`          | `onOrderExpired(order)`                  |
-| `REJECTED`         | `onOrderRejected(order, /*reason*/ "")`  |
-| `REPLACED`         | `onOrderReplaced(order, newOrder)`       |
+| Type               | Dispatched Method                              |
+| ------------------ | ---------------------------------------------- |
+| `SUBMITTED`        | `onOrderSubmitted(order)`                      |
+| `ACCEPTED`         | `onOrderAccepted(order)`                       |
+| `PARTIALLY_FILLED` | `onOrderPartiallyFilled(order, fillQty)`       |
+| `FILLED`           | `onOrderFilled(order)`                         |
+| `PENDING_CANCEL`   | `onOrderPendingCancel(order)`                  |
+| `CANCELED`         | `onOrderCanceled(order)`                       |
+| `EXPIRED`          | `onOrderExpired(order)`                        |
+| `REJECTED`         | `onOrderRejected(order, rejectReason)`         |
+| `REPLACED`         | `onOrderReplaced(order, newOrder)`             |
 
 ## Notes
 

docs/components/util/memory/pool.md

@@ -63,6 +63,26 @@ h->tickSequence = 123;
 * Objects are returned to the pool via an `SPSCQueue<T*>`.
 * Backed by a `monotonic_buffer_resource` and `unsynchronized_pool_resource` for internal vector-like allocations.
 
+## Exhaustion Handling
+
+The pool provides callbacks and statistics for monitoring pool usage:
+
+```cpp
+pool.setExhaustionCallback([](size_t capacity, size_t inUse) {
+  LOG_WARN("Pool exhausted: capacity={}, inUse={}", capacity, inUse);
+});
+```
+
+| Method             | Description                                           |
+| ------------------ | ----------------------------------------------------- |
+| `capacity()`       | Returns the pool's maximum capacity.                  |
+| `inUse()`          | Returns the number of currently acquired objects.     |
+| `exhaustionCount()`| Returns how many times `acquire()` failed.            |
+| `acquireCount()`   | Returns total number of successful acquisitions.      |
+| `releaseCount()`   | Returns total number of releases back to pool.        |
+
+The exhaustion callback is invoked each time `acquire()` returns `nullopt` due to pool exhaustion.
+
 ## Notes
 
 * Zero allocations in steady-state operation.

include/flox/book/nlevel_order_book.h

@@ -537,6 +537,35 @@ class NLevelOrderBook : public IOrderBook
 
   [[nodiscard]] inline Price tickSize() const noexcept { return _tickSize; }
 
+  [[nodiscard]] inline bool isCrossed() const noexcept
+  {
+    if (_bestBidTick < 0 || _bestAskTick < 0)
+    {
+      return false;
+    }
+    return _bestBidTick >= _bestAskTick;
+  }
+
+  [[nodiscard]] inline std::optional<Price> spread() const noexcept
+  {
+    if (_bestBidTick < 0 || _bestAskTick < 0)
+    {
+      return std::nullopt;
+    }
+    int64_t spreadTicks = _bestAskTick - _bestBidTick;
+    return Price::fromRaw(_tickSize.raw() * spreadTicks);
+  }
+
+  [[nodiscard]] inline std::optional<Price> mid() const noexcept
+  {
+    if (_bestBidTick < 0 || _bestAskTick < 0)
+    {
+      return std::nullopt;
+    }
+    int64_t midTick2 = _bestBidTick + _bestAskTick;
+    return Price::fromRaw((_tickSize.raw() * midTick2) / 2);
+  }
+
   void clear() noexcept
   {
     _bids.fill({});

include/flox/connector/abstract_exchange_connector.h

@@ -11,21 +11,28 @@
 
 #include "flox/book/events/book_update_event.h"
 #include "flox/book/events/trade_event.h"
+#include "flox/common.h"
 #include "flox/engine/abstract_subsystem.h"
 #include "flox/util/base/move_only_function.h"
 
 #include <string>
+#include <string_view>
 
 namespace flox
 {
 
-class IExchangeConnector : public ISubsystem
+class IExchangeConnector : public ISubsystem, public IDrainable
 {
  public:
   virtual ~IExchangeConnector() = default;
 
+  bool drain(uint32_t /*timeoutMs*/) override { return true; }
+
   using BookUpdateCallback = MoveOnlyFunction<void(const BookUpdateEvent&)>;
   using TradeCallback = MoveOnlyFunction<void(const TradeEvent&)>;
+  using DisconnectCallback = MoveOnlyFunction<void(std::string_view reason)>;
+  using SequenceGapCallback = MoveOnlyFunction<void(uint64_t expected, uint64_t received)>;
+  using StaleDataCallback = MoveOnlyFunction<void(SymbolId symbol, uint64_t lastUpdateMs)>;
 
   virtual std::string exchangeId() const = 0;
 
@@ -35,6 +42,15 @@ class IExchangeConnector : public ISubsystem
     _onTrade = std::move(onTrade);
   }
 
+  virtual void setErrorCallbacks(DisconnectCallback onDisconnect,
+                                 SequenceGapCallback onSequenceGap,
+                                 StaleDataCallback onStaleData)
+  {
+    _onDisconnect = std::move(onDisconnect);
+    _onSequenceGap = std::move(onSequenceGap);
+    _onStaleData = std::move(onStaleData);
+  }
+
  protected:
   void emitBookUpdate(const BookUpdateEvent& bu)
   {
@@ -52,9 +68,36 @@ class IExchangeConnector : public ISubsystem
     }
   }
 
+  void emitDisconnect(std::string_view reason)
+  {
+    if (_onDisconnect)
+    {
+      _onDisconnect(reason);
+    }
+  }
+
+  void emitSequenceGap(uint64_t expected, uint64_t received)
+  {
+    if (_onSequenceGap)
+    {
+      _onSequenceGap(expected, received);
+    }
+  }
+
+  void emitStaleData(SymbolId symbol, uint64_t lastUpdateMs)
+  {
+    if (_onStaleData)
+    {
+      _onStaleData(symbol, lastUpdateMs);
+    }
+  }
+
  private:
   BookUpdateCallback _onBookUpdate;
   TradeCallback _onTrade;
+  DisconnectCallback _onDisconnect;
+  SequenceGapCallback _onSequenceGap;
+  StaleDataCallback _onStaleData;
 };
 
 }  // namespace flox

include/flox/engine/abstract_subsystem.h

@@ -9,6 +9,8 @@
 
 #pragma once
 
+#include <cstdint>
+
 namespace flox
 {
 
@@ -17,8 +19,15 @@ class ISubsystem
  public:
   virtual ~ISubsystem() = default;
 
-  virtual void start() {};
-  virtual void stop() {};
+  virtual void start() {}
+  virtual void stop() {}
+};
+
+class IDrainable
+{
+ public:
+  virtual ~IDrainable() = default;
+  virtual bool drain(uint32_t timeoutMs) = 0;
 };
 
 }  // namespace flox