feat: integrate HTTP connection pooling for Lighthouse API calls (Patrick-Ehimen#75)

Wire the existing ConnectionPool into LighthouseAISDK so that direct
HTTP calls (e.g. the uploadViDirectAPI fallback) reuse pooled
connections with keep-alive instead of creating new ones per request.

- Add optional `pool` config to LighthouseConfig (default enabled,
  set false to disable)
- Create ConnectionPool in SDK constructor with configurable max
  connections, timeouts, and keep-alive
- Add executeHttpRequest helper that routes through pool or falls back
  to direct axios
- Expose pool metrics via getConnectionPoolStats()
- Add env var support (LIGHTHOUSE_POOL_MAX_CONNECTIONS, etc.) in
  MCP server config
- Add 32 new tests covering pool lifecycle, queuing, and SDK integration

Closes Patrick-Ehimen#54

Author: Patrick-Ehimen

apps/mcp-server/src/config/server-config.ts

@@ -7,6 +7,13 @@ import { MultiTenancyConfig, OrganizationSettings, UsageQuota } from "@lighthous
 import * as path from "path";
 import * as os from "os";
 
+export interface ConnectionPoolServerConfig {
+  maxConnections: number;
+  idleTimeoutMs: number;
+  requestTimeoutMs: number;
+  keepAlive: boolean;
+}
+
 export interface ServerConfig {
   name: string;
   version: string;
@@ -19,6 +26,7 @@ export interface ServerConfig {
   authentication?: AuthConfig;
   performance?: PerformanceConfig;
   multiTenancy?: MultiTenancyConfig;
+  connectionPool?: ConnectionPoolServerConfig;
 }
 
 /**
@@ -73,6 +81,13 @@ export const DEFAULT_PERFORMANCE_CONFIG: PerformanceConfig = {
   concurrentRequestLimit: 100,
 };
 
+export const DEFAULT_CONNECTION_POOL_CONFIG: ConnectionPoolServerConfig = {
+  maxConnections: parseInt(process.env.LIGHTHOUSE_POOL_MAX_CONNECTIONS || "10", 10),
+  idleTimeoutMs: parseInt(process.env.LIGHTHOUSE_POOL_IDLE_TIMEOUT || "60000", 10),
+  requestTimeoutMs: parseInt(process.env.LIGHTHOUSE_POOL_REQUEST_TIMEOUT || "30000", 10),
+  keepAlive: process.env.LIGHTHOUSE_POOL_KEEP_ALIVE !== "false",
+};
+
 export const DEFAULT_ORGANIZATION_SETTINGS: OrganizationSettings = {
   defaultStorageQuota: 10 * 1024 * 1024 * 1024, // 10GB
   defaultRateLimit: 1000, // 1000 requests per minute
@@ -131,6 +146,7 @@ export function getDefaultServerConfig(): ServerConfig {
     authentication: getDefaultAuthConfig(),
     performance: DEFAULT_PERFORMANCE_CONFIG,
     multiTenancy: DEFAULT_MULTI_TENANCY_CONFIG,
+    connectionPool: DEFAULT_CONNECTION_POOL_CONFIG,
   };
 }
 
@@ -149,6 +165,7 @@ export const DEFAULT_SERVER_CONFIG: ServerConfig = {
   authentication: DEFAULT_AUTH_CONFIG,
   performance: DEFAULT_PERFORMANCE_CONFIG,
   multiTenancy: DEFAULT_MULTI_TENANCY_CONFIG,
+  connectionPool: DEFAULT_CONNECTION_POOL_CONFIG,
 };
 
 /**

apps/mcp-server/src/services/LighthouseService.ts

@@ -5,6 +5,7 @@
 import {
   LighthouseAISDK,
   EnhancedAccessCondition,
+  ConnectionPoolConfig,
   BatchUploadOptions,
   BatchDownloadOptions,
   BatchOperationResult,
@@ -29,7 +30,7 @@ export class LighthouseService implements ILighthouseService {
   private fileCache: Map<string, StoredFile> = new Map();
   private datasetCache: Map<string, Dataset> = new Map();
 
-  constructor(apiKey: string, logger?: Logger, dbPath?: string) {
+  constructor(apiKey: string, logger?: Logger, dbPath?: string, poolConfig?: ConnectionPoolConfig) {
     this.logger = logger || Logger.getInstance({ level: "info", component: "LighthouseService" });
     this.dbPath = dbPath;
 
@@ -41,6 +42,7 @@ export class LighthouseService implements ILighthouseService {
       timeout: 30000,
       maxRetries: 3,
       debug: false,
+      pool: poolConfig,
     });
 
     // Set up event listeners for progress tracking
@@ -450,6 +452,7 @@ export class LighthouseService implements ILighthouseService {
       activeOperations: this.sdk.getActiveOperations(),
       errorMetrics: this.sdk.getErrorMetrics(),
       circuitBreaker: this.sdk.getCircuitBreakerStatus(),
+      connectionPool: this.sdk.getConnectionPoolStats(),
     };
   }
 

packages/sdk-wrapper/src/LighthouseAISDK.ts

@@ -2,13 +2,14 @@ import { EventEmitter } from "eventemitter3";
 import lighthouse from "@lighthouse-web3/sdk";
 import { readFileSync, createWriteStream, promises as fsPromises } from "fs";
 import { dirname } from "path";
-import axios from "axios";
+import axios, { AxiosRequestConfig, AxiosResponse } from "axios";
 import { AuthenticationManager } from "./auth/AuthenticationManager";
 import { ProgressTracker } from "./progress/ProgressTracker";
 import { ErrorHandler } from "./errors/ErrorHandler";
 import { CircuitBreaker } from "./errors/CircuitBreaker";
 import { EncryptionManager } from "./encryption/EncryptionManager";
 import { RateLimiter } from "./utils/RateLimiter";
+import { ConnectionPool, ConnectionPoolConfig } from "./pool";
 import {
   LighthouseConfig,
   UploadOptions,
@@ -78,6 +79,7 @@ export class LighthouseAISDK extends EventEmitter {
   private circuitBreaker: CircuitBreaker;
   private encryption: EncryptionManager;
   private rateLimiter: RateLimiter;
+  private connectionPool: ConnectionPool | null;
   private memoryManager: MemoryManager;
   private config: LighthouseConfig;
 
@@ -101,6 +103,22 @@ export class LighthouseAISDK extends EventEmitter {
       autoCleanup: true,
     });
 
+    // Initialize connection pool (unless explicitly disabled)
+    if (config.pool === false) {
+      this.connectionPool = null;
+    } else {
+      const poolConfig: ConnectionPoolConfig = {
+        maxConnections: 10,
+        acquireTimeout: 5000,
+        idleTimeout: 60000,
+        requestTimeout: config.timeout || 30000,
+        keepAlive: true,
+        maxSockets: 50,
+        ...(typeof config.pool === "object" ? config.pool : {}),
+      };
+      this.connectionPool = new ConnectionPool(poolConfig);
+    }
+
     // Forward authentication events
     this.auth.on("auth:error", (error) => this.emit("auth:error", error));
     this.auth.on("auth:refresh", () => this.emit("auth:refresh"));
@@ -145,6 +163,15 @@ export class LighthouseAISDK extends EventEmitter {
       this.emit("encryption:access:control:error", event),
     );
 
+    // Forward connection pool events
+    if (this.connectionPool) {
+      this.connectionPool.on("acquire", (event) => this.emit("pool:acquire", event));
+      this.connectionPool.on("create", (event) => this.emit("pool:create", event));
+      this.connectionPool.on("queue", (event) => this.emit("pool:queue", event));
+      this.connectionPool.on("release", (event) => this.emit("pool:release", event));
+      this.connectionPool.on("cleanup", (event) => this.emit("pool:cleanup", event));
+    }
+
     // Forward memory manager events
     this.memoryManager.on("backpressure:start", (event) =>
       this.emit("memory:backpressure:start", event),
@@ -363,6 +390,25 @@ Maximum file size may be exceeded. Try uploading a smaller file.`);
     ]);
   }
 
+  /**
+   * Execute an HTTP request using the connection pool if available,
+   * otherwise fall back to a direct axios call.
+   */
+  private async executeHttpRequest<T = any>(config: AxiosRequestConfig): Promise<AxiosResponse<T>> {
+    if (this.connectionPool) {
+      const instance = await this.connectionPool.acquire();
+      try {
+        return await instance.request<T>(config);
+      } finally {
+        this.connectionPool.release(instance);
+      }
+    } else {
+      const axiosLib: { request: (config: AxiosRequestConfig) => Promise<AxiosResponse<T>> } =
+        eval("require")("axios");
+      return axiosLib.request(config);
+    }
+  }
+
   /**
    * Upload file via direct API call as fallback when SDK fails
    */
@@ -375,12 +421,14 @@ Maximum file size may be exceeded. Try uploading a smaller file.`);
     // when the standard SDK fails (usually due to node.lighthouse.storage being down)
 
     const FormData = eval("require")("form-data");
-    const axios = eval("require")("axios");
 
     const formData = new FormData();
     formData.append("file", fileBuffer, fileName);
 
-    const response = await axios.post("https://api.lighthouse.storage/api/v0/add", formData, {
+    const response = await this.executeHttpRequest({
+      method: "POST",
+      url: "https://api.lighthouse.storage/api/v0/add",
+      data: formData,
       headers: {
         ...formData.getHeaders(),
         Authorization: `Bearer ${apiKey}`,
@@ -932,6 +980,24 @@ Check your internet connection and try again.`);
     return this.circuitBreaker.getMetrics();
   }
 
+  /**
+   * Get connection pool statistics.
+   * Returns null if the connection pool is disabled.
+   */
+  getConnectionPoolStats(): {
+    totalConnections: number;
+    activeConnections: number;
+    idleConnections: number;
+    queuedRequests: number;
+    totalRequests: number;
+    averageWaitTime: number;
+  } | null {
+    if (!this.connectionPool) {
+      return null;
+    }
+    return this.connectionPool.getStats();
+  }
+
   /**
    * Reset error metrics
    */
@@ -1627,6 +1693,9 @@ Check your internet connection and try again.`);
     this.auth.destroy();
     this.progress.cleanup();
     this.encryption.destroy();
+    if (this.connectionPool) {
+      this.connectionPool.destroy();
+    }
     this.memoryManager.destroy();
     this.removeAllListeners();
   }