adding rate limit, cachin mechanism and smart context filtering

Author: STetsing

libs/remix-ai-core/src/inferencers/remote/remoteInference.ts

@@ -26,7 +26,7 @@ export class RemoteInferencer implements ICompletions, IGeneration {
     const requestURL = rType === AIRequestType.COMPLETION ? this.completion_url : this.api_url
 
     try {
-      const options = { headers: { 'Content-Type': 'application/json', } }
+      const options = { headers: { 'Content-Type': 'application/json', }, timeout: 2000 }
       const result = await axios.post(requestURL, payload, options)
 
       switch (rType) {
@@ -49,6 +49,7 @@ export class RemoteInferencer implements ICompletions, IGeneration {
     } catch (e) {
       ChatHistory.clearHistory()
       console.error('Error making request to Inference server:', e.message)
+      return ""
     }
     finally {
       this.event.emit("onInferenceDone")
@@ -110,7 +111,7 @@ export class RemoteInferencer implements ICompletions, IGeneration {
   }
 
   async code_completion(prompt, promptAfter, ctxFiles, fileName, options:IParams=CompletionParams): Promise<any> {
-    options.max_tokens = 10
+    options.max_tokens = 30
     const payload = { prompt, 'context':promptAfter, "endpoint":"code_completion",
       'ctxFiles':ctxFiles, 'currentFileName':fileName, ...options }
     return this._makeRequest(payload, AIRequestType.COMPLETION)

libs/remix-ui/editor/src/lib/inlineCompetionsLibs/AdaptiveRateLimiter.ts

@@ -0,0 +1,140 @@
+/**
+ * Adaptive Rate Limiter for Inline Completions
+ *
+ */
+
+interface CompletionHistoryEntry {
+  timestamp: number;
+  accepted: boolean;
+}
+
+interface AdaptiveRateLimiterOptions {
+  minRequestInterval?: number;
+  completionCooldown?: number;
+  historyWindow?: number;
+  baseAdaptiveCooldown?: number;
+  maxAdaptiveCooldown?: number;
+}
+
+interface AdaptiveRateLimiterStats {
+  acceptanceRate: number;
+  totalCompletions: number;
+  acceptedCompletions: number;
+  rejectedCompletions: number;
+  currentCooldown: number;
+}
+
+export class AdaptiveRateLimiter {
+  private lastRequestTime: number = 0;
+  private lastCompletionTime: number = 0;
+  private acceptanceRate: number = 0.5;
+  private totalCompletions: number = 0;
+  private acceptedCompletions: number = 0;
+  private rejectedCompletions: number = 0;
+  private recentCompletionHistory: CompletionHistoryEntry[] = [];
+
+  private readonly minRequestInterval: number = 500;
+  private readonly completionCooldown: number = 2000;
+  private readonly historyWindow: number = 300000; // 5 minutes
+  private readonly baseAdaptiveCooldown: number = 1000;
+  private readonly maxAdaptiveCooldown: number = 10000;
+
+  constructor(options?: AdaptiveRateLimiterOptions) {
+    if (options) {
+      this.minRequestInterval = options.minRequestInterval ?? this.minRequestInterval;
+      this.completionCooldown = options.completionCooldown ?? this.completionCooldown;
+      this.historyWindow = options.historyWindow ?? this.historyWindow;
+      this.baseAdaptiveCooldown = options.baseAdaptiveCooldown ?? this.baseAdaptiveCooldown;
+      this.maxAdaptiveCooldown = options.maxAdaptiveCooldown ?? this.maxAdaptiveCooldown;
+    }
+  }
+
+  shouldAllowRequest(currentTime: number = Date.now()): boolean {
+    const timeSinceLastRequest = currentTime - this.lastRequestTime;
+    const timeSinceLastCompletion = currentTime - this.lastCompletionTime;
+
+    // Check minimum request interval
+    if (timeSinceLastRequest < this.minRequestInterval) {
+      return false;
+    }
+
+    // Check adaptive cooldown
+    const adaptiveCooldown = this.getAdaptiveCooldown();
+    if (timeSinceLastCompletion < adaptiveCooldown) {
+      return false;
+    }
+
+    return true;
+  }
+
+  recordRequest(currentTime: number = Date.now()): void {
+    this.lastRequestTime = currentTime;
+  }
+
+  recordCompletion(currentTime: number = Date.now()): void {
+    this.lastCompletionTime = currentTime;
+  }
+
+  trackCompletionShown(): void {
+    this.totalCompletions++;
+    this.recentCompletionHistory.push({
+      timestamp: Date.now(),
+      accepted: false
+    });
+  }
+
+  trackCompletionAccepted(): void {
+    this.acceptedCompletions++;
+
+    // Update the most recent completion as accepted
+    if (this.recentCompletionHistory.length > 0) {
+      this.recentCompletionHistory[this.recentCompletionHistory.length - 1].accepted = true;
+    }
+  }
+
+  trackCompletionRejected(): void {
+    this.rejectedCompletions++;
+  }
+
+  private getAdaptiveCooldown(): number {
+    this.updateAcceptanceRate();
+    // high fidelity adoption
+    // Higher acceptance rate = shorter cooldown, lower acceptance rate = longer cooldown
+    const adaptiveFactor = Math.max(0.1, 1 - this.acceptanceRate);
+    const adaptiveCooldown = Math.min(
+      this.maxAdaptiveCooldown,
+      this.baseAdaptiveCooldown + (this.baseAdaptiveCooldown * adaptiveFactor * 5)
+    );
+
+    return Math.max(this.completionCooldown, adaptiveCooldown);
+  }
+
+  private updateAcceptanceRate(): void {
+    const currentTime = Date.now();
+
+    // Remove old entries beyond the history window
+    this.recentCompletionHistory = this.recentCompletionHistory.filter(
+      entry => currentTime - entry.timestamp < this.historyWindow
+    );
+
+    // Calculate acceptance rate from recent history
+    if (this.recentCompletionHistory.length > 0) {
+      const recentAccepted = this.recentCompletionHistory.filter(entry => entry.accepted).length;
+      this.acceptanceRate = recentAccepted / this.recentCompletionHistory.length;
+    } else {
+      // Default to 0.5 if no recent history
+      // do not penalize anyone at startup
+      this.acceptanceRate = 0.5;
+    }
+  }
+
+  getStats(): AdaptiveRateLimiterStats {
+    return {
+      acceptanceRate: this.acceptanceRate,
+      totalCompletions: this.totalCompletions,
+      acceptedCompletions: this.acceptedCompletions,
+      rejectedCompletions: this.rejectedCompletions,
+      currentCooldown: this.getAdaptiveCooldown()
+    };
+  }
+}

libs/remix-ui/editor/src/lib/inlineCompetionsLibs/CompletionCache.ts

@@ -0,0 +1,174 @@
+/**
+ * Completion Cache for Inline Completions
+ *
+ */
+
+import { monacoTypes } from '@remix-ui/editor';
+
+interface CacheEntry {
+  result: any;
+  timestamp: number;
+}
+
+interface CompletionCacheOptions {
+  cacheTimeout?: number;
+  maxCacheSize?: number;
+}
+
+interface CompletionCacheStats {
+  cacheSize: number;
+  pendingRequests: number;
+  hitRate: number;
+}
+
+export class CompletionCache {
+  private cache: Map<string, CacheEntry> = new Map();
+  private pendingRequests: Map<string, Promise<any>> = new Map();
+
+  private readonly cacheTimeout: number = 30000; // 30 seconds
+  private readonly maxCacheSize: number = 100;
+
+  constructor(options?: CompletionCacheOptions) {
+    if (options) {
+      this.cacheTimeout = options.cacheTimeout ?? this.cacheTimeout;
+      this.maxCacheSize = options.maxCacheSize ?? this.maxCacheSize;
+    }
+  }
+
+  createCacheKey(
+    word: string,
+    wordAfter: string,
+    position: monacoTypes.Position,
+    task?: string
+  ): string {
+    // truncate context
+    const contextHash = `${word.slice(-100)}_${wordAfter.slice(0, 100)}_${position.lineNumber}_${position.column}`;
+    return task ? `${task}_${contextHash}` : contextHash;
+  }
+
+  /**
+   * Get cached result if available and not expired
+   */
+  getCachedResult(cacheKey: string): any | null {
+    const cached = this.cache.get(cacheKey);
+    if (cached && Date.now() - cached.timestamp < this.cacheTimeout) {
+      return cached.result;
+    }
+
+    // Remove expired cache entry
+    if (cached) {
+      this.cache.delete(cacheKey);
+    }
+
+    return null;
+  }
+
+  /**
+   * Cache a completion result
+   */
+  cacheResult(cacheKey: string, result: any): void {
+    // Clean up old cache entries periodically
+    if (this.cache.size >= this.maxCacheSize) {
+      this.cleanupExpiredEntries();
+    }
+
+    // If still at capacity, remove oldest entries
+    if (this.cache.size >= this.maxCacheSize) {
+      const oldestKey = this.cache.keys().next().value;
+      if (oldestKey) {
+        this.cache.delete(oldestKey);
+      }
+    }
+
+    this.cache.set(cacheKey, {
+      result: result,
+      timestamp: Date.now()
+    });
+  }
+
+  /**
+   * Check if a request is already pending
+   */
+  isPending(cacheKey: string): boolean {
+    return this.pendingRequests.has(cacheKey);
+  }
+
+  /**
+   * Get pending request promise
+   */
+  getPendingRequest(cacheKey: string): Promise<any> | null {
+    return this.pendingRequests.get(cacheKey) || null;
+  }
+
+  /**
+   * Set a pending request
+   */
+  setPendingRequest(cacheKey: string, promise: Promise<any>): void {
+    this.pendingRequests.set(cacheKey, promise);
+  }
+
+  /**
+   * Remove a pending request
+   */
+  removePendingRequest(cacheKey: string): void {
+    this.pendingRequests.delete(cacheKey);
+  }
+
+  /**
+   * Handle a request with caching and deduplication
+   */
+  async handleRequest<T>(
+    cacheKey: string,
+    requestFn: () => Promise<T>
+  ): Promise<T> {
+    // Check cache first
+    const cachedResult = this.getCachedResult(cacheKey);
+    if (cachedResult) {
+      return cachedResult;
+    }
+
+    // Check if same request is already pending
+    const pendingRequest = this.getPendingRequest(cacheKey);
+    if (pendingRequest) {
+      return await pendingRequest;
+    }
+
+    // Create and store pending request
+    const promise = requestFn();
+    this.setPendingRequest(cacheKey, promise);
+
+    try {
+      const result = await promise;
+      this.cacheResult(cacheKey, result);
+      return result;
+    } finally {
+      this.removePendingRequest(cacheKey);
+    }
+  }
+
+  private cleanupExpiredEntries(): void {
+    const now = Date.now();
+    for (const [key, entry] of this.cache.entries()) {
+      if (now - entry.timestamp > this.cacheTimeout) {
+        this.cache.delete(key);
+      }
+    }
+  }
+
+  clear(): void {
+    this.cache.clear();
+    this.pendingRequests.clear();
+  }
+
+  getStats(): CompletionCacheStats {
+    return {
+      cacheSize: this.cache.size,
+      pendingRequests: this.pendingRequests.size,
+      hitRate: 0 // Would need to track hits/misses to calculate this
+    };
+  }
+
+  cleanup(): void {
+    this.cleanupExpiredEntries();
+  }
+}