test: fix circuit breaker close after shutdown

Author: gitcommitshow

README.md

@@ -1,6 +1,6 @@
 # ResilientLLM
 
-A robust LLM integration layer designed to ensure reliable, seamless interactions across multiple APIs by intelligently handling failures and rate limits.
+A simple but robust LLM integration layer designed to ensure reliable, seamless interactions across multiple APIs by intelligently handling failures and rate limits.
 
 ## Motivation
 
@@ -34,7 +34,9 @@ const llm = new ResilientLLM({
   rateLimitConfig: {
     requestsPerMinute: 60,      // Limit to 60 requests per minute
     llmTokensPerMinute: 90000   // Limit to 90,000 LLM tokens per minute
-  }
+  },
+  retries: 3,
+  backoffFactor: 2
 });
 
 const conversationHistory = [

ResilientOperation.js

@@ -244,10 +244,10 @@ class ResilientOperation {
      */
     async _executeBasic(asyncFn, config, ...args) {
 
-        let attempt = 0;
+        let retryAttempt = 0;
         let delay = 1000;
 
-        while (attempt <= config.retries) {
+        while (retryAttempt <= config.retries) {
             try {
                 // Check circuit breaker first
                 if (this.circuitBreaker.isCircuitOpen()) {
@@ -275,13 +275,8 @@ class ResilientOperation {
                 this.circuitBreaker.recordSuccess();
 
                 // Log success with retry information
-                if (attempt > 0) {
-                    const status = this.circuitBreaker.getStatus();
-                    console.log(`[ResilientOperation][${this.id}] Operation succeeded after ${attempt} retries. Current fail count: ${status.failCount}/${status.failureThreshold}`);
-                } else {
-                    const status = this.circuitBreaker.getStatus();
-                    console.log(`[ResilientOperation][${this.id}] Operation succeeded on first attempt. Current fail count: ${status.failCount}/${status.failureThreshold}`);
-                }
+                const status = this.circuitBreaker.getStatus();
+                console.log(`[ResilientOperation][${this.id}] Operation succeeded after ${retryAttempt} retries. Current fail count: ${status.failCount}/${status.failureThreshold}`);
 
                 return result;
             } catch (err) {
@@ -303,17 +298,17 @@ class ResilientOperation {
                 this.circuitBreaker.recordFailure();
 
                 // Log retry attempt with circuit breaker status
-                const remainingRetries = config.retries - attempt;
+                const remainingRetries = config.retries - retryAttempt;
                 const status = this.circuitBreaker.getStatus();
 
-                console.log(`[ResilientOperation][${this.id}] Attempt ${attempt + 1} failed: ${err.message}. Retries remaining: ${remainingRetries}. Circuit breaker fail count: ${status.failCount}/${status.failureThreshold}`);
+                console.log(`[ResilientOperation][${this.id}] Attempt ${retryAttempt + 1} failed: ${err.message}. Retries remaining: ${remainingRetries}. Circuit breaker fail count: ${status.failCount}/${status.failureThreshold}`);
                 if(status?.isOpen) {
                     console.log(`[ResilientOperation][${this.id}] Circuit breaker is open. Cooldown remaining: ${status.cooldownRemaining}ms`);
                 }
 
-                if (!this._shouldRetry(err) || attempt >= config.retries) {
+                if (!this._shouldRetry(err) || retryAttempt >= config.retries) {
                     // Log final failure - this operation has exhausted all retries
-                    console.log(`[ResilientOperation][${this.id}] Operation failed after ${attempt + 1} attempts. Circuit breaker fail count: ${status.failCount}/${status.failureThreshold}`);
+                    console.log(`[ResilientOperation][${this.id}] Operation failed after ${retryAttempt + 1} attempts. Circuit breaker fail count: ${status.failCount}/${status.failureThreshold}`);
                     throw err;
                 }
 
@@ -323,7 +318,7 @@ class ResilientOperation {
                 this.nextRetryDelay = null;
                 await sleep(waitTime, this._abortController.signal);
                 delay *= config.backoffFactor;
-                attempt++;
+                retryAttempt++;
             }
         }
         console.log(`[ResilientOperation][${this.id}] Exiting execution attempt loop`);

test/resilient-operation.e2e.test.js

@@ -234,40 +234,45 @@ describe('ResilientOperation E2E Tests', () => {
     }).timeout(50000);
 
     // Circuit breaker close test
-    it('should close circuit breaker after cooldown period', async () => {
+    it.only('should close circuit breaker after cooldown period', async () => {
       // Create a ResilientOperation with short cooldown for testing
       const testResilientOp = new ResilientOperation({
         bucketId: 'cooldown-test',
         circuitBreakerConfig: { failureThreshold: 3, cooldownPeriod: 3000 }, // 1 second cooldown
         retries: 0, // Disable retries to see pure circuit breaker behavior
       });
 
-      const mockAsyncFn = sinon.stub().callsFake(async () => {
+      const failingMocAsyncFn = sinon.stub().callsFake(async () => {
         const error = new Error('Service down');
         error.response = { status: 500 };
         throw error;
       });
 
       // Make enough calls to open the circuit breaker
       const promises = [];
-      for (let i = 0; i < 4; i++) {
-        promises.push(testResilientOp.execute(mockAsyncFn).catch(err => err));
+      for (let i = 0; i < 3; i++) {
+        promises.push(testResilientOp.execute(failingMocAsyncFn).catch(err => err));
       }
 
       await Promise.all(promises);
 
       // Circuit breaker should be open
       let status = testResilientOp.circuitBreaker.getStatus();
       expect(status.isOpen).to.be.true;
-      expect(status.failCount).to.be.at.least(3);
+      expect(status.failCount).to.be.equal(3);
 
       // Wait for cooldown period to expire
-      await new Promise(resolve => setTimeout(resolve, 3100));
+      await new Promise(resolve => setTimeout(resolve, 4100));
+
+      const successMocAsyncFn = sinon.stub().callsFake(async () => {
+        return { data: 'success' };
+      });
+      await testResilientOp.execute(successMocAsyncFn).catch(err => err)
 
       // Circuit breaker should automatically close
       status = testResilientOp.circuitBreaker.getStatus();
       expect(status.isOpen).to.be.false;
-      expect(status.failCount).to.equal(0);
+      expect(status.failCount).to.be.equal(0);
     }).timeout(10000);
 
     // Circuit breaker open test