From 530bc4cbad24f561e5c429aabbfdec9533eb8348 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Mon, 9 Dec 2024 15:39:25 -0700 Subject: [PATCH 1/2] Use common function for service policy boilerplate We would like to use the Cockatiel library in our service classes to ensure that requests are retried using the circuit breaker pattern. Some of our service classes do this already, but we are copying and pasting the code around. This commit extracts the boilerplate code to a new function in the `@metamask/controller-utils` package, `createServicePolicy`, so that we no longer have to do this. --- packages/controller-utils/CHANGELOG.md | 4 + packages/controller-utils/package.json | 2 + .../src/create-service-policy.test.ts | 2703 +++++++++++++++++ .../src/create-service-policy.ts | 181 ++ packages/controller-utils/src/index.test.ts | 75 + packages/controller-utils/src/index.ts | 1 + yarn.lock | 2 + 7 files changed, 2968 insertions(+) create mode 100644 packages/controller-utils/src/create-service-policy.test.ts create mode 100644 packages/controller-utils/src/create-service-policy.ts create mode 100644 packages/controller-utils/src/index.test.ts diff --git a/packages/controller-utils/CHANGELOG.md b/packages/controller-utils/CHANGELOG.md index 2bc867e5b33..bc50f1682e3 100644 --- a/packages/controller-utils/CHANGELOG.md +++ b/packages/controller-utils/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add `createServicePolicy` function to assist with reducing boilerplate for service classes ([#5053](https://github.com/MetaMask/core/pull/5053)) + ## [11.4.4] ### Fixed diff --git a/packages/controller-utils/package.json b/packages/controller-utils/package.json index 3fb36cf2abf..8de5139bbf9 100644 --- a/packages/controller-utils/package.json +++ b/packages/controller-utils/package.json @@ -55,6 +55,7 @@ "@types/bn.js": "^5.1.5", "bignumber.js": "^9.1.2", "bn.js": "^5.2.1", + "cockatiel": "^3.1.2", "eth-ens-namehash": "^2.0.8", "fast-deep-equal": "^3.1.3" }, @@ -65,6 +66,7 @@ "deepmerge": "^4.2.2", "jest": "^27.5.1", "nock": "^13.3.1", + "sinon": "^9.2.4", "ts-jest": "^27.1.4", "typedoc": "^0.24.8", "typedoc-plugin-missing-exports": "^2.0.0", diff --git a/packages/controller-utils/src/create-service-policy.test.ts b/packages/controller-utils/src/create-service-policy.test.ts new file mode 100644 index 00000000000..ce16427596d --- /dev/null +++ b/packages/controller-utils/src/create-service-policy.test.ts @@ -0,0 +1,2703 @@ +import { useFakeTimers } from 'sinon'; +import type { SinonFakeTimers } from 'sinon'; + +import { + createServicePolicy, + DEFAULT_CIRCUIT_BREAK_DURATION, + DEFAULT_DEGRADED_THRESHOLD, + DEFAULT_MAX_CONSECUTIVE_FAILURES, + DEFAULT_MAX_RETRIES, +} from './create-service-policy'; + +describe('createServicePolicy', () => { + let clock: SinonFakeTimers; + + beforeEach(() => { + clock = useFakeTimers(); + }); + + afterEach(() => { + clock.restore(); + }); + + describe('wrapping a service that succeeds on the first try', () => { + it('returns a policy that returns what the service returns', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const policy = createServicePolicy(); + + const returnValue = await policy.execute(mockService); + + expect(returnValue).toStrictEqual({ some: 'data' }); + }); + + it('only calls the service once before returning', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const policy = createServicePolicy(); + + await policy.execute(mockService); + + expect(mockService).toHaveBeenCalledTimes(1); + }); + + it('does not call the onBreak callback, since the circuit never opens', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const onBreak = jest.fn(); + const policy = createServicePolicy({ onBreak }); + + await policy.execute(mockService); + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + await policy.execute(mockService); + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + const mockService = jest.fn(() => { + return new Promise((resolve) => { + setTimeout(() => resolve({ some: 'data' }), delay); + }); + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + const promise = policy.execute(mockService); + clock.tick(delay); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time below the threshold', async () => { + const degradedThreshold = 2000; + const mockService = jest.fn(() => ({ some: 'data' })); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ degradedThreshold, onDegraded }); + + await policy.execute(mockService); + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time beyond the threshold', async () => { + const degradedThreshold = 2000; + const delay = degradedThreshold + 1; + const mockService = jest.fn(() => { + return new Promise((resolve) => { + setTimeout(() => resolve({ some: 'data' }), delay); + }); + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ degradedThreshold, onDegraded }); + + const promise = policy.execute(mockService); + clock.tick(delay); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('wrapping a service that always fails', () => { + describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { + it(`calls the service a total of ${ + 1 + DEFAULT_MAX_RETRIES + } times, delaying each retry using a backoff formula`, async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy(); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(mockService).toHaveBeenCalledTimes(1 + DEFAULT_MAX_RETRIES); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + it('throws what the service throws', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy(); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call the onBreak callback, since the max number of consecutive failures is never reached', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once, since the circuit is still closed', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the service is executed again', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('using a custom max number of retries', () => { + it(`calls the service a total of 1 + times, delaying each retry using a backoff formula`, async () => { + const maxRetries = 5; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(366.8287823691078).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(731.8792783578953).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(mockService).toHaveBeenCalledTimes(1 + maxRetries); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call the onBreak callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const mockService = jest.fn(() => { + throw new Error('failure'); + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + }); + }); + }); + }); + + describe('wrapping a service that fails continuously and then succeeds on the final try', () => { + describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { + it(`calls the service a total of ${ + 1 + DEFAULT_MAX_RETRIES + } times, delaying each retry using a backoff formula`, async () => { + let invocationCounter = 0; + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }); + const policy = createServicePolicy(); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(mockService).toHaveBeenCalledTimes(1 + DEFAULT_MAX_RETRIES); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + it('returns what the service returns', async () => { + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback, since the max number of consecutive failures is never reached', async () => { + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + let invocationCounter = 0; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + let invocationCounter = 0; + const delay = degradedThreshold + 1; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('does not call the onDegraded callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + describe(`using the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the BrokenCircuitError + } + clock.tick(DEFAULT_CIRCUIT_BREAK_DURATION); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + + describe('using a custom circuit break duration', () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + // This has to be high enough to exceed the exponential backoff + const circuitBreakDuration = 5_000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxConsecutiveFailures, + circuitBreakDuration, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the BrokenCircuitError + } + clock.tick(circuitBreakDuration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + }); + }); + }); + + describe('using a custom max number of retries', () => { + it(`calls the service a total of 1 + times, delaying each retry using a backoff formula`, async () => { + const maxRetries = 5; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(366.8287823691078).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(731.8792783578953).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(mockService).toHaveBeenCalledTimes(1 + maxRetries); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('does not call the onDegraded callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + describe(`using the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the BrokenCircuitError + } + clock.tick(DEFAULT_CIRCUIT_BREAK_DURATION); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + + describe('using a custom circuit break duration', () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + // This has to be high enough to exceed the exponential backoff + const circuitBreakDuration = 50_000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + circuitBreakDuration, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await expect(firstExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + clock.tick(circuitBreakDuration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('does not call the onDegraded callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + describe(`using the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the BrokenCircuitError + } + clock.tick(DEFAULT_CIRCUIT_BREAK_DURATION); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + + describe('using a custom circuit break duration', () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + // This has to be high enough to exceed the exponential backoff + const circuitBreakDuration = 5_000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + circuitBreakDuration, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await expect(firstExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + clock.tick(circuitBreakDuration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + }); + }); + }); + }); +}); diff --git a/packages/controller-utils/src/create-service-policy.ts b/packages/controller-utils/src/create-service-policy.ts new file mode 100644 index 00000000000..210516d124c --- /dev/null +++ b/packages/controller-utils/src/create-service-policy.ts @@ -0,0 +1,181 @@ +import { + circuitBreaker, + ConsecutiveBreaker, + ExponentialBackoff, + handleAll, + retry, + wrap, + CircuitState, +} from 'cockatiel'; +import type { IPolicy } from 'cockatiel'; + +/** + * The maximum number of times that a failing action should be re-run before + * giving up. + */ +export const DEFAULT_MAX_RETRIES = 3; + +/** + * The maximum number of times that the action is allowed to fail before pausing + * further retries. + */ +export const DEFAULT_MAX_CONSECUTIVE_FAILURES = (1 + DEFAULT_MAX_RETRIES) * 3; + +/** + * The default length of time (in milliseconds) to temporarily pause retries of + * the action after enough consecutive failures. + */ +export const DEFAULT_CIRCUIT_BREAK_DURATION = 30 * 60 * 1000; + +/** + * The default length of time (in milliseconds) that governs when an action is + * regarded as degraded (affecting when `onDegraded` is called). + */ +export const DEFAULT_DEGRADED_THRESHOLD = 5_000; + +/** + * Constructs an object exposing an `execute` method which, given a function, + * will retry it with ever increasing delays until it succeeds. If it detects + * too many consecutive failures, it will block further retries until a + * designated time period has passed; this particular behavior is primarily + * designed for services that wrap API calls so as not to make needless HTTP + * requests when the API is down and to be able to recover when the API comes + * back up. In addition, hooks allow for responding to certain events, one of + * which can be used to detect when an HTTP request is performing slowly. + * + * Internally, the executor object makes use of the retry and circuit breaker + * policies from the [Cockatiel](https://www.npmjs.com/package/cockatiel) + * library; see there for more. + * + * @param options - The options to this function. + * @param options.maxRetries - The maximum number of times that a failing action + * should be re-run before giving up. Defaults to 3. + * @param options.maxConsecutiveFailures - The maximum number of times that + * the action is allowed to fail before pausing further retries. Defaults to 12. + * @param options.circuitBreakDuration - The length of time (in milliseconds) to + * pause retries of the action after the number of failures reaches + * `maxConsecutiveFailures`. + * @param options.degradedThreshold - The length of time (in milliseconds) that + * governs when an action is regarded as degraded (affecting when `onDegraded` + * is called). Defaults to 5 seconds. + * @param options.onBreak - A function which is called when the action fails too + * many times in a row. + * @param options.onDegraded - A function which is called when the action + * succeeds before `maxConsecutiveFailures` is reached, but takes more time + * than the `degradedThreshold` to run. + * @returns A Cockatiel policy object that can be used to run an arbitrary + * action (a function). + * @example + * To use the policy, call `execute` on it and pass a function: + * ``` ts + * const policy = createServicePolicy({ + * maxRetries: 3, + * maxConsecutiveFailures: 3, + * circuitBreakDuration: 5000, + * degradedThreshold: 2000, + * onBreak: () => { + * console.log('Circuit broke'); + * }, + * onDegraded: () => { + * console.log('Service is degraded'); + * } + * }); + * + * await policy.execute(async () => { + * const response = await fetch('https://some/url'); + * return await response.json(); + * }); + * ``` + * You may wish to store `policy` in a single place and reuse it each time you + * want to execute your action. For instance: + * ``` ts + * class Service { + * constructor() { + * this.#policy = createServicePolicy({ + * maxRetries: 3, + * maxConsecutiveFailures: 3, + * circuitBreakDuration: 5000, + * degradedThreshold: 2000, + * }); + * } + * + * async fetch() { + * return await this.#policy.execute(async () => { + * const response = await fetch('https://some/url'); + * return await response.json(); + * }); + * } + * } + * ``` + */ +export function createServicePolicy({ + maxRetries = DEFAULT_MAX_RETRIES, + maxConsecutiveFailures = DEFAULT_MAX_CONSECUTIVE_FAILURES, + circuitBreakDuration = DEFAULT_CIRCUIT_BREAK_DURATION, + degradedThreshold = DEFAULT_DEGRADED_THRESHOLD, + onBreak = () => { + // do nothing + }, + onDegraded = () => { + // do nothing + }, +}: { + maxRetries?: number; + maxConsecutiveFailures?: number; + degradedThreshold?: number; + circuitBreakDuration?: number; + onBreak?: () => void; + onDegraded?: () => void; +} = {}): IPolicy { + const retryPolicy = retry(handleAll, { + maxAttempts: maxRetries, + // Retries of the action passed to the policy will be padded by increasing + // delays, determined by a formula. + backoff: new ExponentialBackoff(), + }); + + const circuitBreakerPolicy = circuitBreaker(handleAll, { + // While the circuit is open, any additional invocations of the action + // passed to the policy (either via automatic retries or by manually + // executing the policy again) will result in a BrokenCircuitError. The + // circuit will transition to a half-open state after the + // `circuitBreakDuration` passes, after which the action will be allowed to + // run again. If the action succeeds, the circuit will close, otherwise it + // will open again. + halfOpenAfter: circuitBreakDuration, + breaker: new ConsecutiveBreaker(maxConsecutiveFailures), + }); + + // The `onBreak` callback will be called if the number of times the action + // consistently throws exceeds the maximum consecutive number of failures. + // Combined with the retry policy, this can happen if: + // - `maxRetries` > `maxConsecutiveFailures` and the policy is executed once + // - `maxRetries` <= `maxConsecutiveFailures` but the policy is executed + // multiple times, enough for the total number of retries to exceed + // `maxConsecutiveFailures` + circuitBreakerPolicy.onBreak(onBreak); + + retryPolicy.onGiveUp(() => { + if (circuitBreakerPolicy.state === CircuitState.Closed) { + // The `onDegraded` callback will be called if the number of retries is + // exceeded and the maximum number of consecutive failures has not been + // reached yet (whether the policy is called once or multiple times). + onDegraded(); + } + }); + retryPolicy.onSuccess(({ duration }) => { + if ( + circuitBreakerPolicy.state === CircuitState.Closed && + duration > degradedThreshold + ) { + // The `onDegraded` callback will also be called if the action passed to + // the policy does not throw, but the time it takes for the action to run + // exceeds the `degradedThreshold`. + onDegraded(); + } + }); + + // Each time the retry policy retries, it will execute the circuit breaker + // policy. + return wrap(retryPolicy, circuitBreakerPolicy); +} diff --git a/packages/controller-utils/src/index.test.ts b/packages/controller-utils/src/index.test.ts new file mode 100644 index 00000000000..61ef841826f --- /dev/null +++ b/packages/controller-utils/src/index.test.ts @@ -0,0 +1,75 @@ +import * as allExports from '.'; + +describe('@metamask/controller-utils', () => { + it('has expected JavaScript exports', () => { + expect(Object.keys(allExports)).toMatchInlineSnapshot(` + Array [ + "createServicePolicy", + "BNToHex", + "convertHexToDecimal", + "fetchWithErrorHandling", + "fractionBN", + "fromHex", + "getBuyURL", + "gweiDecToWEIBN", + "handleFetch", + "hexToBN", + "hexToText", + "isNonEmptyArray", + "isPlainObject", + "isSafeChainId", + "isSafeDynamicKey", + "isSmartContractCode", + "isValidJson", + "isValidHexAddress", + "normalizeEnsName", + "query", + "safelyExecute", + "safelyExecuteWithTimeout", + "successfulFetch", + "timeoutFetch", + "toChecksumHexAddress", + "toHex", + "weiHexToGweiDec", + "isEqualCaseInsensitive", + "RPC", + "FALL_BACK_VS_CURRENCY", + "IPFS_DEFAULT_GATEWAY_URL", + "GANACHE_CHAIN_ID", + "MAX_SAFE_CHAIN_ID", + "ERC721", + "ERC1155", + "ERC20", + "ERC721_INTERFACE_ID", + "ERC721_METADATA_INTERFACE_ID", + "ERC721_ENUMERABLE_INTERFACE_ID", + "ERC1155_INTERFACE_ID", + "ERC1155_METADATA_URI_INTERFACE_ID", + "ERC1155_TOKEN_RECEIVER_INTERFACE_ID", + "GWEI", + "ASSET_TYPES", + "TESTNET_TICKER_SYMBOLS", + "BUILT_IN_NETWORKS", + "OPENSEA_PROXY_URL", + "NFT_API_BASE_URL", + "NFT_API_VERSION", + "NFT_API_TIMEOUT", + "ORIGIN_METAMASK", + "ApprovalType", + "CHAIN_ID_TO_ETHERS_NETWORK_NAME_MAP", + "InfuraNetworkType", + "NetworkType", + "isNetworkType", + "isInfuraNetworkType", + "BuiltInNetworkName", + "ChainId", + "NetworksTicker", + "BlockExplorerUrl", + "NetworkNickname", + "parseDomainParts", + "isValidSIWEOrigin", + "detectSIWE", + ] + `); + }); +}); diff --git a/packages/controller-utils/src/index.ts b/packages/controller-utils/src/index.ts index 3d35d62c0a0..b3bd8821e12 100644 --- a/packages/controller-utils/src/index.ts +++ b/packages/controller-utils/src/index.ts @@ -1,3 +1,4 @@ +export { createServicePolicy } from './create-service-policy'; export * from './constants'; export type { NonEmptyArray } from './util'; export { diff --git a/yarn.lock b/yarn.lock index 7b7447656d5..44af54cfa42 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2523,11 +2523,13 @@ __metadata: "@types/jest": "npm:^27.4.1" bignumber.js: "npm:^9.1.2" bn.js: "npm:^5.2.1" + cockatiel: "npm:^3.1.2" deepmerge: "npm:^4.2.2" eth-ens-namehash: "npm:^2.0.8" fast-deep-equal: "npm:^3.1.3" jest: "npm:^27.5.1" nock: "npm:^13.3.1" + sinon: "npm:^9.2.4" ts-jest: "npm:^27.1.4" typedoc: "npm:^0.24.8" typedoc-plugin-missing-exports: "npm:^2.0.0" From 2a7186ca76c0154be155d962ce91173672b7209f Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Fri, 10 Jan 2025 11:36:13 -0700 Subject: [PATCH 2/2] Add onRetry and retryFilterPolicy --- .../src/create-service-policy.test.ts | 1409 +++++++++-------- .../src/create-service-policy.ts | 26 +- 2 files changed, 777 insertions(+), 658 deletions(-) diff --git a/packages/controller-utils/src/create-service-policy.test.ts b/packages/controller-utils/src/create-service-policy.test.ts index ce16427596d..cc976dc9be9 100644 --- a/packages/controller-utils/src/create-service-policy.test.ts +++ b/packages/controller-utils/src/create-service-policy.test.ts @@ -1,3 +1,4 @@ +import { handleWhen } from 'cockatiel'; import { useFakeTimers } from 'sinon'; import type { SinonFakeTimers } from 'sinon'; @@ -111,28 +112,59 @@ describe('createServicePolicy', () => { }); describe('wrapping a service that always fails', () => { - describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { - it(`calls the service a total of ${ - 1 + DEFAULT_MAX_RETRIES - } times, delaying each retry using a backoff formula`, async () => { + describe('if a custom retry filter policy is given and the retry filter policy filters out the thrown error', () => { + it('throws what the service throws', async () => { const error = new Error('failure'); const mockService = jest.fn(() => { throw error; }); - const policy = createServicePolicy(); - // Each retry delay is randomized using a decorrelated jitter formula, - // so we need to prevent that - jest.spyOn(Math, 'random').mockReturnValue(0); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); const promise = policy.execute(mockService); - // These values were found by logging them - clock.tickAsync(0).catch(() => { - // ignore any errors - adding to the promise queue is enough + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the service once and only once', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; }); - clock.tickAsync(176.27932892814937).catch(() => { - // ignore any errors - adding to the promise queue is enough + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), }); - clock.tickAsync(186.8886145345685).catch(() => { + + const promise = policy.execute(mockService); + try { + await promise; + } catch { + // ignore the error + } + + expect(mockService).toHaveBeenCalledTimes(1); + }); + + it('does not call the onBreak callback', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { // ignore any errors - adding to the promise queue is enough }); try { @@ -141,56 +173,59 @@ describe('createServicePolicy', () => { // ignore the error } - expect(mockService).toHaveBeenCalledTimes(1 + DEFAULT_MAX_RETRIES); + expect(onBreak).not.toHaveBeenCalled(); }); - describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { - it('throws what the service throws', async () => { - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy(); - - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough - }); + it('does not call the onDegraded callback', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + onDegraded, + }); - await expect(promise).rejects.toThrow(error); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough }); + try { + await promise; + } catch { + // ignore the error + } - it('does not call the onBreak callback, since the max number of consecutive failures is never reached', async () => { + expect(onDegraded).not.toHaveBeenCalled(); + }); + }); + + describe('using the default retry filter policy (which retries all errors)', () => { + describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { + it(`calls the service a total of ${ + 1 + DEFAULT_MAX_RETRIES + } times, delaying each retry using a backoff formula`, async () => { const error = new Error('failure'); const mockService = jest.fn(() => { throw error; }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ onBreak }); + const policy = createServicePolicy(); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { + // These values were found by logging them + clock.tickAsync(0).catch(() => { // ignore any errors - adding to the promise queue is enough }); - try { - await promise; - } catch { - // ignore the error - } - - expect(onBreak).not.toHaveBeenCalled(); - }); - - it('calls the onDegraded callback once, since the circuit is still closed', async () => { - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ onDegraded }); - - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { + clock.tickAsync(186.8886145345685).catch(() => { // ignore any errors - adding to the promise queue is enough }); try { @@ -199,23 +234,16 @@ describe('createServicePolicy', () => { // ignore the error } - expect(onDegraded).toHaveBeenCalledTimes(1); + expect(mockService).toHaveBeenCalledTimes(1 + DEFAULT_MAX_RETRIES); }); - }); - describe('using a custom max number of consecutive failures', () => { - describe('if the initial run + retries is less than the max number of consecutive failures', () => { + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { it('throws what the service throws', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; const error = new Error('failure'); const mockService = jest.fn(() => { throw error; }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ - maxConsecutiveFailures, - onBreak, - }); + const policy = createServicePolicy(); const promise = policy.execute(mockService); clock.runAllAsync().catch(() => { @@ -225,17 +253,13 @@ describe('createServicePolicy', () => { await expect(promise).rejects.toThrow(error); }); - it('does not call the onBreak callback', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + it('does not call the onBreak callback, since the max number of consecutive failures is never reached', async () => { const error = new Error('failure'); const mockService = jest.fn(() => { throw error; }); const onBreak = jest.fn(); - const policy = createServicePolicy({ - maxConsecutiveFailures, - onBreak, - }); + const policy = createServicePolicy({ onBreak }); const promise = policy.execute(mockService); clock.runAllAsync().catch(() => { @@ -250,17 +274,13 @@ describe('createServicePolicy', () => { expect(onBreak).not.toHaveBeenCalled(); }); - it('calls the onDegraded callback once', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + it('calls the onDegraded callback once, since the circuit is still closed', async () => { const error = new Error('failure'); const mockService = jest.fn(() => { throw error; }); const onDegraded = jest.fn(); - const policy = createServicePolicy({ - maxConsecutiveFailures, - onDegraded, - }); + const policy = createServicePolicy({ onDegraded }); const promise = policy.execute(mockService); clock.runAllAsync().catch(() => { @@ -276,702 +296,776 @@ describe('createServicePolicy', () => { }); }); - describe('if the initial run + retries is equal to the max number of consecutive failures', () => { - it('throws what the service throws', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ - maxConsecutiveFailures, - }); - - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough - }); - - await expect(promise).rejects.toThrow(error); - }); + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); - it('calls the onBreak callback once with the error', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ - maxConsecutiveFailures, - onBreak, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + await expect(promise).rejects.toThrow(error); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onBreak).toHaveBeenCalledTimes(1); - expect(onBreak).toHaveBeenCalledWith({ error }); - }); + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); - it('never calls the onDegraded callback, since the circuit is open', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ - maxConsecutiveFailures, - onDegraded, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onBreak).not.toHaveBeenCalled(); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onDegraded).not.toHaveBeenCalled(); - }); + it('calls the onDegraded callback once', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); - it('throws a BrokenCircuitError instead of whatever error the service produces if the service is executed again', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ - maxConsecutiveFailures, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const firstExecution = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onDegraded).toHaveBeenCalledTimes(1); }); - try { - await firstExecution; - } catch { - // ignore the error - } - - const secondExecution = policy.execute(mockService); - await expect(secondExecution).rejects.toThrow( - new Error( - 'Execution prevented because the circuit breaker is open', - ), - ); }); - }); - describe('if the initial run + retries is greater than the max number of consecutive failures', () => { - it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ - maxConsecutiveFailures, - }); + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); }); - await expect(promise).rejects.toThrow( - new Error( - 'Execution prevented because the circuit breaker is open', - ), - ); - }); + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); - it('calls the onBreak callback once with the error', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ - maxConsecutiveFailures, - onBreak, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onBreak).toHaveBeenCalledTimes(1); - expect(onBreak).toHaveBeenCalledWith({ error }); - }); + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); - it('never calls the onDegraded callback, since the circuit is open', async () => { - const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ - maxConsecutiveFailures, - onDegraded, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onDegraded).not.toHaveBeenCalled(); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onDegraded).not.toHaveBeenCalled(); - }); - }); - }); - }); + it('throws a BrokenCircuitError instead of whatever error the service produces if the service is executed again', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); - describe('using a custom max number of retries', () => { - it(`calls the service a total of 1 + times, delaying each retry using a backoff formula`, async () => { - const maxRetries = 5; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ maxRetries }); - // Each retry delay is randomized using a decorrelated jitter formula, - // so we need to prevent that - jest.spyOn(Math, 'random').mockReturnValue(0); + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - // These values were found by logging them - clock.tickAsync(0).catch(() => { - // ignore any errors - adding to the promise queue is enough - }); - clock.tickAsync(176.27932892814937).catch(() => { - // ignore any errors - adding to the promise queue is enough - }); - clock.tickAsync(186.8886145345685).catch(() => { - // ignore any errors - adding to the promise queue is enough - }); - clock.tickAsync(366.8287823691078).catch(() => { - // ignore any errors - adding to the promise queue is enough - }); - clock.tickAsync(731.8792783578953).catch(() => { - // ignore any errors - adding to the promise queue is enough - }); - try { - await promise; - } catch { - // ignore the error - } + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); - expect(mockService).toHaveBeenCalledTimes(1 + maxRetries); - }); + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); - describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { - describe('if the initial run + retries is less than the max number of consecutive failures', () => { - it('throws what the service throws', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ maxRetries }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); }); - await expect(promise).rejects.toThrow(error); - }); + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); - it('does not call the onBreak callback', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ maxRetries, onBreak }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onBreak).not.toHaveBeenCalled(); - }); + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); - it('calls the onDegraded callback once', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ maxRetries, onDegraded }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onDegraded).not.toHaveBeenCalled(); }); - try { - await promise; - } catch { - // ignore the error - } + }); + }); + }); - expect(onDegraded).toHaveBeenCalledTimes(1); + describe('using a custom max number of retries', () => { + it(`calls the service a total of 1 + times, delaying each retry using a backoff formula`, async () => { + const maxRetries = 5; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(366.8287823691078).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(731.8792783578953).catch(() => { + // ignore any errors - adding to the promise queue is enough }); + try { + await promise; + } catch { + // ignore the error + } + + expect(mockService).toHaveBeenCalledTimes(1 + maxRetries); }); - describe('if the initial run + retries is equal to the max number of consecutive failures', () => { - it('throws what the service throws', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ maxRetries }); + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); }); - await expect(promise).rejects.toThrow(error); - }); + it('does not call the onBreak callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); - it('calls the onBreak callback once with the error', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ maxRetries, onBreak }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onBreak).not.toHaveBeenCalled(); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onBreak).toHaveBeenCalledTimes(1); - expect(onBreak).toHaveBeenCalledWith({ error }); - }); + it('calls the onDegraded callback once', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); - it('never calls the onDegraded callback, since the circuit is open', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ maxRetries, onDegraded }); + }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onDegraded).not.toHaveBeenCalled(); - }); + it('calls the onBreak callback once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); - it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ maxRetries, onDegraded }); - const firstExecution = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); }); - try { - await firstExecution; - } catch { - // ignore the error - } - const secondExecution = policy.execute(mockService); - await expect(secondExecution).rejects.toThrow( - new Error( - 'Execution prevented because the circuit breaker is open', - ), - ); - }); - }); + it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); - describe('if the initial run + retries is greater than the max number of consecutive failures', () => { - it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; - const mockService = jest.fn(() => { - throw new Error('failure'); - }); - const policy = createServicePolicy({ maxRetries }); + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); }); - - await expect(promise).rejects.toThrow( - new Error( - 'Execution prevented because the circuit breaker is open', - ), - ); }); - it('calls the onBreak callback once with the error', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ maxRetries, onBreak }); + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const mockService = jest.fn(() => { + throw new Error('failure'); + }); + const policy = createServicePolicy({ maxRetries }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onBreak).toHaveBeenCalledTimes(1); - expect(onBreak).toHaveBeenCalledWith({ error }); - }); + it('calls the onBreak callback once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); - it('never calls the onDegraded callback, since the circuit is open', async () => { - const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ maxRetries, onDegraded }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onDegraded).not.toHaveBeenCalled(); + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); }); }); - }); - describe('using a custom max number of consecutive failures', () => { - describe('if the initial run + retries is less than the max number of consecutive failures', () => { - it('throws what the service throws', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures - 2; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - }); + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); }); - await expect(promise).rejects.toThrow(error); - }); + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); - it('does not call the onBreak callback', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures - 2; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - onBreak, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onBreak).not.toHaveBeenCalled(); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onBreak).not.toHaveBeenCalled(); - }); + it('calls the onDegraded callback once', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); - it('calls the onDegraded callback once', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures - 2; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - onDegraded, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onDegraded).toHaveBeenCalledTimes(1); }); - try { - await promise; - } catch { - // ignore the error - } - - expect(onDegraded).toHaveBeenCalledTimes(1); }); - }); - describe('if the initial run + retries is equal to the max number of consecutive failures', () => { - it('throws what the service throws', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures - 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - }); + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); }); - await expect(promise).rejects.toThrow(error); - }); + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); - it('calls the onBreak callback once with the error', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures - 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - onBreak, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onBreak).toHaveBeenCalledTimes(1); - expect(onBreak).toHaveBeenCalledWith({ error }); - }); + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); - it('never calls the onDegraded callback, since the circuit is open', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures - 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - onDegraded, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onDegraded).not.toHaveBeenCalled(); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onDegraded).not.toHaveBeenCalled(); - }); + it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); - it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures - 1; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - }); + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } - const firstExecution = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); }); - try { - await firstExecution; - } catch { - // ignore the error - } - - const secondExecution = policy.execute(mockService); - await expect(secondExecution).rejects.toThrow( - new Error( - 'Execution prevented because the circuit breaker is open', - ), - ); }); - }); - describe('if the initial run + retries is greater than the max number of consecutive failures', () => { - it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - }); + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); }); - await expect(promise).rejects.toThrow( - new Error( - 'Execution prevented because the circuit breaker is open', - ), - ); - }); + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); - it('calls the onBreak callback once with the error', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onBreak = jest.fn(); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - onBreak, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); }); - try { - await promise; - } catch { - // ignore the error - } - expect(onBreak).toHaveBeenCalledTimes(1); - expect(onBreak).toHaveBeenCalledWith({ error }); - }); + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); - it('never calls the onDegraded callback, since the circuit is open', async () => { - const maxConsecutiveFailures = 5; - const maxRetries = maxConsecutiveFailures; - const error = new Error('failure'); - const mockService = jest.fn(() => { - throw error; - }); - const onDegraded = jest.fn(); - const policy = createServicePolicy({ - maxRetries, - maxConsecutiveFailures, - onDegraded, - }); + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } - const promise = policy.execute(mockService); - clock.runAllAsync().catch(() => { - // ignore any errors - adding to the promise queue is enough + expect(onDegraded).not.toHaveBeenCalled(); }); - try { - await promise; - } catch { - // ignore the error - } - - expect(onDegraded).not.toHaveBeenCalled(); }); }); }); @@ -979,6 +1073,9 @@ describe('createServicePolicy', () => { }); describe('wrapping a service that fails continuously and then succeeds on the final try', () => { + // NOTE: Using a custom retry filter policy is not tested here since the + // same thing would happen as above if the error is filtered out + describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { it(`calls the service a total of ${ 1 + DEFAULT_MAX_RETRIES diff --git a/packages/controller-utils/src/create-service-policy.ts b/packages/controller-utils/src/create-service-policy.ts index 210516d124c..a24d850e897 100644 --- a/packages/controller-utils/src/create-service-policy.ts +++ b/packages/controller-utils/src/create-service-policy.ts @@ -7,7 +7,9 @@ import { wrap, CircuitState, } from 'cockatiel'; -import type { IPolicy } from 'cockatiel'; +import type { IPolicy, Policy } from 'cockatiel'; + +export type { IPolicy as IServicePolicy }; /** * The maximum number of times that a failing action should be re-run before @@ -50,6 +52,10 @@ export const DEFAULT_DEGRADED_THRESHOLD = 5_000; * @param options - The options to this function. * @param options.maxRetries - The maximum number of times that a failing action * should be re-run before giving up. Defaults to 3. + * @param options.retryFilterPolicy - The policy used to control when the + * function should be retried based on either the result of the function or an + * error that it throws. For instance, you could use this to retry only certain + * errors. See `handleWhen` and friends from Cockatiel for more. * @param options.maxConsecutiveFailures - The maximum number of times that * the action is allowed to fail before pausing further retries. Defaults to 12. * @param options.circuitBreakDuration - The length of time (in milliseconds) to @@ -63,6 +69,9 @@ export const DEFAULT_DEGRADED_THRESHOLD = 5_000; * @param options.onDegraded - A function which is called when the action * succeeds before `maxConsecutiveFailures` is reached, but takes more time * than the `degradedThreshold` to run. + * @param options.onRetry - A function which will be called the moment the + * policy kicks off a timer to re-run the function passed to the policy. This + * is primarily useful in tests where we are mocking timers. * @returns A Cockatiel policy object that can be used to run an arbitrary * action (a function). * @example @@ -70,6 +79,9 @@ export const DEFAULT_DEGRADED_THRESHOLD = 5_000; * ``` ts * const policy = createServicePolicy({ * maxRetries: 3, + * retryFilterPolicy: handleWhen((error) => { + * return error.message.includes('oops'); + * }), * maxConsecutiveFailures: 3, * circuitBreakDuration: 5000, * degradedThreshold: 2000, @@ -93,6 +105,9 @@ export const DEFAULT_DEGRADED_THRESHOLD = 5_000; * constructor() { * this.#policy = createServicePolicy({ * maxRetries: 3, + * retryFilterPolicy: handleWhen((error) => { + * return error.message.includes('oops'); + * }), * maxConsecutiveFailures: 3, * circuitBreakDuration: 5000, * degradedThreshold: 2000, @@ -110,6 +125,7 @@ export const DEFAULT_DEGRADED_THRESHOLD = 5_000; */ export function createServicePolicy({ maxRetries = DEFAULT_MAX_RETRIES, + retryFilterPolicy = handleAll, maxConsecutiveFailures = DEFAULT_MAX_CONSECUTIVE_FAILURES, circuitBreakDuration = DEFAULT_CIRCUIT_BREAK_DURATION, degradedThreshold = DEFAULT_DEGRADED_THRESHOLD, @@ -119,15 +135,20 @@ export function createServicePolicy({ onDegraded = () => { // do nothing }, + onRetry = () => { + // do nothing + }, }: { maxRetries?: number; + retryFilterPolicy?: Policy; maxConsecutiveFailures?: number; degradedThreshold?: number; circuitBreakDuration?: number; onBreak?: () => void; onDegraded?: () => void; + onRetry?: () => void; } = {}): IPolicy { - const retryPolicy = retry(handleAll, { + const retryPolicy = retry(retryFilterPolicy, { maxAttempts: maxRetries, // Retries of the action passed to the policy will be padded by increasing // delays, determined by a formula. @@ -155,6 +176,7 @@ export function createServicePolicy({ // `maxConsecutiveFailures` circuitBreakerPolicy.onBreak(onBreak); + retryPolicy.onRetry(onRetry); retryPolicy.onGiveUp(() => { if (circuitBreakerPolicy.state === CircuitState.Closed) { // The `onDegraded` callback will be called if the number of retries is