diff --git a/packages/controller-utils/CHANGELOG.md b/packages/controller-utils/CHANGELOG.md index 2bc867e5b33..bc50f1682e3 100644 --- a/packages/controller-utils/CHANGELOG.md +++ b/packages/controller-utils/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add `createServicePolicy` function to assist with reducing boilerplate for service classes ([#5053](https://github.com/MetaMask/core/pull/5053)) + ## [11.4.4] ### Fixed diff --git a/packages/controller-utils/package.json b/packages/controller-utils/package.json index 3fb36cf2abf..8de5139bbf9 100644 --- a/packages/controller-utils/package.json +++ b/packages/controller-utils/package.json @@ -55,6 +55,7 @@ "@types/bn.js": "^5.1.5", "bignumber.js": "^9.1.2", "bn.js": "^5.2.1", + "cockatiel": "^3.1.2", "eth-ens-namehash": "^2.0.8", "fast-deep-equal": "^3.1.3" }, @@ -65,6 +66,7 @@ "deepmerge": "^4.2.2", "jest": "^27.5.1", "nock": "^13.3.1", + "sinon": "^9.2.4", "ts-jest": "^27.1.4", "typedoc": "^0.24.8", "typedoc-plugin-missing-exports": "^2.0.0", diff --git a/packages/controller-utils/src/create-service-policy.test.ts b/packages/controller-utils/src/create-service-policy.test.ts new file mode 100644 index 00000000000..cc976dc9be9 --- /dev/null +++ b/packages/controller-utils/src/create-service-policy.test.ts @@ -0,0 +1,2800 @@ +import { handleWhen } from 'cockatiel'; +import { useFakeTimers } from 'sinon'; +import type { SinonFakeTimers } from 'sinon'; + +import { + createServicePolicy, + DEFAULT_CIRCUIT_BREAK_DURATION, + DEFAULT_DEGRADED_THRESHOLD, + DEFAULT_MAX_CONSECUTIVE_FAILURES, + DEFAULT_MAX_RETRIES, +} from './create-service-policy'; + +describe('createServicePolicy', () => { + let clock: SinonFakeTimers; + + beforeEach(() => { + clock = useFakeTimers(); + }); + + afterEach(() => { + clock.restore(); + }); + + describe('wrapping a service that succeeds on the first try', () => { + it('returns a policy that returns what the service returns', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const policy = createServicePolicy(); + + const returnValue = await policy.execute(mockService); + + expect(returnValue).toStrictEqual({ some: 'data' }); + }); + + it('only calls the service once before returning', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const policy = createServicePolicy(); + + await policy.execute(mockService); + + expect(mockService).toHaveBeenCalledTimes(1); + }); + + it('does not call the onBreak callback, since the circuit never opens', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const onBreak = jest.fn(); + const policy = createServicePolicy({ onBreak }); + + await policy.execute(mockService); + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + await policy.execute(mockService); + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + const mockService = jest.fn(() => { + return new Promise((resolve) => { + setTimeout(() => resolve({ some: 'data' }), delay); + }); + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + const promise = policy.execute(mockService); + clock.tick(delay); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time below the threshold', async () => { + const degradedThreshold = 2000; + const mockService = jest.fn(() => ({ some: 'data' })); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ degradedThreshold, onDegraded }); + + await policy.execute(mockService); + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time beyond the threshold', async () => { + const degradedThreshold = 2000; + const delay = degradedThreshold + 1; + const mockService = jest.fn(() => { + return new Promise((resolve) => { + setTimeout(() => resolve({ some: 'data' }), delay); + }); + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ degradedThreshold, onDegraded }); + + const promise = policy.execute(mockService); + clock.tick(delay); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('wrapping a service that always fails', () => { + describe('if a custom retry filter policy is given and the retry filter policy filters out the thrown error', () => { + it('throws what the service throws', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); + + const promise = policy.execute(mockService); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the service once and only once', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); + + const promise = policy.execute(mockService); + try { + await promise; + } catch { + // ignore the error + } + + expect(mockService).toHaveBeenCalledTimes(1); + }); + + it('does not call the onBreak callback', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('does not call the onDegraded callback', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + }); + + describe('using the default retry filter policy (which retries all errors)', () => { + describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { + it(`calls the service a total of ${ + 1 + DEFAULT_MAX_RETRIES + } times, delaying each retry using a backoff formula`, async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy(); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(mockService).toHaveBeenCalledTimes(1 + DEFAULT_MAX_RETRIES); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + it('throws what the service throws', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy(); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call the onBreak callback, since the max number of consecutive failures is never reached', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once, since the circuit is still closed', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the service is executed again', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('using a custom max number of retries', () => { + it(`calls the service a total of 1 + times, delaying each retry using a backoff formula`, async () => { + const maxRetries = 5; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(366.8287823691078).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(731.8792783578953).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(mockService).toHaveBeenCalledTimes(1 + maxRetries); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call the onBreak callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const mockService = jest.fn(() => { + throw new Error('failure'); + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the error + } + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('never calls the onDegraded callback, since the circuit is open', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + }); + }); + }); + }); + }); + + describe('wrapping a service that fails continuously and then succeeds on the final try', () => { + // NOTE: Using a custom retry filter policy is not tested here since the + // same thing would happen as above if the error is filtered out + + describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { + it(`calls the service a total of ${ + 1 + DEFAULT_MAX_RETRIES + } times, delaying each retry using a backoff formula`, async () => { + let invocationCounter = 0; + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }); + const policy = createServicePolicy(); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(mockService).toHaveBeenCalledTimes(1 + DEFAULT_MAX_RETRIES); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + it('returns what the service returns', async () => { + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback, since the max number of consecutive failures is never reached', async () => { + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + let invocationCounter = 0; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + let invocationCounter = 0; + const delay = degradedThreshold + 1; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('does not call the onDegraded callback', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + describe(`using the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the BrokenCircuitError + } + clock.tick(DEFAULT_CIRCUIT_BREAK_DURATION); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + + describe('using a custom circuit break duration', () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + // This has to be high enough to exceed the exponential backoff + const circuitBreakDuration = 5_000; + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxConsecutiveFailures, + circuitBreakDuration, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the BrokenCircuitError + } + clock.tick(circuitBreakDuration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + }); + }); + }); + + describe('using a custom max number of retries', () => { + it(`calls the service a total of 1 + times, delaying each retry using a backoff formula`, async () => { + const maxRetries = 5; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // These values were found by logging them + clock.tickAsync(0).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(176.27932892814937).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(186.8886145345685).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(366.8287823691078).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + clock.tickAsync(731.8792783578953).catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(mockService).toHaveBeenCalledTimes(1 + maxRetries); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ maxRetries, onBreak }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('does not call the onDegraded callback', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ maxRetries, onDegraded }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + describe(`using the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the BrokenCircuitError + } + clock.tick(DEFAULT_CIRCUIT_BREAK_DURATION); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + + describe('using a custom circuit break duration', () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + // This has to be high enough to exceed the exponential backoff + const circuitBreakDuration = 50_000; + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + circuitBreakDuration, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await expect(firstExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + clock.tick(circuitBreakDuration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call the onBreak callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onBreak).not.toHaveBeenCalled(); + }); + + describe(`using the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + + describe('using a custom degraded threshold', () => { + it('does not call the onDegraded callback if the service execution time is below the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + it('calls the onDegraded callback once if the service execution time is beyond the threshold', async () => { + const degradedThreshold = 2000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const delay = degradedThreshold + 1; + let invocationCounter = 0; + const mockService = () => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + degradedThreshold, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await promise; + + expect(onDegraded).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls the onBreak callback once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreak = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onBreak, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onBreak).toHaveBeenCalledTimes(1); + expect(onBreak).toHaveBeenCalledWith({ error }); + }); + + it('does not call the onDegraded callback', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegraded = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + onDegraded, + }); + + const promise = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await promise; + } catch { + // ignore the error + } + + expect(onDegraded).not.toHaveBeenCalled(); + }); + + describe(`using the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + try { + await firstExecution; + } catch { + // ignore the BrokenCircuitError + } + clock.tick(DEFAULT_CIRCUIT_BREAK_DURATION); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + + describe('using a custom circuit break duration', () => { + it('returns what the service returns if it is successfully called again after the circuit break duration has elapsed', async () => { + // This has to be high enough to exceed the exponential backoff + const circuitBreakDuration = 5_000; + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = () => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + circuitBreakDuration, + }); + + const firstExecution = policy.execute(mockService); + clock.runAllAsync().catch(() => { + // ignore any errors - adding to the promise queue is enough + }); + await expect(firstExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + clock.tick(circuitBreakDuration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + }); + }); + }); + }); + }); +}); diff --git a/packages/controller-utils/src/create-service-policy.ts b/packages/controller-utils/src/create-service-policy.ts new file mode 100644 index 00000000000..a24d850e897 --- /dev/null +++ b/packages/controller-utils/src/create-service-policy.ts @@ -0,0 +1,203 @@ +import { + circuitBreaker, + ConsecutiveBreaker, + ExponentialBackoff, + handleAll, + retry, + wrap, + CircuitState, +} from 'cockatiel'; +import type { IPolicy, Policy } from 'cockatiel'; + +export type { IPolicy as IServicePolicy }; + +/** + * The maximum number of times that a failing action should be re-run before + * giving up. + */ +export const DEFAULT_MAX_RETRIES = 3; + +/** + * The maximum number of times that the action is allowed to fail before pausing + * further retries. + */ +export const DEFAULT_MAX_CONSECUTIVE_FAILURES = (1 + DEFAULT_MAX_RETRIES) * 3; + +/** + * The default length of time (in milliseconds) to temporarily pause retries of + * the action after enough consecutive failures. + */ +export const DEFAULT_CIRCUIT_BREAK_DURATION = 30 * 60 * 1000; + +/** + * The default length of time (in milliseconds) that governs when an action is + * regarded as degraded (affecting when `onDegraded` is called). + */ +export const DEFAULT_DEGRADED_THRESHOLD = 5_000; + +/** + * Constructs an object exposing an `execute` method which, given a function, + * will retry it with ever increasing delays until it succeeds. If it detects + * too many consecutive failures, it will block further retries until a + * designated time period has passed; this particular behavior is primarily + * designed for services that wrap API calls so as not to make needless HTTP + * requests when the API is down and to be able to recover when the API comes + * back up. In addition, hooks allow for responding to certain events, one of + * which can be used to detect when an HTTP request is performing slowly. + * + * Internally, the executor object makes use of the retry and circuit breaker + * policies from the [Cockatiel](https://www.npmjs.com/package/cockatiel) + * library; see there for more. + * + * @param options - The options to this function. + * @param options.maxRetries - The maximum number of times that a failing action + * should be re-run before giving up. Defaults to 3. + * @param options.retryFilterPolicy - The policy used to control when the + * function should be retried based on either the result of the function or an + * error that it throws. For instance, you could use this to retry only certain + * errors. See `handleWhen` and friends from Cockatiel for more. + * @param options.maxConsecutiveFailures - The maximum number of times that + * the action is allowed to fail before pausing further retries. Defaults to 12. + * @param options.circuitBreakDuration - The length of time (in milliseconds) to + * pause retries of the action after the number of failures reaches + * `maxConsecutiveFailures`. + * @param options.degradedThreshold - The length of time (in milliseconds) that + * governs when an action is regarded as degraded (affecting when `onDegraded` + * is called). Defaults to 5 seconds. + * @param options.onBreak - A function which is called when the action fails too + * many times in a row. + * @param options.onDegraded - A function which is called when the action + * succeeds before `maxConsecutiveFailures` is reached, but takes more time + * than the `degradedThreshold` to run. + * @param options.onRetry - A function which will be called the moment the + * policy kicks off a timer to re-run the function passed to the policy. This + * is primarily useful in tests where we are mocking timers. + * @returns A Cockatiel policy object that can be used to run an arbitrary + * action (a function). + * @example + * To use the policy, call `execute` on it and pass a function: + * ``` ts + * const policy = createServicePolicy({ + * maxRetries: 3, + * retryFilterPolicy: handleWhen((error) => { + * return error.message.includes('oops'); + * }), + * maxConsecutiveFailures: 3, + * circuitBreakDuration: 5000, + * degradedThreshold: 2000, + * onBreak: () => { + * console.log('Circuit broke'); + * }, + * onDegraded: () => { + * console.log('Service is degraded'); + * } + * }); + * + * await policy.execute(async () => { + * const response = await fetch('https://some/url'); + * return await response.json(); + * }); + * ``` + * You may wish to store `policy` in a single place and reuse it each time you + * want to execute your action. For instance: + * ``` ts + * class Service { + * constructor() { + * this.#policy = createServicePolicy({ + * maxRetries: 3, + * retryFilterPolicy: handleWhen((error) => { + * return error.message.includes('oops'); + * }), + * maxConsecutiveFailures: 3, + * circuitBreakDuration: 5000, + * degradedThreshold: 2000, + * }); + * } + * + * async fetch() { + * return await this.#policy.execute(async () => { + * const response = await fetch('https://some/url'); + * return await response.json(); + * }); + * } + * } + * ``` + */ +export function createServicePolicy({ + maxRetries = DEFAULT_MAX_RETRIES, + retryFilterPolicy = handleAll, + maxConsecutiveFailures = DEFAULT_MAX_CONSECUTIVE_FAILURES, + circuitBreakDuration = DEFAULT_CIRCUIT_BREAK_DURATION, + degradedThreshold = DEFAULT_DEGRADED_THRESHOLD, + onBreak = () => { + // do nothing + }, + onDegraded = () => { + // do nothing + }, + onRetry = () => { + // do nothing + }, +}: { + maxRetries?: number; + retryFilterPolicy?: Policy; + maxConsecutiveFailures?: number; + degradedThreshold?: number; + circuitBreakDuration?: number; + onBreak?: () => void; + onDegraded?: () => void; + onRetry?: () => void; +} = {}): IPolicy { + const retryPolicy = retry(retryFilterPolicy, { + maxAttempts: maxRetries, + // Retries of the action passed to the policy will be padded by increasing + // delays, determined by a formula. + backoff: new ExponentialBackoff(), + }); + + const circuitBreakerPolicy = circuitBreaker(handleAll, { + // While the circuit is open, any additional invocations of the action + // passed to the policy (either via automatic retries or by manually + // executing the policy again) will result in a BrokenCircuitError. The + // circuit will transition to a half-open state after the + // `circuitBreakDuration` passes, after which the action will be allowed to + // run again. If the action succeeds, the circuit will close, otherwise it + // will open again. + halfOpenAfter: circuitBreakDuration, + breaker: new ConsecutiveBreaker(maxConsecutiveFailures), + }); + + // The `onBreak` callback will be called if the number of times the action + // consistently throws exceeds the maximum consecutive number of failures. + // Combined with the retry policy, this can happen if: + // - `maxRetries` > `maxConsecutiveFailures` and the policy is executed once + // - `maxRetries` <= `maxConsecutiveFailures` but the policy is executed + // multiple times, enough for the total number of retries to exceed + // `maxConsecutiveFailures` + circuitBreakerPolicy.onBreak(onBreak); + + retryPolicy.onRetry(onRetry); + retryPolicy.onGiveUp(() => { + if (circuitBreakerPolicy.state === CircuitState.Closed) { + // The `onDegraded` callback will be called if the number of retries is + // exceeded and the maximum number of consecutive failures has not been + // reached yet (whether the policy is called once or multiple times). + onDegraded(); + } + }); + retryPolicy.onSuccess(({ duration }) => { + if ( + circuitBreakerPolicy.state === CircuitState.Closed && + duration > degradedThreshold + ) { + // The `onDegraded` callback will also be called if the action passed to + // the policy does not throw, but the time it takes for the action to run + // exceeds the `degradedThreshold`. + onDegraded(); + } + }); + + // Each time the retry policy retries, it will execute the circuit breaker + // policy. + return wrap(retryPolicy, circuitBreakerPolicy); +} diff --git a/packages/controller-utils/src/index.test.ts b/packages/controller-utils/src/index.test.ts new file mode 100644 index 00000000000..61ef841826f --- /dev/null +++ b/packages/controller-utils/src/index.test.ts @@ -0,0 +1,75 @@ +import * as allExports from '.'; + +describe('@metamask/controller-utils', () => { + it('has expected JavaScript exports', () => { + expect(Object.keys(allExports)).toMatchInlineSnapshot(` + Array [ + "createServicePolicy", + "BNToHex", + "convertHexToDecimal", + "fetchWithErrorHandling", + "fractionBN", + "fromHex", + "getBuyURL", + "gweiDecToWEIBN", + "handleFetch", + "hexToBN", + "hexToText", + "isNonEmptyArray", + "isPlainObject", + "isSafeChainId", + "isSafeDynamicKey", + "isSmartContractCode", + "isValidJson", + "isValidHexAddress", + "normalizeEnsName", + "query", + "safelyExecute", + "safelyExecuteWithTimeout", + "successfulFetch", + "timeoutFetch", + "toChecksumHexAddress", + "toHex", + "weiHexToGweiDec", + "isEqualCaseInsensitive", + "RPC", + "FALL_BACK_VS_CURRENCY", + "IPFS_DEFAULT_GATEWAY_URL", + "GANACHE_CHAIN_ID", + "MAX_SAFE_CHAIN_ID", + "ERC721", + "ERC1155", + "ERC20", + "ERC721_INTERFACE_ID", + "ERC721_METADATA_INTERFACE_ID", + "ERC721_ENUMERABLE_INTERFACE_ID", + "ERC1155_INTERFACE_ID", + "ERC1155_METADATA_URI_INTERFACE_ID", + "ERC1155_TOKEN_RECEIVER_INTERFACE_ID", + "GWEI", + "ASSET_TYPES", + "TESTNET_TICKER_SYMBOLS", + "BUILT_IN_NETWORKS", + "OPENSEA_PROXY_URL", + "NFT_API_BASE_URL", + "NFT_API_VERSION", + "NFT_API_TIMEOUT", + "ORIGIN_METAMASK", + "ApprovalType", + "CHAIN_ID_TO_ETHERS_NETWORK_NAME_MAP", + "InfuraNetworkType", + "NetworkType", + "isNetworkType", + "isInfuraNetworkType", + "BuiltInNetworkName", + "ChainId", + "NetworksTicker", + "BlockExplorerUrl", + "NetworkNickname", + "parseDomainParts", + "isValidSIWEOrigin", + "detectSIWE", + ] + `); + }); +}); diff --git a/packages/controller-utils/src/index.ts b/packages/controller-utils/src/index.ts index 3d35d62c0a0..b3bd8821e12 100644 --- a/packages/controller-utils/src/index.ts +++ b/packages/controller-utils/src/index.ts @@ -1,3 +1,4 @@ +export { createServicePolicy } from './create-service-policy'; export * from './constants'; export type { NonEmptyArray } from './util'; export { diff --git a/yarn.lock b/yarn.lock index c92dafbd90a..fd2b200780f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2584,11 +2584,13 @@ __metadata: "@types/jest": "npm:^27.4.1" bignumber.js: "npm:^9.1.2" bn.js: "npm:^5.2.1" + cockatiel: "npm:^3.1.2" deepmerge: "npm:^4.2.2" eth-ens-namehash: "npm:^2.0.8" fast-deep-equal: "npm:^3.1.3" jest: "npm:^27.5.1" nock: "npm:^13.3.1" + sinon: "npm:^9.2.4" ts-jest: "npm:^27.1.4" typedoc: "npm:^0.24.8" typedoc-plugin-missing-exports: "npm:^2.0.0"