feat: Add cacheTtlMs option (#760)

When the SDK retrieves data from the EdgeKV, it does so using a single
sub-request per call to `variation`, `variationDetail`, or
`allFlagsState`. The problem is that Akamai imposes a limit of 4
sub-requests per handler event.

If a customer evaluates more than 4 distinct flags during the handling
of a single event, subsequent flag lookups would fail. To combat this,
we now cache the flag values for a specified amount of time.

---------

Co-authored-by: Ryan Lamb <[email protected]>

Author: keelerm84

packages/sdk/akamai-edgekv/__tests__/index.test.ts

@@ -1,11 +1,13 @@
 import EdgeKVProvider from '../src/edgekv/edgeKVProvider';
-import { init as initWithEdgeKV, LDClient, LDContext } from '../src/index';
+import { init as initWithEdgeKV, LDClient, LDContext, LDLogger } from '../src/index';
 import * as testData from './testData.json';
 
 jest.mock('../src/edgekv/edgekv', () => ({
   EdgeKV: jest.fn(),
 }));
 
+let logger: LDLogger;
+
 const sdkKey = 'test-sdk-key';
 const flagKey1 = 'testFlag1';
 const flagKey2 = 'testFlag2';
@@ -17,11 +19,22 @@ describe('init', () => {
 
   describe('init with Edge KV', () => {
     beforeAll(async () => {
-      ldClient = initWithEdgeKV({ namespace: 'akamai-test', group: 'Akamai', sdkKey });
+      ldClient = initWithEdgeKV({
+        namespace: 'akamai-test',
+        group: 'Akamai',
+        sdkKey,
+        options: { logger },
+      });
       await ldClient.waitForInitialization();
     });
 
     beforeEach(() => {
+      logger = {
+        error: jest.fn(),
+        warn: jest.fn(),
+        info: jest.fn(),
+        debug: jest.fn(),
+      };
       jest
         .spyOn(EdgeKVProvider.prototype, 'get')
         .mockImplementation(() => Promise.resolve(JSON.stringify(testData)));
@@ -31,6 +44,12 @@ describe('init', () => {
       ldClient.close();
     });
 
+    it('should not log a warning about initialization', async () => {
+      const spy = jest.spyOn(logger, 'warn');
+      await ldClient.variation(flagKey1, context, false);
+      expect(spy).not.toHaveBeenCalled();
+    });
+
     describe('flags', () => {
       it('variation default', async () => {
         const value = await ldClient.variation(flagKey1, context, false);

packages/sdk/akamai-edgekv/src/index.ts

@@ -34,12 +34,13 @@ export const init = ({
   sdkKey,
 }: AkamaiLDClientParams): LDClient => {
   const logger = options.logger ?? BasicLogger.get();
+  const cacheTtlMs = options.cacheTtlMs ?? 100;
 
   const edgekvProvider = new EdgeKVProvider({ namespace, group, logger });
 
   return initEdge({
     sdkKey,
-    options: { ...options, logger },
+    options: { ...options, logger, cacheTtlMs },
     featureStoreProvider: edgekvProvider,
     platformName: 'Akamai EdgeWorker',
     sdkName: '@launchdarkly/akamai-server-edgekv-sdk',

packages/shared/akamai-edgeworker-sdk/__tests__/featureStore/cacheableStore.test.ts

@@ -0,0 +1,101 @@
+import { EdgeProvider } from '../../src/featureStore';
+import CacheableStoreProvider from '../../src/featureStore/cacheableStoreProvider';
+import * as testData from '../testData.json';
+
+describe('given a mock edge provider with test data', () => {
+  const mockEdgeProvider: EdgeProvider = {
+    get: jest.fn(),
+  };
+  const mockGet = mockEdgeProvider.get as jest.Mock;
+
+  beforeEach(() => {
+    jest.useFakeTimers();
+    mockGet.mockImplementation(() => Promise.resolve(JSON.stringify(testData)));
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  describe('without cache TTL', () => {
+    it('caches initial request', async () => {
+      const cacheProvider = new CacheableStoreProvider(mockEdgeProvider, 'rootKey');
+      await cacheProvider.get('rootKey');
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+    });
+
+    it('can force a refresh', async () => {
+      const cacheProvider = new CacheableStoreProvider(mockEdgeProvider, 'rootKey');
+      await cacheProvider.get('rootKey');
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+
+      await cacheProvider.prefetchPayloadFromOriginStore();
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('with infinite cache ttl', () => {
+    it('caches initial request', async () => {
+      const cacheProvider = new CacheableStoreProvider(mockEdgeProvider, 'rootKey', 0);
+      await cacheProvider.get('rootKey');
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+    });
+
+    it('does not reset on prefetch', async () => {
+      const cacheProvider = new CacheableStoreProvider(mockEdgeProvider, 'rootKey', 0);
+      await cacheProvider.get('rootKey');
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+
+      await cacheProvider.prefetchPayloadFromOriginStore();
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('with finite cache ttl', () => {
+    it('caches initial request', async () => {
+      const cacheProvider = new CacheableStoreProvider(mockEdgeProvider, 'rootKey', 50);
+      await cacheProvider.get('rootKey');
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+    });
+
+    it('caches expires after duration', async () => {
+      jest.spyOn(Date, 'now').mockImplementation(() => 0);
+      const cacheProvider = new CacheableStoreProvider(mockEdgeProvider, 'rootKey', 50);
+      await cacheProvider.get('rootKey');
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+
+      jest.spyOn(Date, 'now').mockImplementation(() => 20);
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+
+      jest.spyOn(Date, 'now').mockImplementation(() => 50);
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(2);
+    });
+
+    it('prefetch respects cache TTL', async () => {
+      jest.spyOn(Date, 'now').mockImplementation(() => 0);
+      const cacheProvider = new CacheableStoreProvider(mockEdgeProvider, 'rootKey', 50);
+      await cacheProvider.get('rootKey');
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+
+      await cacheProvider.prefetchPayloadFromOriginStore();
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(1);
+
+      jest.spyOn(Date, 'now').mockImplementation(() => 50);
+      await cacheProvider.prefetchPayloadFromOriginStore();
+      await cacheProvider.get('rootKey');
+      expect(mockGet).toHaveBeenCalledTimes(2);
+    });
+  });
+});

packages/shared/akamai-edgeworker-sdk/src/api/LDClient.ts

@@ -34,6 +34,10 @@ class LDClient extends LDClientImpl {
     this._cacheableStoreProvider = storeProvider;
   }
 
+  override initialized(): boolean {
+    return true;
+  }
+
   override waitForInitialization(): Promise<LDClientType> {
     // we need to resolve the promise immediately because Akamai's runtime doesnt
     // have a setimeout so everything executes synchronously.

packages/shared/akamai-edgeworker-sdk/src/featureStore/cacheableStoreProvider.ts

@@ -1,15 +1,34 @@
 import { EdgeProvider } from '.';
 
 /**
- * Wraps around an edge provider to cache a copy of the sdk payload locally an explicit request is made to refetch data from the origin.
- * The wrapper is neccessary to ensure that we dont make redundant sub-requests from Akamai to fetch an entire environment payload.
+ * Wraps around an edge provider to cache a copy of the SDK payload locally.
+ *
+ * If a cacheTtlMs is specified, then the cacheable store provider will cache
+ * results for that specified duration. If the data lookup fails after that
+ * interval, previously stored values will be retained. The lookup will be
+ * retried again after the TTL.
+ *
+ * If no cacheTtlMs is specified, the cache will be stored for the lifetime of
+ * the object. The cache can be manually refreshed by calling
+ * `prefetchPayloadFromOriginStore`.
+ *
+ * The wrapper is necessary to ensure that we don't make redundant sub-requests
+ * from Akamai to fetch an entire environment payload. At the time of this writing,
+ * the Akamai documentation (https://techdocs.akamai.com/edgeworkers/docs/resource-tier-limitations)
+ * limits the number of sub-requests to:
+ *
+ * - 2 for basic compute
+ * - 4 for dynamic compute
+ * - 10 for enterprise
  */
 export default class CacheableStoreProvider implements EdgeProvider {
-  cache: string | null | undefined;
+  cache: Promise<string | null | undefined> | null | undefined;
+  cachedAt: number | undefined;
 
   constructor(
     private readonly _edgeProvider: EdgeProvider,
     private readonly _rootKey: string,
+    private readonly _cacheTtlMs?: number,
   ) {}
 
   /**
@@ -18,22 +37,47 @@ export default class CacheableStoreProvider implements EdgeProvider {
    * @returns
    */
   async get(rootKey: string): Promise<string | null | undefined> {
-    if (!this.cache) {
-      this.cache = await this._edgeProvider.get(rootKey);
+    if (!this._isCacheValid()) {
+      this.cache = this._edgeProvider.get(rootKey);
+      this.cachedAt = Date.now();
     }
 
     return this.cache;
   }
 
   /**
-   * Invalidates cache and fetch environment payload data from origin. The result of this data is cached in memory.
+   * Fetches environment payload data from the origin in accordance with the caching configuration.
+   *
    * You should only call this function within a feature store to pre-fetch and cache payload data in environments
    * where its expensive to make multiple outbound requests to the origin
    * @param rootKey
    * @returns
    */
   async prefetchPayloadFromOriginStore(rootKey?: string): Promise<string | null | undefined> {
-    this.cache = undefined; // clear the cache so that new data can be fetched from the origin
+    if (this._cacheTtlMs === undefined) {
+      this.cache = undefined; // clear the cache so that new data can be fetched from the origin
+    }
+
     return this.get(rootKey || this._rootKey);
   }
+
+  /**
+   * Internal helper to determine if the cached values are still considered valid.
+   */
+  private _isCacheValid(): boolean {
+    // If we don't have a cache, or we don't know how old the cache is, we have
+    // to consider it is invalid.
+    if (!this.cache || this.cachedAt === undefined) {
+      return false;
+    }
+
+    // If the cache provider was configured without a TTL, then the cache is
+    // always considered valid.
+    if (!this._cacheTtlMs) {
+      return true;
+    }
+
+    // Otherwise, it all depends on the time.
+    return Date.now() - this.cachedAt < this._cacheTtlMs;
+  }
 }

packages/shared/akamai-edgeworker-sdk/src/index.ts

@@ -12,7 +12,13 @@ import { validateOptions } from './utils';
  * supported. sendEvents is unsupported and is only included as a beta
  * preview.
  */
-type LDOptions = Pick<LDOptionsCommon, 'logger' | 'sendEvents'>;
+type LDOptions = {
+  /**
+   * The time-to-live for the cache in milliseconds. The default is 100ms. A
+   * value of 0 will cache indefinitely.
+   */
+  cacheTtlMs?: number;
+} & Pick<LDOptionsCommon, 'logger' | 'sendEvents'>;
 
 /**
  * The internal options include featureStore because that's how the LDClient
@@ -33,13 +39,22 @@ type BaseSDKParams = {
 };
 
 export const init = (params: BaseSDKParams): LDClient => {
-  const { sdkKey, options = {}, featureStoreProvider, platformName, sdkName, sdkVersion } = params;
+  const {
+    sdkKey,
+    options: inputOptions = {},
+    featureStoreProvider,
+    platformName,
+    sdkName,
+    sdkVersion,
+  } = params;
 
-  const logger = options.logger ?? BasicLogger.get();
+  const logger = inputOptions.logger ?? BasicLogger.get();
+  const { cacheTtlMs, ...options } = inputOptions as any;
 
   const cachableStoreProvider = new CacheableStoreProvider(
     featureStoreProvider,
     buildRootKey(sdkKey),
+    cacheTtlMs,
   );
   const featureStore = new EdgeFeatureStore(cachableStoreProvider, sdkKey, 'Akamai', logger);