compress configuration cache local storage values; migrate by dropping existing keys (FFL-718) (#245)

* compress configuration cache local storage values; migrate by dropping existing keys (FFL-718)

* docs

* leave configuration meta uncompressed

* remove console spy

* base64 encoding

* 3.17.0

* alpha

* fix spec

* merge main

* fix tests after merge

* merge fixes

* merge artifact

* v3.17.0

---------

Co-authored-by: Tyler Potter <[email protected]>

Author: leoromanovsky

js-client-sdk.api.md

@@ -172,6 +172,10 @@ export interface IClientConfigSync {
     // (undocumented)
     banditLogger?: IBanditLogger;
     // (undocumented)
+    configFetchedAt?: string;
+    // (undocumented)
+    configPublishedAt?: string;
+    // (undocumented)
     enableOverrides?: boolean;
     // (undocumented)
     flagsConfiguration: Record<string, Flag | ObfuscatedFlag>;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eppo/js-client-sdk",
-  "version": "3.16.2",
+  "version": "3.17.0",
   "description": "Eppo SDK for client-side JavaScript applications",
   "main": "dist/index.js",
   "files": [
@@ -59,8 +59,9 @@
     "webpack-cli": "^6.0.1"
   },
   "dependencies": {
+    "@eppo/js-client-sdk-common": "4.15.1",
     "@types/chrome": "^0.0.313",
-    "@eppo/js-client-sdk-common": "4.15.1"
+    "lz-string": "^1.5.0"
   },
   "packageManager": "[email protected]+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

src/local-storage-engine.spec.ts

@@ -1,8 +1,14 @@
+/**
+ * @jest-environment jsdom
+ */
+
+import * as LZString from 'lz-string';
+
 import { LocalStorageEngine } from './local-storage-engine';
 import { StorageFullUnableToWrite, LocalStorageUnknownFailure } from './string-valued.store';
 
 describe('LocalStorageEngine', () => {
-  let mockLocalStorage: Storage;
+  let mockLocalStorage: Storage & { _length: number };
   let engine: LocalStorageEngine;
 
   beforeEach(() => {
@@ -17,16 +23,27 @@ describe('LocalStorageEngine', () => {
       removeItem: jest.fn(),
       setItem: jest.fn(),
     } as Storage & { _length: number };
+
+    // Setup: migration already completed to avoid interference with basic functionality tests
+    (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+      if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+      return null;
+    });
+
     engine = new LocalStorageEngine(mockLocalStorage, 'test');
   });
 
-  describe('setContentsJsonString', () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('Basic Functionality', () => {
     it('should set item successfully when no error occurs', async () => {
       await engine.setContentsJsonString('test-config');
 
       expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
         'eppo-configuration-test',
-        'test-config',
+        LZString.compressToBase64('test-config'),
       );
     });
 
@@ -59,7 +76,7 @@ describe('LocalStorageEngine', () => {
       expect(mockLocalStorage.setItem).toHaveBeenCalledTimes(2);
       expect(mockLocalStorage.setItem).toHaveBeenLastCalledWith(
         'eppo-configuration-test',
-        'test-config',
+        LZString.compressToBase64('test-config'),
       );
     });
 
@@ -156,4 +173,234 @@ describe('LocalStorageEngine', () => {
       expect(mockLocalStorage.removeItem).toHaveBeenCalledTimes(3);
     });
   });
+
+  describe('Compression Migration', () => {
+    let migrationEngine: LocalStorageEngine;
+
+    beforeEach(() => {
+      // Reset mocks for migration tests
+      jest.clearAllMocks();
+      mockLocalStorage = {
+        get length() {
+          return this._length || 0;
+        },
+        _length: 0,
+        clear: jest.fn(),
+        getItem: jest.fn(),
+        key: jest.fn(),
+        removeItem: jest.fn(),
+        setItem: jest.fn(),
+      } as Storage & { _length: number };
+    });
+
+    describe('Migration', () => {
+      it('should run migration on first construction', () => {
+        // Setup: no global meta exists (first time)
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return null;
+          return null;
+        });
+
+        // Mock localStorage.length and key() for iteration
+        (mockLocalStorage as Storage & { _length: number })._length = 3;
+        (mockLocalStorage.key as jest.Mock).mockImplementation((index) => {
+          const keys = ['eppo-configuration-abc123', 'eppo-configuration-meta-def456', 'other-key'];
+          return keys[index] || null;
+        });
+
+        migrationEngine = new LocalStorageEngine(mockLocalStorage, 'test');
+
+        // Should have removed configuration keys
+        expect(mockLocalStorage.removeItem).toHaveBeenCalledWith('eppo-configuration-abc123');
+        expect(mockLocalStorage.removeItem).toHaveBeenCalledWith('eppo-configuration-meta-def456');
+        expect(mockLocalStorage.removeItem).not.toHaveBeenCalledWith('other-key');
+
+        // Should have set global meta
+        expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+          'eppo-meta',
+          expect.stringContaining('"version":1'),
+        );
+      });
+
+      it('should skip migration if already completed', () => {
+        // Setup: migration already done
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+          return null;
+        });
+
+        migrationEngine = new LocalStorageEngine(mockLocalStorage, 'test');
+
+        // Should not have removed any keys
+        expect(mockLocalStorage.removeItem).not.toHaveBeenCalled();
+      });
+
+      it('should handle migration errors gracefully', () => {
+        // Setup: no global meta, but error during migration
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return null;
+          return null;
+        });
+
+        // Make removeItem throw an error
+        (mockLocalStorage.removeItem as jest.Mock).mockImplementation(() => {
+          throw new Error('Storage error');
+        });
+
+        (mockLocalStorage as Storage & { _length: number })._length = 1;
+        (mockLocalStorage.key as jest.Mock).mockReturnValue('eppo-configuration-test');
+
+        // Should not throw error, just continue silently
+        expect(() => new LocalStorageEngine(mockLocalStorage, 'test')).not.toThrow();
+      });
+    });
+
+    describe('Compression', () => {
+      beforeEach(() => {
+        // Setup: migration already completed
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+          return null;
+        });
+
+        migrationEngine = new LocalStorageEngine(mockLocalStorage, 'test');
+      });
+
+      it('should compress data when storing', async () => {
+        const testData = JSON.stringify({ flag: 'test-flag', value: 'test-value' });
+
+        await migrationEngine.setContentsJsonString(testData);
+
+        expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+          'eppo-configuration-test',
+          LZString.compressToBase64(testData),
+        );
+      });
+
+      it('should decompress data when reading', async () => {
+        const testData = JSON.stringify({ flag: 'test-flag', value: 'test-value' });
+        const compressedData = LZString.compressToBase64(testData);
+
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+          if (key === 'eppo-configuration-test') return compressedData;
+          return null;
+        });
+
+        const result = await migrationEngine.getContentsJsonString();
+
+        expect(result).toBe(testData);
+      });
+
+      it('should handle decompression errors gracefully', async () => {
+        // Mock LZString.decompress to throw an error
+        const decompressSpy = jest
+          .spyOn(LZString, 'decompressFromBase64')
+          .mockImplementation(() => {
+            throw new Error('Decompression failed');
+          });
+
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+          if (key === 'eppo-configuration-test') return 'corrupted-data';
+          return null;
+        });
+
+        const result = await migrationEngine.getContentsJsonString();
+
+        expect(result).toBe(null);
+        expect(mockLocalStorage.removeItem).toHaveBeenCalledWith('eppo-configuration-test');
+
+        decompressSpy.mockRestore();
+      });
+
+      it('should store and retrieve meta data without compression', async () => {
+        const metaData = JSON.stringify({ lastUpdated: Date.now() });
+
+        await migrationEngine.setMetaJsonString(metaData);
+
+        // Meta data should be stored uncompressed
+        expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+          'eppo-configuration-meta-test',
+          metaData,
+        );
+
+        // Test reading back
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+          if (key === 'eppo-configuration-meta-test') return metaData;
+          return null;
+        });
+
+        const result = await migrationEngine.getMetaJsonString();
+        expect(result).toBe(metaData);
+      });
+
+      it('should return null for non-existent data', async () => {
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+          return null;
+        });
+
+        const contentsResult = await migrationEngine.getContentsJsonString();
+        const metaResult = await migrationEngine.getMetaJsonString();
+
+        expect(contentsResult).toBe(null);
+        expect(metaResult).toBe(null);
+      });
+    });
+
+    describe('Global Meta Management', () => {
+      it('should parse valid global meta', () => {
+        const validMeta = { version: 1, migratedAt: Date.now() };
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return JSON.stringify(validMeta);
+          return null;
+        });
+
+        new LocalStorageEngine(mockLocalStorage, 'test');
+
+        expect(mockLocalStorage.getItem).toHaveBeenCalledWith('eppo-meta');
+      });
+
+      it('should handle corrupted global meta', () => {
+        (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+          if (key === 'eppo-meta') return 'invalid-json';
+          return null;
+        });
+
+        (mockLocalStorage as Storage & { _length: number })._length = 0;
+
+        // Should not throw error, just continue silently with default version
+        expect(() => new LocalStorageEngine(mockLocalStorage, 'test')).not.toThrow();
+      });
+    });
+
+    describe('Space Optimization', () => {
+      it('should actually compress large configuration data', () => {
+        // Create a large configuration object with repetitive data
+        const largeConfig = {
+          flags: Array.from({ length: 100 }, (_, i) => ({
+            flagKey: `test-flag-${i}`,
+            variationType: 'STRING',
+            allocations: [
+              { key: 'control', value: 'control-value' },
+              { key: 'treatment', value: 'treatment-value' },
+            ],
+            rules: [{ conditions: [{ attribute: 'userId', operator: 'MATCHES', value: '.*' }] }],
+          })),
+        };
+
+        const originalJson = JSON.stringify(largeConfig);
+        const compressedData = LZString.compressToBase64(originalJson);
+
+        // Verify compression actually reduces size
+        expect(compressedData.length).toBeLessThan(originalJson.length);
+
+        // Verify compression ratio is reasonable (should be significant for repetitive JSON)
+        const compressionRatio = compressedData.length / originalJson.length;
+        expect(compressionRatio).toBeLessThan(0.5); // At least 50% compression
+      });
+    });
+  });
 });