compress configuration cache local storage values; migrate by dropping existing keys (FFL-718)

Author: leoromanovsky

js-client-sdk.api.md

@@ -172,6 +172,10 @@ export interface IClientConfigSync {
     // (undocumented)
     banditLogger?: IBanditLogger;
     // (undocumented)
+    configFetchedAt?: string;
+    // (undocumented)
+    configPublishedAt?: string;
+    // (undocumented)
     enableOverrides?: boolean;
     // (undocumented)
     flagsConfiguration: Record<string, Flag | ObfuscatedFlag>;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eppo/js-client-sdk",
-  "version": "3.16.1",
+  "version": "3.17.0-alpha.1",
   "description": "Eppo SDK for client-side JavaScript applications",
   "main": "dist/index.js",
   "files": [
@@ -59,8 +59,9 @@
     "webpack-cli": "^6.0.1"
   },
   "dependencies": {
+    "@eppo/js-client-sdk-common": "4.15.1",
     "@types/chrome": "^0.0.313",
-    "@eppo/js-client-sdk-common": "4.15.1"
+    "lz-string": "^1.5.0"
   },
   "packageManager": "[email protected]+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

src/local-storage-engine.spec.ts

@@ -0,0 +1,274 @@
+/**
+ * @jest-environment jsdom
+ */
+
+import * as LZString from 'lz-string';
+
+import { LocalStorageEngine } from './local-storage-engine';
+
+describe('LocalStorageEngine Compression Migration', () => {
+  let mockLocalStorage: Storage;
+
+  beforeEach(() => {
+    mockLocalStorage = {
+      getItem: jest.fn(),
+      setItem: jest.fn(),
+      removeItem: jest.fn(),
+      clear: jest.fn(),
+      get length() {
+        return this._length || 0;
+      },
+      key: jest.fn(),
+      _length: 0,
+    } as any;
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('Migration', () => {
+    it('should run migration on first construction', () => {
+      // Setup: no global meta exists (first time)
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return null;
+        return null;
+      });
+
+      // Mock localStorage.length and key() for iteration
+      (mockLocalStorage as any)._length = 3;
+      (mockLocalStorage.key as jest.Mock).mockImplementation((index) => {
+        const keys = ['eppo-configuration-abc123', 'eppo-configuration-meta-def456', 'other-key'];
+        return keys[index] || null;
+      });
+
+      const consoleSpy = jest.spyOn(console, 'log').mockImplementation();
+
+      new LocalStorageEngine(mockLocalStorage, 'test');
+
+      // Should have logged migration
+      expect(consoleSpy).toHaveBeenCalledWith('Running storage migration from v0 to v1');
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Configuration cleanup completed - fresh configs will be compressed',
+      );
+
+      // Should have removed configuration keys
+      expect(mockLocalStorage.removeItem).toHaveBeenCalledWith('eppo-configuration-abc123');
+      expect(mockLocalStorage.removeItem).toHaveBeenCalledWith('eppo-configuration-meta-def456');
+      expect(mockLocalStorage.removeItem).not.toHaveBeenCalledWith('other-key');
+
+      // Should have set global meta
+      expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+        'eppo-meta',
+        expect.stringContaining('"version":1'),
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it('should skip migration if already completed', () => {
+      // Setup: migration already done
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+        return null;
+      });
+
+      const consoleSpy = jest.spyOn(console, 'log').mockImplementation();
+
+      new LocalStorageEngine(mockLocalStorage, 'test');
+
+      // Should not have logged migration
+      expect(consoleSpy).not.toHaveBeenCalledWith(
+        expect.stringContaining('Running storage migration'),
+      );
+
+      // Should not have removed any keys
+      expect(mockLocalStorage.removeItem).not.toHaveBeenCalled();
+
+      consoleSpy.mockRestore();
+    });
+
+    it('should handle migration errors gracefully', () => {
+      // Setup: no global meta, but error during migration
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return null;
+        return null;
+      });
+
+      // Make removeItem throw an error
+      (mockLocalStorage.removeItem as jest.Mock).mockImplementation(() => {
+        throw new Error('Storage error');
+      });
+
+      (mockLocalStorage as any)._length = 1;
+      (mockLocalStorage.key as jest.Mock).mockReturnValue('eppo-configuration-test');
+
+      const consoleSpy = jest.spyOn(console, 'warn').mockImplementation();
+
+      new LocalStorageEngine(mockLocalStorage, 'test');
+
+      expect(consoleSpy).toHaveBeenCalledWith('Migration failed:', expect.any(Error));
+
+      consoleSpy.mockRestore();
+    });
+  });
+
+  describe('Compression', () => {
+    let engine: LocalStorageEngine;
+
+    beforeEach(() => {
+      // Setup: migration already completed
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+        return null;
+      });
+
+      engine = new LocalStorageEngine(mockLocalStorage, 'test');
+    });
+
+    it('should compress data when storing', async () => {
+      const testData = JSON.stringify({ flag: 'test-flag', value: 'test-value' });
+
+      await engine.setContentsJsonString(testData);
+
+      expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+        'eppo-configuration-test',
+        LZString.compress(testData),
+      );
+    });
+
+    it('should decompress data when reading', async () => {
+      const testData = JSON.stringify({ flag: 'test-flag', value: 'test-value' });
+      const compressedData = LZString.compress(testData);
+
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+        if (key === 'eppo-configuration-test') return compressedData;
+        return null;
+      });
+
+      const result = await engine.getContentsJsonString();
+
+      expect(result).toBe(testData);
+    });
+
+    it('should handle decompression errors gracefully', async () => {
+      const consoleSpy = jest.spyOn(console, 'warn').mockImplementation();
+
+      // Mock LZString.decompress to throw an error
+      const decompressSpy = jest.spyOn(LZString, 'decompress').mockImplementation(() => {
+        throw new Error('Decompression failed');
+      });
+
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+        if (key === 'eppo-configuration-test') return 'corrupted-data';
+        return null;
+      });
+
+      const result = await engine.getContentsJsonString();
+
+      expect(result).toBe(null);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Failed to decompress configuration, removing corrupted data',
+      );
+      expect(mockLocalStorage.removeItem).toHaveBeenCalledWith('eppo-configuration-test');
+
+      decompressSpy.mockRestore();
+      consoleSpy.mockRestore();
+    });
+
+    it('should compress and decompress meta data', async () => {
+      const metaData = JSON.stringify({ lastUpdated: Date.now() });
+
+      await engine.setMetaJsonString(metaData);
+
+      expect(mockLocalStorage.setItem).toHaveBeenCalledWith(
+        'eppo-configuration-meta-test',
+        LZString.compress(metaData),
+      );
+
+      // Test reading back
+      const compressedMeta = LZString.compress(metaData);
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+        if (key === 'eppo-configuration-meta-test') return compressedMeta;
+        return null;
+      });
+
+      const result = await engine.getMetaJsonString();
+      expect(result).toBe(metaData);
+    });
+
+    it('should return null for non-existent data', async () => {
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return JSON.stringify({ version: 1, migratedAt: Date.now() });
+        return null;
+      });
+
+      const contentsResult = await engine.getContentsJsonString();
+      const metaResult = await engine.getMetaJsonString();
+
+      expect(contentsResult).toBe(null);
+      expect(metaResult).toBe(null);
+    });
+  });
+
+  describe('Global Meta Management', () => {
+    it('should parse valid global meta', () => {
+      const validMeta = { version: 1, migratedAt: Date.now() };
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return JSON.stringify(validMeta);
+        return null;
+      });
+
+      new LocalStorageEngine(mockLocalStorage, 'test');
+
+      expect(mockLocalStorage.getItem).toHaveBeenCalledWith('eppo-meta');
+    });
+
+    it('should handle corrupted global meta', () => {
+      const consoleSpy = jest.spyOn(console, 'warn').mockImplementation();
+
+      (mockLocalStorage.getItem as jest.Mock).mockImplementation((key) => {
+        if (key === 'eppo-meta') return 'invalid-json';
+        return null;
+      });
+
+      (mockLocalStorage as any)._length = 0;
+
+      new LocalStorageEngine(mockLocalStorage, 'test');
+
+      expect(consoleSpy).toHaveBeenCalledWith('Failed to parse global meta:', expect.any(Error));
+
+      consoleSpy.mockRestore();
+    });
+  });
+
+  describe('Space Optimization', () => {
+    it('should actually compress large configuration data', () => {
+      // Create a large configuration object with repetitive data
+      const largeConfig = {
+        flags: Array.from({ length: 100 }, (_, i) => ({
+          flagKey: `test-flag-${i}`,
+          variationType: 'STRING',
+          allocations: [
+            { key: 'control', value: 'control-value' },
+            { key: 'treatment', value: 'treatment-value' },
+          ],
+          rules: [{ conditions: [{ attribute: 'userId', operator: 'MATCHES', value: '.*' }] }],
+        })),
+      };
+
+      const originalJson = JSON.stringify(largeConfig);
+      const compressedData = LZString.compress(originalJson);
+
+      // Verify compression actually reduces size
+      expect(compressedData.length).toBeLessThan(originalJson.length);
+
+      // Verify compression ratio is reasonable (should be significant for repetitive JSON)
+      const compressionRatio = compressedData.length / originalJson.length;
+      expect(compressionRatio).toBeLessThan(0.5); // At least 50% compression
+    });
+  });
+});