feat: Option to use gzip to compress event

contract-tests/index.js

@@ -39,6 +39,8 @@ app.get('/', (req, res) => {
       'evaluation-hooks',
       'wrapper',
       'client-prereq-events',
+      'event-gzip',
+      'optional-event-gzip',
     ],
   });
 });

contract-tests/sdkClientEntity.js

@@ -45,6 +45,7 @@ export function makeSdkConfig(options, tag) {
     cf.diagnosticOptOut = !options.events.enableDiagnostics;
     cf.flushInterval = maybeTime(options.events.flushIntervalMs);
     cf.privateAttributes = options.events.globalPrivateAttributes;
+    cf.enableEventCompression = options.events.enableGzip;
   }
   if (options.tags) {
     cf.application = {

packages/sdk/server-node/__tests__/platform/NodeRequests.test.ts

@@ -8,73 +8,76 @@ const TEXT_RESPONSE = 'Test Text';
 const JSON_RESPONSE = '{"text": "value"}';
 
 interface TestRequestData {
-  body: string;
+  body: string | Buffer;
   method: string | undefined;
   headers: http.IncomingHttpHeaders;
 }
 
-describe('given a default instance of NodeRequests', () => {
-  let resolve: (value: TestRequestData | PromiseLike<TestRequestData>) => void;
-  let promise: Promise<TestRequestData>;
-  let server: http.Server;
-  let resetResolve: () => void;
-  let resetPromise: Promise<void>;
-
-  beforeEach(() => {
-    resetPromise = new Promise((res) => {
-      resetResolve = res;
-    });
+let resolve: (value: TestRequestData | PromiseLike<TestRequestData>) => void;
+let promise: Promise<TestRequestData>;
+let server: http.Server;
+let resetResolve: () => void;
+let resetPromise: Promise<void>;
 
-    promise = new Promise<TestRequestData>((res) => {
-      resolve = res;
+beforeEach(() => {
+  resetPromise = new Promise((res) => {
+    resetResolve = res;
+  });
+
+  promise = new Promise<TestRequestData>((res) => {
+    resolve = res;
+  });
+  server = http.createServer({ keepAlive: false }, (req, res) => {
+    const chunks: any[] = [];
+    req.on('data', (chunk) => {
+      chunks.push(chunk);
     });
-    server = http.createServer({ keepAlive: false }, (req, res) => {
-      const chunks: any[] = [];
-      req.on('data', (chunk) => {
-        chunks.push(chunk);
-      });
-      req.on('end', () => {
-        resolve({
-          method: req.method,
-          body: Buffer.concat(chunks).toString(),
-          headers: req.headers,
-        });
+    req.on('end', () => {
+      resolve({
+        method: req.method,
+        body:
+          req.headers['content-encoding'] === 'gzip'
+            ? Buffer.concat(chunks)
+            : Buffer.concat(chunks).toString(),
+        headers: req.headers,
       });
+    });
+    res.statusCode = 200;
+    res.setHeader('Content-Type', 'text/plain');
+    res.setHeader('Connection', 'close');
+    if ((req.url?.indexOf('json') || -1) >= 0) {
+      res.end(JSON_RESPONSE);
+    } else if ((req.url?.indexOf('interrupt') || -1) >= 0) {
+      res.destroy();
+    } else if ((req.url?.indexOf('404') || -1) >= 0) {
+      res.statusCode = 404;
+      res.end();
+    } else if ((req.url?.indexOf('reset') || -1) >= 0) {
       res.statusCode = 200;
-      res.setHeader('Content-Type', 'text/plain');
-      res.setHeader('Connection', 'close');
-      if ((req.url?.indexOf('json') || -1) >= 0) {
-        res.end(JSON_RESPONSE);
-      } else if ((req.url?.indexOf('interrupt') || -1) >= 0) {
+      res.flushHeaders();
+      res.write('potato');
+      setTimeout(() => {
         res.destroy();
-      } else if ((req.url?.indexOf('404') || -1) >= 0) {
-        res.statusCode = 404;
-        res.end();
-      } else if ((req.url?.indexOf('reset') || -1) >= 0) {
-        res.statusCode = 200;
-        res.flushHeaders();
-        res.write('potato');
-        setTimeout(() => {
-          res.destroy();
-          resetResolve();
-        }, 0);
-      } else if ((req.url?.indexOf('gzip') || -1) >= 0) {
-        res.setHeader('Content-Encoding', 'gzip');
-        res.end(zlib.gzipSync(Buffer.from(JSON_RESPONSE, 'utf8')));
-      } else {
-        res.end(TEXT_RESPONSE);
-      }
-    });
-    server.listen(PORT);
+        resetResolve();
+      }, 0);
+    } else if ((req.url?.indexOf('gzip') || -1) >= 0) {
+      res.setHeader('Content-Encoding', 'gzip');
+      res.end(zlib.gzipSync(Buffer.from(JSON_RESPONSE, 'utf8')));
+    } else {
+      res.end(TEXT_RESPONSE);
+    }
   });
+  server.listen(PORT);
+});
 
-  afterEach(
-    async () =>
-      new Promise((resolveClose) => {
-        server.close(resolveClose);
-      }),
-  );
+afterEach(
+  async () =>
+    new Promise((resolveClose) => {
+      server.close(resolveClose);
+    }),
+);
 
+describe('given a default instance of NodeRequests', () => {
   const requests = new NodeRequests();
   it('can make a basic get request', async () => {
     const res = await requests.fetch(`http://localhost:${PORT}`);
@@ -120,6 +123,17 @@ describe('given a default instance of NodeRequests', () => {
     expect(serverResult.body).toEqual('BODY TEXT');
   });
 
+  it('can make a basic post ignoring compressBodyIfPossible', async () => {
+    await requests.fetch(`http://localhost:${PORT}`, {
+      method: 'POST',
+      body: 'BODY TEXT',
+      compressBodyIfPossible: true,
+    });
+    const serverResult = await promise;
+    expect(serverResult.method).toEqual('POST');
+    expect(serverResult.body).toEqual('BODY TEXT');
+  });
+
   it('can make a request with headers', async () => {
     await requests.fetch(`http://localhost:${PORT}`, {
       method: 'POST',
@@ -166,3 +180,30 @@ describe('given a default instance of NodeRequests', () => {
     expect(serverResult.body).toEqual('');
   });
 });
+
+describe('given an instance of NodeRequests with enableEventCompression turned on', () => {
+  const requests = new NodeRequests(undefined, undefined, undefined, true);
+  it('can make a basic post with compressBodyIfPossible enabled', async () => {
+    await requests.fetch(`http://localhost:${PORT}`, {
+      method: 'POST',
+      body: 'BODY TEXT',
+      compressBodyIfPossible: true,
+    });
+    const serverResult = await promise;
+    expect(serverResult.method).toEqual('POST');
+    expect(serverResult.headers['content-encoding']).toEqual('gzip');
+    expect(serverResult.body).toEqual(zlib.gzipSync('BODY TEXT'));
+  });
+
+  it('can make a basic post with compressBodyIfPossible disabled', async () => {
+    await requests.fetch(`http://localhost:${PORT}`, {
+      method: 'POST',
+      body: 'BODY TEXT',
+      compressBodyIfPossible: false,
+    });
+    const serverResult = await promise;
+    expect(serverResult.method).toEqual('POST');
+    expect(serverResult.headers['content-encoding']).toBeUndefined();
+    expect(serverResult.body).toEqual('BODY TEXT');
+  });
+});

packages/sdk/server-node/src/platform/NodePlatform.ts

@@ -16,6 +16,11 @@ export default class NodePlatform implements platform.Platform {
 
   constructor(options: LDOptions) {
     this.info = new NodeInfo(options);
-    this.requests = new NodeRequests(options.tlsParams, options.proxyOptions, options.logger);
+    this.requests = new NodeRequests(
+      options.tlsParams,
+      options.proxyOptions,
+      options.logger,
+      options.enableEventCompression,
+    );
   }
 }

packages/sdk/server-node/src/platform/NodeRequests.ts

@@ -5,6 +5,8 @@ import { HttpsProxyAgentOptions } from 'https-proxy-agent';
 // No types for the event source.
 // @ts-ignore
 import { EventSource as LDEventSource } from 'launchdarkly-eventsource';
+import { promisify } from 'util';
+import * as zlib from 'zlib';
 
 import {
   EventSourceCapabilities,
@@ -16,6 +18,8 @@ import {
 
 import NodeResponse from './NodeResponse';
 
+const gzip = promisify(zlib.gzip);
+
 function processTlsOptions(tlsOptions: LDTLSOptions): https.AgentOptions {
   const options: https.AgentOptions & { [index: string]: any } = {
     ca: tlsOptions.ca,
@@ -101,25 +105,44 @@ export default class NodeRequests implements platform.Requests {
 
   private _hasProxyAuth: boolean = false;
 
-  constructor(tlsOptions?: LDTLSOptions, proxyOptions?: LDProxyOptions, logger?: LDLogger) {
+  private _enableEventCompression: boolean = false;
+
+  constructor(
+    tlsOptions?: LDTLSOptions,
+    proxyOptions?: LDProxyOptions,
+    logger?: LDLogger,
+    enableEventCompression?: boolean,
+  ) {
     this._agent = createAgent(tlsOptions, proxyOptions, logger);
     this._hasProxy = !!proxyOptions;
     this._hasProxyAuth = !!proxyOptions?.auth;
+    this._enableEventCompression = !!enableEventCompression;
   }
 
-  fetch(url: string, options: platform.Options = {}): Promise<platform.Response> {
+  async fetch(url: string, options: platform.Options = {}): Promise<platform.Response> {
     const isSecure = url.startsWith('https://');
     const impl = isSecure ? https : http;
 
+    const headers = { ...options.headers };
+    let bodyData: String | Buffer | undefined = options.body;
+
     // For get requests we are going to automatically support compressed responses.
     // Note this does not affect SSE as the event source is not using this fetch implementation.
-    const headers =
-      options.method?.toLowerCase() === 'get'
-        ? {
-            ...options.headers,
-            'accept-encoding': 'gzip',
-          }
-        : options.headers;
+    if (options.method?.toLowerCase() === 'get') {
+      headers['accept-encoding'] = 'gzip';
+    }
+    // For post requests we are going to support compressed post bodies if the
+    // enableEventCompression config setting is true and the compressBodyIfPossible
+    // option is true.
+    else if (
+      this._enableEventCompression &&
+      !!options.compressBodyIfPossible &&
+      options.method?.toLowerCase() === 'post' &&
+      options.body
+    ) {
+      headers['content-encoding'] = 'gzip';
+      bodyData = await gzip(Buffer.from(options.body, 'utf8'));
+    }
 
     return new Promise((resolve, reject) => {
       const req = impl.request(
@@ -133,8 +156,8 @@ export default class NodeRequests implements platform.Requests {
         (res) => resolve(new NodeResponse(res)),
       );
 
-      if (options.body) {
-        req.write(options.body);
+      if (bodyData) {
+        req.write(bodyData);
       }
 
       req.on('error', (err) => {

packages/shared/common/__tests__/internal/events/EventSender.test.ts

@@ -133,6 +133,7 @@ describe('given an event sender', () => {
     expect(mockFetch).toHaveBeenCalledTimes(1);
     expect(mockFetch).toHaveBeenCalledWith(`${basicConfig.serviceEndpoints.events}/bulk`, {
       body: JSON.stringify(testEventData1),
+      compressBodyIfPossible: true,
       headers: analyticsHeaders(uuid),
       method: 'POST',
       keepalive: true,
@@ -150,6 +151,7 @@ describe('given an event sender', () => {
     expect(mockFetch).toHaveBeenCalledTimes(2);
     expect(mockFetch).toHaveBeenNthCalledWith(1, `${basicConfig.serviceEndpoints.events}/bulk`, {
       body: JSON.stringify(testEventData1),
+      compressBodyIfPossible: true,
       headers: analyticsHeaders(uuid),
       method: 'POST',
       keepalive: true,
@@ -159,6 +161,7 @@ describe('given an event sender', () => {
       `${basicConfig.serviceEndpoints.events}/diagnostic`,
       {
         body: JSON.stringify(testEventData2),
+        compressBodyIfPossible: true,
         headers: diagnosticHeaders,
         method: 'POST',
         keepalive: true,

packages/shared/common/src/api/platform/Requests.ts

@@ -75,6 +75,11 @@ export interface Options {
   headers?: Record<string, string>;
   method?: string;
   body?: string;
+  /**
+   * Gzip compress the post body only if the underlying SDK framework supports it
+   * and the config option enableEventCompression is set to true.
+   */
+  compressBodyIfPossible?: boolean;
   timeout?: number;
   /**
    * For use in browser environments. Platform support will be best effort for this field.

packages/shared/common/src/internal/events/EventSender.ts

@@ -64,6 +64,7 @@ export default class EventSender implements LDEventSender {
       const { status, headers: resHeaders } = await this._requests.fetch(uri, {
         headers,
         body: JSON.stringify(events),
+        compressBodyIfPossible: true,
         method: 'POST',
         // When sending events from browser environments the request should be completed even
         // if the user is navigating away from the page.

packages/shared/sdk-server/src/api/options/LDOptions.ts

@@ -304,4 +304,14 @@ export interface LDOptions {
    * ```
    */
   hooks?: Hook[];
+
+  /**
+   * Set to true to opt in to compressing event payloads if the SDK supports it, since the
+   * compression library may not be supported in the underlying SDK framework.  If the compression
+   * library is not supported then event payloads will not be compressed even if this option
+   * is enabled.
+   *
+   * Defaults to false.
+   */
+  enableEventCompression?: boolean;
 }

packages/shared/sdk-server/src/options/Configuration.ts

@@ -57,6 +57,7 @@ const validations: Record<string, TypeValidator> = {
   application: TypeValidators.Object,
   payloadFilterKey: TypeValidators.stringMatchingRegex(/^[a-zA-Z0-9](\w|\.|-)*$/),
   hooks: TypeValidators.createTypeArray('Hook[]', {}),
+  enableEventCompression: TypeValidators.Boolean,
 };
 
 /**
@@ -82,6 +83,7 @@ export const defaultValues: ValidatedOptions = {
   diagnosticOptOut: false,
   diagnosticRecordingInterval: 900,
   featureStore: () => new InMemoryFeatureStore(),
+  enableEventCompression: false,
 };
 
 function validateTypesAndNames(options: LDOptions): {
@@ -215,6 +217,8 @@ export default class Configuration {
 
   public readonly hooks?: Hook[];
 
+  public readonly enableEventCompression: boolean;
+
   constructor(options: LDOptions = {}, internalOptions: internal.LDInternalOptions = {}) {
     // The default will handle undefined, but not null.
     // Because we can be called from JS we need to be extra defensive.
@@ -283,5 +287,6 @@ export default class Configuration {
     }
 
     this.hooks = validatedOptions.hooks;
+    this.enableEventCompression = validatedOptions.enableEventCompression;
   }
 }

packages/shared/sdk-server/src/options/ValidatedOptions.ts

@@ -41,4 +41,5 @@ export interface ValidatedOptions {
   [index: string]: any;
   bigSegments?: LDBigSegmentsOptions;
   hooks?: Hook[];
+  enableEventCompression: boolean;
 }