Add support for configuring zstdCompressorCallback (#811)

* add support for ZstdCompressorCallback

* Update .generator/src/generator/templates/configuration.j2

Co-authored-by: NouemanKHAL <[email protected]>

* include zstdCompressorCallback in test config

* update licenses

* pre-commit fixes

* update readme

* code review suggestion

Co-authored-by: NouemanKHAL <[email protected]>
Co-authored-by: ci.datadog-api-spec <[email protected]>

Author: 3 people

.generator/src/generator/templates/configuration.j2

@@ -1,4 +1,4 @@
-import { HttpLibrary, HttpConfiguration, RequestContext } from "./http/http";
+import { HttpLibrary, HttpConfiguration, RequestContext, ZstdCompressorCallback } from "./http/http";
 import { IsomorphicFetchHttpLibrary as DefaultHttpLibrary } from "./http/isomorphic-fetch";
 import { BaseServerConfiguration, server1, servers, operationServers } from "./servers";
 import { configureAuthMethods, AuthMethods, AuthMethodsConfiguration } from "./auth";
@@ -47,6 +47,10 @@ export interface ConfigurationParameters {
    * Flag to enable requests tracing
    */
   debug?: boolean
+  /**
+   * Callback method to compress string body with zstd
+   */
+  zstdCompressorCallback?: ZstdCompressorCallback
 }
 
 /**
@@ -101,6 +105,7 @@ export function createConfiguration(conf: ConfigurationParameters = {}): Configu
     httpConfig: conf.httpConfig || {},
     debug: conf.debug
   };
+  configuration.httpApi.zstdCompressorCallback = conf.zstdCompressorCallback
   configuration.httpApi.debug = configuration.debug;
   return configuration;
 }

.generator/src/generator/templates/http/http.j2

@@ -234,7 +234,10 @@ export class ResponseContext {
   }
 }
 
+export type ZstdCompressorCallback = (body: string) => Buffer;
+
 export interface HttpLibrary {
   debug?: boolean;
+  zstdCompressorCallback?: ZstdCompressorCallback;
   send(request: RequestContext): Promise<ResponseContext>;
 }

.generator/src/generator/templates/http/isomorphic-fetch.j2

@@ -1,10 +1,11 @@
-import { HttpLibrary, RequestContext, ResponseContext } from "./http";
+import { HttpLibrary, RequestContext, ResponseContext, ZstdCompressorCallback } from "./http";
 import fetch from "cross-fetch";
 import pako from "pako";
 import bufferFrom from "buffer-from";
 
 export class IsomorphicFetchHttpLibrary implements HttpLibrary {
   public debug = false;
+  public zstdCompressorCallback: ZstdCompressorCallback | undefined;
 
   public send(request: RequestContext): Promise<ResponseContext> {
     if (this.debug) {
@@ -24,6 +25,12 @@ export class IsomorphicFetchHttpLibrary implements HttpLibrary {
         body = bufferFrom(pako.gzip(body).buffer);
       } else if (headers["Content-Encoding"] == "deflate") {
         body = bufferFrom(pako.deflate(body).buffer);
+      } else if (headers["Content-Encoding"] == "zstd1") {
+        if (this.zstdCompressorCallback) {
+          body = this.zstdCompressorCallback(body);
+        } else {
+          throw new Error("zstdCompressorCallback method missing")
+        }
       }
     }
 

LICENSE-3rdparty.csv

@@ -39,3 +39,4 @@ typedoc,Apache-2.0,Copyright (c) 2015 Sebastian Lenz Copyright (c) 2016-2021 Typ
 typescript,Apache-2.0,Copyright (c) Microsoft Corporation.
 url-parse,MIT,Copyright (c) 2015 Unshift.io, Arnout Kazemier,  the Contributors.
 loglevel,MIT,Copyright (c) 2013 Tim Perry
+zstd.ts,BSD-2-Clause,Copyright (c) [2021] [Beeno Tung (Tung Cheung Leong)]

README.md

@@ -161,9 +161,54 @@ async function main() {
 main();
 ```
 
+### Zstd compression
+
+Zstd compression support requires users to supply their own zstd compressor callback function.
+The callback should accept string arg and return compressed Buffer data.
+Callback signature `(body: string) => Buffer`.
+For example, using `zstd.ts` package:
+
+```typescript
+import { compressSync } from 'zstd.ts'
+import { client, v2 } from "@datadog/datadog-api-client";
+
+async function main() {
+  const configurationOpts = {
+    zstdCompressorCallback: (body: string) => compressSync({input: Buffer.from(body, "utf8")})
+  }
+  const configuration = client.createConfiguration(configurationOpts);
+  const apiInstance = new v2.MetricsApi(configuration);
+  const params: v2.MetricsApiSubmitMetricsRequest = {
+      body: {
+          series: [
+              {
+                  metric: "system.load.1",
+                  type: 0,
+                  points: [
+                      {
+                          timestamp: Math.round(new Date().getTime() / 1000),
+                          value: 0.7,
+                      },
+                  ],
+              },
+          ],
+      },
+      contentEncoding: "zstd1",
+  };
+
+  apiInstance.submitMetrics(params).then((data: v2.IntakePayloadAccepted) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  }).catch((error: any) => console.error(error));
+}
+
+main();
+```
+
 ## Documentation
 
-Documentation for API endpoints can be found in in [Github pages][github pages].
+Documentation for API endpoints can be found in [GitHub pages][github pages].
 
 ## Contributing
 

features/support/given.ts

@@ -1,4 +1,5 @@
 import { Given } from "@cucumber/cucumber";
+import { compressSync } from 'zstd.ts'
 
 import fs from "fs";
 import path from "path";
@@ -43,7 +44,9 @@ for (const apiVersion of Versions) {
           appKeyAuth: process.env.DD_TEST_CLIENT_APP_KEY,
         },
         httpConfig: { compress: false },
+        zstdCompressorCallback: (body: string) => compressSync({input: Buffer.from(body, "utf8")}),
       };
+
       if (process.env.DD_TEST_SITE) {
         const server = datadogApiClient.client.servers[2];
         const serverConf = server.getConfiguration();

package.json

@@ -105,7 +105,8 @@
     "ts-jest": "^27.0.4",
     "ts-node": "^9.1.1",
     "typedoc": "^0.22.11",
-    "typescript": "4.5.5"
+    "typescript": "4.5.5",
+    "zstd.ts": "^1.1.3"
   },
   "engines": {
     "node": ">=12.0.0"

packages/datadog-api-client-common/configuration.ts

@@ -1,4 +1,9 @@
-import { HttpLibrary, HttpConfiguration, RequestContext } from "./http/http";
+import {
+  HttpLibrary,
+  HttpConfiguration,
+  RequestContext,
+  ZstdCompressorCallback,
+} from "./http/http";
 import { IsomorphicFetchHttpLibrary as DefaultHttpLibrary } from "./http/isomorphic-fetch";
 import {
   BaseServerConfiguration,
@@ -55,6 +60,10 @@ export interface ConfigurationParameters {
    * Flag to enable requests tracing
    */
   debug?: boolean;
+  /**
+   * Callback method to compress string body with zstd
+   */
+  zstdCompressorCallback?: ZstdCompressorCallback;
 }
 
 /**
@@ -129,6 +138,7 @@ export function createConfiguration(
     httpConfig: conf.httpConfig || {},
     debug: conf.debug,
   };
+  configuration.httpApi.zstdCompressorCallback = conf.zstdCompressorCallback;
   configuration.httpApi.debug = configuration.debug;
   return configuration;
 }

packages/datadog-api-client-common/http/http.ts

@@ -234,7 +234,10 @@ export class ResponseContext {
   }
 }
 
+export type ZstdCompressorCallback = (body: string) => Buffer;
+
 export interface HttpLibrary {
   debug?: boolean;
+  zstdCompressorCallback?: ZstdCompressorCallback;
   send(request: RequestContext): Promise<ResponseContext>;
 }

packages/datadog-api-client-common/http/isomorphic-fetch.ts

@@ -1,10 +1,16 @@
-import { HttpLibrary, RequestContext, ResponseContext } from "./http";
+import {
+  HttpLibrary,
+  RequestContext,
+  ResponseContext,
+  ZstdCompressorCallback,
+} from "./http";
 import fetch from "cross-fetch";
 import pako from "pako";
 import bufferFrom from "buffer-from";
 
 export class IsomorphicFetchHttpLibrary implements HttpLibrary {
   public debug = false;
+  public zstdCompressorCallback: ZstdCompressorCallback | undefined;
 
   public send(request: RequestContext): Promise<ResponseContext> {
     if (this.debug) {
@@ -24,6 +30,12 @@ export class IsomorphicFetchHttpLibrary implements HttpLibrary {
         body = bufferFrom(pako.gzip(body).buffer);
       } else if (headers["Content-Encoding"] == "deflate") {
         body = bufferFrom(pako.deflate(body).buffer);
+      } else if (headers["Content-Encoding"] == "zstd1") {
+        if (this.zstdCompressorCallback) {
+          body = this.zstdCompressorCallback(body);
+        } else {
+          throw new Error("zstdCompressorCallback method missing");
+        }
       }
     }