Apps plugin

.github/CODEOWNERS

@@ -40,3 +40,6 @@ packages/plugins/async-queue                                          @yoannmoin
 
 # Output
 packages/plugins/output                                               @yoannmoinet
+
+# Apps
+packages/plugins/apps                                                 @yoannmoinet

LICENSES-3rdparty.csv

@@ -478,6 +478,7 @@ http-signature,npm,MIT,Joyent  Inc (https://github.com/joyent/node-http-signatur
 human-signals,npm,Apache-2.0,ehmicky (https://git.io/JeluP)
 husky,npm,MIT,Typicode (https://github.com/typicode/husky#readme)
 ignore,npm,MIT,kael (https://www.npmjs.com/package/ignore)
+immediate,npm,MIT,(https://www.npmjs.com/package/immediate)
 import-fresh,npm,MIT,Sindre Sorhus (https://sindresorhus.com)
 import-local,npm,MIT,Sindre Sorhus (https://sindresorhus.com)
 imurmurhash,npm,MIT,Jens Taylor (https://github.com/jensyt/imurmurhash-js)
@@ -563,9 +564,11 @@ json-stream-stringify,npm,MIT,Faleij (https://github.com/faleij)
 json-stringify-safe,npm,ISC,Isaac Z. Schlueter (https://github.com/isaacs/json-stringify-safe)
 json5,npm,MIT,Aseem Kishore (http://json5.org/)
 jsprim,npm,MIT,(https://www.npmjs.com/package/jsprim)
+jszip,npm,(MIT OR GPL-3.0-or-later),Stuart Knightley (https://www.npmjs.com/package/jszip)
 keyv,npm,MIT,Jared Wray (https://github.com/jaredwray/keyv)
 leven,npm,MIT,Sindre Sorhus (sindresorhus.com)
 levn,npm,MIT,George Zahariev (https://github.com/gkz/levn)
+lie,npm,MIT,(https://www.npmjs.com/package/lie)
 lines-and-columns,npm,MIT,Brian Donovan (https://github.com/eventualbuddha/lines-and-columns#readme)
 lint-staged,npm,MIT,Andrey Okonetchnikov (https://www.npmjs.com/package/lint-staged)
 listr2,npm,MIT,Cenk Kilic (https://srcs.kilic.dev)
@@ -636,6 +639,7 @@ p-timeout,npm,MIT,Sindre Sorhus (sindresorhus.com)
 p-try,npm,MIT,Sindre Sorhus (sindresorhus.com)
 package-json-from-dist,npm,BlueOak-1.0.0,Isaac Z. Schlueter (https://izs.me)
 pad,npm,BSD-3-Clause,David Worms (https://github.com/adaltas/node-pad)
+pako,npm,(MIT AND Zlib),(https://github.com/nodeca/pako)
 parent-module,npm,MIT,Sindre Sorhus (sindresorhus.com)
 parse-json,npm,MIT,Sindre Sorhus (https://sindresorhus.com)
 path-exists,npm,MIT,Sindre Sorhus (sindresorhus.com)
@@ -705,6 +709,7 @@ serialize-javascript,npm,BSD-3-Clause,Eric Ferraiuolo (https://github.com/yahoo/
 set-blocking,npm,ISC,Ben Coe (https://github.com/yargs/set-blocking#readme)
 set-function-length,npm,MIT,Jordan Harband (https://github.com/ljharb/set-function-length#readme)
 set-function-name,npm,MIT,Jordan Harband (https://github.com/ljharb/set-function-name#readme)
+setimmediate,npm,MIT,YuzuJS (https://www.npmjs.com/package/setimmediate)
 shebang-command,npm,MIT,Kevin Mårtensson (github.com/kevva)
 shebang-regex,npm,MIT,Sindre Sorhus (sindresorhus.com)
 side-channel,npm,MIT,Jordan Harband (https://github.com/ljharb/side-channel#readme)

README.md

@@ -100,7 +100,7 @@ Follow the specific documentation for each bundler:
     logLevel?: 'debug' | 'info' | 'warn' | 'error' | 'none',
     metadata?: {
         name?: string;
-    };;
+    };
     errorTracking?: {
         enable?: boolean;
         sourcemaps?: {

packages/core/src/helpers/env.ts

@@ -16,9 +16,17 @@ const yellow = chalk.bold.yellow;
 //   - DATADOG_APP_KEY
 //   - DD_SOURCEMAP_INTAKE_URL
 //   - DATADOG_SOURCEMAP_INTAKE_URL
+//   - DD_APPS_INTAKE_URL
+//   - DATADOG_APPS_INTAKE_URL
 //   - DD_SITE
 //   - DATADOG_SITE
-const OVERRIDE_VARIABLES = ['API_KEY', 'APP_KEY', 'SOURCEMAP_INTAKE_URL', 'SITE'] as const;
+const OVERRIDE_VARIABLES = [
+    'API_KEY',
+    'APP_KEY',
+    'SOURCEMAP_INTAKE_URL',
+    'APPS_INTAKE_URL',
+    'SITE',
+] as const;
 type ENV_KEY = (typeof OVERRIDE_VARIABLES)[number];
 
 // Return the environment variable that would be prefixed with either DATADOG_ or DD_.

packages/core/src/helpers/request.ts

@@ -3,10 +3,50 @@
 // Copyright 2019-Present Datadog, Inc.
 
 import retry from 'async-retry';
+import { Readable } from 'stream';
 import type { RequestInit } from 'undici-types';
+import type { Gzip } from 'zlib';
+import { createGzip } from 'zlib';
 
 import type { RequestOpts } from '../types';
 
+export const getOriginHeaders = (opts: { bundler: string; plugin: string; version: string }) => {
+    return {
+        'DD-EVP-ORIGIN': `${opts.bundler}-build-plugin_${opts.plugin}`,
+        'DD-EVP-ORIGIN-VERSION': opts.version,
+    };
+};
+
+export type GzipFormData = {
+    data: Gzip;
+    headers: Record<string, string>;
+};
+
+export type FormBuilder = (form: FormData) => Promise<void> | void;
+
+export const createGzipFormData = async (
+    builder: FormBuilder,
+    defaultHeaders: Record<string, string> = {},
+): Promise<GzipFormData> => {
+    const form = new FormData();
+    await builder(form);
+
+    const gz = createGzip();
+    // Serialize FormData through Request to get a streaming body and auto-generated headers
+    // (boundary) that we can forward while piping through gzip.
+    const req = new Request('fake://url', { method: 'POST', body: form });
+    const formStream = Readable.fromWeb(req.body!);
+    const data = formStream.pipe(gz);
+
+    const headers = {
+        'Content-Encoding': 'gzip',
+        ...defaultHeaders,
+        ...Object.fromEntries(req.headers.entries()),
+    };
+
+    return { data, headers };
+};
+
 export const ERROR_CODES_NO_RETRY = [400, 403, 413];
 export const NB_RETRIES = 5;
 // Do a retriable fetch.

packages/core/src/helpers/strings.ts

@@ -2,6 +2,8 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2019-Present Datadog, Inc.
 
+import chalk from 'chalk';
+
 // Format a duration 0h 0m 0s 0ms
 export const formatDuration = (duration: number) => {
     const days = Math.floor(duration / 1000 / 60 / 60 / 24);
@@ -60,5 +62,35 @@ export const filterSensitiveInfoFromRepositoryUrl = (repositoryUrl: string = '')
     }
 };
 
+const formatValue = (value: unknown) => {
+    if (value === undefined) {
+        return 'undefined';
+    }
+
+    if (value === null) {
+        return 'null';
+    }
+
+    if (Array.isArray(value)) {
+        return value.join(', ');
+    }
+
+    if (typeof value === 'object') {
+        try {
+            return JSON.stringify(value, null, 2);
+        } catch {
+            return String(value);
+        }
+    }
+
+    return value?.toString() ?? '';
+};
+
+export const prettyObject = (obj: any) => {
+    return Object.entries(obj)
+        .map(([key, value]) => `    - ${key}: ${chalk.bold.green(formatValue(value))}`)
+        .join('\n');
+};
+
 let index = 0;
 export const getUniqueId = () => `${Date.now()}.${performance.now()}.${++index}`;

packages/core/src/types.ts

@@ -8,6 +8,8 @@
 import type { TrackedFilesMatcher } from '@dd/internal-git-plugin/trackedFilesMatcher';
 /* eslint-disable arca/import-ordering */
 // #imports-injection-marker
+import type { AppsOptions } from '@dd/apps-plugin/types';
+import type * as apps from '@dd/apps-plugin';
 import type { ErrorTrackingOptions } from '@dd/error-tracking-plugin/types';
 import type * as errorTracking from '@dd/error-tracking-plugin';
 import type { MetricsOptions } from '@dd/metrics-plugin/types';
@@ -254,6 +256,7 @@ export interface BaseOptions {
 export interface Options extends BaseOptions {
     // Each product should have a unique entry.
     // #types-injection-marker
+    [apps.CONFIG_KEY]?: AppsOptions;
     [errorTracking.CONFIG_KEY]?: ErrorTrackingOptions;
     [metrics.CONFIG_KEY]?: MetricsOptions;
     [output.CONFIG_KEY]?: OutputOptions;

packages/factory/package.json

@@ -19,6 +19,7 @@
         "typecheck": "tsc --noEmit"
     },
     "dependencies": {
+        "@dd/apps-plugin": "workspace:*",
         "@dd/core": "workspace:*",
         "@dd/error-tracking-plugin": "workspace:*",
         "@dd/internal-analytics-plugin": "workspace:*",

packages/factory/src/helpers/context.test.ts

@@ -4,7 +4,6 @@
 
 import type { Options, GlobalContext } from '@dd/core/types';
 import { BUNDLER_VERSIONS } from '@dd/tests/_jest/helpers/constants';
-import { cleanEnv } from '@dd/tests/_jest/helpers/env';
 import { defaultPluginOptions } from '@dd/tests/_jest/helpers/mocks';
 import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers';
 
@@ -13,7 +12,6 @@ describe('Factory Helpers', () => {
     const initialContexts: Record<string, GlobalContext> = {};
     const buildRoots: Record<string, string> = {};
     let workingDir: string;
-    let restoreEnv: () => void;
 
     beforeAll(async () => {
         const pluginConfig: Options = {
@@ -41,15 +39,10 @@ describe('Factory Helpers', () => {
             },
         };
 
-        restoreEnv = cleanEnv();
         const result = await runBundlers(pluginConfig);
         workingDir = result.workingDir;
     });
 
-    afterAll(() => {
-        restoreEnv();
-    });
-
     describe('getContext', () => {
         describe.each(BUNDLERS)('[$name|$version]', ({ name, version }) => {
             test('Should have the right initial context.', () => {

packages/factory/src/index.ts

@@ -35,6 +35,7 @@ import { wrapGetPlugins } from './helpers/wrapPlugins';
 import { ALL_ENVS, HOST_NAME } from '@dd/core/constants';
 import { notifyOnEnvOverrides } from '@dd/core/helpers/env';
 // #imports-injection-marker
+import * as apps from '@dd/apps-plugin';
 import * as errorTracking from '@dd/error-tracking-plugin';
 import * as metrics from '@dd/metrics-plugin';
 import * as output from '@dd/output-plugin';
@@ -49,6 +50,7 @@ import { getInjectionPlugins } from '@dd/internal-injection-plugin';
 import { getTrueEndPlugins } from '@dd/internal-true-end-plugin';
 // #imports-injection-marker
 // #types-export-injection-marker
+export type { types as AppsTypes } from '@dd/apps-plugin';
 export type { types as ErrorTrackingTypes } from '@dd/error-tracking-plugin';
 export type { types as MetricsTypes } from '@dd/metrics-plugin';
 export type { types as OutputTypes } from '@dd/output-plugin';
@@ -159,6 +161,7 @@ export const buildPluginFactory = ({
         // Add the customer facing plugins.
         pluginsToAdd.push(
             // #configs-injection-marker
+            ['apps', apps.getPlugins],
             ['error-tracking', errorTracking.getPlugins],
             ['metrics', metrics.getPlugins],
             ['output', output.getPlugins],

packages/plugins/apps/README.md

@@ -0,0 +1,67 @@
+# Apps Plugin <!-- #omit in toc -->
+
+A plugin to upload assets to Datadog's storage
+
+> [!WARNING]
+> The Apps plugin is in **alpha** and is likely to break in most setups.
+> Use it only for experimentation; behavior and APIs may change without notice.
+
+<!-- The title and the following line will both be added to the root README.md  -->
+
+## Table of content <!-- #omit in toc -->
+
+<!-- This is auto generated with yarn cli integrity -->
+
+<!-- #toc -->
+-   [Configuration](#configuration)
+-   [Assets Upload](#assets-upload)
+    -   [apps.dryRun](#appsdryrun)
+    -   [apps.enable](#appsenable)
+    -   [apps.include](#appsinclude)
+    -   [apps.identifier](#appsidentifier)
+<!-- #toc -->
+
+## Configuration
+
+```ts
+apps?: {
+    dryRun?: boolean;
+    enable?: boolean;
+    include?: string[];
+    identifier?: string;
+}
+```
+
+## Assets Upload
+
+Upload built assets to Datadog storage as a compressed archive.
+
+> [!NOTE]
+> You can override the domain used in the request with the `DATADOG_SITE` environment variable or the `auth.site` options (eg. `datadoghq.eu`).
+> You can override the full intake URL by setting the `DATADOG_APPS_INTAKE_URL` environment variable (eg. `https://apps-intake.datadoghq.com/api/v1/apps`).
+
+### apps.dryRun
+
+> default: `false`
+
+Prepare the archive and log the upload summary without sending anything to Datadog.
+
+### apps.enable
+
+> default: `true` when an `apps` config block is present
+
+Enable or disable the plugin without removing its configuration.
+
+### apps.include
+
+> default: `[]`
+
+Additional glob patterns (relative to the project root) to include in the uploaded archive. The bundler output directory is always included.
+
+### apps.identifier
+
+> default: an internal computation between the `name` and `repository` fields in `package.json` or from the `git` plugin.
+
+Override the app's identifier used to identify the current app against the assets upload API.
+
+Can be useful to enforce a static identifier instead of relying on possibly changing information like app's name and repository's url.

packages/plugins/apps/package.json

@@ -0,0 +1,34 @@
+{
+    "name": "@dd/apps-plugin",
+    "packageManager": "[email protected]",
+    "license": "MIT",
+    "private": true,
+    "author": "Datadog",
+    "description": "A plugin to upload assets to Datadog's storage",
+    "homepage": "https://github.com/DataDog/build-plugins/tree/main/packages/plugins/apps#readme",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/DataDog/build-plugins",
+        "directory": "packages/plugins/apps"
+    },
+    "buildPlugin": {
+        "hideFromRootReadme": true
+    },
+    "exports": {
+        ".": "./src/index.ts",
+        "./*": "./src/*.ts"
+    },
+    "scripts": {
+        "typecheck": "tsc --noEmit"
+    },
+    "dependencies": {
+        "@dd/core": "workspace:*",
+        "chalk": "2.3.1",
+        "glob": "11.0.0",
+        "jszip": "3.10.1",
+        "pretty-bytes": "5.6.0"
+    },
+    "devDependencies": {
+        "typescript": "5.4.3"
+    }
+}