[Security Solution] Implement prebuilt rules package generation tool (#237561)

**Partially addresses:** elastic/security-team#12849, #188090

## Summary

This PR adds a toolkit to generate mock Prebuilt Rule Assets Fleet package a.k.a. `security_detection_engine` for performance testing purposes.

## Details

The toolkit lib was added to `@kbn/security-solution-test-api-clients` security solution package. It allows to generate packages of arbitrary size and composition. The only limitation that it uses a single mock "thick" prebuilt rule asset template to produce prebuilt rule assets and their historical versions. To mimic the real package behavior the template contains large setup and investigation guides along the other fields having values copied from the real rules in the real recent package version.

The toolkit could be run with the following command

```bash
node ./x-pack/solutions/security/packages/test-api-clients/scripts/prebuilt_rules/generate_package.js --packageSemver 99.0.0 --numOfRules 3000 --numOfHistoricalVersions 2
```

and it produces the following logs

```bash
 info 🪄 Generating prebuilt rules package...
 debg Total 9000 prebuilt rules assets will be generated (3000 latest version prebuilt rules assets + 6000 historical prebuilt rules assets)
 succ 📦 Generated package has been written to <home path>/security_detection_engine-99.0.0.zip (173.93 MB)
```

and it generates a package `security_detection_engine-99.0.0.zip` in the current user's home folder.

Author: maximpn

.eslintrc.js

@@ -1275,6 +1275,15 @@ module.exports = {
         ],
       },
     },
+    // Allow node.js imports only for the prebuilt rules package generation scripts
+    {
+      files: [
+        'x-pack/solutions/security/packages/test-api-clients/prebuilt_rules_package_generation/*.{js,mjs,ts,tsx}',
+      ],
+      rules: {
+        'import/no-nodejs-modules': 'off',
+      },
+    },
     {
       // typescript only for front and back end, but excludes the test files.
       // We use this section to add rules in which we do not want to apply to test files.

package.json

@@ -940,7 +940,6 @@
     "@kbn/security-solution-serverless": "link:x-pack/solutions/security/plugins/security_solution_serverless",
     "@kbn/security-solution-side-nav": "link:x-pack/solutions/security/packages/side-nav",
     "@kbn/security-solution-storybook-config": "link:x-pack/solutions/security/packages/storybook/config",
-    "@kbn/security-solution-test-api-clients": "link:x-pack/solutions/security/packages/test-api-clients",
     "@kbn/security-solution-upselling": "link:x-pack/solutions/security/packages/upselling",
     "@kbn/security-test-endpoints-plugin": "link:x-pack/platform/test/security_functional/plugins/test_endpoints",
     "@kbn/security-ui-components": "link:x-pack/platform/packages/private/security/ui_components",
@@ -1642,6 +1641,7 @@
     "@kbn/scout-reporting": "link:src/platform/packages/private/kbn-scout-reporting",
     "@kbn/scout-security": "link:x-pack/solutions/security/packages/kbn-scout-security",
     "@kbn/security-api-integration-helpers": "link:x-pack/platform/test/security_api_integration/packages/helpers",
+    "@kbn/security-solution-test-api-clients": "link:x-pack/solutions/security/packages/test-api-clients",
     "@kbn/serverless-storybook-config": "link:src/platform/packages/shared/serverless/storybook/config",
     "@kbn/set-map": "link:packages/kbn-set-map",
     "@kbn/shared-ux-card-no-data-mocks": "link:src/platform/packages/shared/shared-ux/card/no_data/mocks",

x-pack/solutions/security/packages/test-api-clients/kibana.jsonc

@@ -1,9 +1,8 @@
 {
   "id": "@kbn/security-solution-test-api-clients",
   "type": "shared-common",
-  "owner": [
-    "@elastic/security-detection-rule-management"
-  ],
+  "owner": ["@elastic/security-detection-rule-management"],
   "group": "security",
-  "visibility": "private"
-}
+  "visibility": "private",
+  "devOnly": true
+}

x-pack/solutions/security/packages/test-api-clients/prebuilt_rules_package_generation/README.md

@@ -0,0 +1,36 @@
+# Prebuilt Rules Package Generation
+
+This package provides a small toolkit to generate mock prebuilt detection rule assets packed in a Fleet package compatible with the Security prebuilt rules workflow. It could be used to generate payload for stress and performance testing.
+
+# Details
+
+The toolkit uses a mock prebuilt rule asset mimicking a quite heavy real one with large setup and investigation guides. This prebuilt rule asset is multiplied by the desired N with an ability to add M historical prebuilt rule assets.
+
+# Running the script
+
+The script requires the following parameters
+
+- `--packageSemver` - package version in a semver format. e.g. `9.2.0`
+- `--numOfRules` - the total number of prebuilt rule assets with the recent version, e.g. `3000`
+
+and the following optional parameters could be specified
+
+- `--packageName` - a desired package name, by default `security_detection_engine` is used
+- `--numOfHistoricalVersions` - a number of historical prebuilt rule assets per each recent version prebuilt rule asset, zero by default
+- `--output` or `-o` - output directory path, by default current user's home directory is used
+
+For example the following command
+
+```bash
+node ./x-pack/solutions/security/packages/test-api-clients/scripts/prebuilt_rules/generate_package.js --packageSemver 99.0.0 --numOfRules 3000 --numOfHistoricalVersions 2
+```
+
+would output the following logs
+
+```bash
+ info 🪄 Generating prebuilt rules package...
+ debg Total 9000 prebuilt rules assets will be generated (3000 latest version prebuilt rules assets + 6000 historical prebuilt rules assets)
+ succ 📦 Generated package has been written to <home path>/security_detection_engine-99.0.0.zip (173.93 MB)
+```
+
+and it'd generate a package `security_detection_engine-99.0.0.zip` in the current user's home folder.

x-pack/solutions/security/packages/test-api-clients/prebuilt_rules_package_generation/cli.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { normalize } from 'path';
+import { homedir } from 'os';
+import yargs from 'yargs/yargs';
+import { ToolingLog } from '@kbn/tooling-log';
+import chalk from 'chalk';
+import { statSync } from 'fs';
+import { generatePrebuiltRuleAssets } from './generate_prebuilt_rule_assets';
+import { generatePrebuiltRulesPackageZipFile } from './generate_prebuilt_rules_package_zip_file';
+
+const DEFAULT_PREBUILT_RULES_PACKAGE_NAME = 'security_detection_engine';
+
+export function runCli() {
+  yargs(process.argv.slice(2))
+    .command(
+      '*',
+      'Generate Prebuilt Rules mock Fleet package',
+      (y) =>
+        y
+          .option('packageName', {
+            describe: `Package name (default: ${DEFAULT_PREBUILT_RULES_PACKAGE_NAME})`,
+            default: DEFAULT_PREBUILT_RULES_PACKAGE_NAME,
+            string: true,
+          })
+          .option('packageSemver', {
+            describe: 'Package version in semver format (e.g. 9.2.0)',
+            demandOption: true,
+            string: true,
+          })
+          .option('numOfRules', {
+            describe: 'A number of mock prebuilt rule assets to generate',
+            demandOption: true,
+            number: true,
+          })
+          .option('numOfHistoricalVersions', {
+            describe:
+              'A number of historical mock prebuilt rule assets per each rule to generate. TotalNumOfRules = numOfRules * (1 + numOfHistoricalVersions)',
+            default: 0,
+            number: true,
+          })
+          .option('output', {
+            describe: 'Output directory path',
+            default: homedir(),
+            string: true,
+            alias: 'o',
+          })
+          .showHelpOnFail(true),
+      async ({ packageName, packageSemver, numOfRules, numOfHistoricalVersions, output }) => {
+        const logger = new ToolingLog({
+          level: 'debug',
+          writeTo: process.stdout,
+        });
+        const outputFilePath = normalize(`${output}/${packageName}-${packageSemver}.zip`);
+        const totalNumOfPrebuiltRulesAssets = numOfRules * (1 + numOfHistoricalVersions);
+
+        logger.info('🪄 Generating prebuilt rules package...');
+
+        logger.debug(
+          `Total ${chalk.blue(
+            totalNumOfPrebuiltRulesAssets
+          )} prebuilt rules assets will be generated (${chalk.bold(
+            numOfRules
+          )} latest version prebuilt rules assets + ${chalk.bold(
+            numOfRules * numOfHistoricalVersions
+          )} historical prebuilt rules assets)`
+        );
+
+        const prebuiltRuleAssets = generatePrebuiltRuleAssets({
+          numOfRules,
+          numOfHistoricalVersions,
+          packageVersion: packageSemver,
+        });
+
+        await generatePrebuiltRulesPackageZipFile({
+          packageName,
+          packageSemver,
+          prebuiltRuleAssets,
+          filePath: outputFilePath,
+        });
+
+        const packageStats = statSync(outputFilePath);
+        const packageSizeInMB = Math.ceil((packageStats.size / (1024 * 1024)) * 100) / 100;
+
+        logger.success(
+          `📦 Generated package has been written to ${chalk.bold(outputFilePath)} (${chalk.magenta(
+            packageSizeInMB
+          )} ${chalk.magenta('MB')})`
+        );
+      }
+    )
+    .parse();
+}

x-pack/solutions/security/packages/test-api-clients/prebuilt_rules_package_generation/generate_prebuilt_rule_assets.ts

@@ -5,53 +5,74 @@
  * 2.0.
  */
 
-import AdmZip from 'adm-zip';
+import archiver from 'archiver';
 import { dump } from 'js-yaml';
 import semver from 'semver';
 import type { PackageSpecManifest } from '@kbn/fleet-plugin/common';
 import type { PrebuiltRuleAsset } from '@kbn/security-solution-plugin/server/lib/detection_engine/prebuilt_rules';
 
-interface CreatePrebuiltRulesPackageParams {
+export interface GeneratePrebuiltRulesPackageBaseParams {
   packageName: string;
   packageSemver: string;
   prebuiltRuleAssets: PrebuiltRuleAsset[];
 }
 
-export function createPrebuiltRulesPackage({
+interface GeneratePrebuiltRulesPackageParams extends GeneratePrebuiltRulesPackageBaseParams {
+  output: NodeJS.WritableStream;
+}
+
+/**
+ * Generates a prebuilt rules package with the provided name, version, and rule assets.
+ *
+ * `output` allows to use any writable stream, e.g. a file stream or a PassThrough stream to get a Buffer.
+ */
+export function generatePrebuiltRulesPackage({
   packageName,
   packageSemver,
   prebuiltRuleAssets,
-}: CreatePrebuiltRulesPackageParams): AdmZip {
+  output,
+}: GeneratePrebuiltRulesPackageParams): void {
   validateSemver(packageSemver);
 
+  const prebuiltRulesPackage = archiver('zip', {
+    zlib: { level: 9 },
+    forceZip64: true,
+  });
+
+  prebuiltRulesPackage.pipe(output);
+
   const packageRootFolder = `${packageName}-${packageSemver}`;
-  const prebuiltRulesPackage = new AdmZip();
 
-  prebuiltRulesPackage.addFile(
-    `${packageRootFolder}/manifest.yml`,
-    createPackageManifest(packageName, packageSemver)
-  );
+  prebuiltRulesPackage.append(createPackageManifest(packageName, packageSemver), {
+    name: `${packageRootFolder}/manifest.yml`,
+  });
+
+  prebuiltRulesPackage.append(createReadmeFileContent(packageName, packageSemver), {
+    name: `${packageRootFolder}/docs/README.md`,
+  });
 
   for (const prebuiltRuleAsset of prebuiltRuleAssets) {
     const assetContent = JSON.stringify({
       attributes: prebuiltRuleAsset,
-      id: prebuiltRuleAsset.rule_id,
+      id: `${prebuiltRuleAsset.rule_id}_${prebuiltRuleAsset.version}`,
       type: 'security-rule',
     });
     const assetFileName = `${packageRootFolder}/kibana/security_rule/rules/${prebuiltRuleAsset.rule_id}_${prebuiltRuleAsset.version}.json`;
 
-    prebuiltRulesPackage.addFile(assetFileName, Buffer.from(assetContent, 'utf8'));
+    prebuiltRulesPackage.append(Buffer.from(assetContent, 'utf8'), { name: assetFileName });
   }
 
-  return prebuiltRulesPackage;
+  prebuiltRulesPackage.finalize();
 }
 
 function createPackageManifest(packageName: string, packageSemver: string): Buffer {
   const packageManifest: PackageSpecManifest = {
     name: packageName,
+    description: 'Prebuilt detection rules for Elastic Security',
     title: 'Prebuilt Security Detection Rules',
     version: packageSemver,
     owner: { github: 'elastic/protections' },
+    format_version: '3.0.0',
   };
 
   const yamlContent = dump(packageManifest, {
@@ -62,6 +83,12 @@ function createPackageManifest(packageName: string, packageSemver: string): Buff
   return Buffer.from(yamlContent, 'utf8');
 }
 
+function createReadmeFileContent(packageName: string, packageSemver: string): Buffer {
+  const readmeContent = `# Mock ${packageName} - ${packageSemver}`;
+
+  return Buffer.from(readmeContent, 'utf8');
+}
+
 function validateSemver(version: string): void {
   if (!semver.valid(version)) {
     throw new Error(

x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/create_prebuilt_rules_package.ts renamed to x-pack/solutions/security/packages/test-api-clients/prebuilt_rules_package_generation/generate_prebuilt_rules_package.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PassThrough } from 'stream';
+import type { GeneratePrebuiltRulesPackageBaseParams } from './generate_prebuilt_rules_package';
+import { generatePrebuiltRulesPackage } from './generate_prebuilt_rules_package';
+
+export async function generatePrebuiltRulesPackageBuffer({
+  packageName,
+  packageSemver,
+  prebuiltRuleAssets,
+}: GeneratePrebuiltRulesPackageBaseParams): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const output = new PassThrough();
+    const chunks: Uint8Array[] = [];
+
+    output.on('data', (chunk) => chunks.push(chunk));
+    output.on('end', () => resolve(Buffer.concat(chunks)));
+    output.on('error', reject);
+
+    generatePrebuiltRulesPackage({
+      packageName,
+      packageSemver,
+      prebuiltRuleAssets,
+      output,
+    });
+  });
+}

x-pack/solutions/security/packages/test-api-clients/prebuilt_rules_package_generation/generate_prebuilt_rules_package_buffer.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createWriteStream, mkdirSync } from 'fs';
+import { dirname } from 'path';
+import type { GeneratePrebuiltRulesPackageBaseParams } from './generate_prebuilt_rules_package';
+import { generatePrebuiltRulesPackage } from './generate_prebuilt_rules_package';
+
+interface GeneratePrebuiltRulesPackageZipFileParams extends GeneratePrebuiltRulesPackageBaseParams {
+  filePath: string;
+}
+
+export async function generatePrebuiltRulesPackageZipFile({
+  packageName,
+  packageSemver,
+  prebuiltRuleAssets,
+  filePath,
+}: GeneratePrebuiltRulesPackageZipFileParams): Promise<void> {
+  return new Promise((resolve, reject) => {
+    // Make sure the path exists
+    mkdirSync(dirname(filePath), { recursive: true });
+
+    const fileStream = createWriteStream(filePath);
+
+    fileStream.on('close', resolve);
+    fileStream.on('error', reject);
+
+    generatePrebuiltRulesPackage({
+      packageName,
+      packageSemver,
+      prebuiltRuleAssets,
+      output: fileStream,
+    });
+  });
+}

x-pack/solutions/security/packages/test-api-clients/prebuilt_rules_package_generation/generate_prebuilt_rules_package_zip_file.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './cli';
+export * from './generate_prebuilt_rules_package_buffer';
+export * from './generate_prebuilt_rules_package_zip_file';