[OAS] Reduce CI time for OAS extraction (elastic#235336)

Author: jloleysens

packages/kbn-capture-oas-snapshot-cli/src/capture_oas_snapshot.ts

@@ -8,17 +8,13 @@
  */
 
 import fs from 'node:fs/promises';
-import { encode } from 'node:querystring';
 import type { ChildProcess } from 'node:child_process';
-import fetch from 'node-fetch';
 import * as Rx from 'rxjs';
 import { startTSWorker } from '@kbn/dev-utils';
-import { createTestEsCluster } from '@kbn/test';
 import type { ToolingLog } from '@kbn/tooling-log';
-import { createTestServerlessInstances } from '@kbn/core-test-helpers-kbn-server';
 import type { Result } from './kibana_worker';
 import { sortAndPrettyPrint } from './run_capture_oas_snapshot_cli';
-import { buildFlavourEnvArgName } from './common';
+import { buildFlavourEnvArgName, filtersJsonEnvArgName } from './common';
 
 interface CaptureOasSnapshotArgs {
   log: ToolingLog;
@@ -42,68 +38,51 @@ export async function captureOasSnapshot({
   outputFile,
 }: CaptureOasSnapshotArgs): Promise<void> {
   const { excludePathsMatching = [], pathStartsWith } = filters;
-  // internal consts
-  const port = 5622;
   // We are only including /api/status for now
   excludePathsMatching.push(
     '/{path*}',
     // Our internal asset paths
     '/XXXXXXXXXXXX/'
   );
 
-  let esCluster: undefined | { stop(): Promise<void> };
-  let kbWorker: undefined | ChildProcess;
-
   try {
-    log.info('Starting es...');
-    esCluster = await log.indent(4, async () => {
-      if (buildFlavour === 'serverless') {
-        const { startES } = createTestServerlessInstances();
-        return await startES();
-      }
-      const cluster = createTestEsCluster({ log });
-      await cluster.start();
-      return { stop: () => cluster.cleanup() };
-    });
-
     log.info('Starting Kibana...');
-    kbWorker = await log.indent(4, async () => {
+
+    const currentOas = await log.indent(4, async () => {
       log.info('Loading core with all plugins enabled so that we can capture OAS for all...');
-      const { msg$, proc } = startTSWorker<Result>({
-        log,
-        src: require.resolve('./kibana_worker'),
-        env: { ...process.env, [buildFlavourEnvArgName]: buildFlavour },
-      });
-      await Rx.firstValueFrom(
-        msg$.pipe(
-          Rx.map((msg) => {
-            if (msg !== 'ready')
-              throw new Error(`received unexpected message from worker (expected "ready"): ${msg}`);
-          })
-        )
-      );
-      return proc;
+      let proc: undefined | ChildProcess;
+      try {
+        const worker = startTSWorker<Result>({
+          log,
+          src: require.resolve('./kibana_worker'),
+          env: {
+            ...process.env,
+            [buildFlavourEnvArgName]: buildFlavour,
+            [filtersJsonEnvArgName]: JSON.stringify({
+              access: 'public',
+              version: '2023-10-31', // hard coded for now, we can make this configurable later
+              pathStartsWith,
+              excludePathsMatching,
+            }),
+          },
+        });
+        proc = worker.proc;
+        return await Rx.firstValueFrom(
+          worker.msg$.pipe(
+            Rx.map((result) => {
+              try {
+                return JSON.parse(result);
+              } catch (e) {
+                throw new Error('expected JSON, received:' + result);
+              }
+            })
+          )
+        );
+      } finally {
+        proc?.kill('SIGKILL');
+      }
     });
 
-    const qs = encode({
-      access: 'public',
-      version: '2023-10-31', // hard coded for now, we can make this configurable later
-      pathStartsWith,
-      excludePathsMatching,
-    });
-    const url = `http://localhost:${port}/api/oas?${qs}`;
-    log.info(`Fetching OAS at ${url}...`);
-    const result = await fetch(url, {
-      headers: {
-        'kbn-xsrf': 'kbn-oas-snapshot',
-        authorization: `Basic ${Buffer.from('elastic:changeme').toString('base64')}`,
-      },
-    });
-    if (result.status !== 200) {
-      log.error(`Failed to fetch OAS: ${JSON.stringify(result, null, 2)}`);
-      throw new Error(`Failed to fetch OAS: ${result.status}`);
-    }
-    const currentOas = await result.json();
     log.info(`Recieved OAS, writing to ${outputFile}...`);
     if (update) {
       await fs.writeFile(outputFile, sortAndPrettyPrint(currentOas));
@@ -117,10 +96,7 @@ export async function captureOasSnapshot({
       );
     }
   } catch (err) {
-    log.error(`Failed to capture OAS: ${JSON.stringify(err, null, 2)}`);
+    log.error(`Failed to capture OAS: ${err}`);
     throw err;
-  } finally {
-    kbWorker?.kill('SIGILL');
-    await esCluster?.stop();
   }
 }

packages/kbn-capture-oas-snapshot-cli/src/common.ts

@@ -7,4 +7,5 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+export const filtersJsonEnvArgName = 'CAPTURE_OAS_SNAPSHOT_WORKER_FILTERS_JSON';
 export const buildFlavourEnvArgName = 'CAPTURE_OAS_SNAPSHOT_WORKER_BUILD_FLAVOR';

packages/kbn-capture-oas-snapshot-cli/src/kibana_worker.ts

@@ -9,18 +9,22 @@
 
 import {
   createRootWithCorePlugins,
-  createTestServerlessInstances,
+  createServerlessKibana,
 } from '@kbn/core-test-helpers-kbn-server';
 import { set } from '@kbn/safer-lodash-set';
+import type { Root } from '@kbn/core-root-server-internal';
 import { PLUGIN_SYSTEM_ENABLE_ALL_PLUGINS_CONFIG_PATH } from '@kbn/core-plugins-server-internal/src/constants';
-import { buildFlavourEnvArgName } from './common';
+import { buildFlavourEnvArgName, filtersJsonEnvArgName } from './common';
 
-export type Result = 'ready';
+export type Result = string;
 
 (async () => {
   if (!process.send) {
     throw new Error('worker must be run in a node.js fork');
   }
+  const filtersString = process.env[filtersJsonEnvArgName];
+  if (!filtersString) throw new Error(`env arg ${filtersJsonEnvArgName} must be provided`);
+  const filters = JSON.parse(filtersString);
   const buildFlavour = process.env[buildFlavourEnvArgName];
   if (!buildFlavour) throw new Error(`env arg ${buildFlavourEnvArgName} must be provided`);
 
@@ -52,25 +56,25 @@ export type Result = 'ready';
     watch: false,
   };
 
+  let root: Root;
+
+  set(settings, 'migrations.skip', true);
+  set(settings, 'elasticsearch.skipStartupConnectionCheck', true);
+
   if (serverless) {
     // Satisfy spaces config for serverless:
     set(settings, 'xpack.spaces.allowFeatureVisibility', false);
     set(settings, 'xpack.spaces.allowSolutionVisibility', false);
-    const { startKibana } = createTestServerlessInstances({
-      kibana: { settings, cliArgs },
-    });
-    await startKibana();
+    root = createServerlessKibana(settings, cliArgs);
   } else {
-    const root = createRootWithCorePlugins(settings, cliArgs);
-    await root.preboot();
-    await root.setup();
-    await root.start();
+    root = createRootWithCorePlugins(settings, cliArgs);
   }
-
-  const result: Result = 'ready';
-
-  process.send(result);
+  await root.preboot();
+  await root.setup();
+  const { http } = await root.start();
+  const oas = await http.generateOas({ baseUrl: 'http://localhost:5622', filters });
+  process.send(JSON.stringify(oas));
 })().catch((error) => {
-  process.stderr.write(`UNHANDLED ERROR: ${error.stack}`);
+  process.stderr.write(`UNHANDLED ERROR: ${error}`);
   process.exit(1);
 });

packages/kbn-capture-oas-snapshot-cli/tsconfig.json

@@ -19,8 +19,8 @@
     "@kbn/safer-lodash-set",
     "@kbn/core-plugins-server-internal",
     "@kbn/dev-cli-runner",
-    "@kbn/test",
     "@kbn/dev-utils",
     "@kbn/tooling-log",
+    "@kbn/core-root-server-internal",
   ]
 }

packages/kbn-check-saved-objects-cli/src/compatibility/extract_mappings_from_plugins.ts

@@ -35,7 +35,7 @@ export async function extractMappingsFromPlugins(
     msg$.pipe(
       Rx.map((result) => {
         log.debug('message received from worker', result);
-        proc.kill('SIGILL');
+        proc.kill('SIGKILL');
         return result.mappings;
       }),
       Rx.defaultIfEmpty(undefined)

packages/kbn-check-saved-objects-cli/src/mappings_additions/extract_field_lists_from_plugins.ts

@@ -32,7 +32,7 @@ export async function extractFieldListsFromPlugins(log: SomeDevLog): Promise<Res
     msg$.pipe(
       Rx.map((outcome) => {
         log.debug('message received from worker', outcome);
-        proc.kill('SIGILL');
+        proc.kill('SIGKILL');
         return outcome;
       }),
       Rx.defaultIfEmpty(undefined)

src/core/packages/http/server-internal/src/http_service.ts

@@ -48,6 +48,7 @@ import type {
   InternalHttpServicePreboot,
   InternalHttpServiceSetup,
   InternalHttpServiceStart,
+  GenerateOasArgs,
 } from './types';
 import { registerCoreHandlers } from './register_lifecycle_handlers';
 import type { ExternalUrlConfigType } from './external_url';
@@ -226,11 +227,12 @@ export class HttpService
     return this.internalSetup;
   }
 
-  // this method exists because we need the start contract to create the `CoreStart` used to start
+  // this method exists because we need the start contract to create `CoreStart` used to start
   // the `plugin` and `legacy` services.
   public getStartContract(): InternalHttpServiceStart {
     return {
       ...pick(this.internalSetup!, ['auth', 'basePath', 'getServerInfo', 'staticAssets']),
+      generateOas: (args: GenerateOasArgs) => this.generateOas(args),
       isListening: () => this.httpServer.isListening(),
     };
   }
@@ -259,6 +261,24 @@ export class HttpService
     return this.getStartContract();
   }
 
+  private generateOas({ pluginId, baseUrl, filters }: GenerateOasArgs) {
+    // Potentially quite expensive
+    return firstValueFrom(
+      of(1).pipe(
+        HttpService.generateOasSemaphore.acquire(),
+        mergeMap(async () => {
+          return generateOpenApiDocument(this.httpServer.getRouters({ pluginId }), {
+            baseUrl,
+            title: 'Kibana HTTP APIs',
+            version: '0.0.0', // TODO get a better version here
+            filters,
+            env: { serverless: this.env.packageInfo.buildFlavor === 'serverless' },
+          });
+        })
+      )
+    );
+  }
+
   private registerOasApi(config: HttpConfig) {
     const basePath = this.internalSetup?.basePath;
     const server = this.internalSetup?.server;
@@ -305,32 +325,19 @@ export class HttpService
         } catch (e) {
           return h.response({ message: e.message }).code(400);
         }
-        return await firstValueFrom(
-          of(1).pipe(
-            HttpService.generateOasSemaphore.acquire(),
-            mergeMap(async () => {
-              try {
-                // Potentially quite expensive
-                const result = await generateOpenApiDocument(
-                  this.httpServer.getRouters({ pluginId: query.pluginId }),
-                  {
-                    baseUrl,
-                    title: 'Kibana HTTP APIs',
-                    version: '0.0.0', // TODO get a better version here
-                    filters,
-                    env: { serverless: this.env.packageInfo.buildFlavor === 'serverless' },
-                  }
-                );
-                return h.response(result);
-              } catch (e) {
-                this.log.error(e);
-                return h.response({ message: e.message }).code(500);
-              }
-            })
-          )
-        );
+        try {
+          const result = await this.generateOas({
+            baseUrl,
+            filters,
+            pluginId: query.pluginId,
+          });
+          return h.response(result);
+        } catch (e) {
+          return h.response({ message: e.message }).code(500);
+        }
       },
       options: {
+        auth: false,
         app: {
           access: 'public',
           security: {
@@ -340,7 +347,6 @@ export class HttpService
             },
           },
         },
-        auth: false,
         cache: {
           privacy: 'public',
           otherwise: 'must-revalidate',

src/core/packages/http/server-internal/src/types.ts

@@ -20,6 +20,7 @@ import type {
 } from '@kbn/core-http-server';
 import type { CoreKibanaRequest } from '@kbn/core-http-router-server-internal';
 import type { PostValidationMetadata } from '@kbn/core-http-server';
+import type { GenerateOpenApiDocumentOptionsFilters } from '@kbn/router-to-openapispec';
 import type { HttpServerSetup } from './http_server';
 import type { ExternalUrlConfig } from './external_url';
 import type { InternalStaticAssets } from './static_assets';
@@ -81,6 +82,14 @@ export interface InternalHttpServiceSetup
 /** @internal */
 export interface InternalHttpServiceStart extends Omit<HttpServiceStart, 'staticAssets'> {
   staticAssets: InternalStaticAssets;
+  generateOas: (args: GenerateOasArgs) => Promise<object>;
   /** Indicates if the http server is listening on the configured port */
   isListening: () => boolean;
 }
+
+/** @internal */
+export interface GenerateOasArgs {
+  pluginId?: string;
+  baseUrl: string;
+  filters?: GenerateOpenApiDocumentOptionsFilters;
+}

src/core/packages/http/server-mocks/src/http_service.mock.ts

@@ -253,6 +253,7 @@ const createInternalStartContractMock = () => {
   const basePath = createBasePathMock();
   const mock: InternalHttpServiceStartMock = {
     ...createStartContractMock(),
+    generateOas: jest.fn(),
     staticAssets: createInternalStaticAssetsMock(basePath),
     isListening: jest.fn(),
   };

src/core/test-helpers/kbn-server/index.ts

@@ -14,6 +14,7 @@ export {
   createRootWithCorePlugins,
   createTestServers,
   createTestServerlessInstances,
+  createServerlessKibana,
   request,
 } from './src';