Skip to content

Commit 7f23710

Browse files
bhapasbhapas
authored
[8.x] [Automatic Import] Remove pipeline tests from the package (#211223) (#211393)
# Backport This will backport the following commits from `main` to `8.x`: - [[Automatic Import] Remove pipeline tests from the package (#211223)](#211223) <!--- Backport version: 9.6.4 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Bharat Pasupula","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-02-17T09:28:57Z","message":"[Automatic Import] Remove pipeline tests from the package (#211223)\n\n## Summary\r\n\r\nThis PR removes unused pipeline tests from the packaging of integration.\r\n\r\nThe pipeline tests are not run today when the integration is built.\r\nHence removing them for now.","sha":"54b6e65a2047cdcc727032e1734170e7c8307aff","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:prev-minor","backport:prev-major","ci:build-cloud-image","ci:cloud-deploy","Team:Security-Scalability","Feature:AutomaticImport","v9.1.0"],"title":"[Automatic Import] Remove pipeline tests from the package","number":211223,"url":"https://github.com/elastic/kibana/pull/211223","mergeCommit":{"message":"[Automatic Import] Remove pipeline tests from the package (#211223)\n\n## Summary\r\n\r\nThis PR removes unused pipeline tests from the packaging of integration.\r\n\r\nThe pipeline tests are not run today when the integration is built.\r\nHence removing them for now.","sha":"54b6e65a2047cdcc727032e1734170e7c8307aff"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/211223","number":211223,"mergeCommit":{"message":"[Automatic Import] Remove pipeline tests from the package (#211223)\n\n## Summary\r\n\r\nThis PR removes unused pipeline tests from the packaging of integration.\r\n\r\nThe pipeline tests are not run today when the integration is built.\r\nHence removing them for now.","sha":"54b6e65a2047cdcc727032e1734170e7c8307aff"}}]}] BACKPORT-->
1 parent 7a5b6ea commit 7f23710

File tree

11 files changed

+106
-108
lines changed

11 files changed

+106
-108
lines changed

‎x-pack/platform/plugins/private/translations/translations/fr-FR.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48333,8 +48333,8 @@
4833348333
"xpack.upgradeAssistant.overview.pageDescription": "Préparez-vous pour la prochaine version de la Suite Elastic !",
4833448334
"xpack.upgradeAssistant.overview.pageTitle": "Assistant de mise à niveau",
4833548335
"xpack.upgradeAssistant.overview.snapshotRestoreLink": "Créer un snapshot",
48336-
"xpack.upgradeAssistant.overview.systemIndices.body.hiddenIndicesLink": "index masqués",
4833748336
"xpack.upgradeAssistant.overview.systemIndices.body": "Préparez les index système qui stockent des informations internes pour la mise à niveau. Cette préparation est requise uniquement lors des mises à jour des versions majeures. Tous les {hiddenIndicesLink} devant être réindexés seront affichés à l'étape suivante.",
48337+
"xpack.upgradeAssistant.overview.systemIndices.body.hiddenIndicesLink": "index masqués",
4833848338
"xpack.upgradeAssistant.overview.systemIndices.errorLabel": "Échec de la migration",
4833948339
"xpack.upgradeAssistant.overview.systemIndices.featureNameTableColumn": "Fonctionnalité",
4834048340
"xpack.upgradeAssistant.overview.systemIndices.flyoutCloseButtonLabel": "Fermer",

‎x-pack/platform/plugins/private/translations/translations/ja-JP.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48179,8 +48179,8 @@
4817948179
"xpack.upgradeAssistant.overview.pageDescription": "次のバージョンのElastic Stackをお待ちください。",
4818048180
"xpack.upgradeAssistant.overview.pageTitle": "アップグレードアシスタント",
4818148181
"xpack.upgradeAssistant.overview.snapshotRestoreLink": "スナップショットの作成",
48182-
"xpack.upgradeAssistant.overview.systemIndices.body.hiddenIndicesLink": "非表示のインデックス",
4818348182
"xpack.upgradeAssistant.overview.systemIndices.body": "アップグレードの内部情報を格納するシステムインデックスを準備します。これはメジャーバージョンアップグレード中にのみ必要です。インデックスを再作成する必要があるすべての{hiddenIndicesLink}は次のステップで表示されます。",
48183+
"xpack.upgradeAssistant.overview.systemIndices.body.hiddenIndicesLink": "非表示のインデックス",
4818448184
"xpack.upgradeAssistant.overview.systemIndices.errorLabel": "移行失敗",
4818548185
"xpack.upgradeAssistant.overview.systemIndices.featureNameTableColumn": "機能",
4818648186
"xpack.upgradeAssistant.overview.systemIndices.flyoutCloseButtonLabel": "閉じる",

‎x-pack/platform/plugins/private/translations/translations/zh-CN.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48286,8 +48286,8 @@
4828648286
"xpack.upgradeAssistant.overview.pageDescription": "准备使用下一版 Elastic Stack！",
4828748287
"xpack.upgradeAssistant.overview.pageTitle": "升级助手",
4828848288
"xpack.upgradeAssistant.overview.snapshotRestoreLink": "创建快照",
48289-
"xpack.upgradeAssistant.overview.systemIndices.body.hiddenIndicesLink": "隐藏的索引",
4829048289
"xpack.upgradeAssistant.overview.systemIndices.body": "为升级准备存储内部信息的系统索引。仅在主要版本升级期间需要此项。将在下一步中显示任何需要重新索引的{hiddenIndicesLink}。",
48290+
"xpack.upgradeAssistant.overview.systemIndices.body.hiddenIndicesLink": "隐藏的索引",
4829148291
"xpack.upgradeAssistant.overview.systemIndices.errorLabel": "迁移失败",
4829248292
"xpack.upgradeAssistant.overview.systemIndices.featureNameTableColumn": "功能",
4829348293
"xpack.upgradeAssistant.overview.systemIndices.flyoutCloseButtonLabel": "关闭",

‎x-pack/platform/plugins/shared/integration_assistant/common/constants.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export enum GenerationErrorCode {
3636
UNSUPPORTED_LOG_SAMPLES_FORMAT = 'unsupported-log-samples-format',
3737
UNPARSEABLE_CSV_DATA = 'unparseable-csv-data',
3838
CEF_ERROR = 'cef-not-supported',
39+
BUILD_INTEGRATION_ERROR = 'build-integration-error',
3940
}
4041

4142
// Size limits

‎x-pack/platform/plugins/shared/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,12 @@ export const GENERATION_ERROR_TRANSLATION: Record<
232232
defaultMessage:
233233
'CEF format detected. Please decode the CEF logs into JSON format using filebeat decode_cef processor.',
234234
}),
235+
[GenerationErrorCode.BUILD_INTEGRATION_ERROR]: i18n.translate(
236+
'xpack.integrationAssistant.errors.buildIntegrationError',
237+
{
238+
defaultMessage: 'An error occurred while building the integration package. Please try again.',
239+
}
240+
),
235241
[GenerationErrorCode.UNPARSEABLE_CSV_DATA]: (attributes) => {
236242
if (
237243
attributes.underlyingMessages !== undefined &&

‎x-pack/platform/plugins/shared/integration_assistant/server/graphs/ecs/graph.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ describe('EcsGraph', () => {
8585
throw Error(`getEcsGraph threw an error: ${error}`);
8686
}
8787

88-
expect(response.results).toStrictEqual(ecsMappingExpectedResults);
88+
expect(response.results).toEqual(ecsMappingExpectedResults);
8989

9090
// Check if the functions were called
9191
expect(handleEcsMapping).toHaveBeenCalled();

‎x-pack/platform/plugins/shared/integration_assistant/server/integration_builder/build_integration.ts‎

Lines changed: 42 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import { createFieldMapping } from './fields';
1919
import { createPipeline } from './pipeline';
2020
import { createReadme } from './readme_files';
2121
import { Field, flattenObjectsList } from '../util/samples';
22+
import { BuildIntegrationError } from '../lib/errors/build_integration_error';
2223

2324
const initialVersion = '1.0.0';
2425

@@ -36,46 +37,55 @@ export async function buildPackage(integration: Integration): Promise<Buffer> {
3637
configureNunjucks();
3738

3839
if (!isValidName(integration.name)) {
39-
throw new Error(
40+
throw new BuildIntegrationError(
4041
`Invalid integration name: ${integration.name}, Should only contain letters, numbers and underscores`
4142
);
4243
}
43-
4444
const workingDir = joinPath(getDataPath(), `integration-assistant-${generateUniqueId()}`);
45-
const packageDirectoryName = `${integration.name}-${initialVersion}`;
46-
const packageDir = createDirectories(workingDir, integration, packageDirectoryName);
47-
48-
const dataStreamsDir = joinPath(packageDir, 'data_stream');
49-
const fieldsPerDatastream = integration.dataStreams.map((dataStream) => {
50-
const dataStreamName = dataStream.name;
51-
if (!isValidName(dataStreamName)) {
52-
throw new Error(
53-
`Invalid datastream name: ${dataStreamName}, Should only contain letters, numbers and underscores`
45+
try {
46+
const packageDirectoryName = `${integration.name}-${initialVersion}`;
47+
const packageDir = createDirectories(workingDir, integration, packageDirectoryName);
48+
49+
const dataStreamsDir = joinPath(packageDir, 'data_stream');
50+
const fieldsPerDatastream = integration.dataStreams.map((dataStream) => {
51+
const dataStreamName = dataStream.name;
52+
if (!isValidName(dataStreamName)) {
53+
throw new BuildIntegrationError(
54+
`Invalid datastream name: ${dataStreamName}, Should only contain letters, numbers and underscores`
55+
);
56+
}
57+
const specificDataStreamDir = joinPath(dataStreamsDir, dataStreamName);
58+
59+
const dataStreamFields = createDataStream(
60+
integration.name,
61+
specificDataStreamDir,
62+
dataStream
63+
);
64+
createAgentInput(specificDataStreamDir, dataStream.inputTypes);
65+
createPipeline(specificDataStreamDir, dataStream.pipeline);
66+
const fields = createFieldMapping(
67+
integration.name,
68+
dataStreamName,
69+
specificDataStreamDir,
70+
dataStream.docs
5471
);
55-
}
56-
const specificDataStreamDir = joinPath(dataStreamsDir, dataStreamName);
57-
58-
const dataStreamFields = createDataStream(integration.name, specificDataStreamDir, dataStream);
59-
createAgentInput(specificDataStreamDir, dataStream.inputTypes);
60-
createPipeline(specificDataStreamDir, dataStream.pipeline);
61-
const fields = createFieldMapping(
62-
integration.name,
63-
dataStreamName,
64-
specificDataStreamDir,
65-
dataStream.docs
66-
);
6772

68-
return {
69-
datastream: dataStreamName,
70-
fields: mergeAndSortFields(fields, dataStreamFields),
71-
};
72-
});
73+
return {
74+
datastream: dataStreamName,
75+
fields: mergeAndSortFields(fields, dataStreamFields),
76+
};
77+
});
7378

74-
createReadme(packageDir, integration.name, integration.dataStreams, fieldsPerDatastream);
75-
const zipBuffer = await createZipArchive(integration, workingDir, packageDirectoryName);
79+
createReadme(packageDir, integration.name, integration.dataStreams, fieldsPerDatastream);
80+
const zipBuffer = await createZipArchive(integration, workingDir, packageDirectoryName);
7681

77-
removeDirSync(workingDir);
78-
return zipBuffer;
82+
removeDirSync(workingDir);
83+
return zipBuffer;
84+
} catch (error) {
85+
throw new BuildIntegrationError('Building the Integration failed');
86+
} finally {
87+
removeDirSync(workingDir);
88+
}
7989
}
8090
export function isValidName(input: string): boolean {
8191
return input.length > 0 && NAME_REGEX_PATTERN.test(input);

‎x-pack/platform/plugins/shared/integration_assistant/server/integration_builder/data_stream.test.ts‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -85,17 +85,6 @@ describe('createDataStream', () => {
8585
// dataStream files
8686
expect(copySync).toHaveBeenCalledWith(expect.any(String), `${dataStreamPath}/fields`);
8787

88-
// test files
89-
expect(ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/_dev/test/pipeline`);
90-
expect(copySync).toHaveBeenCalledWith(
91-
expect.any(String),
92-
`${dataStreamPath}/_dev/test/pipeline/test-common-config.yml`
93-
);
94-
expect(createSync).toHaveBeenCalledWith(
95-
`${dataStreamPath}/_dev/test/pipeline/test-${packageName}-datastream-1.log`,
96-
samples
97-
);
98-
9988
// // Manifest files
10089
expect(createSync).toHaveBeenCalledWith(`${dataStreamPath}/manifest.yml`, undefined);
10190
expect(render).toHaveBeenCalledWith(`filestream_manifest.yml.njk`, expect.anything());

‎x-pack/platform/plugins/shared/integration_assistant/server/integration_builder/data_stream.ts‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@ export function createDataStream(
2727

2828
ensureDirSync(specificDataStreamDir);
2929
const fields = createDataStreamFolders(specificDataStreamDir, pipelineDir);
30-
createPipelineTests(specificDataStreamDir, dataStream.rawSamples, packageName, dataStreamName);
31-
3230
const dataStreams: string[] = [];
3331
for (const inputType of dataStream.inputTypes) {
3432
let mappedValues = {
@@ -103,27 +101,3 @@ function loadFieldsFromFiles(sourcePath: string, files: string[]): Field[] {
103101
return safeLoad(content) as Field[];
104102
});
105103
}
106-
107-
function createPipelineTests(
108-
specificDataStreamDir: string,
109-
rawSamples: string[],
110-
packageName: string,
111-
dataStreamName: string
112-
): void {
113-
const pipelineTestTemplatesDir = joinPath(__dirname, '../templates/pipeline_tests');
114-
const pipelineTestsDir = joinPath(specificDataStreamDir, '_dev/test/pipeline');
115-
ensureDirSync(pipelineTestsDir);
116-
const items = listDirSync(pipelineTestTemplatesDir);
117-
for (const item of items) {
118-
const s = joinPath(pipelineTestTemplatesDir, item);
119-
const d = joinPath(pipelineTestsDir, item.replaceAll('_', '-'));
120-
copySync(s, d);
121-
}
122-
const formattedPackageName = packageName.replace(/_/g, '-');
123-
const formattedDataStreamName = dataStreamName.replace(/_/g, '-');
124-
const testFileName = joinPath(
125-
pipelineTestsDir,
126-
`test-${formattedPackageName}-${formattedDataStreamName}.log`
127-
);
128-
createSync(testFileName, rawSamples.join('\n'));
129-
}

‎x-pack/platform/plugins/shared/integration_assistant/server/lib/errors/build_integration_error.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { KibanaResponseFactory } from '@kbn/core/server';
9+
import { ErrorThatHandlesItsOwnResponse } from './types';
10+
import { GenerationErrorCode } from '../../../common/constants';
11+
12+
export class BuildIntegrationError extends Error implements ErrorThatHandlesItsOwnResponse {
13+
private readonly errorCode: GenerationErrorCode = GenerationErrorCode.BUILD_INTEGRATION_ERROR;
14+
15+
public sendResponse(res: KibanaResponseFactory) {
16+
return res.badRequest({
17+
body: { message: this.message, attributes: { errorCode: this.errorCode } },
18+
});
19+
}
20+
}

0 commit comments

Comments
 (0)