Skip to content

Commit 903c1f0

Browse files
csrwayneseymour
andauthored
[8.18] [FTR] Move Scout reporter events upload to pipeline scripts (#220277) (#236293)
# Backport This will backport the following commits from `main` to `8.18`: - [[FTR] Move Scout reporter events upload to pipeline scripts (#220277)](#220277) <!--- Backport version: 10.1.0 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Tre","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-07-28T13:36:46Z","message":"[FTR] Move Scout reporter events upload to pipeline scripts (#220277)\n\nThis PR:\n\n* Refactors the existing `upload-events` Scout CLI command to centralize\nand streamline the event log upload logic.\n* Introduces a new helper function `uploadAllEventsFromPath` that:\n* Accepts a path to either a single `.ndjson` event file or a directory.\n* Recursively scans directories to find and upload all `.ndjson` event\nfiles.\n* Updates the CLI logic to:\n* Default to the `SCOUT_REPORT_OUTPUT_ROOT` path\n(`<KIBANA_ROOT>/.scout/reports`) when `--eventLogPath` is not provided.\n* Log and optionally suppress errors using the `--dontFailOnError` flag,\nallowing flexible handling in CI pipelines.\n\n### Clarifications on `--eventLogPath`:\n\n* The `--eventLogPath` flag supports both file and directory paths.\n * If the provided does not exist, an error is thrown.\n* If the path points to a file, and the file doesn't end with `.ndjson`,\nan error is thrown.\n* If the path points to a directory, and the directory doesn't contain\nany file ending with `.ndjson`, we exit early and log a warning.\n* If the flag is omitted the CLI defaults to the\n`SCOUT_REPORT_OUTPUT_ROOT` directory.\n\n---------\n\nCo-authored-by: Cesare de Cal <[email protected]>","sha":"3402b5e0fc939300fff33769210b72f96fc41250","branchLabelMapping":{"^v9.2.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:skip","FTR","v9.2.0"],"title":"[FTR] Move Scout reporter events upload to pipeline scripts","number":220277,"url":"https://github.com/elastic/kibana/pull/220277","mergeCommit":{"message":"[FTR] Move Scout reporter events upload to pipeline scripts (#220277)\n\nThis PR:\n\n* Refactors the existing `upload-events` Scout CLI command to centralize\nand streamline the event log upload logic.\n* Introduces a new helper function `uploadAllEventsFromPath` that:\n* Accepts a path to either a single `.ndjson` event file or a directory.\n* Recursively scans directories to find and upload all `.ndjson` event\nfiles.\n* Updates the CLI logic to:\n* Default to the `SCOUT_REPORT_OUTPUT_ROOT` path\n(`<KIBANA_ROOT>/.scout/reports`) when `--eventLogPath` is not provided.\n* Log and optionally suppress errors using the `--dontFailOnError` flag,\nallowing flexible handling in CI pipelines.\n\n### Clarifications on `--eventLogPath`:\n\n* The `--eventLogPath` flag supports both file and directory paths.\n * If the provided does not exist, an error is thrown.\n* If the path points to a file, and the file doesn't end with `.ndjson`,\nan error is thrown.\n* If the path points to a directory, and the directory doesn't contain\nany file ending with `.ndjson`, we exit early and log a warning.\n* If the flag is omitted the CLI defaults to the\n`SCOUT_REPORT_OUTPUT_ROOT` directory.\n\n---------\n\nCo-authored-by: Cesare de Cal <[email protected]>","sha":"3402b5e0fc939300fff33769210b72f96fc41250"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.2.0","branchLabelMappingKey":"^v9.2.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/220277","number":220277,"mergeCommit":{"message":"[FTR] Move Scout reporter events upload to pipeline scripts (#220277)\n\nThis PR:\n\n* Refactors the existing `upload-events` Scout CLI command to centralize\nand streamline the event log upload logic.\n* Introduces a new helper function `uploadAllEventsFromPath` that:\n* Accepts a path to either a single `.ndjson` event file or a directory.\n* Recursively scans directories to find and upload all `.ndjson` event\nfiles.\n* Updates the CLI logic to:\n* Default to the `SCOUT_REPORT_OUTPUT_ROOT` path\n(`<KIBANA_ROOT>/.scout/reports`) when `--eventLogPath` is not provided.\n* Log and optionally suppress errors using the `--dontFailOnError` flag,\nallowing flexible handling in CI pipelines.\n\n### Clarifications on `--eventLogPath`:\n\n* The `--eventLogPath` flag supports both file and directory paths.\n * If the provided does not exist, an error is thrown.\n* If the path points to a file, and the file doesn't end with `.ndjson`,\nan error is thrown.\n* If the path points to a directory, and the directory doesn't contain\nany file ending with `.ndjson`, we exit early and log a warning.\n* If the flag is omitted the CLI defaults to the\n`SCOUT_REPORT_OUTPUT_ROOT` directory.\n\n---------\n\nCo-authored-by: Cesare de Cal <[email protected]>","sha":"3402b5e0fc939300fff33769210b72f96fc41250"}}]}] BACKPORT--> --------- Co-authored-by: Tre <[email protected]>
1 parent 4c32995 commit 903c1f0

File tree

8 files changed

+332
-75
lines changed

8 files changed

+332
-75
lines changed

.buildkite/scripts/steps/code_coverage/jest_parallel.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,5 +86,9 @@ mv target/kibana-coverage/jest-combined/coverage-final.json \
8686
echo "--- Jest [$TEST_TYPE] configs complete"
8787
printf "%s\n" "${results[@]}"
8888

89+
# Scout reporter
90+
echo "--- Upload Scout reporter events to AppEx QA's team cluster"
91+
node scripts/scout upload-events --dontFailOnError
92+
8993
# Force exit 0 to ensure the next build step starts.
9094
exit 0

.buildkite/scripts/steps/test/ftr_configs.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,4 +114,8 @@ echo "--- FTR configs complete"
114114
printf "%s\n" "${results[@]}"
115115
echo ""
116116

117+
# Scout reporter
118+
echo "--- Upload Scout reporter events to AppEx QA's team cluster"
119+
node scripts/scout upload-events --dontFailOnError
120+
117121
exit $exitCode

.buildkite/scripts/steps/test/jest_parallel.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,4 +114,8 @@ echo "--- Jest configs complete"
114114
printf "%s\n" "${results[@]}"
115115
echo ""
116116

117+
# Scout reporter
118+
echo "--- Upload Scout reporter events to AppEx QA's team cluster"
119+
node scripts/scout upload-events --dontFailOnError
120+
117121
exit $exitCode
Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
import fs from 'node:fs';
11+
12+
import { uploadAllEventsFromPath } from './upload_events';
13+
import { ToolingLog } from '@kbn/tooling-log';
14+
15+
jest.mock('node:fs');
16+
17+
jest.mock('@kbn/scout-info', () => ({
18+
SCOUT_REPORT_OUTPUT_ROOT: 'scout/reports/directory',
19+
}));
20+
21+
jest.mock('../helpers/elasticsearch', () => ({
22+
getValidatedESClient: jest.fn(),
23+
}));
24+
25+
const mockAddEventsFromFile = jest.fn();
26+
27+
jest.mock('../reporting/report/events', () => ({
28+
ScoutReportDataStream: jest.fn().mockImplementation(() => {
29+
return {
30+
addEventsFromFile: mockAddEventsFromFile,
31+
};
32+
}),
33+
}));
34+
35+
describe('uploadAllEventsFromPath', () => {
36+
let log: jest.Mocked<ToolingLog>;
37+
38+
beforeEach(() => {
39+
log = {
40+
info: jest.fn(),
41+
error: jest.fn(),
42+
warning: jest.fn(),
43+
} as any;
44+
});
45+
46+
afterEach(() => {
47+
jest.clearAllMocks();
48+
});
49+
50+
it('should throw an error if the provided eventLogPath does not exist', async () => {
51+
jest.spyOn(fs, 'existsSync').mockReturnValue(false);
52+
53+
await expect(
54+
uploadAllEventsFromPath('non_existent_path', {
55+
esURL: 'esURL',
56+
esAPIKey: 'esAPIKey',
57+
verifyTLSCerts: true,
58+
log,
59+
})
60+
).rejects.toThrowErrorMatchingInlineSnapshot(
61+
`"The provided event log path 'non_existent_path' does not exist."`
62+
);
63+
});
64+
65+
it('should throw an error if the provided eventLogPath is a file and it does not end with .ndjson', async () => {
66+
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
67+
jest.spyOn(fs, 'statSync').mockReturnValue({
68+
isDirectory: () => false,
69+
} as unknown as fs.Stats);
70+
71+
await expect(
72+
uploadAllEventsFromPath('invalid_event_log.txt', {
73+
esURL: 'esURL',
74+
esAPIKey: 'esAPIKey',
75+
verifyTLSCerts: true,
76+
log,
77+
})
78+
).rejects.toThrowErrorMatchingInlineSnapshot(
79+
`"The provided event log file 'invalid_event_log.txt' must end with .ndjson."`
80+
);
81+
});
82+
83+
it('should log a warning if the provided eventLogPath is a directory and it does not contain any .ndjson file', async () => {
84+
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
85+
86+
// Simulate directory contents: 1 .txt file
87+
(fs.readdirSync as jest.Mock).mockImplementation((directoryPath: string) => {
88+
if (directoryPath === 'mocked_directory') {
89+
return ['not_events.txt'];
90+
}
91+
return [];
92+
});
93+
94+
(fs.statSync as jest.Mock).mockImplementation((filePath: string) => {
95+
if (filePath === 'mocked_directory') {
96+
return { isDirectory: () => true, isFile: () => false };
97+
}
98+
return { isDirectory: () => false, isFile: () => true };
99+
});
100+
101+
await uploadAllEventsFromPath('mocked_directory', {
102+
esURL: 'esURL',
103+
esAPIKey: 'esAPIKey',
104+
verifyTLSCerts: true,
105+
log,
106+
});
107+
108+
expect(log.warning).toHaveBeenCalledWith(
109+
`No .ndjson event log files found in directory 'mocked_directory'.`
110+
);
111+
});
112+
113+
it('should upload the event log file if the provided eventLogPath if a file and ends with .ndjson', async () => {
114+
jest.spyOn(fs, 'statSync').mockReturnValue({
115+
isDirectory: () => true,
116+
} as unknown as fs.Stats);
117+
jest.spyOn(fs, 'readdirSync').mockReturnValue(['file.txt' as unknown as fs.Dirent]);
118+
119+
// assume the provided event log path exists
120+
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
121+
122+
// the provided event log path is not a directory
123+
jest.spyOn(fs, 'statSync').mockReturnValue({
124+
isDirectory: () => false,
125+
} as unknown as fs.Stats);
126+
127+
await uploadAllEventsFromPath('existing_event_log.ndjson', {
128+
esURL: 'esURL',
129+
esAPIKey: 'esAPIKey',
130+
verifyTLSCerts: true,
131+
log,
132+
});
133+
134+
expect(mockAddEventsFromFile).toHaveBeenCalledWith('existing_event_log.ndjson');
135+
});
136+
137+
it('should find event log files recursively', async () => {
138+
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
139+
140+
(fs.readdirSync as jest.Mock).mockImplementation((directoryPath: string) => {
141+
if (directoryPath === 'mocked_directory') {
142+
return ['sub_directory', 'no_events_here.txt'];
143+
}
144+
145+
if (directoryPath === 'mocked_directory/sub_directory') {
146+
return ['events.ndjson'];
147+
}
148+
149+
return [];
150+
});
151+
152+
(fs.statSync as jest.Mock).mockImplementation((filePath: string) => {
153+
if (filePath === 'mocked_directory') {
154+
return { isDirectory: () => true, isFile: () => false } as fs.Stats;
155+
}
156+
if (filePath === 'mocked_directory/sub_directory') {
157+
return { isDirectory: () => true, isFile: () => false } as fs.Stats;
158+
}
159+
return { isDirectory: () => false, isFile: () => true } as fs.Stats;
160+
});
161+
162+
await uploadAllEventsFromPath('mocked_directory/sub_directory', {
163+
esURL: 'esURL',
164+
esAPIKey: 'esAPIKey',
165+
verifyTLSCerts: true,
166+
log,
167+
});
168+
169+
expect(mockAddEventsFromFile).toHaveBeenCalledWith(
170+
'mocked_directory/sub_directory/events.ndjson'
171+
);
172+
173+
expect(log.info.mock.calls).toEqual([
174+
['Connecting to Elasticsearch at esURL'],
175+
["Recursively found 1 .ndjson event log file in directory 'mocked_directory/sub_directory'."],
176+
]);
177+
});
178+
179+
it('should upload multiple event log files if the provided eventLogPath is a directory and contains .ndjson files', async () => {
180+
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
181+
182+
// Simulate directory contents: 2 .ndjson files, 1 .txt file
183+
(fs.readdirSync as jest.Mock).mockImplementation((directoryPath: string) => {
184+
if (directoryPath === 'mocked_directory') {
185+
return ['file1.ndjson', 'file2.ndjson', 'not_events.txt'];
186+
}
187+
return [];
188+
});
189+
190+
(fs.statSync as jest.Mock).mockImplementation((filePath: string) => {
191+
if (filePath === 'mocked_directory') {
192+
return { isDirectory: () => true, isFile: () => false };
193+
}
194+
return { isDirectory: () => false, isFile: () => true };
195+
});
196+
197+
await uploadAllEventsFromPath('mocked_directory', {
198+
esURL: 'esURL',
199+
esAPIKey: 'esAPIKey',
200+
verifyTLSCerts: true,
201+
log,
202+
});
203+
204+
expect(mockAddEventsFromFile).toHaveBeenCalledWith('mocked_directory/file1.ndjson');
205+
expect(mockAddEventsFromFile).toHaveBeenCalledWith('mocked_directory/file2.ndjson');
206+
expect(mockAddEventsFromFile).not.toHaveBeenCalledWith('mocked_directory/not_events.txt');
207+
208+
expect(log.info.mock.calls).toEqual([
209+
['Connecting to Elasticsearch at esURL'],
210+
["Recursively found 2 .ndjson event log files in directory 'mocked_directory'."],
211+
]);
212+
});
213+
});

0 commit comments

Comments
 (0)