Skip to content

Commit 8954b65

Browse files
brianseedersbrianseeders
authored
[buildkite] Add more logging and debug information to PR pipeline generation (#100709) (#100711)
(cherry picked from commit 81a441e)
1 parent 7c61e7f commit 8954b65

File tree

4 files changed

+182
-149
lines changed

4 files changed

+182
-149
lines changed

‎.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap‎

Lines changed: 159 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -3,183 +3,201 @@
33
exports[`generatePipelines should generate correct pipelines with a non-docs change 1`] = `
44
[
55
{
6-
"steps": [
7-
{
8-
"group": "bwc-snapshots",
9-
"steps": [
10-
{
11-
"agents": {
12-
"buildDirectory": "/dev/shm/bk",
13-
"image": "family/elasticsearch-ubuntu-2004",
14-
"machineType": "custom-32-98304",
15-
"provider": "gcp",
16-
},
17-
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
18-
"env": {
19-
"BWC_VERSION": "{{matrix.BWC_VERSION}}",
20-
},
21-
"label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
22-
"matrix": {
23-
"setup": {
24-
"BWC_VERSION": [
25-
"7.17.14",
26-
"8.10.3",
27-
"8.11.0",
28-
],
6+
"name": "bwc-snapshots",
7+
"pipeline": {
8+
"steps": [
9+
{
10+
"group": "bwc-snapshots",
11+
"steps": [
12+
{
13+
"agents": {
14+
"buildDirectory": "/dev/shm/bk",
15+
"image": "family/elasticsearch-ubuntu-2004",
16+
"machineType": "custom-32-98304",
17+
"provider": "gcp",
18+
},
19+
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
20+
"env": {
21+
"BWC_VERSION": "{{matrix.BWC_VERSION}}",
2922
},
23+
"label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
24+
"matrix": {
25+
"setup": {
26+
"BWC_VERSION": [
27+
"7.17.14",
28+
"8.10.3",
29+
"8.11.0",
30+
],
31+
},
32+
},
33+
"timeout_in_minutes": 300,
3034
},
31-
"timeout_in_minutes": 300,
32-
},
33-
],
34-
},
35-
],
35+
],
36+
},
37+
],
38+
},
3639
},
3740
{
38-
"env": {
39-
"CUSTOM_ENV_VAR": "value",
40-
},
41-
"steps": [
42-
{
43-
"command": "echo 'hello world'",
44-
"label": "test-step",
41+
"name": "using-defaults",
42+
"pipeline": {
43+
"env": {
44+
"CUSTOM_ENV_VAR": "value",
4545
},
46-
],
46+
"steps": [
47+
{
48+
"command": "echo 'hello world'",
49+
"label": "test-step",
50+
},
51+
],
52+
},
4753
},
4854
]
4955
`;
5056

5157
exports[`generatePipelines should generate correct pipelines with only docs changes 1`] = `
5258
[
5359
{
54-
"steps": [
55-
{
56-
"agents": {
57-
"buildDirectory": "/dev/shm/bk",
58-
"image": "family/elasticsearch-ubuntu-2004",
59-
"machineType": "custom-32-98304",
60-
"provider": "gcp",
60+
"name": "docs-check",
61+
"pipeline": {
62+
"steps": [
63+
{
64+
"agents": {
65+
"buildDirectory": "/dev/shm/bk",
66+
"image": "family/elasticsearch-ubuntu-2004",
67+
"machineType": "custom-32-98304",
68+
"provider": "gcp",
69+
},
70+
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check",
71+
"label": "docs-check",
72+
"timeout_in_minutes": 300,
6173
},
62-
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check",
63-
"label": "docs-check",
64-
"timeout_in_minutes": 300,
65-
},
66-
],
74+
],
75+
},
6776
},
6877
]
6978
`;
7079

7180
exports[`generatePipelines should generate correct pipelines with full BWC expansion 1`] = `
7281
[
7382
{
74-
"steps": [
75-
{
76-
"group": "bwc",
77-
"steps": [
78-
{
79-
"agents": {
80-
"buildDirectory": "/dev/shm/bk",
81-
"image": "family/elasticsearch-ubuntu-2004",
82-
"machineType": "custom-32-98304",
83-
"provider": "gcp",
84-
},
85-
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest",
86-
"env": {
87-
"BWC_VERSION": "7.0.0",
88-
},
89-
"key": "full-bwc:7_0_0",
90-
"label": "7.0.0 / bwc",
91-
"timeout_in_minutes": 300,
92-
},
93-
{
94-
"agents": {
95-
"buildDirectory": "/dev/shm/bk",
96-
"image": "family/elasticsearch-ubuntu-2004",
97-
"machineType": "custom-32-98304",
98-
"provider": "gcp",
99-
},
100-
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest",
101-
"env": {
102-
"BWC_VERSION": "7.0.1",
103-
},
104-
"key": "full-bwc:7_0_1",
105-
"label": "7.0.1 / bwc",
106-
"timeout_in_minutes": 300,
107-
},
108-
{
109-
"agents": {
110-
"buildDirectory": "/dev/shm/bk",
111-
"image": "family/elasticsearch-ubuntu-2004",
112-
"machineType": "custom-32-98304",
113-
"provider": "gcp",
114-
},
115-
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest",
116-
"env": {
117-
"BWC_VERSION": "7.1.0",
83+
"name": "full-bwc",
84+
"pipeline": {
85+
"steps": [
86+
{
87+
"group": "bwc",
88+
"steps": [
89+
{
90+
"agents": {
91+
"buildDirectory": "/dev/shm/bk",
92+
"image": "family/elasticsearch-ubuntu-2004",
93+
"machineType": "custom-32-98304",
94+
"provider": "gcp",
95+
},
96+
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest",
97+
"env": {
98+
"BWC_VERSION": "7.0.0",
99+
},
100+
"key": "full-bwc:7_0_0",
101+
"label": "7.0.0 / bwc",
102+
"timeout_in_minutes": 300,
118103
},
119-
"key": "full-bwc:7_1_0",
120-
"label": "7.1.0 / bwc",
121-
"timeout_in_minutes": 300,
122-
},
123-
{
124-
"agents": {
125-
"buildDirectory": "/dev/shm/bk",
126-
"image": "family/elasticsearch-ubuntu-2004",
127-
"machineType": "custom-32-98304",
128-
"provider": "gcp",
104+
{
105+
"agents": {
106+
"buildDirectory": "/dev/shm/bk",
107+
"image": "family/elasticsearch-ubuntu-2004",
108+
"machineType": "custom-32-98304",
109+
"provider": "gcp",
110+
},
111+
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest",
112+
"env": {
113+
"BWC_VERSION": "7.0.1",
114+
},
115+
"key": "full-bwc:7_0_1",
116+
"label": "7.0.1 / bwc",
117+
"timeout_in_minutes": 300,
129118
},
130-
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest",
131-
"env": {
132-
"BWC_VERSION": "8.10.0",
119+
{
120+
"agents": {
121+
"buildDirectory": "/dev/shm/bk",
122+
"image": "family/elasticsearch-ubuntu-2004",
123+
"machineType": "custom-32-98304",
124+
"provider": "gcp",
125+
},
126+
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest",
127+
"env": {
128+
"BWC_VERSION": "7.1.0",
129+
},
130+
"key": "full-bwc:7_1_0",
131+
"label": "7.1.0 / bwc",
132+
"timeout_in_minutes": 300,
133133
},
134-
"key": "full-bwc:8_10_0",
135-
"label": "8.10.0 / bwc",
136-
"timeout_in_minutes": 300,
137-
},
138-
{
139-
"agents": {
140-
"buildDirectory": "/dev/shm/bk",
141-
"image": "family/elasticsearch-ubuntu-2004",
142-
"machineType": "custom-32-98304",
143-
"provider": "gcp",
134+
{
135+
"agents": {
136+
"buildDirectory": "/dev/shm/bk",
137+
"image": "family/elasticsearch-ubuntu-2004",
138+
"machineType": "custom-32-98304",
139+
"provider": "gcp",
140+
},
141+
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest",
142+
"env": {
143+
"BWC_VERSION": "8.10.0",
144+
},
145+
"key": "full-bwc:8_10_0",
146+
"label": "8.10.0 / bwc",
147+
"timeout_in_minutes": 300,
144148
},
145-
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest",
146-
"env": {
147-
"BWC_VERSION": "8.11.0",
149+
{
150+
"agents": {
151+
"buildDirectory": "/dev/shm/bk",
152+
"image": "family/elasticsearch-ubuntu-2004",
153+
"machineType": "custom-32-98304",
154+
"provider": "gcp",
155+
},
156+
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest",
157+
"env": {
158+
"BWC_VERSION": "8.11.0",
159+
},
160+
"key": "full-bwc:8_11_0",
161+
"label": "8.11.0 / bwc",
162+
"timeout_in_minutes": 300,
148163
},
149-
"key": "full-bwc:8_11_0",
150-
"label": "8.11.0 / bwc",
151-
"timeout_in_minutes": 300,
152-
},
153-
],
154-
},
155-
],
164+
],
165+
},
166+
],
167+
},
156168
},
157169
{
158-
"env": {
159-
"CUSTOM_ENV_VAR": "value",
160-
},
161-
"steps": [
162-
{
163-
"command": "echo 'hello world'",
164-
"label": "test-step",
170+
"name": "using-defaults",
171+
"pipeline": {
172+
"env": {
173+
"CUSTOM_ENV_VAR": "value",
165174
},
166-
],
175+
"steps": [
176+
{
177+
"command": "echo 'hello world'",
178+
"label": "test-step",
179+
},
180+
],
181+
},
167182
},
168183
]
169184
`;
170185

171186
exports[`generatePipelines should generate correct pipeline when using a trigger comment for it 1`] = `
172187
[
173188
{
174-
"env": {
175-
"CUSTOM_ENV_VAR": "value",
176-
},
177-
"steps": [
178-
{
179-
"command": "echo 'hello world'",
180-
"label": "test-step",
189+
"name": "using-defaults",
190+
"pipeline": {
191+
"env": {
192+
"CUSTOM_ENV_VAR": "value",
181193
},
182-
],
194+
"steps": [
195+
{
196+
"command": "echo 'hello world'",
197+
"label": "test-step",
198+
},
199+
],
200+
},
183201
},
184202
]
185203
`;

‎.buildkite/scripts/pull-request/pipeline.generate.ts‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,19 @@ import { generatePipelines } from "./pipeline";
66
const pipelines = generatePipelines();
77

88
for (const pipeline of pipelines) {
9-
if (!process.env.CI) {
10-
// Just for local debugging purposes
9+
const yaml = stringify(pipeline.pipeline);
10+
11+
console.log(`--- Generated pipeline: ${pipeline.name}`);
12+
console.log(yaml);
13+
14+
// Only do the pipeline upload if we're actually in CI
15+
// This lets us run the tool locally and see the output
16+
if (process.env.CI) {
1117
console.log("");
12-
console.log(stringify(pipeline));
13-
} else {
18+
console.log("Uploading pipeline...");
19+
1420
execSync(`buildkite-agent pipeline upload`, {
15-
input: stringify(pipeline),
21+
input: yaml,
1622
stdio: ["pipe", "inherit", "inherit"],
1723
});
1824
}

‎.buildkite/scripts/pull-request/pipeline.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,8 @@
22

33
set -euo pipefail
44

5+
echo --- Installing bun
56
npm install -g bun
7+
8+
echo --- Generating pipeline
69
bun .buildkite/scripts/pull-request/pipeline.generate.ts

0 commit comments

Comments
 (0)