1313import subprocess
1414from pathlib import Path
1515
16- DEFAULT_INSTANCES = [
17- "c5n.metal" , # Intel Skylake
18- "m5n.metal" , # Intel Cascade Lake
19- "m6i.metal" , # Intel Icelake
20- "m6a.metal" , # AMD Milan
21- "m6g.metal" , # Graviton2
22- "m7g.metal" , # Graviton3
23- ]
16+ DEFAULT_INSTANCES = {
17+ "c5n.metal" : "x86_64" , # Intel Skylake
18+ "m5n.metal" : "x86_64" , # Intel Cascade Lake
19+ "m6i.metal" : "x86_64" , # Intel Icelake
20+ "m6a.metal" : "x86_64" , # AMD Milan
21+ "m6g.metal" : "aarch64" , # Graviton2
22+ "m7g.metal" : "aarch64" , # Graviton3
23+ }
2424
2525DEFAULT_PLATFORMS = [
2626 ("al2" , "linux_5.10" ),
@@ -145,7 +145,7 @@ def __call__(self, parser, namespace, value, option_string=None):
145145 "--instances" ,
146146 required = False ,
147147 nargs = "+" ,
148- default = DEFAULT_INSTANCES ,
148+ default = DEFAULT_INSTANCES . keys () ,
149149)
150150COMMON_PARSER .add_argument (
151151 "--platforms" ,
@@ -188,6 +188,7 @@ def ab_revision_build(revision):
188188 "git branch $$branch_name $$commitish" ,
189189 f"git clone -b $$branch_name . build/{ revision } ,
190190 f"cd build/{ revision } ,
191+ "git branch -D $$branch_name" ,
191192 ]
192193
193194
@@ -204,7 +205,9 @@ def shared_build():
204205 if rev_a is not None :
205206 rev_b = os .environ .get ("REVISION_B" )
206207 assert rev_b is not None , "REVISION_B environment variable not set"
207- build_cmds = ab_revision_build (rev_a ) + ab_revision_build (rev_b )
208+ build_cmds = ab_revision_build (rev_a )
209+ if rev_a != rev_b :
210+ build_cmds += ab_revision_build (rev_b )
208211 elif os .environ .get ("BUILDKITE_PULL_REQUEST" , "false" ) != "false" :
209212 build_cmds = ab_revision_build (
210213 os .environ .get ("BUILDKITE_PULL_REQUEST_BASE_BRANCH" , "main" )
@@ -229,7 +232,7 @@ class BKPipeline:
229232
230233 parser = COMMON_PARSER
231234
232- def __init__ (self , initial_steps = None , with_build_step = True , ** kwargs ):
235+ def __init__ (self , with_build_step = True , ** kwargs ):
233236 self .steps = []
234237 self .args = args = self .parser .parse_args ()
235238 # Retry one time if agent was lost. This can happen if we terminate the
@@ -254,33 +257,22 @@ def __init__(self, initial_steps=None, with_build_step=True, **kwargs):
254257 # Build sharing
255258 if with_build_step :
256259 build_cmds , self .shared_build = shared_build ()
257- step_build = group ("🏗️ Build" , build_cmds , ** self .per_arch )
258- self .steps += [step_build , "wait" ]
260+ self .build_group_per_arch (
261+ "🏗️ Build" , build_cmds , depends_on_build = False , set_key = True
262+ )
259263 else :
260264 self .shared_build = None
261265
262- # If we run initial_steps before the "wait" step above, then a failure of the initial steps
263- # would result in the build not progressing past the "wait" step (as buildkite only proceeds past a wait step
264- # if everything before it passed). Thus put the initial steps after the "wait" step, but set `"depends_on": null`
265- # to start running them immediately (e.g. without waiting for the "wait" step to unblock).
266- #
267- # See also https://buildkite.com/docs/pipelines/dependencies#explicit-dependencies-in-uploaded-steps
268- if initial_steps :
269- for step in initial_steps :
270- step ["depends_on" ] = None
271-
272- self .steps += initial_steps
273-
274- def add_step (self , step , decorate = True ):
266+ def add_step (self , step , depends_on_build = True ):
275267 """
276268 Add a step to the pipeline.
277269
278270 https://buildkite.com/docs/pipelines/step-reference
279271
280272 :param step: a Buildkite step
281- :param decorate : inject needed commands for sharing builds
273+ :param depends_on_build : inject needed commands for sharing builds
282274 """
283- if decorate and isinstance (step , dict ):
275+ if depends_on_build and isinstance (step , dict ):
284276 step = self ._adapt_group (step )
285277 self .steps .append (step )
286278 return step
@@ -303,6 +295,10 @@ def _adapt_group(self, group):
303295
304296 for step in group ["steps" ]:
305297 step ["command" ] = prepend + step ["command" ]
298+ if self .shared_build is not None :
299+ step ["depends_on" ] = self .build_key (
300+ DEFAULT_INSTANCES [step ["agents" ]["instance" ]]
301+ )
306302 return group
307303
308304 def build_group (self , * args , ** kwargs ):
@@ -311,15 +307,34 @@ def build_group(self, *args, **kwargs):
311307
312308 https://buildkite.com/docs/pipelines/group-step
313309 """
310+ depends_on_build = kwargs .pop ("depends_on_build" , True )
314311 combined = overlay_dict (self .per_instance , kwargs )
315- return self .add_step (group (* args , ** combined ))
312+ return self .add_step (
313+ group (* args , ** combined ), depends_on_build = depends_on_build
314+ )
315+
316+ def build_key (self , arch ):
317+ """Return the Buildkite key for the build step, for the specified arch"""
318+ return self .shared_build .replace ("$(uname -m)" , arch ).replace (".tar.gz" , "" )
316319
317- def build_group_per_arch (self , * args , ** kwargs ):
320+ def build_group_per_arch (self , label , * args , ** kwargs ):
318321 """
319322 Build a group, parametrizing over the architectures only.
323+
324+ kwargs consumed by this method and not passed down to `group`:
325+ - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
326+ - `set_key`: If True, causes the generated steps to have a "key" field
320327 """
328+ depends_on_build = kwargs .pop ("depends_on_build" , True )
329+ set_key = kwargs .pop ("set_key" , None )
321330 combined = overlay_dict (self .per_arch , kwargs )
322- return self .add_step (group (* args , ** combined ))
331+ grp = group (label , * args , ** combined )
332+ if set_key :
333+ for step in grp ["steps" ]:
334+ step ["key" ] = self .build_key (
335+ DEFAULT_INSTANCES [step ["agents" ]["instance" ]]
336+ )
337+ return self .add_step (grp , depends_on_build = depends_on_build )
323338
324339 def to_dict (self ):
325340 """Render the pipeline as a dictionary."""
0 commit comments