Skip to content

Commit 729471a

Browse files
georgeeegeorgeee
authored
Merge branch 'compatible' into georgeee/use-stricter-error-type-in-work-paritioner
2 parents 6ff41f6 + 40c8136 commit 729471a

File tree

61 files changed

+1002
-327
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+1002
-327
lines changed

‎buildkite/scripts/debian/update.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,7 @@ APT_SOURCES_DIR="/etc/apt/sources.list.d"
2929
export DEBIAN_FRONTEND=noninteractive
3030

3131
# Configuration
32-
BLACKLISTED_REPOS=( helm-stable-debian.list ) # Default blacklisted repositories
33-
# BLACKLISTED_REPOS=() # Uncomment to start with no blacklisted repos
32+
BLACKLISTED_REPOS=() # Uncomment to start with no blacklisted repos
3433
VERBOSE=false
3534
DRY_RUN=false
3635
SUDO_CMD=""

‎buildkite/src/Command/ArchiveNodeTest.dhall‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,3 @@
1-
let B = ../External/Buildkite.dhall
2-
3-
let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
4-
51
let Artifacts = ../Constants/Artifacts.dhall
62

73
let Command = ./Base.dhall
@@ -35,7 +31,6 @@ in { step =
3531
, label = "Archive: Node Test"
3632
, key = key
3733
, target = Size.Large
38-
, soft_fail = Some (B/SoftFail.Boolean True)
3934
, depends_on = dependsOn
4035
}
4136
}

‎buildkite/src/Command/DockerImage.dhall‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ let VerifyDockers = ../Command/Packages/VerifyDockers.dhall
3030

3131
let Extensions = ../Lib/Extensions.dhall
3232

33+
let Arch = ../Constants/Arch.dhall
34+
3335
let ReleaseSpec =
3436
{ Type =
3537
{ deps : List Command.TaggedKey.Type
@@ -38,6 +40,7 @@ let ReleaseSpec =
3840
, version : Text
3941
, branch : Text
4042
, repo : Text
43+
, arch : Arch.Type
4144
, no_cache : Bool
4245
, no_debian : Bool
4346
, deb_codename : DebianVersions.DebVersion
@@ -56,6 +59,7 @@ let ReleaseSpec =
5659
, default =
5760
{ deps = [] : List Command.TaggedKey.Type
5861
, network = Network.Type.Berkeley
62+
, arch = Arch.Type.Amd64
5963
, version = "\\\${MINA_DOCKER_TAG}"
6064
, service = Artifacts.Type.Daemon
6165
, branch = "\\\${BUILDKITE_BRANCH}"
@@ -88,11 +92,14 @@ let stepLabel =
8892
spec.network} ${DebianVersions.capitalName
8993
spec.deb_codename} ${Profiles.toSuffixUppercase
9094
spec.deb_profile} ${BuildFlags.toSuffixUppercase
91-
spec.build_flags}"
95+
spec.build_flags} ${Arch.capitalName
96+
spec.arch}"
9297

9398
let generateStep =
9499
\(spec : ReleaseSpec.Type)
95-
-> let exportMinaDebCmd =
100+
-> let installBuildx = "./scripts/docker/setup_buildx.sh"
101+
102+
let exportMinaDebCmd =
96103
"export MINA_DEB_CODENAME=${DebianVersions.lowerName
97104
spec.deb_codename}"
98105

@@ -152,6 +159,7 @@ let generateStep =
152159
, version = spec.deb_version
153160
, codenames = [ spec.deb_codename ]
154161
, suffix = suffix
162+
, arch = spec.arch
155163
}
156164

157165
else ""
@@ -177,6 +185,7 @@ let generateStep =
177185
++ " --deb-legacy-version ${spec.deb_legacy_version}"
178186
++ debSuffix
179187
++ " --repo ${spec.repo}"
188+
++ " --platform ${Arch.platform spec.arch}"
180189

181190
let releaseDockerCmd =
182191
if DockerPublish.shouldPublish
@@ -194,6 +203,7 @@ let generateStep =
194203
spec.deb_profile}"
195204
++ " --deb-build-flags ${BuildFlags.lowerName
196205
spec.build_flags}"
206+
++ " --platform ${Arch.platform spec.arch}"
197207

198208
else " echo In order to ensure storage optimization, skipping publishing docker as this is not essential one or publishing is disabled . Docker publish setting is set to ${DockerPublish.show
199209
spec.docker_publish}."
@@ -217,7 +227,9 @@ let generateStep =
217227
, Stable = remoteRepoCmds
218228
, Local =
219229
[ Cmd.run
220-
( exportMinaDebCmd
230+
( installBuildx
231+
++ " && "
232+
++ exportMinaDebCmd
221233
++ " && "
222234
++ pruneDockerImages
223235
++ maybeStartDebianRepo

‎buildkite/src/Command/MinaArtifact.dhall‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ let Artifacts = ../Constants/Artifacts.dhall
4444

4545
let Toolchain = ../Constants/Toolchain.dhall
4646

47+
let Arch = ../Constants/Arch.dhall
48+
4749
let MinaBuildSpec =
4850
{ Type =
4951
{ prefix : Text
@@ -59,6 +61,7 @@ let MinaBuildSpec =
5961
, channel : DebianChannel.Type
6062
, debianRepo : DebianRepo.Type
6163
, buildScript : Text
64+
, arch : Arch.Type
6265
, deb_legacy_version : Text
6366
, suffix : Optional Text
6467
, if : Optional B/If
@@ -71,14 +74,15 @@ let MinaBuildSpec =
7174
, profile = Profiles.Type.Devnet
7275
, buildFlags = BuildFlags.Type.None
7376
, network = Network.Type.Berkeley
74-
, toolchainSelectMode = Toolchain.SelectionMode.ByDebian
77+
, toolchainSelectMode = Toolchain.SelectionMode.ByDebianAndArch
7578
, tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
7679
, scope = PipelineScope.Full
7780
, channel = DebianChannel.Type.Unstable
7881
, debianRepo = DebianRepo.Type.Unstable
7982
, extraBuildEnvs = [] : List Text
8083
, suffix = None Text
8184
, deb_legacy_version = "3.1.1-alpha1-compatible-14a8b92"
85+
, arch = Arch.Type.Amd64
8286
, if = None B/If
8387
}
8488
}
@@ -90,7 +94,8 @@ let labelSuffix
9094
spec.debVersion} ${Network.capitalName
9195
spec.network} ${Profiles.toSuffixUppercase
9296
spec.profile} ${BuildFlags.toSuffixUppercase
93-
spec.buildFlags}"
97+
spec.buildFlags}${Arch.labelSuffix
98+
spec.arch}"
9499

95100
let nameSuffix
96101
: MinaBuildSpec.Type -> Text
@@ -99,7 +104,8 @@ let nameSuffix
99104
spec.debVersion}${Network.capitalName
100105
spec.network}${Profiles.toSuffixUppercase
101106
spec.profile}${BuildFlags.toSuffixUppercase
102-
spec.buildFlags}"
107+
spec.buildFlags}${Arch.nameSuffix
108+
spec.arch}"
103109

104110
let build_artifacts
105111
: MinaBuildSpec.Type -> Command.Type
@@ -110,13 +116,15 @@ let build_artifacts
110116
Toolchain.select
111117
spec.toolchainSelectMode
112118
spec.debVersion
119+
spec.arch
113120
( [ "DUNE_PROFILE=${Profiles.duneProfile spec.profile}"
114121
, "AWS_ACCESS_KEY_ID"
115122
, "AWS_SECRET_ACCESS_KEY"
116123
, "MINA_BRANCH=\$BUILDKITE_BRANCH"
117124
, "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT"
118125
, "MINA_DEB_CODENAME=${DebianVersions.lowerName
119126
spec.debVersion}"
127+
, "ARCHITECTURE=${Arch.lowerName spec.arch}"
120128
, Network.buildMainnetEnv spec.network
121129
]
122130
# BuildFlags.buildEnvs spec.buildFlags
@@ -156,6 +164,7 @@ let docker_step
156164
, build_flag = spec.buildFlags
157165
, step = step_dep_name
158166
, prefix = spec.prefix
167+
, arch = spec.arch
159168
}
160169

161170
let docker_publish = DockerPublish.Type.Essential
@@ -173,6 +182,7 @@ let docker_step
173182
, deb_repo = DebianRepo.Type.Local
174183
, deb_legacy_version = spec.deb_legacy_version
175184
, verify = True
185+
, arch = spec.arch
176186
, if = spec.if
177187
}
178188
]
@@ -216,6 +226,7 @@ let docker_step
216226
, docker_publish = docker_publish
217227
, deb_repo = DebianRepo.Type.Local
218228
, deb_legacy_version = spec.deb_legacy_version
229+
, arch = spec.arch
219230
}
220231
]
221232
, TestExecutive = [] : List DockerImage.ReleaseSpec.Type
@@ -232,6 +243,7 @@ let docker_step
232243
, docker_publish = docker_publish
233244
, deb_repo = DebianRepo.Type.Local
234245
, deb_legacy_version = spec.deb_legacy_version
246+
, arch = spec.arch
235247
, if = spec.if
236248
}
237249
]
@@ -247,6 +259,7 @@ let docker_step
247259
, deb_repo = DebianRepo.Type.Local
248260
, deb_legacy_version = spec.deb_legacy_version
249261
, verify = True
262+
, arch = spec.arch
250263
, if = spec.if
251264
}
252265
]
@@ -261,6 +274,7 @@ let docker_step
261274
, deb_repo = DebianRepo.Type.Local
262275
, deb_legacy_version = spec.deb_legacy_version
263276
, verify = True
277+
, arch = spec.arch
264278
, if = spec.if
265279
}
266280
]
@@ -274,6 +288,7 @@ let docker_step
274288
, deb_profile = spec.profile
275289
, deb_codename = spec.debVersion
276290
, deb_legacy_version = spec.deb_legacy_version
291+
, arch = spec.arch
277292
, if = spec.if
278293
}
279294
]
@@ -288,6 +303,7 @@ let docker_step
288303
, deb_repo = DebianRepo.Type.Local
289304
, deb_profile = spec.profile
290305
, deb_legacy_version = spec.deb_legacy_version
306+
, arch = spec.arch
291307
, if = spec.if
292308
}
293309
]

‎buildkite/src/Command/MinaArtifactToolchain.dhall‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
let Pipeline = ../Pipeline/Dsl.dhall
2+
3+
let PipelineTag = ../Pipeline/Tag.dhall
4+
5+
let JobSpec = ../Pipeline/JobSpec.dhall
6+
7+
let S = ../Lib/SelectFiles.dhall
8+
9+
let DockerImage = ../Command/DockerImage.dhall
10+
11+
let DebianVersions = ../Constants/DebianVersions.dhall
12+
13+
let Arch = ../Constants/Arch.dhall
14+
15+
let toolchainPipeline =
16+
\(spec : DockerImage.ReleaseSpec.Type)
17+
-> Pipeline.build
18+
Pipeline.Config::{
19+
, spec = JobSpec::{
20+
, dirtyWhen =
21+
[ S.strictlyStart (S.contains "dockerfiles/stages/1-")
22+
, S.strictlyStart (S.contains "dockerfiles/stages/2-")
23+
, S.strictlyStart (S.contains "dockerfiles/stages/3-")
24+
, S.strictlyStart
25+
( S.contains
26+
"buildkite/src/Jobs/Release/MinaToolchainArtifact"
27+
)
28+
, S.strictly (S.contains "opam.export")
29+
, S.strictlyEnd (S.contains "rust-toolchain.toml")
30+
]
31+
, path = "Release"
32+
, name =
33+
"MinaToolchainArtifact${DebianVersions.capitalName
34+
spec.deb_codename}${Arch.nameSuffix
35+
spec.arch}"
36+
, tags = [ PipelineTag.Type.Toolchain ]
37+
}
38+
, steps = [ DockerImage.generateStep spec ]
39+
}
40+
41+
in { pipeline = toolchainPipeline }

‎buildkite/src/Command/Packages/VerifyDockers.dhall‎

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ let Artifact = ../../Constants/Artifacts.dhall
1414

1515
let DebianVersions = ../../Constants/DebianVersions.dhall
1616

17+
let Arch = ../../Constants/Arch.dhall
18+
1719
let Spec =
1820
{ Type =
1921
{ artifacts : List Artifact.Type
@@ -22,6 +24,7 @@ let Spec =
2224
, codenames : List DebianVersions.DebVersion
2325
, published_to_docker_io : Bool
2426
, suffix : Optional Text
27+
, arch : Arch.Type
2528
}
2629
, default =
2730
{ artifacts = [] : List Package.Type
@@ -32,6 +35,7 @@ let Spec =
3235
]
3336
, published_to_docker_io = False
3437
, suffix = None Text
38+
, arch = Arch.Type.Amd64
3539
}
3640
}
3741

@@ -68,17 +72,38 @@ let joinCodenames
6872
let verify
6973
: Spec.Type -> Text
7074
= \(spec : Spec.Type)
71-
-> ". ./buildkite/scripts/export-git-env-vars.sh && "
72-
++ "./buildkite/scripts/release/manager.sh verify "
73-
++ "--artifacts ${joinArtifacts spec} "
74-
++ "--networks ${joinNetworks spec} "
75-
++ "--version ${spec.version} "
76-
++ "--codenames ${joinCodenames spec} "
77-
++ merge
78-
{ None = ""
79-
, Some = \(suffix : Text) -> "--docker-suffix ${suffix} "
80-
}
81-
spec.suffix
82-
++ "--only-dockers "
75+
-> let arch = Arch.toOptional spec.arch
76+
77+
let suffixAndArchFlag =
78+
let archFlag =
79+
Prelude.Optional.map
80+
Text
81+
Text
82+
(\(archValue : Text) -> "--arch ${archValue} ")
83+
arch
84+
85+
let suffixFlag =
86+
Prelude.Optional.map
87+
Text
88+
Text
89+
(\(suffix : Text) -> "--docker-suffix ${suffix} ")
90+
spec.suffix
91+
92+
in Prelude.Optional.default Text "" archFlag
93+
++ Prelude.Optional.default Text "" suffixFlag
94+
95+
in ". ./buildkite/scripts/export-git-env-vars.sh && "
96+
++ "./buildkite/scripts/release/manager.sh verify "
97+
++ suffixAndArchFlag
98+
++ "--artifacts ${joinArtifacts spec} "
99+
++ "--networks ${joinNetworks spec} "
100+
++ "--version ${spec.version} "
101+
++ "--codenames ${joinCodenames spec} "
102+
++ merge
103+
{ None = ""
104+
, Some = \(suffix : Text) -> "--docker-suffix ${suffix} "
105+
}
106+
spec.suffix
107+
++ "--only-dockers "
83108

84109
in { verify = verify, Spec = Spec }

0 commit comments

Comments
 (0)