Skip to content

Commit 1d00af8

Browse files
dkijaniadkijania
authored
Merge branch 'compatible' into dkijania/regenerate_archive_data
2 parents 6845a33 + b2e50b6 commit 1d00af8

File tree

45 files changed

+736
-368
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+736
-368
lines changed

‎buildkite/scripts/debian/install.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ fi
88

99
if [ -z $MINA_DEB_CODENAME ]; then
1010
echo "MINA_DEB_CODENAME env var is not defined"
11-
exit -1
11+
exit 1
1212
fi
1313

1414
DEBS=$1
@@ -29,8 +29,9 @@ source ./buildkite/scripts/export-git-env-vars.sh
2929
# Download required debians from bucket locally
3030
if [ -z "$DEBS" ]; then
3131
echo "DEBS env var is empty. It should contains comma delimitered names of debians to install"
32-
exit -1
32+
exit 1
3333
else
34+
# shellcheck disable=SC2206
3435
debs=(${DEBS//,/ })
3536
for i in "${debs[@]}"; do
3637
case $i in

‎buildkite/scripts/release/manager.sh‎

Lines changed: 185 additions & 116 deletions
Large diffs are not rendered by default.

‎buildkite/scripts/tests/archive-node-unit-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ if [[ -z "$user" || -z "$password" || -z "$db" || -z "$command_key" ]]; then
1717
exit 1
1818
fi
1919

20-
eval $(opam config env)
20+
eval "$(opam config env)"
2121

2222
echo "Setting up database for archive node tests..."
2323

‎buildkite/src/Command/Bench/Base.dhall‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ let JobSpec = ../../Pipeline/JobSpec.dhall
1010

1111
let DebianVersions = ../../Constants/DebianVersions.dhall
1212

13+
let BuildFlags = ../../Constants/BuildFlags.dhall
14+
1315
let RunInToolchain = ../../Command/RunInToolchain.dhall
1416

1517
let Command = ../../Command/Base.dhall
@@ -44,7 +46,12 @@ let Spec =
4446
, default =
4547
{ mode = PipelineMode.Type.PullRequest
4648
, size = Size.Perf
47-
, dependsOn = DebianVersions.dependsOn DebianVersions.DepsSpec::{=}
49+
, dependsOn =
50+
DebianVersions.dependsOn
51+
DebianVersions.DepsSpec::{
52+
, build_flag = BuildFlags.Type.Instrumented
53+
}
54+
# DebianVersions.dependsOn DebianVersions.DepsSpec::{=}
4855
, additionalDirtyWhen = [] : List SelectFiles.Type
4956
, yellowThreshold = 0.1
5057
, redThreshold = 0.2

‎buildkite/src/Command/DockerImage.dhall‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ let Network = ../Constants/Network.dhall
2626

2727
let DockerPublish = ../Constants/DockerPublish.dhall
2828

29+
let VerifyDockers = ../Command/Packages/VerifyDockers.dhall
30+
31+
let Extensions = ../Lib/Extensions.dhall
32+
2933
let ReleaseSpec =
3034
{ Type =
3135
{ deps : List Command.TaggedKey.Type
@@ -45,6 +49,7 @@ let ReleaseSpec =
4549
, build_flags : BuildFlags.Type
4650
, step_key_suffix : Text
4751
, docker_publish : DockerPublish.Type
52+
, verify : Bool
4853
, if : Optional B/If
4954
}
5055
, default =
@@ -65,6 +70,7 @@ let ReleaseSpec =
6570
, no_cache = False
6671
, no_debian = False
6772
, step_key_suffix = "-docker-image"
73+
, verify = False
6874
, if = None B/If
6975
}
7076
}
@@ -105,6 +111,43 @@ let generateStep =
105111

106112
else " && ./scripts/debian/aptly.sh stop"
107113

114+
let suffix =
115+
Extensions.joinOptionals
116+
"-"
117+
[ merge
118+
{ Mainnet = None Text
119+
, Devnet = None Text
120+
, Dev = None Text
121+
, Lightnet = Some
122+
"${Profiles.toSuffixLowercase spec.deb_profile}"
123+
}
124+
spec.deb_profile
125+
, merge
126+
{ None = None Text
127+
, Instrumented = Some
128+
"${BuildFlags.toSuffixLowercase spec.build_flags}"
129+
}
130+
spec.build_flags
131+
]
132+
133+
let maybeVerify =
134+
if spec.verify
135+
&& DockerPublish.shouldPublish
136+
spec.docker_publish
137+
spec.service
138+
139+
then " && "
140+
++ VerifyDockers.verify
141+
VerifyDockers.Spec::{
142+
, artifacts = [ spec.service ]
143+
, networks = [ spec.network ]
144+
, version = spec.deb_version
145+
, codenames = [ spec.deb_codename ]
146+
, suffix = suffix
147+
}
148+
149+
else ""
150+
108151
let buildDockerCmd =
109152
"./scripts/docker/build.sh"
110153
++ " --service ${Artifacts.dockerName spec.service}"
@@ -151,6 +194,7 @@ let generateStep =
151194
++ buildDockerCmd
152195
++ " && "
153196
++ releaseDockerCmd
197+
++ maybeVerify
154198
)
155199
]
156200

@@ -169,6 +213,7 @@ let generateStep =
169213
++ " && "
170214
++ releaseDockerCmd
171215
++ maybeStopDebianRepo
216+
++ maybeVerify
172217
)
173218
]
174219
}

‎buildkite/src/Command/MinaArtifact.dhall‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ let docker_step
163163
, docker_publish = docker_publish
164164
, deb_repo = DebianRepo.Type.Local
165165
, deb_legacy_version = spec.deb_legacy_version
166+
, verify = True
166167
, if = spec.if
167168
}
168169
]
@@ -213,6 +214,7 @@ let docker_step
213214
, docker_publish = docker_publish
214215
, deb_repo = DebianRepo.Type.Local
215216
, deb_legacy_version = spec.deb_legacy_version
217+
, verify = True
216218
, if = spec.if
217219
}
218220
]
@@ -226,6 +228,7 @@ let docker_step
226228
, docker_publish = docker_publish
227229
, deb_repo = DebianRepo.Type.Local
228230
, deb_legacy_version = spec.deb_legacy_version
231+
, verify = True
229232
, if = spec.if
230233
}
231234
]

‎buildkite/src/Command/PublishPackages.dhall‎ renamed to ‎buildkite/src/Command/Packages/Publish.dhall‎

Lines changed: 22 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,38 @@
1-
let Prelude = ../External/Prelude.dhall
1+
let Prelude = ../../External/Prelude.dhall
22

3-
let Optional/map = Prelude.Optional.map
4-
5-
let Optional/default = Prelude.Optional.default
6-
7-
let List/drop = Prelude.List.drop
8-
9-
let List/concatMap = Prelude.List.concatMap
3+
let Extensions = ../../Lib/Extensions.dhall
104

11-
let Text/concat = Prelude.Text.concat
5+
let join = Extensions.join
126

13-
let Artifacts = ../Constants/Artifacts.dhall
14-
15-
let Size = ../Command/Size.dhall
16-
17-
let Package = ../Constants/DebianPackage.dhall
7+
let Optional/map = Prelude.Optional.map
188

19-
let Network = ../Constants/Network.dhall
9+
let Optional/default = Prelude.Optional.default
2010

21-
let DebianChannel = ../Constants/DebianChannel.dhall
11+
let Artifacts = ../../Constants/Artifacts.dhall
2212

23-
let Profiles = ../Constants/Profiles.dhall
13+
let Size = ../../Command/Size.dhall
2414

25-
let Artifact = ../Constants/Artifacts.dhall
15+
let Package = ../../Constants/DebianPackage.dhall
2616

27-
let DebianVersions = ../Constants/DebianVersions.dhall
17+
let Network = ../../Constants/Network.dhall
2818

29-
let DebianRepo = ../Constants/DebianRepo.dhall
19+
let DebianChannel = ../../Constants/DebianChannel.dhall
3020

31-
let Toolchain = ../Constants/Toolchain.dhall
21+
let Profiles = ../../Constants/Profiles.dhall
3222

33-
let ContainerImages = ../Constants/ContainerImages.dhall
23+
let DebianVersions = ../../Constants/DebianVersions.dhall
3424

35-
let Command = ./Base.dhall
25+
let DebianRepo = ../../Constants/DebianRepo.dhall
3626

37-
let Cmd = ../Lib/Cmds.dhall
27+
let ContainerImages = ../../Constants/ContainerImages.dhall
3828

39-
let Mina = ./Mina.dhall
29+
let Command = ../Base.dhall
4030

41-
let join =
42-
\(sep : Text)
43-
-> \(xs : List Text)
44-
-> let concatWithSepAtStart =
45-
List/concatMap Text Text (\(item : Text) -> [ sep, item ]) xs
31+
let Cmd = ../../Lib/Cmds.dhall
4632

47-
let concat = List/drop 1 Text concatWithSepAtStart
33+
let Mina = ../Mina.dhall
4834

49-
in Text/concat concat
35+
let Artifact = ../../Constants/Artifacts.dhall
5036

5137
let Spec =
5238
{ Type =
@@ -179,7 +165,8 @@ let publish
179165
( "git config --global --add safe.directory /workdir && "
180166
++ ". ./buildkite/scripts/export-git-env-vars.sh && "
181167
++ " gpg --import /var/secrets/debian/key.gpg && "
182-
++ "./buildkite/scripts/release/manager.sh publish "
168+
++ " mkdir -p ./cache && "
169+
++ "DEBIAN_CACHE_FOLDER=/workdir/cache ./buildkite/scripts/release/manager.sh publish "
183170
++ "--artifacts ${artifacts} "
184171
++ "--networks ${networks} "
185172
++ "--buildkite-build-id ${spec.build_id} "
@@ -223,9 +210,7 @@ let publish
223210
-> Command.build
224211
Command.Config::{
225212
, commands =
226-
Toolchain.runner
227-
DebianVersions.DebVersion.Bullseye
228-
[ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" ]
213+
[ Cmd.run
229214
( ". ./buildkite/scripts/export-git-env-vars.sh && "
230215
++ "./buildkite/scripts/release/manager.sh publish "
231216
++ "--artifacts ${artifacts} "
@@ -240,6 +225,7 @@ let publish
240225
++ "--codenames ${codenames} "
241226
++ "--only-dockers "
242227
)
228+
]
243229
, label = "Docker Packages Publishing"
244230
, key = "publish-dockers-${Natural/show r.index}"
245231
, target = Size.Small

‎buildkite/src/Command/Packages/VerifyDockers.dhall‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
let Prelude = ../../External/Prelude.dhall
2+
3+
let Extensions = ../../Lib/Extensions.dhall
4+
5+
let join = Extensions.join
6+
7+
let Artifacts = ../../Constants/Artifacts.dhall
8+
9+
let Package = ../../Constants/DebianPackage.dhall
10+
11+
let Network = ../../Constants/Network.dhall
12+
13+
let Artifact = ../../Constants/Artifacts.dhall
14+
15+
let DebianVersions = ../../Constants/DebianVersions.dhall
16+
17+
let Spec =
18+
{ Type =
19+
{ artifacts : List Artifact.Type
20+
, networks : List Network.Type
21+
, version : Text
22+
, codenames : List DebianVersions.DebVersion
23+
, published_to_docker_io : Bool
24+
, suffix : Optional Text
25+
}
26+
, default =
27+
{ artifacts = [] : List Package.Type
28+
, networks = [ Network.Type.Mainnet, Network.Type.Devnet ]
29+
, codenames =
30+
[ DebianVersions.DebVersion.Focal
31+
, DebianVersions.DebVersion.Bullseye
32+
]
33+
, published_to_docker_io = False
34+
, suffix = None Text
35+
}
36+
}
37+
38+
let joinArtifacts
39+
: Spec.Type -> Text
40+
= \(spec : Spec.Type) -> join "," (Artifacts.dockerNames spec.artifacts)
41+
42+
let joinNetworks
43+
: Spec.Type -> Text
44+
= \(spec : Spec.Type)
45+
-> join
46+
","
47+
( Prelude.List.map
48+
Network.Type
49+
Text
50+
(\(network : Network.Type) -> Network.lowerName network)
51+
spec.networks
52+
)
53+
54+
let joinCodenames
55+
: Spec.Type -> Text
56+
= \(spec : Spec.Type)
57+
-> join
58+
","
59+
( Prelude.List.map
60+
DebianVersions.DebVersion
61+
Text
62+
( \(debian : DebianVersions.DebVersion)
63+
-> DebianVersions.lowerName debian
64+
)
65+
spec.codenames
66+
)
67+
68+
let verify
69+
: Spec.Type -> Text
70+
= \(spec : Spec.Type)
71+
-> ". ./buildkite/scripts/export-git-env-vars.sh && "
72+
++ "./buildkite/scripts/release/manager.sh verify "
73+
++ "--artifacts ${joinArtifacts spec} "
74+
++ "--networks ${joinNetworks spec} "
75+
++ "--version ${spec.version} "
76+
++ "--codenames ${joinCodenames spec} "
77+
++ merge
78+
{ None = ""
79+
, Some = \(suffix : Text) -> "--docker-suffix ${suffix} "
80+
}
81+
spec.suffix
82+
++ "--only-dockers "
83+
84+
in { verify = verify, Spec = Spec }

‎buildkite/src/Entrypoints/PublishPackages.dhall‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
let S = ../Lib/SelectFiles.dhall
22

3-
let PublishPackages = ../Command/PublishPackages.dhall
3+
let PublishPackages = ../Command/Packages/Publish.dhall
44

55
let Profile = ../Constants/Profiles.dhall
66

‎buildkite/src/Jobs/Lint/Bash.dhall‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
let B = ../../External/Buildkite.dhall
2-
31
let S = ../../Lib/SelectFiles.dhall
42

53
let Pipeline = ../../Pipeline/Dsl.dhall
@@ -16,8 +14,6 @@ let Size = ../../Command/Size.dhall
1614

1715
let RunInToolchain = ../../Command/RunInToolchain.dhall
1816

19-
let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
20-
2117
in Pipeline.build
2218
Pipeline.Config::{
2319
, spec = JobSpec::{
@@ -47,7 +43,6 @@ in Pipeline.build
4743
, label = "Bash: shellcheck"
4844
, key = "check-bash"
4945
, target = Size.Multi
50-
, soft_fail = Some (B/SoftFail.Boolean True)
5146
, docker = None Docker.Type
5247
}
5348
]

0 commit comments

Comments
 (0)