Merge pull request #17461 from MinaProtocol/dkijania/verify_dockers_in_ci

dkijania · web-flow · commit 9c65e1b41834 · 2025-07-10T19:08:03.000+02:00
Post build Verification for dockers and debians in CI
diff --git a/buildkite/scripts/release/manager.sh b/buildkite/scripts/release/manager.sh
diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall
@@ -26,6 +26,10 @@ let Network = ../Constants/Network.dhall
 
 let DockerPublish = ../Constants/DockerPublish.dhall
 
+let VerifyDockers = ../Command/Packages/VerifyDockers.dhall
+
+let Extensions = ../Lib/Extensions.dhall
+
 let ReleaseSpec =
       { Type =
           { deps : List Command.TaggedKey.Type
@@ -45,6 +49,7 @@ let ReleaseSpec =
           , build_flags : BuildFlags.Type
           , step_key_suffix : Text
           , docker_publish : DockerPublish.Type
+          , verify : Bool
           , if : Optional B/If
           }
       , default =
@@ -65,6 +70,7 @@ let ReleaseSpec =
           , no_cache = False
           , no_debian = False
           , step_key_suffix = "-docker-image"
+          , verify = False
           , if = None B/If
           }
       }
@@ -105,6 +111,43 @@ let generateStep =
 
                 else  " && ./scripts/debian/aptly.sh stop"
 
+          let suffix =
+                Extensions.joinOptionals
+                  "-"
+                  [ merge
+                      { Standard = None Text
+                      , Mainnet = None Text
+                      , Dev = None Text
+                      , Lightnet = Some
+                          "${Profiles.toSuffixLowercase spec.deb_profile}"
+                      }
+                      spec.deb_profile
+                  , merge
+                      { None = None Text
+                      , Instrumented = Some
+                          "${BuildFlags.toSuffixLowercase spec.build_flags}"
+                      }
+                      spec.build_flags
+                  ]
+
+          let maybeVerify =
+                      if     spec.verify
+                         &&  DockerPublish.shouldPublish
+                               spec.docker_publish
+                               spec.service
+
+                then      " && "
+                      ++  VerifyDockers.verify
+                            VerifyDockers.Spec::{
+                            , artifacts = [ spec.service ]
+                            , networks = [ spec.network ]
+                            , version = spec.deb_version
+                            , codenames = [ spec.deb_codename ]
+                            , suffix = suffix
+                            }
+
+                else  ""
+
           let buildDockerCmd =
                     "./scripts/docker/build.sh"
                 ++  " --service ${Artifacts.dockerName spec.service}"
@@ -151,6 +194,7 @@ let generateStep =
                       ++  buildDockerCmd
                       ++  " && "
                       ++  releaseDockerCmd
+                      ++  maybeVerify
                     )
                 ]
 
@@ -169,6 +213,7 @@ let generateStep =
                           ++  " && "
                           ++  releaseDockerCmd
                           ++  maybeStopDebianRepo
+                          ++  maybeVerify
                         )
                     ]
                   }
diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall
@@ -163,6 +163,7 @@ let docker_step
                     , docker_publish = docker_publish
                     , deb_repo = DebianRepo.Type.Local
                     , deb_legacy_version = spec.deb_legacy_version
+                    , verify = True
                     , if = spec.if
                     }
                   ]
@@ -213,6 +214,7 @@ let docker_step
                     , docker_publish = docker_publish
                     , deb_repo = DebianRepo.Type.Local
                     , deb_legacy_version = spec.deb_legacy_version
+                    , verify = True
                     , if = spec.if
                     }
                   ]
@@ -226,6 +228,7 @@ let docker_step
                     , docker_publish = docker_publish
                     , deb_repo = DebianRepo.Type.Local
                     , deb_legacy_version = spec.deb_legacy_version
+                    , verify = True
                     , if = spec.if
                     }
                   ]
diff --git a/buildkite/src/Command/Packages/Publish.dhall b/buildkite/src/Command/Packages/Publish.dhall
@@ -1,52 +1,40 @@
-let Prelude = ../External/Prelude.dhall
+let Prelude = ../../External/Prelude.dhall
 
-let Optional/map = Prelude.Optional.map
-
-let Optional/default = Prelude.Optional.default
-
-let List/drop = Prelude.List.drop
+let Extensions = ../../Lib/Extensions.dhall
 
-let List/concatMap = Prelude.List.concatMap
+let join = Extensions.join
 
-let Text/concat = Prelude.Text.concat
-
-let Artifacts = ../Constants/Artifacts.dhall
-
-let Size = ../Command/Size.dhall
+let Optional/map = Prelude.Optional.map
 
-let Package = ../Constants/DebianPackage.dhall
+let Optional/default = Prelude.Optional.default
 
-let Network = ../Constants/Network.dhall
+let Artifacts = ../../Constants/Artifacts.dhall
 
-let DebianChannel = ../Constants/DebianChannel.dhall
+let Size = ../../Command/Size.dhall
 
-let Profiles = ../Constants/Profiles.dhall
+let Package = ../../Constants/DebianPackage.dhall
 
-let Artifact = ../Constants/Artifacts.dhall
+let Network = ../../Constants/Network.dhall
 
-let DebianVersions = ../Constants/DebianVersions.dhall
+let DebianChannel = ../../Constants/DebianChannel.dhall
 
-let DebianRepo = ../Constants/DebianRepo.dhall
+let Profiles = ../../Constants/Profiles.dhall
 
-let Toolchain = ../Constants/Toolchain.dhall
+let Artifact = ../../Constants/Artifacts.dhall
 
-let ContainerImages = ../Constants/ContainerImages.dhall
+let DebianVersions = ../../Constants/DebianVersions.dhall
 
-let Command = ./Base.dhall
+let DebianRepo = ../../Constants/DebianRepo.dhall
 
-let Cmd = ../Lib/Cmds.dhall
+let Toolchain = ../../Constants/Toolchain.dhall
 
-let Mina = ./Mina.dhall
+let ContainerImages = ../../Constants/ContainerImages.dhall
 
-let join =
-          \(sep : Text)
-      ->  \(xs : List Text)
-      ->  let concatWithSepAtStart =
-                List/concatMap Text Text (\(item : Text) -> [ sep, item ]) xs
+let Command = ../Base.dhall
 
-          let concat = List/drop 1 Text concatWithSepAtStart
+let Cmd = ../../Lib/Cmds.dhall
 
-          in  Text/concat concat
+let Mina = ../Mina.dhall
 
 let Spec =
       { Type =
diff --git a/buildkite/src/Command/Packages/VerifyDockers.dhall b/buildkite/src/Command/Packages/VerifyDockers.dhall
@@ -0,0 +1,84 @@
+let Prelude = ../../External/Prelude.dhall
+
+let Extensions = ../../Lib/Extensions.dhall
+
+let join = Extensions.join
+
+let Artifacts = ../../Constants/Artifacts.dhall
+
+let Package = ../../Constants/DebianPackage.dhall
+
+let Network = ../../Constants/Network.dhall
+
+let Artifact = ../../Constants/Artifacts.dhall
+
+let DebianVersions = ../../Constants/DebianVersions.dhall
+
+let Spec =
+      { Type =
+          { artifacts : List Artifact.Type
+          , networks : List Network.Type
+          , version : Text
+          , codenames : List DebianVersions.DebVersion
+          , published_to_docker_io : Bool
+          , suffix : Optional Text
+          }
+      , default =
+          { artifacts = [] : List Package.Type
+          , networks = [ Network.Type.Mainnet, Network.Type.Devnet ]
+          , codenames =
+            [ DebianVersions.DebVersion.Focal
+            , DebianVersions.DebVersion.Bullseye
+            ]
+          , published_to_docker_io = False
+          , suffix = None Text
+          }
+      }
+
+let joinArtifacts
+    : Spec.Type -> Text
+    = \(spec : Spec.Type) -> join "," (Artifacts.dockerNames spec.artifacts)
+
+let joinNetworks
+    : Spec.Type -> Text
+    =     \(spec : Spec.Type)
+      ->  join
+            ","
+            ( Prelude.List.map
+                Network.Type
+                Text
+                (\(network : Network.Type) -> Network.lowerName network)
+                spec.networks
+            )
+
+let joinCodenames
+    : Spec.Type -> Text
+    =     \(spec : Spec.Type)
+      ->  join
+            ","
+            ( Prelude.List.map
+                DebianVersions.DebVersion
+                Text
+                (     \(debian : DebianVersions.DebVersion)
+                  ->  DebianVersions.lowerName debian
+                )
+                spec.codenames
+            )
+
+let verify
+    : Spec.Type -> Text
+    =     \(spec : Spec.Type)
+      ->      ". ./buildkite/scripts/export-git-env-vars.sh && "
+          ++  "./buildkite/scripts/release/manager.sh verify "
+          ++  "--artifacts ${joinArtifacts spec} "
+          ++  "--networks ${joinNetworks spec} "
+          ++  "--version ${spec.version} "
+          ++  "--codenames ${joinCodenames spec} "
+          ++  merge
+                { None = ""
+                , Some = \(suffix : Text) -> "--docker-suffix ${suffix} "
+                }
+                spec.suffix
+          ++  "--only-dockers "
+
+in  { verify = verify, Spec = Spec }
diff --git a/buildkite/src/Entrypoints/PublishPackages.dhall b/buildkite/src/Entrypoints/PublishPackages.dhall
@@ -1,6 +1,6 @@
 let S = ../Lib/SelectFiles.dhall
 
-let PublishPackages = ../Command/PublishPackages.dhall
+let PublishPackages = ../Command/Packages/Publish.dhall
 
 let Profile = ../Constants/Profiles.dhall
 
diff --git a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall
@@ -22,7 +22,7 @@ let PipelineTag = ../../Pipeline/Tag.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
 
-let PublishPackages = ../../Command/PublishPackages.dhall
+let PublishPackages = ../../Command/Packages/Publish.dhall
 
 let new_tags =
           \(codename : DebianVersions.DebVersion)
diff --git a/buildkite/src/Lib/Extensions.dhall b/buildkite/src/Lib/Extensions.dhall
@@ -0,0 +1,50 @@
+-- This package provides utility functions which are not part of the
+-- standard Prelude, such as `join` for concatenating lists of text with a separator.
+
+let Prelude = ../External/Prelude.dhall
+
+let List/concatMap = Prelude.List.concatMap
+
+let List/drop = Prelude.List.drop
+
+let List/fold_ = Prelude.List.fold
+
+let Text/concat = Prelude.Text.concat
+
+let join =
+          \(sep : Text)
+      ->  \(xs : List Text)
+      ->  let concatWithSepAtStart =
+                List/concatMap Text Text (\(item : Text) -> [ sep, item ]) xs
+
+          let concat = List/drop 1 Text concatWithSepAtStart
+
+          in  Text/concat concat
+
+let joinOptionals
+    : Text -> List (Optional Text) -> Optional Text
+    =     \(sep : Text)
+      ->  \(xs : List (Optional Text))
+      ->  List/fold_
+            (Optional Text)
+            xs
+            (Optional Text)
+            (     \(acc : Optional Text)
+              ->  \(x : Optional Text)
+              ->  merge
+                    { Some =
+                            \(text : Text)
+                        ->  merge
+                              { None = Some text
+                              , Some =
+                                      \(accText : Text)
+                                  ->  Some (accText ++ sep ++ text)
+                              }
+                              acc
+                    , None = acc
+                    }
+                    x
+            )
+            (None Text)
+
+in  { join = join, joinOptionals = joinOptionals }
diff --git a/changes/17461.md b/changes/17461.md
@@ -0,0 +1 @@
+Added post docker & debian build verification jobs to CI 
diff --git a/scripts/docker/verify.sh b/scripts/docker/verify.sh
@@ -1,5 +1,11 @@
 #!/usr/bin/env bash
-set +x
+
+# Verify that the Docker image for a given package and version is working correctly.
+# This script pulls the Docker image and runs a set of simplest commands (fetch help and version) 
+# to ensure that the package dependencies are correctly resolved.
+# Usage: ./scripts/docker/verify.sh -p <package> -c <codename> [-s <suffix>] [-r <repo>] [-v <version>]
+
+set -eo pipefail
 
 REPO=gcr.io/o1labs-192920
 VERSION=3.0.0-f872d85
@@ -13,9 +19,34 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
-if ! docker pull $REPO/$PACKAGE:$VERSION-${CODENAME}${SUFFIX} ; then
+COMMANDS=(--version --help)
+
+case $PACKAGE in
+  mina-archive) APPS=mina-archive ;;
+  mina-logproc) APPS=(); echo "skipped execution for mina-logproc" ;;
+  mina-rosetta*) APPS=("mina" "mina-archive" "mina-rosetta") ;;
+  mina-*) APPS=("mina");;
+  *) echo "❌  Unknown package passed: $PACKAGE"; exit 1;;
+esac
+
+DOCKER_IMAGE="$REPO/$PACKAGE:$VERSION-${CODENAME}${SUFFIX}"
+
+if ! docker pull "$DOCKER_IMAGE" ; then
   echo "❌ Docker verification for $CODENAME $PACKAGE failed"
+  echo "❌ Please check if the image $DOCKER_IMAGE exists."
   exit 1
-else
-  echo "✅ Docker verification for $CODENAME $PACKAGE passed"
 fi
+
+for APP in "${APPS[@]}"; do
+  for COMMAND in "${COMMANDS[@]}"; do
+    echo "📋  Testing $APP $COMMAND in $DOCKER_IMAGE"
+    # Do not quote $COMMAND, because it may contain spaces or other special characters
+    # shellcheck disable=SC2086
+    if ! docker run --entrypoint "$APP" --rm "$DOCKER_IMAGE" $COMMAND; then
+      echo "❌  KO: ERROR running $APP $COMMAND"
+      exit 1
+    fi
+  done
+done
+
+echo '✅  OK: ALL WORKED FINE!'

Original file line number	Diff line number	Diff line change
`@@ -163,6 +163,7 @@ let docker_step`
`163`	`163`	`, docker_publish = docker_publish`
`164`	`164`	`, deb_repo = DebianRepo.Type.Local`
`165`	`165`	`, deb_legacy_version = spec.deb_legacy_version`
	`166`	`+ , verify = True`
`166`	`167`	`, if = spec.if`
`167`	`168`	`}`
`168`	`169`	`]`
`@@ -213,6 +214,7 @@ let docker_step`
`213`	`214`	`, docker_publish = docker_publish`
`214`	`215`	`, deb_repo = DebianRepo.Type.Local`
`215`	`216`	`, deb_legacy_version = spec.deb_legacy_version`
	`217`	`+ , verify = True`
`216`	`218`	`, if = spec.if`
`217`	`219`	`}`
`218`	`220`	`]`
`@@ -226,6 +228,7 @@ let docker_step`
`226`	`228`	`, docker_publish = docker_publish`
`227`	`229`	`, deb_repo = DebianRepo.Type.Local`
`228`	`230`	`, deb_legacy_version = spec.deb_legacy_version`
	`231`	`+ , verify = True`
`229`	`232`	`, if = spec.if`
`230`	`233`	`}`
`231`	`234`	`]`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Added post docker & debian build verification jobs to CI`