From c2ad35d846b87dcc0f11b504815e0eb247c19ec3 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 23 Feb 2023 15:55:17 -0500 Subject: [PATCH 1/2] jobs/build: gate all rerun logic on uploading We only actually run the jobs when `uploading` is set. So it seems wrong to update the build description. In fact, let's not even waste time looking for missing arches or reading build metadata. Patch best viewed with whitespace ignored. --- jobs/build.Jenkinsfile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/jobs/build.Jenkinsfile b/jobs/build.Jenkinsfile index f5a475588..02d2cb359 100644 --- a/jobs/build.Jenkinsfile +++ b/jobs/build.Jenkinsfile @@ -265,15 +265,16 @@ lock(resource: "build-${params.STREAM}") { // Nothing changed since the latest build. Check if it's missing // some arches and retrigger `build-arch` only for the missing // arches, and the follow-up `release` job. Match the exact src - // config commit that was used. - def builds = readJSON file: "builds/builds.json" - assert buildID == builds.builds[0].id - def missing_arches = additional_arches - builds.builds[0].arches - if (missing_arches) { - def meta = readJSON(text: shwrapCapture("cosa meta --build=${buildID} --dump")) - def rev = meta["coreos-assembler.config-gitrev"] - currentBuild.description = "${build_description} 🔨 ${buildID}" - if (uploading) { + // config commit that was used. Skip if not uploading since it's + // required for multi-arch. + if (uploading) { + def builds = readJSON file: "builds/builds.json" + assert buildID == builds.builds[0].id + def missing_arches = additional_arches - builds.builds[0].arches + if (missing_arches) { + def meta = readJSON(text: shwrapCapture("cosa meta --build=${buildID} --dump")) + def rev = meta["coreos-assembler.config-gitrev"] + currentBuild.description = "${build_description} 🔨 ${buildID}" // Run the mArch jobs and wait. We wait here because if they fail // we don't want to bother running the release job again since the // goal is to get a complete build. From a084590661b92a179c6dba82290e272cd7893558 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 23 Feb 2023 15:41:29 -0500 Subject: [PATCH 2/2] jobs/build: don't start new build until previous build completes Since 973bcf9 ("jobs/build: rerun `build-arch` if previous build is incomplete"), there is a race possible where the `build` job rerun logic could kick in before the `release` jobs initially triggered for that build has finished. We don't want to queue builds in that case. Let's just prevent starting up the build job at all if there are remaining pieces from another build for this stream running. Co-authored-by: Jonathan Lebon --- jobs/build.Jenkinsfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jobs/build.Jenkinsfile b/jobs/build.Jenkinsfile index 02d2cb359..1308932ee 100644 --- a/jobs/build.Jenkinsfile +++ b/jobs/build.Jenkinsfile @@ -121,6 +121,15 @@ if (params.WAIT_FOR_RELEASE_JOB) { timeout_mins += timeout_mins + 30 } +// Before proceeding let's make sure no jobs for this stream are currently +// running. This includes multi-arch jobs and release jobs. +echo "Waiting for a clear runway for ${params.STREAM}" +def locks = additional_arches.collect{[resource: "build-${params.STREAM}-${it}"]} +locks += [resource: "release-${params.STREAM}"] +lock(resource: "build-${params.STREAM}", extra: locks) { + echo "The runway is clear for ${params.STREAM}" +} + lock(resource: "build-${params.STREAM}") { timeout(time: timeout_mins, unit: 'MINUTES') { cosaPod(cpu: "${ncpus}",