Skip to content

Commit 1820d00

Browse files
HannesWellHannesWell
committed
[I/Y-Build] Parallelize result gathering and promotion
Also unify specification of shell options.
1 parent 2b75bbc commit 1820d00

File tree

5 files changed

+167
-167
lines changed

5 files changed

+167
-167
lines changed

‎JenkinsJobs/Builds/build.jenkinsfile‎

Lines changed: 161 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -228,118 +228,180 @@ pipeline {
228228
}
229229
}
230230
}
231-
stage('Gather Eclipse Parts'){
232-
environment {
233-
KEYRING = credentials('secret-subkeys-releng.asc')
234-
KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase')
235-
}
236-
steps {
237-
dir("${CJE_ROOT}/mbscripts") {
238-
sh '''
239-
set -eo pipefail
240-
bash -x ./mb300_gatherEclipseParts.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb300_gatherEclipseParts.sh.log
241-
'''
242-
}
243-
}
244-
}
245-
stage('Gather Equinox Parts'){
246-
environment {
247-
KEYRING = credentials('secret-subkeys-releng.asc')
248-
KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase')
249-
}
250-
steps {
251-
dir("${CJE_ROOT}/mbscripts") {
252-
sh '''
253-
set -eo pipefail
254-
./mb310_gatherEquinoxParts.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb310_gatherEquinoxParts.sh.log
255-
'''
256-
}
257-
}
258-
}
259-
stage('Generate Repo reports'){
260-
steps {
261-
dir("${CJE_ROOT}/mbscripts") {
262-
sh '''
263-
set -eo pipefail
264-
./mb500_createRepoReports.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb500_createRepoReports.sh.log
265-
'''
266-
}
267-
}
268-
}
269-
stage('Generate API tools reports'){
270-
steps {
271-
dir("${CJE_ROOT}/mbscripts") {
272-
sh '''
273-
set -eo pipefail
274-
./mb510_createApiToolsReports.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb510_createApiToolsReports.sh.log
275-
'''
276-
}
277-
}
278-
}
279-
stage('Export environment variables stage 2'){
231+
stage('Check for comparator errors') {
280232
steps {
233+
sh '''#!/bin/bash -xe
234+
source $CJE_ROOT/scripts/common-functions.shsource
235+
source $CJE_ROOT/buildproperties.shsource
236+
237+
# Gather maven properties
238+
cp $CJE_ROOT/$AGG_DIR/eclipse-platform-parent/target/mavenproperties.properties $CJE_ROOT/$DROP_DIR/$BUILD_ID/mavenproperties.properties
239+
240+
# Gather artifactcomparisons
241+
pushd $CJE_ROOT/$AGG_DIR
242+
comparatorlogsDir=${CJE_ROOT}/${DROP_DIR}/${BUILD_ID}/buildlogs/comparatorlogs
243+
mkdir -p $comparatorlogsDir
244+
find . -regex .*target/artifactcomparison -type d -exec zip -r $comparatorlogsDir/artifactcomparisons.zip '{}' \\;
245+
popd
246+
247+
# Verify comparatorlogs
248+
#
249+
# Note: copy mb220_buildSdkPatch.sh.log as mb060_run-maven-build_output.txt for now to avoid changing eclipse_compare.xml
250+
# ToDo: Modify org.eclipse.releng.build.tools.comparator.Extractor to be configurable:
251+
# https://github.com/eclipse-platform/eclipse.platform.releng.buildtools/blob/c5f7ecf1951d44311e24ce7bd6b505189aabb4da/bundles/org.eclipse.releng.build.tools.comparator/src/org/eclipse/releng/build/tools/comparator/Extractor.java#L27
252+
cp ${logDir}/mb220_buildSdkPatch.sh.log $CJE_ROOT/$DROP_DIR/$BUILD_ID/buildlogs/mb060_run-maven-build_output.txt
253+
254+
pushd $CJE_ROOT/$DROP_DIR/$BUILD_ID
255+
java -jar $LAUNCHER_JAR \
256+
-application org.eclipse.ant.core.antRunner \
257+
-buildfile $ECLIPSE_BUILDER_DIR/eclipse/buildScripts/eclipse_compare.xml \
258+
-data $CJE_ROOT/$TMP_DIR/workspace-comparatorLogs \
259+
-DbuildDirectory=$CJE_ROOT/$DROP_DIR/$BUILD_ID \
260+
-Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \
261+
-v \
262+
compare
263+
popd
264+
265+
comparatorLogMinimumSize=350
266+
comparatorLog=${comparatorlogsDir}/buildtimeComparatorUnanticipated.log.txt
267+
logSize=$(stat --format='%s' ${comparatorLog})
268+
if [[ ${logSize} -gt ${comparatorLogMinimumSize} ]]; then
269+
echo -e "DEBUG: found logsize greater an minimum. preparing message using ${link}"
270+
fn-write-property COMPARATOR_ERRORS "true"
271+
fn-write-property COMPARATOR_ERRORS_SUBJECT "\\"- Comparator Errors Found\\""
272+
else
273+
echo -e "DEBUG: comparator logSize of $logSize was not greater than comparatorLogMinimumSize of ${comparatorLogMinimumSize}"
274+
fn-write-property COMPARATOR_ERRORS_SUBJECT "\\"\\""
275+
fi
276+
'''
281277
script {
282278
def buildProps = readBuildProperties("${CJE_ROOT}/buildproperties.properties")
283279
env.COMPARATOR_ERRORS_SUBJECT = buildProps.COMPARATOR_ERRORS_SUBJECT
284280
}
285281
}
286282
}
287-
stage('Archive artifacts'){
288-
steps {
289-
sh '''
290-
source $CJE_ROOT/buildproperties.shsource
291-
cp -r $logDir/* $CJE_ROOT/$DROP_DIR/$BUILD_ID/buildlogs
292-
rm -rf $CJE_ROOT/$DROP_DIR/$BUILD_ID/apitoolingreference
293-
cp $CJE_ROOT/buildproperties.* $CJE_ROOT/$DROP_DIR/$BUILD_ID
294-
cp $CJE_ROOT/$DROP_DIR/$BUILD_ID/buildproperties.* $CJE_ROOT/$EQUINOX_DROP_DIR/$BUILD_ID
295-
'''
296-
}
297-
}
298-
stage('Promote Eclipse platform'){
299-
steps {
300-
dir("${CJE_ROOT}/mbscripts") {
301-
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
302-
sh '''
303-
./mb600_promoteEclipse.sh $CJE_ROOT/buildproperties.shsource
304-
'''
305-
}
306-
build job: 'Releng/updateIndex', wait: false
283+
stage('Promote results') {
284+
parallel {
285+
stage('Eclipse') {
286+
stages {
287+
stage('Gather Eclipse parts') {
288+
environment {
289+
KEYRING = credentials('secret-subkeys-releng.asc')
290+
KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase')
291+
}
292+
steps {
293+
dir("${CJE_ROOT}/mbscripts") {
294+
sh '''#!/bin/bash -xe
295+
set -o pipefail
296+
source $CJE_ROOT/buildproperties.shsource
297+
cp $CJE_ROOT/buildproperties.* $CJE_ROOT/$DROP_DIR/$BUILD_ID
298+
./mb300_gatherEclipseParts.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb300_gatherEclipseParts.sh.log
299+
'''
300+
}
301+
}
302+
}
303+
stage('Generate Repo reports') {
304+
steps {
305+
dir("${CJE_ROOT}/mbscripts") {
306+
sh '''#!/bin/bash -xe
307+
set -eo pipefail
308+
./mb500_createRepoReports.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb500_createRepoReports.sh.log
309+
'''
310+
}
311+
}
312+
}
313+
stage('Generate API tools reports') {
314+
steps {
315+
dir("${CJE_ROOT}/mbscripts") {
316+
sh '''#!/bin/bash -xe
317+
set -eo pipefail
318+
source $CJE_ROOT/buildproperties.shsource
319+
./mb510_createApiToolsReports.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb510_createApiToolsReports.sh.log
320+
rm -rf $CJE_ROOT/$DROP_DIR/$BUILD_ID/apitoolingreference
321+
'''
322+
}
323+
}
324+
}
325+
stage('Promote Eclipse platform') {
326+
steps {
327+
dir("${CJE_ROOT}/mbscripts") {
328+
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
329+
sh '''#!/bin/bash -xe
330+
./mb600_promoteEclipse.sh $CJE_ROOT/buildproperties.shsource
331+
'''
332+
}
333+
build job: 'Releng/updateIndex', wait: false
334+
}
335+
}
336+
}
337+
}
307338
}
308-
}
309-
}
310-
stage('Promote Equinox'){
311-
steps {
312-
dir("${CJE_ROOT}/mbscripts") {
313-
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
314-
sh '''
315-
./mb610_promoteEquinox.sh $CJE_ROOT/buildproperties.shsource
316-
'''
317-
}
339+
stage('Equinox') {
340+
stages {
341+
stage('Gather Equinox parts') {
342+
environment {
343+
KEYRING = credentials('secret-subkeys-releng.asc')
344+
KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase')
345+
}
346+
steps {
347+
dir("${CJE_ROOT}/mbscripts") {
348+
sh '''#!/bin/bash -xe
349+
set -o pipefail
350+
source $CJE_ROOT/buildproperties.shsource
351+
cp $CJE_ROOT/buildproperties.* $CJE_ROOT/$EQUINOX_DROP_DIR/$BUILD_ID
352+
./mb310_gatherEquinoxParts.sh $CJE_ROOT/buildproperties.shsource 2>&1 | tee $logDir/mb310_gatherEquinoxParts.sh.log
353+
'''
354+
}
355+
}
356+
}
357+
stage('Promote Equinox') {
358+
steps {
359+
dir("${CJE_ROOT}/mbscripts") {
360+
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
361+
sh '''#!/bin/bash -xe
362+
./mb610_promoteEquinox.sh $CJE_ROOT/buildproperties.shsource
363+
'''
364+
}
365+
}
366+
}
367+
}
368+
}
318369
}
319-
}
320-
}
321-
stage('Promote Update Site'){
322-
steps {
323-
dir("${CJE_ROOT}/mbscripts") {
324-
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
325-
sh '''
326-
./mb620_promoteUpdateSite.sh $CJE_ROOT/buildproperties.shsource
327-
'''
328-
}
329-
script {
330-
if ("${COMPARATOR_ERRORS_SUBJECT}" != '' && "$BUILD_TYPE" == "I") {
331-
echo 'Skip adding unstable build to composite repository.'
332-
return
370+
stage('Update Site') {
371+
steps {
372+
dir("${CJE_ROOT}/mbscripts") {
373+
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
374+
sh '''#!/bin/bash -xe
375+
./mb620_promoteUpdateSite.sh $CJE_ROOT/buildproperties.shsource
376+
'''
377+
}
378+
}
379+
script {
380+
if ("${COMPARATOR_ERRORS_SUBJECT}" != '' && "${BUILD_TYPE}" == 'I') {
381+
echo 'Skip adding unstable build to composite repository.'
382+
return
383+
}
384+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
385+
string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VER}-${BUILD_TYPE}-builds"),
386+
string(name: 'add', value: "${BUILD_IID}"),
387+
string(name: 'sizeLimit', value: '3')
388+
]
333389
}
334-
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
335-
string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VER}-${BUILD_TYPE}-builds"),
336-
string(name: 'add', value: "${BUILD_IID}"),
337-
string(name: 'sizeLimit', value: '3')
338-
]
339390
}
340391
}
341392
}
342393
}
394+
stage('Archive build logs') {
395+
steps {
396+
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
397+
sh '''#!/bin/bash -xe
398+
source $CJE_ROOT/buildproperties.shsource
399+
dropPath=/home/data/httpd/download.eclipse.org/eclipse/downloads/drops4/${BUILD_ID}
400+
scp -r $logDir/* [email protected]:${dropPath}/buildlogs/
401+
'''
402+
}
403+
}
404+
}
343405
stage('Trigger tests') {
344406
steps {
345407
script {
@@ -412,7 +474,7 @@ def assignEnvVariable(String name, String value) {
412474
println("${name}=${value}")
413475
}
414476

415-
def readBuildProperties(String buildPropertiesFile){
477+
def readBuildProperties(String buildPropertiesFile) {
416478
return readProperties(file: buildPropertiesFile, charset: 'UTF-8').collectEntries{n, v ->
417479
v = v.trim();
418480
return [n, (v.startsWith('"') && v.endsWith('"') ? v.substring(1, v.length() - 1) : v)]

‎cje-production/buildproperties.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#*******************************************************************************
2-
# Copyright (c) 2018, 2024 IBM Corporation and others.
2+
# Copyright (c) 2018, 2025 IBM Corporation and others.
33
#
44
# This program and the accompanying materials
55
# are made available under the terms of the Eclipse Public License 2.0
@@ -23,7 +23,6 @@ STREAMMajor="4"
2323
STREAMMinor="38"
2424
STREAMService="0"
2525
DROP_DIR="siteDir/eclipse/downloads/drops4"
26-
UPDATES_DIR="siteDir/eclipse/updates"
2726
EQUINOX_DROP_DIR="siteDir/equinox/drops"
2827
TMP_DIR="tmp"
2928
DOWNLOAD_HOST="download.eclipse.org"

‎cje-production/mbscripts/mb300_gatherEclipseParts.sh‎

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ source $CJE_ROOT/scripts/common-functions.shsource
2323
source $1
2424

2525
mkdir -p $CJE_ROOT/$DROP_DIR/$BUILD_ID
26-
mkdir -p $CJE_ROOT/$UPDATES_DIR/$BUILD_ID
2726
mkdir -p $CJE_ROOT/$EQUINOX_DROP_DIR/$BUILD_ID
2827
mkdir -p $CJE_ROOT/$DROP_DIR/$BUILD_ID/testresults/consolelogs
2928

@@ -33,15 +32,8 @@ JavaCMD=${JAVA_HOME}/bin/java
3332
cp $CJE_ROOT/$AGG_DIR/eclipse-platform-parent/target/mavenproperties.properties $CJE_ROOT/$DROP_DIR/$BUILD_ID/mavenproperties.properties
3433

3534
# gather repo
36-
REPO_DIR=$PLATFORM_REPO_DIR
3735
REPO_ZIP=$PLATFORM_TARGET_DIR/eclipse.platform.repository-${STREAMMajor}.${STREAMMinor}.${STREAMService}-SNAPSHOT.zip
3836

39-
if [ -d $REPO_DIR ]; then
40-
pushd $REPO_DIR
41-
cp -r * $CJE_ROOT/$UPDATES_DIR/$BUILD_ID
42-
popd
43-
fi
44-
4537
if [ -f $REPO_ZIP ]; then
4638
cp $REPO_ZIP $CJE_ROOT/$DROP_DIR/$BUILD_ID/repository-$BUILD_ID.zip
4739
fi
@@ -142,36 +134,6 @@ if [ -d $CJE_ROOT/$AGG_DIR ]; then
142134
popd
143135
fi
144136

145-
# gather artifactcomparisons
146-
if [ -d $CJE_ROOT/$AGG_DIR ]; then
147-
pushd $CJE_ROOT/$AGG_DIR
148-
comparatorlogsDir=$CJE_ROOT/$DROP_DIR/$BUILD_ID/buildlogs/comparatorlogs
149-
mkdir -p $comparatorlogsDir
150-
find . -regex .*target/artifactcomparison -type d -exec zip -r $comparatorlogsDir/artifactcomparisons.zip '{}' \;
151-
popd
152-
fi
153-
154-
# verify comparatorlogs
155-
#
156-
# Note: copy mb220_buildSdkPatch.sh.log as mb060_run-maven-build_output.txt for now to avoid changing eclipse_compare.xml
157-
# To-do: modify eclipse_compare.xml to use mb220_buildSdkPatch.sh.log after CJE migration
158-
cp $CJE_ROOT/buildlogs/mb220_buildSdkPatch.sh.log $CJE_ROOT/$DROP_DIR/$BUILD_ID/buildlogs/mb060_run-maven-build_output.txt
159-
#
160-
pushd $CJE_ROOT/$DROP_DIR/$BUILD_ID
161-
ANT_SCRIPT=$ECLIPSE_BUILDER_DIR/eclipse/buildScripts/eclipse_compare.xml
162-
$JavaCMD -jar $LAUNCHER_JAR \
163-
-application org.eclipse.ant.core.antRunner \
164-
-buildfile $ANT_SCRIPT \
165-
-data $CJE_ROOT/$TMP_DIR/workspace-comparatorLogs \
166-
-DEBuilderDir=$ECLIPSE_BUILDER_DIR \
167-
-DbuildDirectory=$CJE_ROOT/$DROP_DIR/$BUILD_ID \
168-
-DbuildId=$BUILD_ID \
169-
-DbuildLabel=$BUILD_ID \
170-
-Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \
171-
-v \
172-
compare
173-
popd
174-
175137
# gather compilelogs
176138
if [ -d $CJE_ROOT/$AGG_DIR ]; then
177139
pushd $CJE_ROOT/$AGG_DIR
@@ -248,27 +210,3 @@ $JavaCMD -jar $LAUNCHER_JAR \
248210
-v \
249211
publish
250212
popd
251-
252-
comparatorLogMinimumSize=350
253-
comparatorLog=$CJE_ROOT/$DROP_DIR/$BUILD_ID/buildlogs/comparatorlogs/buildtimeComparatorUnanticipated.log.txt
254-
255-
logSize=0
256-
if [[ -e ${comparatorLog} ]]
257-
then
258-
logSize=$(stat -c '%s' ${comparatorLog} )
259-
echo -e "DEBUG: comparatorLog found at\n\t${comparatorLog}\n\tWith size of $logSize bytes"
260-
else
261-
echo -e "DEBUG: comparatorLog was surprisingly not found at:\n\t${comparatorLog}"
262-
fi
263-
264-
if [[ $logSize -gt ${comparatorLogMinimumSize} ]]
265-
then
266-
echo -e "DEBUG: found logsize greater an minimum. preparing message using ${link}"
267-
268-
fn-write-property COMPARATOR_ERRORS "true"
269-
fn-write-property COMPARATOR_ERRORS_SUBJECT "\"- Comparator Errors Found\""
270-
else
271-
echo -e "DEBUG: comparator logSize of $logSize was not greater than comparatorLogMinimumSize of ${comparatorLogMinimumSize}"
272-
fn-write-property COMPARATOR_ERRORS_SUBJECT "\"\""
273-
fi
274-

0 commit comments

Comments
 (0)