Skip to content

Commit db80a08

Browse files
HannesWellHannesWell
committed
[RelEng] Inline promoteSites script and streamline promotion work
Perform the copy and file-renaming directly on the download/storage-server instead of coping all files to the build-agent and back to the server, just to rename them. Also fix BUILD_LABEL computation when DROP_ID is a stable build.
1 parent 9438eb9 commit db80a08

File tree

2 files changed

+115
-307
lines changed

2 files changed

+115
-307
lines changed

‎JenkinsJobs/Releng/promoteBuild.jenkinsfile

Lines changed: 115 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@ pipeline {
1212
HIDE_SITE = 'true'
1313
// Download Server locations (would very seldom change)
1414
EP_ROOT = '/home/data/httpd/download.eclipse.org'
15-
EP_ECLIPSE_ROOT = "${EP_ROOT}/eclipse"
16-
EP_EQUINOX_ROOT = "${EP_ROOT}/equinox"
1715
}
1816
tools {
1917
jdk 'temurin-jdk21-latest'
@@ -30,29 +28,25 @@ pipeline {
3028
env.CHECKPOINT = readParameter('CHECKPOINT')
3129
env.SIGNOFF_BUG = readParameter('SIGNOFF_BUG')
3230
env.TRAIN_NAME = readParameter('TRAIN_NAME')
31+
//TODO: record the train in buildproperties.shsource and read it here
3332
if (!"${TRAIN_NAME}") {
3433
error("Parameter 'TRAIN_NAME' is not specified")
3534
}
3635
def idMatcher = null
37-
if ((idMatcher = env.DROP_ID =~ /(?<type>I)(?<date>\d{8})-(?<time>\d{4})/).matches()) {
36+
if ((idMatcher = env.DROP_ID =~ /I(?<date>\d{8})-(?<time>\d{4})/).matches()) {
3837
assignEnvVariable('BUILD_LABEL', "${DROP_ID}")
39-
} else if ((idMatcher = env.DROP_ID =~ /(?<type>S)-(?<major>\d+)\.(?<minor>\d+)(?<service>\.\d+)?((M|RC)\d+[a-z]?)?-(?<date>\d{8})(?<time>\d{4})/).matches()) {
40-
assignEnvVariable('BUILD_LABEL', idMatcher.group('major') + '.' + idMatcher.group('minor') + (idMatcher.group('service') ?: ''))
38+
} else if ((idMatcher = env.DROP_ID =~ /S-(?<label>\d+\.\d+(\.\d+)?(M|RC)\d+[a-z]?)-(?<date>\d{8})(?<time>\d{4})/).matches()) {
39+
assignEnvVariable('BUILD_LABEL', idMatcher.group('label'))
4140
if ("${CHECKPOINT}") {
4241
error "Stable build DROP_ID=${DROP_ID} may only be promoted to release CHECKPOINT, which therefore must be empty: ${CHECKPOINT}"
4342
}
4443
} else {
4544
error "DROP_ID, ${DROP_ID}, did not match any expected pattern."
4645
}
47-
assignEnvVariable('BUILD_TYPE', idMatcher.group('type'))
48-
assignEnvVariable('REPO_BUILD_TYPE', 'I')
49-
assignEnvVariable('BUILD_TIMESTAMP', idMatcher.group('date') + idMatcher.group('time'))
46+
def buildTimestamp = idMatcher.group('date') + idMatcher.group('time')
5047
assignEnvVariable('REPO_ID', "I${idMatcher.group('date')}-${idMatcher.group('time')}")
5148
idMatcher = null // release matcher as it's not serializable
5249

53-
assignEnvVariable('BUILD_LABEL_EQ', "${BUILD_LABEL}")
54-
assignEnvVariable('DROP_ID_EQ', "${DROP_ID}")
55-
5650
sh 'curl -o buildproperties.shsource --fail https://download.eclipse.org/eclipse/downloads/drops4/${DROP_ID}/buildproperties.shsource'
5751
def STREAM = sh(returnStdout: true, script: 'source ./buildproperties.shsource && echo ${STREAM}').trim()
5852
def versionMatcher = STREAM =~ /(?<major>\d+)\.(?<minor>\d+).(?<service>\d+)/
@@ -64,81 +58,89 @@ pipeline {
6458
assignEnvVariable('BUILD_SERVICE', versionMatcher.group('service'))
6559
versionMatcher = null // release matcher as it's not serializable
6660

67-
assignEnvVariable('BUILD_REPO_ORIGINAL', "${BUILD_MAJOR}.${BUILD_MINOR}-${REPO_BUILD_TYPE}-builds")
61+
assignEnvVariable('BUILD_REPO_ORIGINAL', "${BUILD_MAJOR}.${BUILD_MINOR}-I-builds")
6862

6963
if ("${CHECKPOINT}" ==~ /M\d+([a-z])?/ || "${CHECKPOINT}" ==~ /RC\d+([a-z])?/) { // milestone or RC promotion
7064
assignEnvVariable('DL_TYPE', 'S')
7165
// REPO_SITE_SEGMENT variale not used in this case
72-
} else if(!"${CHECKPOINT}") { // release promotion
66+
} else if (!"${CHECKPOINT}") { // release promotion
7367
assignEnvVariable('DL_TYPE', 'R')
7468
assignEnvVariable('REPO_SITE_SEGMENT', "${BUILD_MAJOR}.${BUILD_MINOR}")
7569
} else {
7670
error "CHECKPOINT, ${CHECKPOINT}, did not match any expected pattern."
7771
}
78-
assignEnvVariable('NEWS_ID', "${BUILD_MAJOR}.${BUILD_MINOR}")
7972

8073
assignEnvVariable('DL_LABEL', "${BUILD_SERVICE}" == '0' // For initial releases, do not include service in label
8174
? "${BUILD_MAJOR}.${BUILD_MINOR}${CHECKPOINT}"
8275
: "${BUILD_MAJOR}.${BUILD_MINOR}.${BUILD_SERVICE}${CHECKPOINT}"
8376
)
84-
assignEnvVariable('DL_LABEL_EQ', "${DL_LABEL}")
85-
86-
// This is DL_DROP_ID for Eclipse. The one for equinox has DL_LABEL_EQ in middle.
87-
assignEnvVariable('DL_DROP_ID', "${DL_TYPE}-${DL_LABEL}-${BUILD_TIMESTAMP}")
88-
assignEnvVariable('DL_DROP_ID_EQ', "${DL_TYPE}-${DL_LABEL_EQ}-${BUILD_TIMESTAMP}")
77+
// This is DL_DROP_ID for Eclipse and Equinox
78+
assignEnvVariable('DL_DROP_ID', "${DL_TYPE}-${DL_LABEL}-${buildTimestamp}")
8979

9080
if (!env.SIGNOFF_BUG) {
9181
echo '''\
9282
[WARNING] SIGNOFF_BUG was not defined. That is valid if no Unit Tests failures but otherwise should be defined.
9383
Can be added by hand to buildproperties.php in drop site, if in fact there were errors, and simply forgot to specify.
9484
'''.stripIndent()
85+
} else if (env.SIGNOFF_BUG ==~ '\\d+') {
86+
assignEnvVariable('SIGNOFF_BUG', "https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/${SIGNOFF_BUG}")
9587
}
88+
assignEnvVariable('SIGNOFF_BUG_LABEL', env.SIGNOFF_BUG.replace('https://github.com/','').replace('/issues/','#'))
9689

9790
def serviceVersionSegment = (env.CHECKPOINT || env.BUILD_SERVICE != '0') ? ('_' + env.BUILD_SERVICE) : ''
9891
assignEnvVariable('TAG', "${DL_TYPE}${BUILD_MAJOR}_${BUILD_MINOR}${serviceVersionSegment}${env.CHECKPOINT ? ('_' + env.CHECKPOINT) : ''}")
9992
}
10093
}
10194
}
102-
stage('Rename and Promote') {
103-
environment {
104-
BUILDMACHINE_BASE_DL = "${EP_ECLIPSE_ROOT}/downloads/drops4"
105-
BUILDMACHINE_BASE_EQ = "${EP_EQUINOX_ROOT}/drops"
106-
// Eclipse and Equinox drop Site (final segment)
107-
ECLIPSE_DL_DROP_DIR_SEGMENT = "${DL_TYPE}-${DL_LABEL}-${BUILD_TIMESTAMP}"
108-
EQUINOX_DL_DROP_DIR_SEGMENT = "${DL_TYPE}-${DL_LABEL_EQ}-${BUILD_TIMESTAMP}"
109-
}
95+
stage('Move and rename Pages') {
11096
steps {
11197
writeFile(file: "${WORKSPACE}/stage2output${TRAIN_NAME}${CHECKPOINT}/mailtemplate.txt", text: """\
11298
We are pleased to announce that ${TRAIN_NAME} ${CHECKPOINT} is available for download and updates.
11399

114100
Eclipse downloads:
115-
https://download.eclipse.org/eclipse/downloads/drops4/${ECLIPSE_DL_DROP_DIR_SEGMENT}/
101+
https://download.eclipse.org/eclipse/downloads/drops4/${DL_DROP_ID}/
116102

117103
New and Noteworthy:
118-
https://www.eclipse.org/eclipse/news/${NEWS_ID}/
104+
https://www.eclipse.org/eclipse/news/${BUILD_MAJOR}.${BUILD_MINOR}/
119105

120106
Update existing (non-production) installs:
121107
https://download.eclipse.org/eclipse/updates/${DL_TYPE == 'R' ? REPO_SITE_SEGMENT : BUILD_REPO_ORIGINAL}/
122108

123109
Specific repository good for building against:
124-
https://download.eclipse.org/eclipse/updates/${DL_TYPE == 'R' ? (REPO_SITE_SEGMENT + '/' + ECLIPSE_DL_DROP_DIR_SEGMENT) : (BUILD_REPO_ORIGINAL + '/' + DROP_ID)}/
110+
https://download.eclipse.org/eclipse/updates/${DL_TYPE == 'R' ? (REPO_SITE_SEGMENT + '/' + DL_DROP_ID) : (BUILD_REPO_ORIGINAL + '/' + DROP_ID)}/
125111

126112
Equinox specific downloads:
127-
https://download.eclipse.org/equinox/drops/${EQUINOX_DL_DROP_DIR_SEGMENT}/
113+
https://download.eclipse.org/equinox/drops/${DL_DROP_ID}/
128114

129115
Thank you to everyone who made this checkpoint possible.
130116
""".stripIndent())
131117
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
132-
sh '''#!/bin/bash -x
133-
curl -o promoteSites.sh https://download.eclipse.org/eclipse/relengScripts/cje-production/promotion/promoteSites.sh
134-
chmod +x promoteSites.sh
135-
./promoteSites.sh
136-
'''
118+
119+
echo 'Promote Equinox'
120+
dir("${WORKSPACE}/equinox") {
121+
renameBuildDrop('equinox/drops', "${DROP_ID}", "${BUILD_LABEL}", "${DL_DROP_ID}", "${DL_LABEL}")
122+
}
123+
124+
echo 'Promote Eclipse'
125+
dir("${WORKSPACE}/eclipse") {
126+
renameBuildDrop('eclipse/downloads/drops4', "${DROP_ID}", "${BUILD_LABEL}", "${DL_DROP_ID}", "${DL_LABEL}") {
127+
sh '''
128+
echo "\\$NEWS_ID = \\"${BUILD_MAJOR}.${BUILD_MINOR}\\";" >> 'buildproperties.php'
129+
echo "\\$ACK_ID = \\"${BUILD_MAJOR}.${BUILD_MINOR}\\";" >> 'buildproperties.php'
130+
echo "\\$README_ID = \\"${BUILD_MAJOR}.${BUILD_MINOR}\\";" >> 'buildproperties.php'
131+
132+
# SIGNOFF_BUG should not be defined if there are no JUnit failures to investigate and explain
133+
if [[ -n "${SIGNOFF_BUG}" ]]; then
134+
echo -e "<p>Any unit test failures below have been investigated and found to be test-related and do not affect the quality of the build.\\nSee the sign-off page <a href=\\"${SIGNOFF_BUG}\\">${SIGNOFF_BUG_LABEL}</a> for details.</p>" > 'testNotes.html'
135+
fi
136+
'''
137+
}
138+
}
137139
}
138140
build job: 'Releng/tagEclipseRelease', wait: true, propagate: true, parameters: [
139141
string(name: 'tag', value: "${TAG}"),
140142
string(name: 'buildID', value: "${DROP_ID}"),
141-
string(name: 'annotation', value: "${ params.SIGNOFF_BUG ? 'https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/' + params.SIGNOFF_BUG : '' }")
143+
string(name: 'annotation', value: "${SIGNOFF_BUG}")
142144
]
143145
}
144146
}
@@ -177,7 +179,7 @@ pipeline {
177179
}
178180
post {
179181
always {
180-
archiveArtifacts '**/stage2output*/**'
182+
archiveArtifacts '**/*'
181183
}
182184
}
183185
}
@@ -197,3 +199,78 @@ def readParameter(String name) {
197199
}
198200
return value
199201
}
202+
203+
def renameBuildDrop(String baseDropPath, String oldDropID, String oldBuildLabel, String newDropID, String newBuildLabel, Closure extraTasks=null) {
204+
def sourcePath="${EP_ROOT}/${baseDropPath}/${oldDropID}"
205+
def targetPath="${EP_ROOT}/${baseDropPath}/${newDropID}"
206+
207+
sh """#!/bin/bash -xe
208+
209+
# Copy drop-directory to new location
210+
ssh [email protected] mkdir -p '${targetPath}'
211+
if [[ "${HIDE_SITE}" == "true" ]]; then
212+
# Create this marker first to ensure the page is never without
213+
ssh [email protected] touch '${targetPath}/buildHidden'
214+
fi
215+
ssh [email protected] cp -r '${sourcePath}/.' '${targetPath}'
216+
217+
# Rename files
218+
echo 'Rename files at new location from *${oldBuildLabel}* to *${newBuildLabel}*.'
219+
# References to locally defined variables required double escaping
220+
# because they are escaped for groovy and the bash HERE document.
221+
# In general this section is very fragile! Only change with great caution and extensive testing!
222+
ssh [email protected] << EOF
223+
set -xe
224+
cd "${targetPath}"
225+
files=\\\$(find . -mindepth 1 -maxdepth 2 -name "*${oldBuildLabel}*" -print)
226+
echo " \\\$(echo "\\\$files" | wc -l) files found to rename."
227+
228+
for file in \\\${files}; do
229+
if [[ \\\$file =~ (.*)(${oldBuildLabel})(.*) ]]; then
230+
mv "\\\$file" "\\\${BASH_REMATCH[1]}${newBuildLabel}\\\${BASH_REMATCH[3]}"
231+
fi
232+
done
233+
#DO NOT INDENT EOF!
234+
EOF
235+
236+
#Update checksums to new filenames
237+
ssh [email protected] sed --in-place --expression 's/${oldBuildLabel}/${newBuildLabel}/g' ${targetPath}/checksum/*
238+
"""
239+
240+
// Update buildproperties.php to new names
241+
def newTypeName = null
242+
if ("${DL_TYPE}" == 'R') {
243+
newTypeName = 'Release'
244+
} else if (newBuildLabel.contains('RC')) {
245+
newTypeName = 'Release Candidate'
246+
} else if ("${DL_TYPE}" == 'S') {
247+
newTypeName = 'Stable'
248+
} else {
249+
error "Unexpected DL_TYPE value, ${DL_TYPE}"
250+
}
251+
def newProperties = [
252+
BUILD_ID: "${newBuildLabel}",
253+
BUILD_TYPE: "${DL_TYPE}",
254+
BUILD_TYPE_NAME: "${newTypeName}",
255+
BUILD_DIR_SEG: "${newDropID}",
256+
EQ_BUILD_DIR_SEG: "${newDropID}",
257+
]
258+
def buildPropertiesPHP = sh(returnStdout: true, script: "ssh [email protected] cat ${sourcePath}/buildproperties.php")
259+
for (entry in newProperties) {
260+
buildPropertiesPHP = buildPropertiesPHP.replaceFirst('\\$' + entry.key + ' = "[^"]+"', '\\$' + entry.key + ' = "' + entry.value + '"')
261+
}
262+
writeFile(file: 'buildproperties.php', text: buildPropertiesPHP)
263+
264+
if (extraTasks) {
265+
extraTasks()
266+
}
267+
sh """#!/bin/bash -xe
268+
# Copy locally modified files to the download server
269+
scp -r . [email protected]:${targetPath}/
270+
271+
if [[ '${DL_TYPE}' == 'R' ]]; then
272+
echo 'Creating archive'
273+
ssh [email protected] cp -r '${targetPath}' /home/data/httpd/archive.eclipse.org/${baseDropPath}/
274+
fi
275+
"""
276+
}

0 commit comments

Comments
 (0)