Skip to content

Commit ccd0140

Browse files
HannesWellHannesWell
committed
[RelEng] Inline promoteSites script and streamline promotion work
1 parent 9d12a33 commit ccd0140

File tree

2 files changed

+128
-299
lines changed

2 files changed

+128
-299
lines changed

‎JenkinsJobs/Releng/promoteBuild.jenkinsfile‎

Lines changed: 128 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@ pipeline {
1212
HIDE_SITE = 'true'
1313
// Download Server locations (would very seldom change)
1414
EP_ROOT = '/home/data/httpd/download.eclipse.org'
15-
EP_ECLIPSE_ROOT = "${EP_ROOT}/eclipse"
16-
EP_EQUINOX_ROOT = "${EP_ROOT}/equinox"
1715
}
1816
tools {
1917
jdk 'temurin-jdk21-latest'
@@ -50,9 +48,6 @@ pipeline {
5048
assignEnvVariable('REPO_ID', "I${idMatcher.group('date')}-${idMatcher.group('time')}")
5149
idMatcher = null // release matcher as it's not serializable
5250

53-
assignEnvVariable('BUILD_LABEL_EQ', "${BUILD_LABEL}")
54-
assignEnvVariable('DROP_ID_EQ', "${DROP_ID}")
55-
5651
sh 'curl -o buildproperties.shsource --fail https://download.eclipse.org/eclipse/downloads/drops4/${DROP_ID}/buildproperties.shsource'
5752
def STREAM = sh(returnStdout: true, script: 'source ./buildproperties.shsource && echo ${STREAM}').trim()
5853
def versionMatcher = STREAM =~ /(?<major>\d+)\.(?<minor>\d+).(?<service>\d+)/
@@ -69,76 +64,85 @@ pipeline {
6964
if ("${CHECKPOINT}" ==~ /M\d+([a-z])?/ || "${CHECKPOINT}" ==~ /RC\d+([a-z])?/) { // milestone or RC promotion
7065
assignEnvVariable('DL_TYPE', 'S')
7166
// REPO_SITE_SEGMENT variale not used in this case
72-
} else if(!"${CHECKPOINT}") { // release promotion
67+
} else if (!"${CHECKPOINT}") { // release promotion
7368
assignEnvVariable('DL_TYPE', 'R')
7469
assignEnvVariable('REPO_SITE_SEGMENT', "${BUILD_MAJOR}.${BUILD_MINOR}")
7570
} else {
7671
error "CHECKPOINT, ${CHECKPOINT}, did not match any expected pattern."
7772
}
78-
assignEnvVariable('NEWS_ID', "${BUILD_MAJOR}.${BUILD_MINOR}")
7973

8074
assignEnvVariable('DL_LABEL', "${BUILD_SERVICE}" == '0' // For initial releases, do not include service in label
8175
? "${BUILD_MAJOR}.${BUILD_MINOR}${CHECKPOINT}"
8276
: "${BUILD_MAJOR}.${BUILD_MINOR}.${BUILD_SERVICE}${CHECKPOINT}"
8377
)
84-
assignEnvVariable('DL_LABEL_EQ', "${DL_LABEL}")
85-
86-
// This is DL_DROP_ID for Eclipse. The one for equinox has DL_LABEL_EQ in middle.
78+
// This is DL_DROP_ID for Eclipse and Equinox
8779
assignEnvVariable('DL_DROP_ID', "${DL_TYPE}-${DL_LABEL}-${BUILD_TIMESTAMP}")
88-
assignEnvVariable('DL_DROP_ID_EQ', "${DL_TYPE}-${DL_LABEL_EQ}-${BUILD_TIMESTAMP}")
8980

9081
if (!env.SIGNOFF_BUG) {
9182
echo '''\
9283
[WARNING] SIGNOFF_BUG was not defined. That is valid if no Unit Tests failures but otherwise should be defined.
9384
Can be added by hand to buildproperties.php in drop site, if in fact there were errors, and simply forgot to specify.
9485
'''.stripIndent()
86+
} else if (env.SIGNOFF_BUG ==~ '\\d+') {
87+
assignEnvVariable('SIGNOFF_BUG', "https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/${SIGNOFF_BUG}")
9588
}
89+
assignEnvVariable('SIGNOFF_BUG_LABEL', env.SIGNOFF_BUG.replace('https://github.com/','').replace('/issues/','#'))
9690

9791
def serviceVersionSegment = (env.CHECKPOINT || env.BUILD_SERVICE != '0') ? ('_' + env.BUILD_SERVICE) : ''
9892
assignEnvVariable('TAG', "${DL_TYPE}${BUILD_MAJOR}_${BUILD_MINOR}${serviceVersionSegment}${env.CHECKPOINT ? ('_' + env.CHECKPOINT) : ''}")
93+
//TODO: Remove now unused (because previously duplicated) parameters
9994
}
10095
}
10196
}
102-
stage('Rename and Promote') {
103-
environment {
104-
BUILDMACHINE_BASE_DL = "${EP_ECLIPSE_ROOT}/downloads/drops4"
105-
BUILDMACHINE_BASE_EQ = "${EP_EQUINOX_ROOT}/drops"
106-
// Eclipse and Equinox drop Site (final segment)
107-
ECLIPSE_DL_DROP_DIR_SEGMENT = "${DL_TYPE}-${DL_LABEL}-${BUILD_TIMESTAMP}"
108-
EQUINOX_DL_DROP_DIR_SEGMENT = "${DL_TYPE}-${DL_LABEL_EQ}-${BUILD_TIMESTAMP}"
109-
}
97+
stage('Move and rename Pages') {
11098
steps {
11199
writeFile(file: "${WORKSPACE}/stage2output${TRAIN_NAME}${CHECKPOINT}/mailtemplate.txt", text: """\
112100
We are pleased to announce that ${TRAIN_NAME} ${CHECKPOINT} is available for download and updates.
113101

114102
Eclipse downloads:
115-
https://download.eclipse.org/eclipse/downloads/drops4/${ECLIPSE_DL_DROP_DIR_SEGMENT}/
103+
https://download.eclipse.org/eclipse/downloads/drops4/${DL_DROP_ID}/
116104

117105
New and Noteworthy:
118-
https://www.eclipse.org/eclipse/news/${NEWS_ID}/
106+
https://www.eclipse.org/eclipse/news/${BUILD_MAJOR}.${BUILD_MINOR}/
119107

120108
Update existing (non-production) installs:
121109
https://download.eclipse.org/eclipse/updates/${DL_TYPE == 'R' ? REPO_SITE_SEGMENT : BUILD_REPO_ORIGINAL}/
122110

123111
Specific repository good for building against:
124-
https://download.eclipse.org/eclipse/updates/${DL_TYPE == 'R' ? (REPO_SITE_SEGMENT + '/' + ECLIPSE_DL_DROP_DIR_SEGMENT) : (BUILD_REPO_ORIGINAL + '/' + DROP_ID)}/
112+
https://download.eclipse.org/eclipse/updates/${DL_TYPE == 'R' ? (REPO_SITE_SEGMENT + '/' + DL_DROP_ID) : (BUILD_REPO_ORIGINAL + '/' + DROP_ID)}/
125113

126114
Equinox specific downloads:
127-
https://download.eclipse.org/equinox/drops/${EQUINOX_DL_DROP_DIR_SEGMENT}/
115+
https://download.eclipse.org/equinox/drops/${DL_DROP_ID}/
128116

129117
Thank you to everyone who made this checkpoint possible.
130118
""".stripIndent())
131119
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
132-
sh '''#!/bin/bash -x
133-
curl -o promoteSites.sh https://download.eclipse.org/eclipse/relengScripts/cje-production/promotion/promoteSites.sh
134-
chmod +x promoteSites.sh
135-
./promoteSites.sh
136-
'''
120+
121+
echo 'Promote Equinox'
122+
dir("${WORKSPACE}/equinox") {
123+
renameBuildDrop('equinox/drops', "${DROP_ID}", "${BUILD_LABEL}", "${DL_DROP_ID}", "${DL_LABEL}")
124+
}
125+
126+
echo 'Promote Eclipse'
127+
dir("${WORKSPACE}/eclipse") {
128+
renameBuildDrop('eclipse/downloads/drops4', "${DROP_ID}", "${BUILD_LABEL}", "${DL_DROP_ID}", "${DL_LABEL}") {
129+
sh '''
130+
echo "\$NEWS_ID = \"${BUILD_MAJOR}.${BUILD_MINOR}\";" >> 'buildproperties.php'
131+
echo "\$ACK_ID = \"${BUILD_MAJOR}.${BUILD_MINOR}\";" >> 'buildproperties.php'
132+
echo "\$README_ID = \"${BUILD_MAJOR}.${BUILD_MINOR}\";" >> 'buildproperties.php'
133+
134+
# SIGNOFF_BUG should not be defined if there are no JUnit failures to investigate and explain
135+
if [[ -n "${SIGNOFF_BUG}" ]]; then
136+
echo -e "<p>Any unit test failures below have been investigated and found to be test-related and do not affect the quality of the build.\nSee the sign-off page <a href=\"${SIGNOFF_BUG}\">${SIGNOFF_BUG_LABEL}</a> for details.</p>" > 'testNotes.html'
137+
fi
138+
'''
139+
}
140+
}
137141
}
138142
build job: 'Releng/tagEclipseRelease', wait: true, propagate: true, parameters: [
139143
string(name: 'tag', value: "${TAG}"),
140144
string(name: 'buildID', value: "${DROP_ID}"),
141-
string(name: 'annotation', value: "${ params.SIGNOFF_BUG ? 'https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/' + params.SIGNOFF_BUG : '' }")
145+
string(name: 'annotation', value: "${SIGNOFF_BUG}")
142146
]
143147
}
144148
}
@@ -177,7 +181,7 @@ pipeline {
177181
}
178182
post {
179183
always {
180-
archiveArtifacts '**/stage2output*/**'
184+
archiveArtifacts '**/*'
181185
}
182186
}
183187
}
@@ -197,3 +201,97 @@ def readParameter(String name) {
197201
}
198202
return value
199203
}
204+
205+
def renameBuildDrop(String baseDropPath, String oldDropID, String oldBuildLabel, String newDropID, String newBuildLabel, Closure extraTasks=null) {
206+
def sourcePath="${EP_ROOT}/${baseDropPath}/${oldDropID}"
207+
def targetPath="${EP_ROOT}/${baseDropPath}/${newDropID}"
208+
209+
sh """#!/bin/bash -xe
210+
211+
# Copy drop-directory to new location
212+
ssh [email protected] mkdir -p '${targetPath}'
213+
if [[ "${HIDE_SITE}" == "true" ]]; then
214+
# Create this marker first to ensure the page is never without
215+
ssh [email protected] touch '${targetPath}/buildHidden'
216+
fi
217+
ssh [email protected] cp -r '${sourcePath}/.' '${targetPath}'
218+
219+
# Rename files
220+
echo 'Rename files at new location from *${oldBuildLabel}* to *${newBuildLabel}*.'
221+
# References to locally defined variables required double escaping
222+
# because they are escaped for groovy and the bash HERE document.
223+
# In general this section is very fragile! Only change with great caution and extensive testing!
224+
ssh [email protected] << EOF
225+
set -xe
226+
cd "${targetPath}"
227+
files=\\\$(find . -mindepth 1 -maxdepth 2 -name "*${oldBuildLabel}*" -print)
228+
echo " \\\$(echo "\\\$files" | wc -l) files found to rename."
229+
230+
for file in \\\${files}; do
231+
if [[ \\\$file =~ (.*)(${oldBuildLabel})(.*) ]]; then
232+
mv "\\\$file" "\\\${BASH_REMATCH[1]}${newBuildLabel}\\\${BASH_REMATCH[3]}"
233+
fi
234+
done
235+
#DO NOT INDENT EOF!
236+
EOF
237+
238+
#Update checksums to new filenames
239+
ssh [email protected] sed --in-place --expression 's/${oldBuildLabel}/${newBuildLabel}/g' ${targetPath}/checksum/*
240+
"""
241+
242+
// Update buildproperties.php to new names
243+
// sh """#!/bin/bash -xe
244+
// scp [email protected]:${sourcePath}/buildproperties.php .
245+
// sed --in-place --expression "s|BUILD_DIR_SEG = \\"${oldDropID}\\"|BUILD_DIR_SEG = \\"${newDropID}\\"|g" buildproperties.php
246+
// sed --in-place --expression "s|BUILD_ID = \\"${oldBuildLabel}\\"|BUILD_ID = \\"${newBuildLabel}\\"|g" buildproperties.php
247+
// sed --in-place --expression "s|BUILD_TYPE = \\"${BUILD_TYPE}\\"|BUILD_TYPE = \\"${DL_TYPE}\\"|g" buildproperties.php
248+
//
249+
// if [[ "${newBuildLabel}" =~ .*RC.* ]]; then
250+
// newTypeName="Release Candidate"
251+
// elif [[ "${DL_TYPE}" == "R" ]]; then
252+
// newTypeName="Release"
253+
// elif [[ "${DL_TYPE}" == "S" ]]; then
254+
// newTypeName="Stable"
255+
// else
256+
// echo "ERROR: Unexpected DL_TYPE value: ${DL_TYPE}"
257+
// exit 1
258+
// fi
259+
// sed --in-place --expression "s|BUILD_TYPE_NAME = \\"[^\\"]\\+\\"|BUILD_TYPE_NAME = \\"${newTypeName}\\"|g" buildproperties.php
260+
// """
261+
def newTypeName = null
262+
if ("${DL_TYPE}" == 'R') {
263+
newTypeName = 'Release'
264+
} else if (newBuildLabel.contains('RC')) {
265+
newTypeName = 'Release Candidate'
266+
} else if ("${DL_TYPE}" == 'S') {
267+
newTypeName = 'Stable'
268+
} else {
269+
error "Unexpected DL_TYPE value, ${DL_TYPE}"
270+
}
271+
def newProperties = [
272+
BUILD_ID: "${newBuildLabel}",
273+
BUILD_TYPE: "${DL_TYPE}",
274+
BUILD_TYPE_NAME: "${newTypeName}",
275+
BUILD_DIR_SEG: "${newDropID}",
276+
EQ_BUILD_DIR_SEG: "${newDropID}",
277+
]
278+
def buildPropertiesPHP = sh(returnStdout: true, script: "ssh [email protected] cat ${sourcePath}/buildproperties.php")
279+
for (entry in newProperties) {
280+
buildPropertiesPHP = buildPropertiesPHP.replaceFirst('\\$' + entry.key + ' = "[^"]+"', '\\$' + entry.key + ' = "' + entry.value + '"')
281+
}
282+
writeFile(file: 'buildproperties.php', text: buildPropertiesPHP)
283+
284+
if (extraTasks) {
285+
extraTasks()
286+
}
287+
sh """#!/bin/bash -xe
288+
# Copy locally modified files to the download server
289+
scp -r . [email protected]:${targetPath}/
290+
291+
if [[ '${DL_TYPE}' == 'R' ]]; then
292+
echo 'Creating archive'
293+
#FIXME: remove echo
294+
ssh [email protected] cp -r '${targetPath}' /home/data/httpd/archive.eclipse.org/${baseDropPath}/
295+
fi
296+
"""
297+
}

0 commit comments

Comments
 (0)