Skip to content

Commit 8f91eef

Browse files
HannesWellHannesWell
committed
Add dedicated job to modify a p2 composite repository
- unifies the code to modify a composite p2-repository, which was previously written in multiple locations - the new job performs all operations locally within the job's workspace and only copies the resulting files to the download-server. This therefore eliminates the remaining executions of Eclipse directly on the download-server. - A dedicated job, prevents concurrent modifications of the same repository - A dedicated job, allows run a specific modification manually if necessary (for out of order trouble-shooting).
1 parent 65535f1 commit 8f91eef

File tree

11 files changed

+197
-326
lines changed

11 files changed

+197
-326
lines changed

‎JenkinsJobs/Builds/build.jenkinsfile‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,17 @@ spec:
280280
./mb620_promoteUpdateSite.sh $CJE_ROOT/buildproperties.shsource
281281
'''
282282
}
283+
script {
284+
if ("${COMPARATOR_ERRORS_SUBJECT}" != '' && "$BUILD_TYPE" == "I") {
285+
echo 'Skip adding unstable build to composite repository.'
286+
return
287+
}
288+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
289+
string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VER}-I-builds"),
290+
string(name: 'add', value: "${BUILD_IID}"),
291+
string(name: 'sizeLimit', value: '3')
292+
]
293+
}
283294
}
284295
}
285296
}

‎JenkinsJobs/Builds/markBuild.groovy‎

Lines changed: 19 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@ pipeline {
2222
}
2323
stages {
2424
stage('Mark ''' + marker + ''''){
25-
steps { // workspace is not always cleaned by default. Clean before custom tools are installed into workspace.
25+
environment {
26+
RELEASE_VER = readBuildProperty('RELEASE_VER')
27+
}
28+
steps {
2629
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
2730
sh \'''#!/bin/bash -xe
2831
# this function executes command passed as command line parameter and
@@ -40,48 +43,8 @@ pipeline {
4043
echo "IssueUrl is empty! Exiting."
4144
exit 1
4245
fi''' : '') + '''
43-
4446
epDownloadDir=/home/data/httpd/download.eclipse.org/eclipse
45-
dropsPath=${epDownloadDir}/downloads/drops4
46-
p2RepoPath=${epDownloadDir}/updates
47-
buildDir=${dropsPath}/${buildId}
48-
49-
workingDir=${epDownloadDir}/workingDir
50-
51-
workspace=${workingDir}/${JOB_NAME}-${BUILD_NUMBER}
52-
53-
ssh [email protected] rm -rf ${workingDir}/${JOB_NAME}*
54-
55-
ssh [email protected] mkdir -p ${workspace}
56-
ssh [email protected] cd ${workspace}
57-
58-
#get latest Eclipse platform product
59-
epRelDir=$(ssh [email protected] ls -d --format=single-column ${dropsPath}/R-*|sort|tail -1)
60-
ssh [email protected] tar -C ${workspace} -xzf ${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz
61-
62-
#get requisite tools
63-
markingScriptName=\'''' + { switch (marker) {
64-
case 'stable': return 'addToComposite'
65-
case 'unstable': return 'removeFromComposite'
66-
default : throw new IllegalArgumentException("Unknown marker: ${marker}")
67-
} }() + ''''
68-
ssh [email protected] wget -O ${workspace}/${markingScriptName}.xml https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/${markingScriptName}.xml
69-
70-
#triggering ant runner
71-
baseBuilderDir=${workspace}/eclipse
72-
javaCMD=/opt/public/common/java/openjdk/jdk-21_x64-latest/bin/java
73-
74-
launcherJar=$(ssh [email protected] find ${baseBuilderDir}/. -name "org.eclipse.equinox.launcher_*.jar" | sort | head -1 )
75-
76-
scp [email protected]:${buildDir}/buildproperties.shsource .
77-
source ./buildproperties.shsource
78-
repoDir=/home/data/httpd/download.eclipse.org/eclipse/updates/${STREAMMajor}.${STREAMMinor}-${BUILD_TYPE}-builds
79-
80-
devworkspace=${workspace}/workspace-antRunner
81-
devArgs=-Xmx512m
82-
extraArgs="${markingScriptName} -Drepodir=${repoDir} -Dcomplocation=${buildId}"
83-
ssh [email protected] ${javaCMD} -jar ${launcherJar} -nosplash -consolelog -debug -data $devworkspace \\
84-
-application org.eclipse.ant.core.antRunner -file ${workspace}/${markingScriptName}.xml ${extraArgs} -vmargs $devArgs
47+
buildDir="${epDownloadDir}/downloads/drops4/${buildId}"
8548
''' + { switch (marker) {
8649
case 'stable': return '''
8750
#Remove hidden attribute and unstable tags
@@ -98,14 +61,26 @@ pipeline {
9861
'''
9962
default : throw new IllegalArgumentException("Unknown marker: ${marker}")
10063
} }() + '''
101-
ssh [email protected] rm -rf ${workingDir}/${JOB_NAME}*
10264
\'''
103-
build job: 'Releng/updateIndex', wait: false
10465
}
66+
build job: 'Releng/updateIndex', wait: false
67+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
68+
string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VER}-I-builds"),
69+
string(name: \'''' + { switch (marker) {
70+
case 'stable': return 'add'
71+
case 'unstable': return 'remove'
72+
default : throw new IllegalArgumentException("Unknown marker: ${marker}")
73+
} }() + '''\', value: "${buildId}")
74+
]
10575
}
10676
}
10777
}
10878
}
79+
80+
def readBuildProperty(String name) {
81+
def value = sh(script: """curl https://download.eclipse.org/eclipse/downloads/drops4/${buildId}/buildproperties.properties | grep "^${name}" | cut -d'=' -f2""", returnStdout: true).trim()
82+
return value.startsWith('"') && value.endsWith('"') ? value.substring(1, value.length() - 1) : value
83+
}
10984
''')
11085
}
11186
}

‎JenkinsJobs/Releng/FOLDER.groovy‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,29 @@ Releases are published to <a href="https://repo1.maven.org/maven2/org/eclipse/">
6161
}
6262
}
6363

64+
pipelineJob('Releng/modifyP2CompositeRepository'){
65+
displayName('Modify P2 composite repository')
66+
description('Add or remove children from an Eclipse-P2 composite repository.')
67+
parameters {
68+
stringParam('repositoryPath', null, "Relative repository path from https://download.eclipse.org/. E.g. 'eclipse/updates/4.37-I-builds'")
69+
stringParam('add', null, 'Comma separated list of children to add. May be empty')
70+
stringParam('remove', null, 'Comma separated list of children to remove. May be empty')
71+
stringParam('sizeLimit', null, '''
72+
The maximum number of childrem the modified composite should contain.
73+
If the total number of children exceeds this limit (after adding new ones), a corresponding number of children is removed from the beginning of the list.
74+
''')
75+
}
76+
definition {
77+
cpsScm {
78+
lightweight(true)
79+
scm {
80+
github('eclipse-platform/eclipse.platform.releng.aggregator', 'master')
81+
}
82+
scriptPath('JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile')
83+
}
84+
}
85+
}
86+
6487
pipelineJob('Releng/prepareNextDevCycle'){
6588
displayName('Prepare Next Development Cycle')
6689
description('Perform all steps to prepare the next development cycle of Eclipse.')

‎JenkinsJobs/Releng/checkCompositesValidity.groovy‎

Lines changed: 0 additions & 47 deletions
This file was deleted.

‎JenkinsJobs/Releng/makeVisible.jenkinsfile‎

Lines changed: 4 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -62,42 +62,10 @@ pipeline {
6262
environment name: 'RELEASE_TYPE', value: 'R'
6363
}
6464
steps {
65-
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
66-
sh '''
67-
#Repository will be available only for R builds. add it to composite
68-
epDownloadDir=/home/data/httpd/download.eclipse.org/eclipse
69-
dropsPath=${epDownloadDir}/downloads/drops4
70-
workingDir=${epDownloadDir}/workingDir
71-
72-
workspace=${workingDir}/${JOB_NAME}-${BUILD_NUMBER}
73-
74-
ssh [email protected] rm -rf ${workingDir}/${JOB_NAME}*
75-
76-
ssh [email protected] mkdir -p ${workspace}
77-
ssh [email protected] cd ${workspace}
78-
79-
#get latest Eclipse platform product
80-
epRelDir=$(ssh [email protected] ls -d --format=single-column ${dropsPath}/R-*|sort|tail -1)
81-
ssh [email protected] tar -C ${workspace} -xzf ${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz
82-
83-
#get requisite tools
84-
ssh [email protected] wget -O ${workspace}/addToComposite.xml https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/addToComposite.xml
85-
86-
#triggering ant runner
87-
baseBuilderDir=${workspace}/eclipse
88-
javaCMD=/opt/public/common/java/openjdk/jdk-21_x64-latest/bin/java
89-
launcherJar=$(ssh [email protected] find ${baseBuilderDir}/. -name "org.eclipse.equinox.launcher_*.jar" | sort | head -1 )
90-
91-
repoDir=/"home/data/httpd/download.eclipse.org/eclipse/updates/${BUILD_MAJOR}.${BUILD_MINOR}"
92-
93-
devWorkspace=${workspace}/workspace-antRunner
94-
devArgs=-Xmx512m
95-
extraArgs="addToComposite -Drepodir=${repoDir} -Dcomplocation=${RELEASE_ID}"
96-
ssh [email protected] ${javaCMD} -jar ${launcherJar} -nosplash -consolelog -debug -data $devWorkspace -application org.eclipse.ant.core.antRunner -file ${workspace}/addToComposite.xml ${extraArgs} -vmargs $devArgs
97-
98-
ssh [email protected] rm -rf ${workingDir}/${JOB_NAME}*
99-
'''
100-
}
65+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
66+
string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}"),
67+
string(name: 'add', value: "${RELEASE_ID}")
68+
]
10169
}
10270
}
10371
}

‎JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
pipeline {
2+
options {
3+
disableConcurrentBuilds() // prevent concurrent updates of the same repository
4+
timestamps()
5+
skipDefaultCheckout()
6+
timeout(time: 15, unit: 'MINUTES')
7+
buildDiscarder(logRotator(numToKeepStr:'5'))
8+
}
9+
agent {
10+
label 'ubuntu-2404'
11+
}
12+
tools {
13+
jdk 'temurin-jdk21-latest'
14+
}
15+
environment {
16+
ECLIPSE_EXE = installLatestReleasedEclipsePlatformProduct()
17+
}
18+
stages {
19+
stage('Check Composites Validity') {
20+
steps {
21+
sh '''
22+
for repo in ${add//,/ }; do
23+
if [[ $repo != http* ]]; then
24+
repo="https://download.eclipse.org/${repositoryPath}/${repo}"
25+
fi
26+
echo "Validate content consistency of: ${repo}"
27+
${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -application org.eclipse.equinox.p2.director -repository "${repo}" -list
28+
done
29+
'''
30+
}
31+
}
32+
stage('Update composite repository') {
33+
environment {
34+
ANT_TASK_NAME = 'modifyComposite'
35+
OUTPUT_PATH = 'output-repository'
36+
}
37+
steps {
38+
script {
39+
sh "curl -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar"
40+
def childenXPathSet = sh(script: '''#!/bin/sh +xe
41+
unzip -p compositeArtifacts.jar compositeArtifacts.xml |\
42+
xmllint - --xpath '/repository/children/child/@location'
43+
exit 0
44+
''', returnStdout: true).trim()
45+
def compositeChildren = 'XPath set is empty' == childenXPathSet ? []
46+
: parseList(childenXPathSet, '\\s+')
47+
.collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null}
48+
.findAll{c -> c != null}
49+
50+
echo "Current children: ${compositeChildren}"
51+
def toAdd = parseList(params.add, ',').unique()
52+
def toRemove = parseList(params.remove, ',').unique()
53+
toRemove = toRemove.findAll{e -> compositeChildren.contains(e)}
54+
55+
if (params.sizeLimit) {
56+
def sizeRestrictionRemovals = compositeChildren.size() + toAdd.size() - toRemove.size() - Integer.parseInt(params.sizeLimit)
57+
if (sizeRestrictionRemovals > 0) {
58+
toRemove += compositeChildren.subList(0, sizeRestrictionRemovals)
59+
}
60+
}
61+
echo "Children added: ${toAdd}"
62+
echo "Children removed: ${toRemove}"
63+
if (compositeChildren.size() + toAdd.size() - toRemove.size() == 0) {
64+
error('All children are removed and composite repository becomes empty.')
65+
}
66+
def modifyComposite_xml = """\
67+
<?xml version="1.0" encoding="UTF-8"?>
68+
<project default="${ANT_TASK_NAME}" basedir=".">
69+
<target name="${ANT_TASK_NAME}">
70+
<p2.composite.repository>
71+
<source location="https://download.eclipse.org/${repositoryPath}" />
72+
<repository location="${OUTPUT_PATH}" />
73+
""".stripIndent()
74+
for (child in toAdd) {
75+
modifyComposite_xml += """ <add><repository location="${child}"/></add>\n"""
76+
}
77+
for (child in toRemove) {
78+
modifyComposite_xml += """ <remove><repository location="${child}"/></remove>\n"""
79+
}
80+
modifyComposite_xml+= '''\
81+
</p2.composite.repository>
82+
</target>
83+
</project>
84+
'''.stripIndent()
85+
writeFile(file: "${ANT_TASK_NAME}.xml", text: modifyComposite_xml)
86+
87+
sh '''
88+
${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -debug -data ./eclipse-ws \
89+
-application org.eclipse.ant.core.antRunner -file "${ANT_TASK_NAME}.xml" "${ANT_TASK_NAME}"
90+
'''
91+
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
92+
sh '''
93+
epDownloadsDir='/home/data/httpd/download.eclipse.org'
94+
scp -r ${OUTPUT_PATH}/* "[email protected]:${epDownloadsDir}/${repositoryPath}"
95+
'''
96+
}
97+
}
98+
}
99+
}
100+
}
101+
post {
102+
always {
103+
archiveArtifacts allowEmptyArchive: true, artifacts: '**/*', excludes: 'tools/**/*'
104+
}
105+
}
106+
}
107+
108+
@NonCPS
109+
def parseList(String str, String delimiterPattern) {
110+
return str !=null && !str.trim().isEmpty() ? str.trim().split(delimiterPattern).collect{c -> c.trim()} : []
111+
}
112+
113+
def installLatestReleasedEclipsePlatformProduct() {
114+
sh 'curl -o buildproperties.txt https://download.eclipse.org/eclipse/relengScripts/cje-production/buildproperties.txt'
115+
def eclipseURL = sh(script: '''#!/bin/sh +xe
116+
source ./buildproperties.txt
117+
#TODO: Remove this after the next release!
118+
PREVIOUS_RELEASE_ID='S-4.37M1-202507031800'
119+
PREVIOUS_RELEASE_VER='4.37M1'
120+
echo "https://download.eclipse.org/eclipse/downloads/drops4/${PREVIOUS_RELEASE_ID}/eclipse-platform-${PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz"
121+
''', returnStdout: true).trim()
122+
return install('eclipse-platform', eclipseURL) + '/eclipse'
123+
}
124+
125+
def install(String toolType, String url) {
126+
dir("${WORKSPACE}/tools/${toolType}") {
127+
sh "curl -L ${url} | tar -xzf -"
128+
return "${pwd()}/" + sh(script: 'ls', returnStdout: true).trim()
129+
}
130+
}

0 commit comments

Comments
 (0)