Skip to content

Commit d638293

Browse files
HannesWellHannesWell
committed
[RelEng] Unify and simplify composite repository creation
Use the 'Modify P2 Composite Repository' Jenkins job to create and update the required (composite) P2-repositories during the preparation and promotion process of a new release. To enable this, make that job capable to also create new composite repositories.
1 parent 80cb8ce commit d638293

File tree

7 files changed

+84
-214
lines changed

7 files changed

+84
-214
lines changed

‎JenkinsJobs/Releng/FOLDER.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ pipelineJob('Releng/modifyP2CompositeRepository'){
5858
The maximum number of childrem the modified composite should contain.
5959
If the total number of children exceeds this limit (after adding new ones), a corresponding number of children is removed from the beginning of the list.
6060
''')
61+
stringParam('repositoryName', null, 'Optional name attribute of the composite repository to set (if blank the name is not changed)')
6162
}
6263
definition {
6364
cpsScm {

‎JenkinsJobs/Releng/createGenericComposites.groovy

Lines changed: 0 additions & 58 deletions
This file was deleted.

‎JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile

Lines changed: 46 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@ pipeline {
2424
repo="https://download.eclipse.org/${repositoryPath}/${repo}"
2525
fi
2626
echo "Validate content consistency of: ${repo}"
27-
${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -application org.eclipse.equinox.p2.director -repository "${repo}" -list
27+
${ECLIPSE_EXE} -application org.eclipse.equinox.p2.director -repository "${repo}" -list
2828
done
2929
'''
30+
// The resulting composite repository is not tested as it might not exist yet.
3031
}
3132
}
3233
stage('Update composite repository') {
@@ -36,21 +37,25 @@ pipeline {
3637
}
3738
steps {
3839
script {
39-
sh "curl -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar"
40-
def childenXPathSet = sh(script: '''#!/bin/sh +xe
41-
unzip -p compositeArtifacts.jar compositeArtifacts.xml |\
42-
xmllint - --xpath '/repository/children/child/@location'
43-
exit 0
44-
''', returnStdout: true).trim()
45-
def compositeChildren = 'XPath set is empty' == childenXPathSet ? []
46-
: parseList(childenXPathSet, '\\s+')
47-
.collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null}
48-
.findAll{c -> c != null}
40+
def boolean sourceRepositoryExists = sh(script: "curl --fail -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar", returnStatus: true) == 0
41+
def compositeChildren = []
42+
if (sourceRepositoryExists) {
43+
def childenXPathSet = sh(script: '''#!/bin/sh +xe
44+
unzip -p compositeArtifacts.jar compositeArtifacts.xml |\
45+
xmllint - --xpath '/repository/children/child/@location'
46+
exit 0
47+
''', returnStdout: true).trim()
48+
compositeChildren = 'XPath set is empty' == childenXPathSet ? []
49+
: parseList(childenXPathSet, '\\s+')
50+
.collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null}
51+
.findAll{c -> c != null}
52+
}
4953

5054
echo "Current children: ${compositeChildren}"
5155
def toAdd = parseList(params.add, ',').unique()
5256
def toRemove = parseList(params.remove, ',').unique()
5357
toRemove = toRemove.findAll{e -> compositeChildren.contains(e)}
58+
def repositoryName = params.repositoryName.trim()
5459

5560
if (params.sizeLimit) {
5661
def sizeRestrictionRemovals = compositeChildren.size() + toAdd.size() - toRemove.size() - Integer.parseInt(params.sizeLimit)
@@ -60,17 +65,19 @@ pipeline {
6065
}
6166
echo "Children added: ${toAdd}"
6267
echo "Children removed: ${toRemove}"
63-
if (compositeChildren.size() + toAdd.size() - toRemove.size() == 0) {
68+
if (!toRemove.isEmpty() &&compositeChildren.size() + toAdd.size() - toRemove.size() == 0) {
6469
error('All children are removed and composite repository becomes empty.')
6570
}
6671
def modifyComposite_xml = """\
6772
<?xml version="1.0" encoding="UTF-8"?>
6873
<project default="${ANT_TASK_NAME}" basedir=".">
6974
<target name="${ANT_TASK_NAME}">
7075
<p2.composite.repository>
71-
<source location="https://download.eclipse.org/${repositoryPath}" />
72-
<repository location="${OUTPUT_PATH}" />
76+
<repository location="${OUTPUT_PATH}" ${repositoryName ? ('name="' + repositoryName + '"') :''}/>
7377
""".stripIndent()
78+
if (sourceRepositoryExists) {
79+
modifyComposite_xml +=""" <source location="https://download.eclipse.org/${repositoryPath}" />"""
80+
}
7481
for (child in toAdd) {
7582
modifyComposite_xml += """ <add><repository location="${child}"/></add>\n"""
7683
}
@@ -85,12 +92,33 @@ pipeline {
8592
writeFile(file: "${ANT_TASK_NAME}.xml", text: modifyComposite_xml)
8693

8794
sh '''
88-
${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -debug -data ./eclipse-ws \
95+
${ECLIPSE_EXE} -debug -data ./eclipse-ws \
8996
-application org.eclipse.ant.core.antRunner -file "${ANT_TASK_NAME}.xml" "${ANT_TASK_NAME}"
9097
'''
98+
//TODO: Let p2 produce p2.index file
99+
writeFile(file: "${OUTPUT_PATH}/p2.index", text: """\
100+
#${new Date()}
101+
version=1
102+
metadata.repository.factory.order=compositeContent.xml,\\!
103+
artifact.repository.factory.order=compositeArtifacts.xml,\\!
104+
""".stripIndent())
105+
106+
// The composite files are somehow not compressed if the repository is empty...
107+
sh '''
108+
cd "${OUTPUT_PATH}"
109+
if [[ -f compositeArtifacts.xml ]]; then
110+
zip compositeArtifacts.jar compositeArtifacts.xml
111+
rm compositeArtifacts.xml
112+
fi
113+
if [[ -f compositeContent.xml ]]; then
114+
zip compositeContent.jar compositeContent.xml
115+
rm compositeContent.xml
116+
fi
117+
'''
91118
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
92119
sh '''
93120
epDownloadsDir='/home/data/httpd/download.eclipse.org'
121+
ssh [email protected] mkdir -p "${epDownloadsDir}/${repositoryPath}"
94122
scp -r ${OUTPUT_PATH}/* "[email protected]:${epDownloadsDir}/${repositoryPath}"
95123
'''
96124
}
@@ -115,11 +143,11 @@ def installLatestReleasedEclipsePlatformProduct() {
115143
def eclipseURL = sh(script: '''#!/bin/sh +xe
116144
source ./buildproperties.txt
117145
#TODO: Remove this after the next release!
118-
PREVIOUS_RELEASE_ID='S-4.37M1-202507031800'
119-
PREVIOUS_RELEASE_VER='4.37M1'
146+
PREVIOUS_RELEASE_ID='S-4.37M3-202508141800'
147+
PREVIOUS_RELEASE_VER='4.37M3'
120148
echo "https://download.eclipse.org/eclipse/downloads/drops4/${PREVIOUS_RELEASE_ID}/eclipse-platform-${PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz"
121149
''', returnStdout: true).trim()
122-
return install('eclipse-platform', eclipseURL) + '/eclipse'
150+
return install('eclipse-platform', eclipseURL) + '/eclipse -nosplash --launcher.suppressErrors -consolelog'
123151
}
124152

125153
def install(String toolType, String url) {

‎JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -163,28 +163,39 @@ pipeline {
163163
commitAllChangesExcludingSubmodules("Move previous version to ${PREVIOUS_RELEASE_CANDIDATE_TAG} in build scripts")
164164
}
165165
}
166-
stage ('Create New Stream Repos') {
166+
stage ('Create and update Stream Repos') {
167167
when {
168168
not { expression { params.DRY_RUN } }
169169
}
170170
steps {
171-
dir("${WORKSPACE}/target/repositories") {
172-
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
173-
sh '''#!/bin/bash -xe
174-
epUpdatesDir='/home/data/httpd/download.eclipse.org/eclipse/updates'
175-
templateRepo="${epUpdatesDir}/template_repo"
176-
177-
ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}-I-builds
178-
ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}-Y-builds
179-
ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}
180-
'''
181-
}
182-
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
183-
string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds"),
184-
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/")
185-
// Size-limit is not relevant, the repository is initially empty.
186-
]
187-
}
171+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
172+
string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds"),
173+
string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} integration builds"),
174+
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/")
175+
// Size-limit is not relevant, the repository is initially empty.
176+
]
177+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
178+
string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-Y-builds"),
179+
string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} Beta Java builds")
180+
]
181+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
182+
string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}"),
183+
string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} release")
184+
]
185+
// Update generic composite repositories for I/Y-builds (clearing all previous children)
186+
// Note: The stream number is not in the 'name', because once a 'name' is defined in Eclipse's UI, it does not change.
187+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
188+
string(name: 'repositoryPath', value: "eclipse/updates/I-builds"),
189+
string(name: 'repositoryName', value: "Eclipse latest integration builds"),
190+
string(name: 'sizeLimit', value: '1'), // Clear all previous children
191+
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds/"),
192+
]
193+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
194+
string(name: 'repositoryPath', value: "eclipse/updates/Y-builds"),
195+
string(name: 'repositoryName', value: "Eclipse latest Beta Java builds"),
196+
string(name: 'sizeLimit', value: '1'), // Clear all previous children
197+
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${NEXT_RELEASE_VERSION}-Y-builds/"),
198+
]
188199
}
189200
}
190201
stage('Deploy parent-pom and SDK-target') {
@@ -292,13 +303,6 @@ pipeline {
292303
}
293304
}
294305
}
295-
post {
296-
always {
297-
archiveArtifacts allowEmptyArchive: true, artifacts: '\
298-
target/repositories/**,\
299-
'
300-
}
301-
}
302306
}
303307

304308
// --- utility methods

‎JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,14 @@ pipeline {
6666
string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}"),
6767
string(name: 'add', value: "${RELEASE_BUILD_ID}")
6868
]
69+
// Update generic composite repositories for latest release (clearing all previous children)
70+
// Note: The stream number is not in the 'name', because once a 'name' is defined in Eclipse's UI, it does not change.
71+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
72+
string(name: 'repositoryPath', value: "eclipse/updates/latest"),
73+
string(name: 'repositoryName', value: "Eclipse latest release"),
74+
string(name: 'sizeLimit', value: '1'), // Clear all previous children
75+
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}/"),
76+
]
6977
}
7078
}
7179
}

‎RELEASE.md

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -206,12 +206,6 @@ Previously they were created in ther own job:
206206
**General Cleanup**
207207
- In [eclipse.platform.common] search for and clear out all of the forceQualifierUpdate.txt files.
208208
The context here is that the doc builds only check for changes in this repo and so these files need to be changed to trigger a full rebuild.
209-
* #### **Create Generic Composites**
210-
- After First Stable Ibuild move Generic repos to next stream.
211-
- Run the [Create Generic Composites](https://ci.eclipse.org/releng/job/Releng/job/createGenericComposites/) job to recreate the generic build repos for the next release.
212-
- `currentStream`: To clarify this is the next stream, not the one currently being released. If you are releasing 4.32, the 'current' stream is 4.33 so that repos are created for it.
213-
- `previousStream`: The stream being released, which needs to be removed.
214-
- For reference, the generic repositories created are for the [latest GA release](https://download.eclipse.org/eclipse/updates/latest/) and the current (ongoing) [I-builds](https://download.eclipse.org/eclipse/updates/I-builds/), [Y-builds](https://download.eclipse.org/eclipse/updates/Y-builds/) and [P-builds](https://download.eclipse.org/eclipse/updates/P-builds/).
215209

216210
**RC2a Release**
217211
* Sometimes there is a critical issue that requires a fix, if it's decided that one is needed then an RC2a (followed by RC2b, RC2c etc if necessary) is built from the maintenance branch and promoted using the RC2 process.

‎cje-production/scripts/updateGenericComposites.xml

Lines changed: 0 additions & 107 deletions
This file was deleted.

0 commit comments

Comments
 (0)