Skip to content

Commit 148cd9b

Browse files
HannesWellHannesWell
committed
[RelEng] Unify composite repository creation
1 parent bb531ea commit 148cd9b

File tree

7 files changed

+70
-204
lines changed

7 files changed

+70
-204
lines changed

‎JenkinsJobs/Releng/FOLDER.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ pipelineJob('Releng/modifyP2CompositeRepository'){
5858
The maximum number of childrem the modified composite should contain.
5959
If the total number of children exceeds this limit (after adding new ones), a corresponding number of children is removed from the beginning of the list.
6060
''')
61+
stringParam('repositoryName', null, "Optional name attribute of the composite repository to set (if blank the name is not changed)")
6162
}
6263
definition {
6364
cpsScm {

‎JenkinsJobs/Releng/createGenericComposites.groovy

Lines changed: 0 additions & 58 deletions
This file was deleted.

‎JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@ pipeline {
2424
repo="https://download.eclipse.org/${repositoryPath}/${repo}"
2525
fi
2626
echo "Validate content consistency of: ${repo}"
27-
${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -application org.eclipse.equinox.p2.director -repository "${repo}" -list
27+
${ECLIPSE_EXE} -application org.eclipse.equinox.p2.director -repository "${repo}" -list
2828
done
2929
'''
30+
// The resulting composite repository is not tested as it might not exist yet.
3031
}
3132
}
3233
stage('Update composite repository') {
@@ -36,21 +37,25 @@ pipeline {
3637
}
3738
steps {
3839
script {
39-
sh "curl -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar"
40-
def childenXPathSet = sh(script: '''#!/bin/sh +xe
41-
unzip -p compositeArtifacts.jar compositeArtifacts.xml |\
42-
xmllint - --xpath '/repository/children/child/@location'
43-
exit 0
44-
''', returnStdout: true).trim()
45-
def compositeChildren = 'XPath set is empty' == childenXPathSet ? []
46-
: parseList(childenXPathSet, '\\s+')
47-
.collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null}
48-
.findAll{c -> c != null}
40+
def boolean sourceRepositoryExists = sh("curl --fail -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar") == 0
41+
def compositeChildren = []
42+
if (sourceRepositoryExists) {
43+
def childenXPathSet = sh(script: '''#!/bin/sh +xe
44+
unzip -p compositeArtifacts.jar compositeArtifacts.xml |\
45+
xmllint - --xpath '/repository/children/child/@location'
46+
exit 0
47+
''', returnStdout: true).trim()
48+
compositeChildren = 'XPath set is empty' == childenXPathSet ? []
49+
: parseList(childenXPathSet, '\\s+')
50+
.collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null}
51+
.findAll{c -> c != null}
52+
}
4953

5054
echo "Current children: ${compositeChildren}"
5155
def toAdd = parseList(params.add, ',').unique()
5256
def toRemove = parseList(params.remove, ',').unique()
5357
toRemove = toRemove.findAll{e -> compositeChildren.contains(e)}
58+
def repositoryName = params.repositoryName.trim()
5459

5560
if (params.sizeLimit) {
5661
def sizeRestrictionRemovals = compositeChildren.size() + toAdd.size() - toRemove.size() - Integer.parseInt(params.sizeLimit)
@@ -68,9 +73,11 @@ pipeline {
6873
<project default="${ANT_TASK_NAME}" basedir=".">
6974
<target name="${ANT_TASK_NAME}">
7075
<p2.composite.repository>
71-
<source location="https://download.eclipse.org/${repositoryPath}" />
72-
<repository location="${OUTPUT_PATH}" />
76+
<repository location="${OUTPUT_PATH}" ${repositoryName ? ('name="' + repositoryName + '"') :''}/>
7377
""".stripIndent()
78+
if (sourceRepositoryExists) {
79+
modifyComposite_xml +=""" <source location="https://download.eclipse.org/${repositoryPath}" />"""
80+
}
7481
for (child in toAdd) {
7582
modifyComposite_xml += """ <add><repository location="${child}"/></add>\n"""
7683
}
@@ -85,9 +92,17 @@ pipeline {
8592
writeFile(file: "${ANT_TASK_NAME}.xml", text: modifyComposite_xml)
8693

8794
sh '''
88-
${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -debug -data ./eclipse-ws \
95+
${ECLIPSE_EXE} -debug -data ./eclipse-ws \
8996
-application org.eclipse.ant.core.antRunner -file "${ANT_TASK_NAME}.xml" "${ANT_TASK_NAME}"
9097
'''
98+
//TODO: Let p2 produce p2.index file
99+
writeFile(file: "${OUTPUT_PATH}/p2.index", text: """\
100+
#${new Date()}
101+
version=1
102+
metadata.repository.factory.order=compositeContent.xml,\\!
103+
artifact.repository.factory.order=compositeArtifacts.xml,\\!
104+
""".stripIndent())
105+
91106
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
92107
sh '''
93108
epDownloadsDir='/home/data/httpd/download.eclipse.org'
@@ -119,7 +134,7 @@ def installLatestReleasedEclipsePlatformProduct() {
119134
PREVIOUS_RELEASE_VER='4.37M1'
120135
echo "https://download.eclipse.org/eclipse/downloads/drops4/${PREVIOUS_RELEASE_ID}/eclipse-platform-${PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz"
121136
''', returnStdout: true).trim()
122-
return install('eclipse-platform', eclipseURL) + '/eclipse'
137+
return install('eclipse-platform', eclipseURL) + '/eclipse -nosplash --launcher.suppressErrors -consolelog'
123138
}
124139

125140
def install(String toolType, String url) {

‎JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -163,28 +163,41 @@ pipeline {
163163
commitAllChangesExcludingSubmodules("Move previous version to ${PREVIOUS_RELEASE_CANDIDATE_TAG} in build scripts")
164164
}
165165
}
166-
stage ('Create New Stream Repos') {
166+
stage ('Create Stream Repos') {
167167
when {
168168
not { expression { params.DRY_RUN } }
169169
}
170170
steps {
171-
dir("${WORKSPACE}/target/repositories") {
172-
sshagent(['projects-storage.eclipse.org-bot-ssh']) {
173-
sh '''#!/bin/bash -xe
174-
epUpdatesDir='/home/data/httpd/download.eclipse.org/eclipse/updates'
175-
templateRepo="${epUpdatesDir}/template_repo"
176-
177-
ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}-I-builds
178-
ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}-Y-builds
179-
ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}
180-
'''
181-
}
182-
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
183-
string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds"),
184-
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/")
185-
// Size-limit is not relevant, the repository is initially empty.
186-
]
187-
}
171+
//FIXME: define categories!!!
172+
//TODO: Delete template_repo
173+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
174+
string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds"),
175+
string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} integration builds repository"),
176+
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/")
177+
// Size-limit is not relevant, the repository is initially empty.
178+
]
179+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
180+
string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-Y-builds"),
181+
string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} Beta Java builds repository")
182+
]
183+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
184+
string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}"),
185+
string(name: 'repositoryName', value: "Eclipse Project ${NEXT_RELEASE_VERSION} release repository")
186+
]
187+
// Update generice composite repositories for I/Y-builds (clearing all previous children)
188+
// Note: we do not put stream number in 'name', since once a 'name' is defined in Eclipse's UI, it does not change.
189+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
190+
string(name: 'repositoryPath', value: "eclipse/updates/I-builds"),
191+
string(name: 'repositoryName', value: "Eclipse Project current integration builds repository"),
192+
string(name: 'sizeLimit', value: '1'), // Clear all previous children
193+
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds/"),
194+
]
195+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
196+
string(name: 'repositoryPath', value: "eclipse/updates/Y-builds"),
197+
string(name: 'repositoryName', value: "Eclipse Project current Beta Java builds repository"),
198+
string(name: 'sizeLimit', value: '1'), // Clear all previous children
199+
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${NEXT_RELEASE_VERSION}-Y-builds/"),
200+
]
188201
}
189202
}
190203
stage('Deploy parent-pom and SDK-target') {

‎JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,14 @@ pipeline {
6666
string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}"),
6767
string(name: 'add', value: "${RELEASE_BUILD_ID}")
6868
]
69+
// Update generice composite repositories for latest release (clearing all previous children)
70+
// Note: we do not put stream number in 'name', since once a 'name' is defined in Eclipse's UI, it does not change.
71+
build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
72+
string(name: 'repositoryPath', value: "eclipse/updates/latest"),
73+
string(name: 'repositoryName', value: "Eclipse Project latest release repository"),
74+
string(name: 'sizeLimit', value: '1'), // Clear all previous children
75+
string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}/"),
76+
]
6977
}
7078
}
7179
}

‎RELEASE.md

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -207,12 +207,6 @@ Previously they were created in ther own job:
207207
**General Cleanup**
208208
- In [eclipse.platform.common] search for and clear out all of the forceQualifierUpdate.txt files.
209209
The context here is that the doc builds only check for changes in this repo and so these files need to be changed to trigger a full rebuild.
210-
* #### **Create Generic Composites**
211-
- After First Stable Ibuild move Generic repos to next stream.
212-
- Run the [Create Generic Composites](https://ci.eclipse.org/releng/job/Releng/job/createGenericComposites/) job to recreate the generic build repos for the next release.
213-
- `currentStream`: To clarify this is the next stream, not the one currently being released. If you are releasing 4.32, the 'current' stream is 4.33 so that repos are created for it.
214-
- `previousStream`: The stream being released, which needs to be removed.
215-
- For reference, the generic repositories created are for the [latest GA release](https://download.eclipse.org/eclipse/updates/latest/) and the current (ongoing) [I-builds](https://download.eclipse.org/eclipse/updates/I-builds/), [Y-builds](https://download.eclipse.org/eclipse/updates/Y-builds/) and [P-builds](https://download.eclipse.org/eclipse/updates/P-builds/).
216210

217211
**RC2a Release**
218212
* Sometimes there is a critical issue that requires a fix, if it's decided that one is needed then an RC2a (followed by RC2b, RC2c etc if necessary) is built from the maintenance branch and promoted using the RC2 process.

‎cje-production/scripts/updateGenericComposites.xml

Lines changed: 0 additions & 107 deletions
This file was deleted.

0 commit comments

Comments
 (0)