[Build] Simplify git cloning/tagging and submodule branch definition

- Define maintenance branches for submodules in the .gitmodules file
instead of separate text files and fetch from the tip of the remote of
each submodules in I/Y-builds.
- Specify other branches to checkout (like in case of Y-builds), in
the buildConfigurations.json.
- In I/Y-builds clone the same branch from that the current build runs
respectively that's defined in the Jenkins job.

Author: HannesWell

JenkinsJobs/Builds/build.jenkinsfile

@@ -49,7 +49,7 @@ spec:
 					def configurations = readJSON(file: "${WORKSPACE}/JenkinsJobs/buildConfigurations.json")
 					BUILD = configurations["${BUILD_TYPE}"]
 					assignEnvVariable('BUILD_TYPE_NAME', BUILD.typeName)
-					assignEnvVariable('PATCH_OR_BRANCH_LABEL', BUILD.branchLabel)
+					assignEnvVariable('GIT_SUBMODULE_BRANCHES', BUILD.branches ? BUILD.branches.collect{ name, branch -> "${name}:${branch}" }.join(',') : '')
 					assignEnvVariable('TEST_NAME_PREFIX', "ep${matcher.group('major')}${matcher.group('minor')}${BUILD_TYPE}-unit")
 					assignEnvVariable('TEST_CONFIGURATIONS_EXPECTED', BUILD.tests.collect{ c ->
 						"${TEST_NAME_PREFIX}-${c.os}-${c.arch}-java${c.javaVersion}_${c.os}.${c.ws}.${c.arch}_${c.javaVersion}"
@@ -82,12 +82,21 @@ spec:
 						git config --global user.name 'Eclipse Releng Bot'
 
 						# Clone this repo and all submodules into the 'AGG_DIR' directory
-						git clone --branch=$BRANCH --recursive \
+						git clone --branch=$BRANCH --recurse-submodules --remote-submodules \
 							[email protected]:eclipse-platform/eclipse.platform.releng.aggregator.git "$CJE_ROOT/$AGG_DIR"
 
 						cd "$CJE_ROOT/$AGG_DIR"
 
-						git submodule foreach 'git fetch; SUBMODULE_BRANCH=$(grep $name: $CJE_ROOT/streams/repositories_$PATCH_OR_BRANCH_LABEL.txt | cut -f2 -d\\ ); SUBMODULE_BRANCH=${SUBMODULE_BRANCH:-$BRANCH}; echo Checking out $SUBMODULE_BRANCH; git checkout $SUBMODULE_BRANCH; git pull'
+						if [ -n "${GIT_SUBMODULE_BRANCHES}" ]; then
+							for submodule in ${GIT_SUBMODULE_BRANCHES//,/ }; do
+								path=$(echo "$submodule" | cut -d: -f1)
+								branch=$(echo "$submodule" | cut -d: -f2)
+								pushd "${path}"
+									git checkout ${branch}
+									git pull
+								popd
+							done
+						fi
 
 						# Create 'Build input' commit (considering commits potentially submitted to the aggregator BRANCH in the meantime)
 						git checkout $BRANCH
@@ -133,14 +142,14 @@ spec:
 					sshagent (['github-bot-ssh']) {
 						sh '''#!/bin/bash -xe
 							source $CJE_ROOT/buildproperties.shsource
-							function toPushRepo() {
-								from="$1"
-								echo $(sed --expression 's,https://github.com/,[email protected]:,' <<< $from)
+							function tagAndPush {
+								git tag ${BUILD_ID} HEAD
+								pushURL=$(git config remote.origin.url | sed --expression 's,https://github.com/,[email protected]:,')
+								git push "${pushURL}" tag ${BUILD_ID}
 							}
-							export -f toPushRepo
-							git submodule foreach 'if grep "^${name}:" $CJE_ROOT/streams/repositories_$PATCH_OR_BRANCH_LABEL.txt > /dev/null; then git tag $BUILD_ID; PUSH_URL="$(toPushRepo $(git config --get remote.origin.url))"; git push --verbose $PUSH_URL $BUILD_ID; else echo Skipping $name; fi || :'
-							git tag $BUILD_ID
-							git push --verbose origin $BUILD_ID
+							export -f tagAndPush
+							git submodule foreach 'tagAndPush'
+							tagAndPush
 						'''
 					}
 					// Git log creation

JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile

@@ -284,7 +284,6 @@ pipeline {
 							build.branch = env.MAINTENANCE_BRANCH
 							build.schedule = ''
 						}
-						builds.I.branchLabel = env.MAINTENANCE_BRANCH
 					}
 				}
 				replaceInFile('JenkinsJobs/Builds/dockerImages.jenkinsfile', [
@@ -293,13 +292,9 @@ pipeline {
 				replaceInFile('cje-production/buildproperties.txt', [
 					'BRANCH="master"' : "BRANCH=\"${MAINTENANCE_BRANCH}\"",
 				])
-				replaceInFile('cje-production/streams/repositories_java25.txt', [
-					': master' : ": ${MAINTENANCE_BRANCH}",
+				replaceAllInFile('.gitmodules', [
+					'branch\\s*=\\s*master' : "branch = ${MAINTENANCE_BRANCH}",
 				])
-				replaceInFile('cje-production/streams/repositories_master.txt', [
-					': master' : ": ${MAINTENANCE_BRANCH}",
-				])
-				sh "mv cje-production/streams/repositories_master.txt cje-production/streams/repositories_${MAINTENANCE_BRANCH}.txt"
 
 				gitCommitAllExcludingSubmodules("Move ${PREVIOUS_RELEASE_VERSION}-I builds to ${MAINTENANCE_BRANCH} branch")
 

JenkinsJobs/buildConfigurations.json

@@ -1,7 +1,6 @@
 {
 	"I": {
 		"typeName": "Integration",
-		"branchLabel": "master",
 		"streams": {
 			"4.38": {
 				"branch": "master",
@@ -55,14 +54,18 @@
 	},
 	"Y": {
 		"typeName": "Beta Java 25",
-		"branchLabel": "java25",
 		"streams": {
 			"4.38": {
 				"branch": "master",
 				"disabled": "true",
 				"schedule": "0 10 * 8-10 2,4,6\n0 10 1-26 11 2,4,6"
 			}
 		},
+		"branches": {
+			"eclipse.jdt.core": "BETA_JAVA25",
+			"eclipse.jdt.debug": "BETA_JAVA25",
+			"eclipse.jdt.ui": "BETA_JAVA25"
+		},
 		"mailingList": "[email protected]",
 		"testJobFolder": "YBuilds",
 		"tests": [

RELENG.md

@@ -54,8 +54,7 @@ When the JDT team is ready they will raise an issue to create new Y builds and s
 
 **Things to Do:**
   * Update the Y-build configuration in the (buildConfigurations.json)[JenkinsJobs/buildConfigurations.json]
-    - Update `branchLabel` and `typeName` to the name of the new java version.
+    - Update `typeName` to the name of the new java version.
     - Remove the disablement of the current stream in the Y-build configuration (should be the only Y-build stream).
+    - Update `branches` to point to the new BETA branch.
     - Add unit tests for the new java version and remove old ones.
-  * Update and rename the java repository files in (cje-production/streams)[cje-production/streams]
-    - Repos without a `BETA_JAVA##` branch should be set to master

cje-production/mbscripts/mb010_createEnvfiles.sh

@@ -74,7 +74,6 @@ fn-addToPropFiles BUILD_ENV_FILE_PROP "\"$BUILD_ENV_FILE_PROP\""
 # variables in buildproperties.txt are now defined, add other commonly used variables to prop files
 fn-addToPropFiles BUILD_TYPE "\"${BUILD_TYPE}\""
 fn-addToPropFiles BUILD_TYPE_NAME "\"${BUILD_TYPE_NAME}\""
-fn-addToPropFiles PATCH_OR_BRANCH_LABEL "\"${PATCH_OR_BRANCH_LABEL}\""
 fn-addToPropFiles BUILD_ID "\"$BUILD_TYPE$TIMESTAMP\""
 fn-addToPropFiles BUILD_DIR_SEG "\"$BUILD_TYPE$TIMESTAMP\""
 fn-addToPropFiles EQ_BUILD_DIR_SEG "\"$BUILD_TYPE$TIMESTAMP\""