[I/Y-Build] Streamline and parallelize collection of test result

Transfer the test results directly from the individual test-build
Jenkins agents to the storage server upon completion and stop archiving
the result files in the jobs.
In the 'collectTestResults' job only regenerate the test-result
overview page and don't handle the result file transfer anymore.
Consequently rename the job to just 'updateTestResultIndex'.

Author: HannesWell

JenkinsJobs/AutomatedTests/integrationTests.jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
 	parameters {
 		string(name: 'buildId', trim: true, description: 'Build Id to test. For example: <code>I20251010-0150</code>')
 		string(name: 'testSuite', defaultValue: 'all', trim: true, description: '''This can be any ant target from
-			https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/blob/master/production/testScripts/configuration/sdk.tests/testScripts/test.xml
+			the <a href="https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/blob/master/production/testScripts/configuration/sdk.tests/testScripts/test.xml"><code>test.xml</code></a>
 		''')
 		string(name: 'testAgent', trim: true, description: '''The Jenkins build agent this test should run on. If not specified to will be read from the
 			<a href="https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/blob/master/JenkinsJobs/buildConfigurations.json"><code>buildConfigurations.json</code></a>
@@ -42,6 +42,8 @@ pipeline {
 				label "${ params.testAgent ?: TEST_AGENT }"
 			}
 			environment {
+				// Download Server locations (seldomly change)
+				EP_ECLIPSE_DROPS = '/home/data/httpd/download.eclipse.org/eclipse/downloads/drops4'
 				// Declaring a jdk and ant the usual way in the 'tools' section, because of unknown reasons, breaks the usage of system commands like xvnc, pkill and sh
 				ANT_HOME = tool(type:'ant', name:'apache-ant-latest')
 				ANT_OPTS = "-Djava.io.tmpdir=${pathOf(env.WORKSPACE + '/tmp')}"
@@ -83,22 +85,43 @@ pipeline {
 						echo JAVA_HOME: §[JAVA_HOME]
 						echo ANT_HOME: §[ANT_HOME]
 						echo PATH: §[PATH]
-						java -XshowSettings -version 1>javaSettings.txt 2>&1
+						java -XshowSettings -version
 
 						curl --fail --output getEBuilder.xml https://download.eclipse.org/eclipse/relengScripts/testScripts/bootstrap/getEBuilder.xml
 						ant -f getEBuilder.xml -DbuildId=${params.buildId} -Dosgi.os=${test.os} -Dosgi.ws=${test.ws} -Dosgi.arch=${test.arch} -DtestSuite=${params.testSuite}
 					""")
 				}
-				archiveArtifacts '**/eclipse-testing/results/**, *.properties, *.txt'
 				junit keepLongStdio: true, testResults: '**/eclipse-testing/results/xml/*.xml'
-				build job: 'Releng/collectTestResults', wait: false, parameters: [
-					string(name: 'triggeringJob', value: "${JOB_BASE_NAME}"),
-					string(name: 'buildURL', value: "${BUILD_URL}"),
-					string(name: 'buildID', value: "${params.buildId}")
-				]
 			}
 			post {
 				always {
+					script {
+						if (currentBuild.result != 'ABORTED') {
+							// Transfer test results to storage-server (skip for partial results on aborted run)
+							sshagent(['projects-storage.eclipse.org-bot-ssh']) {
+								sh '''#!/bin/bash -xe
+									remoteResultsDirectory="${EP_ECLIPSE_DROPS}/${buildId}/testresults"
+									localResultsDirectory="workarea/${buildId}/eclipse-testing/results"
+									ssh [email protected] mkdir -p ${remoteResultsDirectory}
+									if [ -d "${localResultsDirectory}/xml" ] && [ -z "$(find ${localResultsDirectory}/xml -maxdepth 0 -empty)" ]; then
+										# First delete result files from a previous run of the same configuration (in case that test configuration was run again), then transfer the new results.
+										ssh [email protected] rm -rfv ${remoteResultsDirectory}/${JOB_BASE_NAME}* ${remoteResultsDirectory}/*/*${JOB_BASE_NAME}*
+										scp -r ${localResultsDirectory}/* [email protected]:${remoteResultsDirectory}
+										# Download the result summary directly to the storage server
+										ssh [email protected] \
+											curl --fail --location --output "${remoteResultsDirectory}/${JOB_BASE_NAME}.xml" "${BUILD_URL}/testReport/api/xml?tree=failCount,passCount,skipCount,duration"
+									fi
+								'''
+							}
+						}
+						if (currentBuild.result == 'SUCCESS' || currentBuild.result == 'UNSTABLE') {
+							build job: 'Releng/updateTestResultIndex', wait: false, parameters: [
+								string(name: 'buildID', value: "${params.buildId}")
+							]
+						}
+					}
+				}
+				cleanup {
 					cleanWs()
 				}
 			}

JenkinsJobs/Releng/collectTestResults.jenkinsfile renamed to JenkinsJobs/Releng/updateTestResultIndex.jenkinsfile

@@ -10,8 +10,6 @@ pipeline {
 		disableConcurrentBuilds()
 	}
 	parameters {
-		string(name: 'triggeringJob', trim: true, description: 'Name of the job to collect results from: i.e. \'ep427I-unit-cen64-gtk3-java17\'.')
-		string(name: 'buildURL', trim: true, description: 'Build URL of the triggering job.')
 		string(name: 'buildID', trim: true, description: 'ID of the I-build being tested.')
 	}
 	agent {
@@ -55,36 +53,14 @@ pipeline {
 						testResultsDir="${buildDirectory}/testresults"
 						mkdir -p "${testResultsDir}"
 
-						# ==========================================
-						# Collect results and overview from test-job
-						pushd "${testResultsDir}"
-						
-						curl -L -o "${triggeringJob}.xml" "${buildURL}/testReport/api/xml?tree=failCount,passCount,skipCount,duration"
-						curl -L -o results.zip "${buildURL}/artifact/workarea/${buildID}/eclipse-testing/results/*zip*/results.zip"
-						unzip results.zip
-						# copy them to the expected location (from the subfolder of the zip) and remove all download artifacts.
-						mv results/* .
-						rm -rf results results.zip
-						popd
-						
-						# Fetch previously collected test results and already generated files (that would otherwise be re-generated)
-						allTestResultsDirectory="${EP_ECLIPSE_DROPS}/${buildID}/testresults"
-						if ssh [email protected] "[ -d '${allTestResultsDirectory}/xml' ]"; then
-							# First delete result files from a previous run of the same configuration (in case that test configuration was run again).
-							ssh [email protected] rm -rfv ${allTestResultsDirectory}/${triggeringJob}* ${allTestResultsDirectory}/*/*${triggeringJob}*
-							rsync -avzh [email protected]:${allTestResultsDirectory}/xml ${buildDirectory}/testresults
-						else
-							echo 'Test results of other configurations not yet published.'
-						fi
+						# Fetch previously collected test results of all completed runs
+						rsync -avzh [email protected]:${EP_ECLIPSE_DROPS}/${buildID}/testresults/xml ${testResultsDir}
 
 						#triggering ant runner
-						devworkspace=${WORKSPACE}/workspace-updateTestResults
-						
 						$ECLIPSE -debug -data ${WORKSPACE}/workspace-updateTestResults \
 							-application org.eclipse.ant.core.antRunner \
 							-file "${WORKSPACE}/git-repo/cje-production/scripts/publish.xml" \
 							-DbuildDirectory=${buildDirectory} \
-							-Djob=${triggeringJob} \
 							-DbuildType=${BUILD_TYPE} \
 							"-DtestsConfigExpected=${TEST_CONFIGURATIONS_EXPECTED}" \
 							"-DmanifestFile=${WORKSPACE}/git-repo/eclipse.platform.releng.tychoeclipsebuilder/eclipse/publishingFiles/testManifest.xml"

cje-production/scripts/publish.xml

@@ -25,7 +25,6 @@
 		<taskdef name="indexResults" classname="${class}" />
 
 		<echo message=" = = properties in publish.xml == " />
-		<echo message="     job: ${job}" />
 		<echo message="     isBuildTested: ${isBuildTested}" />
 		<echo message="     buildType:  ${buildType}" />
 		<echo message="     dropTokenList: ${dropTokenList}" />
@@ -63,7 +62,6 @@
 
 	<target name="initResultsVariables">
 		<fail unless="buildDirectory" />
-		<fail unless="job" />
 		<fail unless="buildType" />
 		<fail unless="testsConfigExpected" />
 		<fail unless="manifestFile" />