From 1419aafd600247facf5bda9703adaed93204cb0c Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Mon, 1 Dec 2025 21:22:17 +0100 Subject: [PATCH] [I/Y-Build] Inline scripts for Eclipse base-builder and report creation Also use the same Eclipse installation to generate the repository and API report and don't create a separate installation from the one fetched from the download server. This avoids the need to maintain the simple eclipse invocations in separate files and thus simplifies the pipeline. --- .gitignore | 1 + JenkinsJobs/Builds/build.jenkinsfile | 71 +++++++++++++------ cje-production/buildproperties.txt | 2 - .../mbscripts/mb020_createBaseBuilder.sh | 46 ------------ .../mbscripts/mb300_gatherEclipseParts.sh | 9 +-- .../mbscripts/mb310_gatherEquinoxParts.sh | 3 +- .../mbscripts/mb500_createRepoReports.sh | 41 ----------- .../mbscripts/mb510_createApiToolsReports.sh | 44 ------------ 8 files changed, 56 insertions(+), 161 deletions(-) delete mode 100755 cje-production/mbscripts/mb020_createBaseBuilder.sh delete mode 100755 cje-production/mbscripts/mb500_createRepoReports.sh delete mode 100755 cje-production/mbscripts/mb510_createApiToolsReports.sh diff --git a/.gitignore b/.gitignore index d01a6898f10..f74826f609b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ workspace/ RemoteSystemsTempFiles/ /apiAnalyzer-workspace/ eclipse.platform.common/bundles/org.eclipse.platform.doc.isv/samples/*/doc-html +/tools/ # Maven/Tycho build artifats target/ diff --git a/JenkinsJobs/Builds/build.jenkinsfile b/JenkinsJobs/Builds/build.jenkinsfile index a463f78ac28..56d8aa24110 100644 --- a/JenkinsJobs/Builds/build.jenkinsfile +++ b/JenkinsJobs/Builds/build.jenkinsfile @@ -61,6 +61,8 @@ pipeline { assignEnvVariable('TEST_CONFIGURATIONS_EXPECTED', BUILD.tests.collect{ c -> "${TEST_NAME_PREFIX}-${c.os}-${c.arch}-java${c.javaVersion}_${c.os}.${c.ws}.${c.arch}_${c.javaVersion}" }.join(',')) + + utilities = load 'JenkinsJobs/shared/utilities.groovy' } dir("${CJE_ROOT}") { sh '''#!/bin/bash -xe @@ -73,6 +75,7 @@ pipeline { exportPropertiesAsEnvironmentVariables("${CJE_ROOT}/buildproperties.properties", [ 'BUILD_ID', 'STREAM', 'RELEASE_VER', + 'PREVIOUS_RELEASE_ID', 'PREVIOUS_RELEASE_VER', ]) } } @@ -191,14 +194,21 @@ pipeline { } } } - stage('Create Base builder'){ - steps { - dir("${CJE_ROOT}/mbscripts") { - sshagent(['projects-storage.eclipse.org-bot-ssh']) { - sh '''#!/bin/bash -xe - ./mb020_createBaseBuilder.sh $CJE_ROOT/buildproperties.shsource 2>&1 - ''' - } + stage('Create base builder'){ + steps { + script{ + def eclipsePlatformLatestRelease = "https://download.eclipse.org/eclipse/downloads/drops4/${PREVIOUS_RELEASE_ID}/eclipse-platform-${PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz" + def exe = utilities.installDownloadableTool('eclipse', eclipsePlatformLatestRelease) + '/eclipse' + assignEnvVariable('BASE_BUILDER_ECLIPSE_EXE', "${exe} -nosplash --launcher.suppressErrors") + sh ''' + source $CJE_ROOT/buildproperties.shsource + $BASE_BUILDER_ECLIPSE_EXE \ + -debug -consolelog -data $CJE_ROOT/$TMP_DIR/workspace-toolsinstall \ + -application org.eclipse.equinox.p2.director \ + -repository ${ECLIPSE_RUN_REPO},${BUILDTOOLS_REPO},https://download.eclipse.org/cbi/updates/p2-analyzers/products/nightly/latest \ + -installIU org.eclipse.pde.api.tools,org.eclipse.releng.build.tools.feature.feature.group,org.eclipse.cbi.p2repo.analyzers \ + -profile SDKProfile + ''' } } } @@ -252,7 +262,7 @@ pipeline { cp ${logDir}/mb220_buildSdkPatch.sh.log $CJE_ROOT/$DROP_DIR/$BUILD_ID/buildlogs/mb060_run-maven-build_output.txt pushd $CJE_ROOT/$DROP_DIR/$BUILD_ID - java -jar $LAUNCHER_JAR \ + $BASE_BUILDER_ECLIPSE_EXE \ -application org.eclipse.ant.core.antRunner \ -buildfile $ECLIPSE_BUILDER_DIR/eclipse/buildScripts/eclipse_compare.xml \ -data $CJE_ROOT/$TMP_DIR/workspace-comparatorLogs \ @@ -297,21 +307,39 @@ pipeline { } stage('Generate Repo reports') { steps { - dir("${CJE_ROOT}/mbscripts") { - sh '''#!/bin/bash -xe - ./mb500_createRepoReports.sh $CJE_ROOT/buildproperties.shsource 2>&1 - ''' - } + sh '''#!/bin/bash -xe + source $CJE_ROOT/buildproperties.shsource + $BASE_BUILDER_ECLIPSE_EXE \ + -application org.eclipse.cbi.p2repo.analyzers.repoReport \ + -data $CJE_ROOT/$TMP_DIR/workspace-report -vmargs -Xmx1g \ + -DreferenceRepo=$CJE_ROOT/$TMP_DIR/$BUILD_TO_COMPARE_SITE/$PREVIOUS_RELEASE_VER/$BASEBUILD_ID \ + -DreportRepoDir=${PLATFORM_REPO_DIR} \ + -DreportOutputDir=${CJE_ROOT}/${DROP_DIR}/${BUILD_ID}/buildlogs + ''' } } stage('Generate API tools reports') { steps { - dir("${CJE_ROOT}/mbscripts") { - sh '''#!/bin/bash -xe - ./mb510_createApiToolsReports.sh $CJE_ROOT/buildproperties.shsource 2>&1 - rm -rf $CJE_ROOT/$DROP_DIR/$BUILD_ID/apitoolingreference - ''' - } + sh '''#!/bin/bash -xe + source $CJE_ROOT/buildproperties.shsource + pushd $CJE_ROOT/$DROP_DIR/$BUILD_ID + $BASE_BUILDER_ECLIPSE_EXE \ + -application org.eclipse.ant.core.antRunner \ + -buildfile $ECLIPSE_BUILDER_DIR/eclipse/buildScripts/api-tools-builder.xml \ + -data $CJE_ROOT/$TMP_DIR/workspace-apitoolingsLogs \ + -DEBuilderDir=$ECLIPSE_BUILDER_DIR \ + -DbuildDirectory=$CJE_ROOT/$DROP_DIR/$BUILD_ID \ + -DbuildId=$BUILD_ID \ + -DbuildLabel=$BUILD_ID \ + -DbuildWorkingArea=$CJE_ROOT/$AGG_DIR \ + -DpreviousBaseURL=https://$DOWNLOAD_HOST/eclipse/downloads/drops4/$PREVIOUS_RELEASE_ID/eclipse-SDK-$PREVIOUS_RELEASE_VER-win32-x86_64.zip \ + -DpreviousBaselineName=Eclipse-SDK-$PREVIOUS_RELEASE_VER \ + -DpreviousBaselineFilename=eclipse-SDK-$PREVIOUS_RELEASE_VER-win32-x86_64.zip \ + -Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \ + apiToolsReports + popd + rm -rf $CJE_ROOT/$DROP_DIR/$BUILD_ID/apitoolingreference + ''' } } stage('Promote Eclipse platform') { @@ -481,6 +509,9 @@ pipeline { } } +@groovy.transform.Field +def utilities = null + @NonCPS def assignEnvVariable(String name, String value) { env[name] = value diff --git a/cje-production/buildproperties.txt b/cje-production/buildproperties.txt index e8f00f79888..234a13d9f78 100644 --- a/cje-production/buildproperties.txt +++ b/cje-production/buildproperties.txt @@ -40,6 +40,4 @@ BASEBUILD_ID="I20251126-2330" PREVIOUS_RELEASE_ID="S-4.38RC2-202511262330" BUILDTOOLS_REPO="https://download.eclipse.org/eclipse/updates/buildtools/snapshots" -WEBTOOLS_REPO="https://download.eclipse.org/webtools/downloads/drops/R3.37.0/R-3.37.0-20250303081219/repositoryunittests/" -BASEBUILDER_DIR="tmp/org.eclipse.releng.basebuilder" ECLIPSE_RUN_REPO="https://download.eclipse.org/eclipse/updates/4.39-I-builds/" diff --git a/cje-production/mbscripts/mb020_createBaseBuilder.sh b/cje-production/mbscripts/mb020_createBaseBuilder.sh deleted file mode 100755 index 059faf60215..00000000000 --- a/cje-production/mbscripts/mb020_createBaseBuilder.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -#******************************************************************************* -# Copyright (c) 2019 IBM Corporation and others. -# -# This program and the accompanying materials -# are made available under the terms of the Eclipse Public License 2.0 -# which accompanies this distribution, and is available at -# https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# Sravan Lakkimsetti - initial API and implementation -#******************************************************************************* -set -e - -if [ $# -ne 1 ]; then - echo USAGE: $0 env_file - exit 1 -fi - -source $CJE_ROOT/scripts/common-functions.shsource -source $1 - -mkdir -p $CJE_ROOT/$TMP_DIR -pushd $CJE_ROOT/$TMP_DIR -epDownloadDir=/home/data/httpd/download.eclipse.org/eclipse -dropsPath=${epDownloadDir}/downloads/drops4 -#get latest Eclipse platform product -epRelDir=$(ssh genie.releng@projects-storage.eclipse.org ls -d --format=single-column ${dropsPath}/R-*|sort|tail -1) -scp genie.releng@projects-storage.eclipse.org:${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz eclipsePlatform.tar.gz - -#wget -O eclipsePlatform.tar.gz https://$DOWNLOAD_HOST/eclipse/downloads/drops4/$PREVIOUS_RELEASE_ID/eclipse-platform-${PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz -tar zxf eclipsePlatform.tar.gz -popd - -$CJE_ROOT/$TMP_DIR/eclipse/eclipse -nosplash \ - -debug -consolelog -data $CJE_ROOT/$TMP_DIR/workspace-toolsinstall \ - -application org.eclipse.equinox.p2.director \ - -repository ${ECLIPSE_RUN_REPO},${BUILDTOOLS_REPO},${WEBTOOLS_REPO} \ - -installIU org.eclipse.platform.ide,org.eclipse.pde.api.tools,org.eclipse.releng.build.tools.feature.feature.group,org.eclipse.wtp.releng.tools.feature.feature.group \ - -destination $CJE_ROOT/$BASEBUILDER_DIR \ - -profile SDKProfile -vm ${JAVA_HOME}/bin/java - -fn-write-property LAUNCHER_JAR \"$(find $CJE_ROOT/$BASEBUILDER_DIR -name org.eclipse.equinox.launcher_*.jar | tail -1)\" diff --git a/cje-production/mbscripts/mb300_gatherEclipseParts.sh b/cje-production/mbscripts/mb300_gatherEclipseParts.sh index 17ee0393e5e..87487d09cb7 100755 --- a/cje-production/mbscripts/mb300_gatherEclipseParts.sh +++ b/cje-production/mbscripts/mb300_gatherEclipseParts.sh @@ -24,8 +24,6 @@ source $1 mkdir -p $CJE_ROOT/$DROP_DIR/$BUILD_ID/testresults/consolelogs -JavaCMD=${JAVA_HOME}/bin/java - # gather repo REPO_ZIP=$PLATFORM_TARGET_DIR/eclipse.platform.repository-${STREAMMajor}.${STREAMMinor}.${STREAMService}-SNAPSHOT.zip @@ -94,7 +92,7 @@ fi ANT_SCRIPT=$ECLIPSE_BUILDER_DIR/repos/platformrepo.xml if [ -d $PLATFORM_REPO_DIR ]; then pushd $PLATFORM_REPO_DIR - java -jar $LAUNCHER_JAR \ + $BASE_BUILDER_ECLIPSE_EXE \ -application org.eclipse.ant.core.antRunner \ -buildfile $ANT_SCRIPT \ -data $CJE_ROOT/$TMP_DIR/workspace-buildrepos \ @@ -158,7 +156,7 @@ fi # verify compilelog pushd $CJE_ROOT/$DROP_DIR/$BUILD_ID ANT_SCRIPT=$ECLIPSE_BUILDER_DIR/eclipse/helper.xml -$JavaCMD -jar $LAUNCHER_JAR \ +$BASE_BUILDER_ECLIPSE_EXE \ -application org.eclipse.ant.core.antRunner \ -buildfile $ANT_SCRIPT \ -data $CJE_ROOT/$TMP_DIR/workspace-verifyCompile \ @@ -185,7 +183,7 @@ fi # publish Eclipse pushd $CJE_ROOT ANT_SCRIPT=$ECLIPSE_BUILDER_DIR/eclipse/helper.xml -$JavaCMD -jar $LAUNCHER_JAR \ +$BASE_BUILDER_ECLIPSE_EXE \ -application org.eclipse.ant.core.antRunner \ -buildfile $ANT_SCRIPT \ -data $CJE_ROOT/$TMP_DIR/workspace-publish \ @@ -199,7 +197,6 @@ $JavaCMD -jar $LAUNCHER_JAR \ -DbuildType=$BUILD_TYPE \ -DpublishingContent=$ECLIPSE_BUILDER_DIR/eclipse/publishingFiles \ -DindexFileName=index.php \ - -Dbase.builder=$CJE_ROOT/$BASEBUILDER_DIR \ -Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \ -v \ publish diff --git a/cje-production/mbscripts/mb310_gatherEquinoxParts.sh b/cje-production/mbscripts/mb310_gatherEquinoxParts.sh index 5ea3accc141..8eee7b63908 100755 --- a/cje-production/mbscripts/mb310_gatherEquinoxParts.sh +++ b/cje-production/mbscripts/mb310_gatherEquinoxParts.sh @@ -69,7 +69,7 @@ fi pushd $CJE_ROOT mkdir -p $ECLIPSE_BUILDER_DIR/equinox/$TMP_DIR ANT_SCRIPT=$ECLIPSE_BUILDER_DIR/equinox/helper.xml -${JAVA_HOME}/bin/java -jar $LAUNCHER_JAR \ +$BASE_BUILDER_ECLIPSE_EXE \ -application org.eclipse.ant.core.antRunner \ -buildfile $ANT_SCRIPT \ -data $CJE_ROOT/$TMP_DIR/workspace-publishEquinox \ @@ -84,7 +84,6 @@ ${JAVA_HOME}/bin/java -jar $LAUNCHER_JAR \ -DeqpublishingContent=$ECLIPSE_BUILDER_DIR/equinox/publishingFiles \ -DindexFileName=index.php \ -Dequinox.build.configs=$ECLIPSE_BUILDER_DIR/equinox/buildConfigs \ - -Dbase.builder=$CJE_ROOT/$BASEBUILDER_DIR \ -Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \ -v \ publish diff --git a/cje-production/mbscripts/mb500_createRepoReports.sh b/cje-production/mbscripts/mb500_createRepoReports.sh deleted file mode 100755 index 012e9c29456..00000000000 --- a/cje-production/mbscripts/mb500_createRepoReports.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -#******************************************************************************* -# Copyright (c) 2020, 2025 IBM Corporation and others. -# -# This program and the accompanying materials -# are made available under the terms of the Eclipse Public License 2.0 -# which accompanies this distribution, and is available at -# https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# Kit Lo - initial API and implementation -#******************************************************************************* -set -e - -if [ $# -ne 1 ]; then - echo USAGE: $0 env_file - exit 1 -fi - -wait - -source $CJE_ROOT/scripts/common-functions.shsource -source $1 - -buildToTest=$PLATFORM_REPO_DIR -output_dir=$CJE_ROOT/$DROP_DIR/$BUILD_ID/buildlogs -tar_name=org.eclipse.cbi.p2repo.analyzers.product-linux.gtk.x86_64.tar.gz -report_app_dir=$CJE_ROOT/$TMP_DIR/reportApplication - -wget --no-proxy --no-verbose --no-cache -O $CJE_ROOT/$TMP_DIR/$tar_name https://download.eclipse.org/cbi/updates/p2-analyzers/products/nightly/latest/$tar_name - -mkdir -p $report_app_dir -tar -xf $CJE_ROOT/$TMP_DIR/$tar_name -C $report_app_dir - -$report_app_dir/p2analyze/p2analyze -data $CJE_ROOT/$TMP_DIR/workspace-report -vmargs -Xmx1g \ - -DreferenceRepo=$CJE_ROOT/$TMP_DIR/$BUILD_TO_COMPARE_SITE/$PREVIOUS_RELEASE_VER/$BASEBUILD_ID \ - -DreportRepoDir=$buildToTest \ - -DreportOutputDir=$output_dir diff --git a/cje-production/mbscripts/mb510_createApiToolsReports.sh b/cje-production/mbscripts/mb510_createApiToolsReports.sh deleted file mode 100755 index 7b367ee686e..00000000000 --- a/cje-production/mbscripts/mb510_createApiToolsReports.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -#******************************************************************************* -# Copyright (c) 2019 IBM Corporation and others. -# -# This program and the accompanying materials -# are made available under the terms of the Eclipse Public License 2.0 -# which accompanies this distribution, and is available at -# https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# Kit Lo - initial API and implementation -#******************************************************************************* -set -e - -if [ $# -ne 1 ]; then - echo USAGE: $0 env_file - exit 1 -fi - -source $CJE_ROOT/scripts/common-functions.shsource -source $1 - -ANT_SCRIPT=$ECLIPSE_BUILDER_DIR/eclipse/buildScripts/api-tools-builder.xml - -pushd $CJE_ROOT/$DROP_DIR/$BUILD_ID -${JAVA_HOME}/bin/java -jar $LAUNCHER_JAR \ - -application org.eclipse.ant.core.antRunner \ - -buildfile $ANT_SCRIPT \ - -data $CJE_ROOT/$TMP_DIR/workspace-apitoolingsLogs \ - -DEBuilderDir=$ECLIPSE_BUILDER_DIR \ - -DbuildDirectory=$CJE_ROOT/$DROP_DIR/$BUILD_ID \ - -DbuildId=$BUILD_ID \ - -DbuildLabel=$BUILD_ID \ - -DbuildWorkingArea=$CJE_ROOT/$AGG_DIR \ - -DpreviousBaseURL=https://$DOWNLOAD_HOST/eclipse/downloads/drops4/$PREVIOUS_RELEASE_ID/eclipse-SDK-$PREVIOUS_RELEASE_VER-win32-x86_64.zip \ - -DpreviousBaselineName=Eclipse-SDK-$PREVIOUS_RELEASE_VER \ - -DpreviousBaselineFilename=eclipse-SDK-$PREVIOUS_RELEASE_VER-win32-x86_64.zip \ - -Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \ - $FREEZE_PARAMS \ - apiToolsReports -popd