diff --git a/.project b/.project index 5d7e6a6779a..c6289ce77a2 100644 --- a/.project +++ b/.project @@ -15,7 +15,7 @@ 30 org.eclipse.ui.ide.multiFilter - 1.0-projectRelativePath-matches-true-true-(eclipse.platform.releng.prereqs.sdk|eclipse.platform.releng.tychoeclipsebuilder|oomph|products|sites)/.* + 1.0-projectRelativePath-matches-true-true-(eclipse.platform.releng.prereqs.sdk|eclipse.platform.releng.tychoeclipsebuilder|oomph|products|scripts|sites)/.* diff --git a/JenkinsJobs/Builds/build.jenkinsfile b/JenkinsJobs/Builds/build.jenkinsfile index 4f384e02cb2..d5e82464817 100644 --- a/JenkinsJobs/Builds/build.jenkinsfile +++ b/JenkinsJobs/Builds/build.jenkinsfile @@ -211,8 +211,8 @@ pipeline { $BASE_BUILDER_ECLIPSE_EXE \ -debug -consolelog -data $CJE_ROOT/$TMP_DIR/workspace-toolsinstall \ -application org.eclipse.equinox.p2.director \ - -repository ${ECLIPSE_RUN_REPO},${BUILDTOOLS_REPO},https://download.eclipse.org/cbi/updates/p2-analyzers/products/nightly/latest \ - -installIU org.eclipse.pde.api.tools,org.eclipse.releng.build.tools.feature.feature.group,org.eclipse.cbi.p2repo.analyzers \ + -repository ${ECLIPSE_RUN_REPO},https://download.eclipse.org/cbi/updates/p2-analyzers/products/nightly/latest \ + -installIU org.eclipse.pde.api.tools,org.eclipse.cbi.p2repo.analyzers \ -profile SDKProfile ''' } @@ -251,6 +251,7 @@ pipeline { # Gather maven properties cp ${AGG_DIR}/eclipse-platform-parent/target/mavenproperties.properties ${DROP_DIR}/$BUILD_ID/mavenproperties.properties + comparatorRepo=$(grep '^comparator.repo=' ${DROP_DIR}/${BUILD_ID}/mavenproperties.properties | cut -d'=' -f2-) # Gather artifactcomparisons pushd ${AGG_DIR} @@ -260,23 +261,14 @@ pipeline { popd # Verify comparatorlogs - # - # Note: copy mb220_buildSdkPatch.sh.log as mb060_run-maven-build_output.txt for now to avoid changing eclipse_compare.xml - # ToDo: Modify org.eclipse.releng.build.tools.comparator.Extractor to be configurable: - # https://github.com/eclipse-platform/eclipse.platform.releng.buildtools/blob/c5f7ecf1951d44311e24ce7bd6b505189aabb4da/bundles/org.eclipse.releng.build.tools.comparator/src/org/eclipse/releng/build/tools/comparator/Extractor.java#L27 #TODO: Generally try to avoid the need to capture the build log in a file/for teeing - cp ${logDir}/mb220_buildSdkPatch.sh.log ${DROP_DIR}/$BUILD_ID/buildlogs/mb060_run-maven-build_output.txt + cp ${logDir}/mb220_buildSdkPatch.sh.log ${DROP_DIR}/$BUILD_ID/buildlogs/ - pushd ${DROP_DIR}/$BUILD_ID - $BASE_BUILDER_ECLIPSE_EXE \ - -application org.eclipse.ant.core.antRunner \ - -buildfile $ECLIPSE_BUILDER_DIR/eclipse/buildScripts/eclipse_compare.xml \ - -data $CJE_ROOT/$TMP_DIR/workspace-comparatorLogs \ + java \ -DbuildDirectory=${DROP_DIR}/$BUILD_ID \ + -DcomparatorRepo=${comparatorRepo} \ -Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \ - -v \ - compare - popd + ${WORKSPACE}/scripts/releng/ComparatorSummaryExtractor.java comparatorLogMinimumSize=350 comparatorLog=${comparatorlogsDir}/buildtimeComparatorUnanticipated.log.txt @@ -297,6 +289,9 @@ pipeline { stage('Eclipse') { stages { stage('Gather Eclipse parts') { + tools { + jdk 'temurin-jdk25-latest' + } environment { KEYRING = credentials('secret-subkeys-releng.asc') KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase') @@ -402,19 +397,9 @@ pipeline { popd # Verify compilelog - pushd ${DROP_DIR}/${BUILD_ID} - $BASE_BUILDER_ECLIPSE_EXE \ - -application org.eclipse.ant.core.antRunner \ - -buildfile $ECLIPSE_BUILDER_DIR/eclipse/helper.xml \ - -data $CJE_ROOT/$TMP_DIR/workspace-verifyCompile \ - -DcjeDir=$CJE_ROOT \ - -DEBuilderDir=$ECLIPSE_BUILDER_DIR \ - -DbuildDirectory=${DROP_DIR}/${BUILD_ID} \ - -DbuildLabel=$BUILD_ID \ - -Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \ - -v \ - verifyCompile - popd + java \ + -Dinput=${DROP_DIR}/${BUILD_ID}/compilelogs/plugins \ + ${WORKSPACE}/scripts/releng/CompileLogConverter.java # Generate repository reports $BASE_BUILDER_ECLIPSE_EXE \ @@ -451,25 +436,25 @@ pipeline { done # Publish Eclipse - pushd $CJE_ROOT - $BASE_BUILDER_ECLIPSE_EXE \ - -application org.eclipse.ant.core.antRunner \ - -buildfile $ECLIPSE_BUILDER_DIR/eclipse/helper.xml \ - -data $CJE_ROOT/$TMP_DIR/workspace-publish \ - -DAGGR_DIR=${AGG_DIR} \ - -DcjeDir=$CJE_ROOT \ - -DEBuilderDir=$ECLIPSE_BUILDER_DIR \ - -DbuildDirectory=${DROP_DIR}/${BUILD_ID} \ - -DbuildLabel=$BUILD_ID \ - -DbuildDir=$BUILD_ID \ - -DbuildRepo=$PLATFORM_REPO_DIR \ - -DbuildType=$BUILD_TYPE \ - -DpublishingContent=$ECLIPSE_BUILDER_DIR/eclipse/publishingFiles \ - -DindexFileName=index.php \ - -Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \ - -v \ - publish + + pushd ${DROP_DIR}/${BUILD_ID} + bash ${CJE_ROOT}/scripts/produceChecksum.sh eclipse popd + + cp -r ${ECLIPSE_BUILDER_DIR}/eclipse/publishingFiles/staticDropFiles/. ${DROP_DIR}/${BUILD_ID} + + java \ + -DisBuildTested=true \ + -DbuildType=${BUILD_TYPE} \ + -DdropTokenList='%repository%,%sdk%,%tests%,%runtime%,%jdtc%,%swt%' \ + -DdropHtmlFileName=index.php \ + -DxmlDirectoryName=${DROP_DIR}/${BUILD_ID}/testresults/xml \ + -DdropDirectoryName=${DROP_DIR}/${BUILD_ID} \ + -DdropTemplateFileName=${ECLIPSE_BUILDER_DIR}/eclipse/publishingFiles/templateFiles/index.template.php \ + -DcompileLogsDirectoryName=${DROP_DIR}/${BUILD_ID}/compilelogs/plugins \ + -DtestManifestFileName=${ECLIPSE_BUILDER_DIR}/eclipse/publishingFiles/testManifest.xml \ + -DtestsConfigExpected=${TEST_CONFIGURATIONS_EXPECTED} \ + ${WORKSPACE}/scripts/releng/TestResultsGenerator.java ''' } } @@ -492,6 +477,9 @@ pipeline { stage('Equinox') { stages { stage('Gather Equinox parts') { + tools { + jdk 'temurin-jdk25-latest' + } environment { KEYRING = credentials('secret-subkeys-releng.asc') KEYRING_PASSPHRASE = credentials('secret-subkeys-releng.asc-passphrase') @@ -533,25 +521,40 @@ pipeline { # Publish Equinox pushd $CJE_ROOT mkdir -p $ECLIPSE_BUILDER_DIR/equinox/$TMP_DIR + mkdir -p $CJE_ROOT/$TMP_DIR $BASE_BUILDER_ECLIPSE_EXE \ -application org.eclipse.ant.core.antRunner \ -buildfile $ECLIPSE_BUILDER_DIR/equinox/helper.xml \ -data $CJE_ROOT/$TMP_DIR/workspace-publishEquinox \ -DEBuilderDir=$ECLIPSE_BUILDER_DIR \ -DbuildDir=$BUILD_ID \ - -DbuildDirectory=${EQUINOX_DROP_DIR} \ -DbuildId=$BUILD_ID \ -DbuildRepo=$PLATFORM_REPO_DIR \ - -DbuildType=$BUILD_TYPE \ - -DpostingDirectory=${DROP_DIR}/$BUILD_ID \ -DequinoxPostingDirectory=${EQUINOX_DROP_DIR} \ -DeqpublishingContent=$ECLIPSE_BUILDER_DIR/equinox/publishingFiles \ - -DindexFileName=index.php \ -Dequinox.build.configs=$ECLIPSE_BUILDER_DIR/equinox/buildConfigs \ -Djava.io.tmpdir=$CJE_ROOT/$TMP_DIR \ -v \ publish popd + + pushd ${EQUINOX_DROP_DIR}/${BUILD_ID} + bash ${CJE_ROOT}/scripts/produceChecksum.sh equinox + popd + + cp -r ${ECLIPSE_BUILDER_DIR}/equinox/publishingFiles/staticDropFiles/. ${EQUINOX_DROP_DIR}/${BUILD_ID} + + java \ + -DisBuildTested=false \ + -DbuildType=${BUILD_TYPE} \ + -DdropTokenList='%equinox%,%framework%,%extrabundles%,%other%,%launchers%,%osgistarterkits%' \ + -DdropHtmlFileName=index.php \ + -DxmlDirectoryName=${DROP_DIR}/$BUILD_ID/testresults/xml \ + -DdropDirectoryName=${EQUINOX_DROP_DIR}/${BUILD_ID} \ + -DdropTemplateFileName=${ECLIPSE_BUILDER_DIR}/equinox/publishingFiles/templateFiles/index.template.php \ + -DcompileLogsDirectoryName=${EQUINOX_DROP_DIR}/${BUILD_ID}/compilelogs/plugins \ + -DtestManifestFileName=${ECLIPSE_BUILDER_DIR}/equinox/publishingFiles/testManifest.xml \ + ${WORKSPACE}/scripts/releng/TestResultsGenerator.java ''' } } diff --git a/JenkinsJobs/Releng/updateTestResultIndex.jenkinsfile b/JenkinsJobs/Releng/updateTestResultIndex.jenkinsfile index fcf6c0ba49e..dd9c7f0c544 100644 --- a/JenkinsJobs/Releng/updateTestResultIndex.jenkinsfile +++ b/JenkinsJobs/Releng/updateTestResultIndex.jenkinsfile @@ -16,7 +16,7 @@ pipeline { label 'basic' } tools { - jdk 'temurin-jdk21-latest' + jdk 'temurin-jdk25-latest' } stages { stage('Checkout SCM') { @@ -24,9 +24,8 @@ pipeline { dir("${WORKSPACE}/git-repo") { checkout scmGit(userRemoteConfigs: [[url: "${scm.userRemoteConfigs[0].url}"]], branches: [[name: "${scm.branches[0].name}"]], extensions: [cloneOption(depth: 1, shallow: true, noTags: true), sparseCheckout([ - [path: 'cje-production/'], + [path: 'scripts/releng/'], [path: 'eclipse.platform.releng.tychoeclipsebuilder/eclipse/publishingFiles/testManifest.xml'], - [path: 'JenkinsJobs/shared/utilities.groovy'], ])]) } } @@ -35,7 +34,6 @@ pipeline { environment { // Download Server locations (seldomly change) EP_ECLIPSE_DROPS = '/home/data/httpd/download.eclipse.org/eclipse/downloads/drops4' - ECLIPSE = installLatestEclipse() } steps { sshagent(['projects-storage.eclipse.org-bot-ssh']) { @@ -44,11 +42,6 @@ pipeline { curl -L -o "${WORKSPACE}/buildproperties.shsource" http://download.eclipse.org/eclipse/downloads/drops4/${buildID}/buildproperties.shsource source "${WORKSPACE}/buildproperties.shsource" - $ECLIPSE -data workspace-toolsinstall \ - -application org.eclipse.equinox.p2.director \ - -repository ${ECLIPSE_RUN_REPO},${BUILDTOOLS_REPO} \ - -installIU org.eclipse.releng.build.tools.feature.feature.group - buildDirectory="${WORKSPACE}/postingDir/${buildID}" testResultsDir="${buildDirectory}/testresults" mkdir -p "${testResultsDir}" @@ -56,14 +49,14 @@ pipeline { # Fetch previously collected test results of all completed runs rsync -avzh genie.releng@projects-storage.eclipse.org:${EP_ECLIPSE_DROPS}/${buildID}/testresults/xml ${testResultsDir} - #triggering ant runner - $ECLIPSE -debug -data ${WORKSPACE}/workspace-updateTestResults \ - -application org.eclipse.ant.core.antRunner \ - -file "${WORKSPACE}/git-repo/cje-production/scripts/publish.xml" \ - -DbuildDirectory=${buildDirectory} \ + java \ + -DisBuildTested=true \ -DbuildType=${BUILD_TYPE} \ - "-DtestsConfigExpected=${TEST_CONFIGURATIONS_EXPECTED}" \ - "-DmanifestFile=${WORKSPACE}/git-repo/eclipse.platform.releng.tychoeclipsebuilder/eclipse/publishingFiles/testManifest.xml" + -DdropDirectoryName=${buildDirectory} \ + -DxmlDirectoryName=${buildDirectory}/testresults/xml \ + -DtestManifestFileName=${WORKSPACE}/git-repo/eclipse.platform.releng.tychoeclipsebuilder/eclipse/publishingFiles/testManifest.xml \ + -DtestsConfigExpected=${TEST_CONFIGURATIONS_EXPECTED} \ + ${WORKSPACE}/git-repo/scripts/releng/TestResultsGenerator.java rsync -avzh ${buildDirectory} genie.releng@projects-storage.eclipse.org:${EP_ECLIPSE_DROPS} ''' @@ -73,13 +66,3 @@ pipeline { } } } - -def installLatestEclipse(){ - def props = readProperties(file: "${WORKSPACE}/git-repo/cje-production/buildproperties.txt").collectEntries{n, v -> - v = v.trim(); - return [n, (v.startsWith('"') && v.endsWith('"') ? v.substring(1, v.length() - 1) : v)] - } - def utilities = load "${WORKSPACE}/git-repo/JenkinsJobs/shared/utilities.groovy" - def eclipseURL = "https://download.eclipse.org/eclipse/downloads/drops4/${props.PREVIOUS_RELEASE_ID}/eclipse-platform-${props.PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz" - return utilities.installDownloadableTool('eclipse', eclipseURL) + '/eclipse --launcher.suppressErrors -nosplash -consolelog' -} diff --git a/cje-production/buildproperties.txt b/cje-production/buildproperties.txt index f2b0e2a7399..a887ed0ecc4 100644 --- a/cje-production/buildproperties.txt +++ b/cje-production/buildproperties.txt @@ -38,5 +38,4 @@ BASEBUILD_ID="R-4.38-202512010920" #release id for downloading eclipse PREVIOUS_RELEASE_ID="R-4.38-202512010920" -BUILDTOOLS_REPO="https://download.eclipse.org/eclipse/updates/buildtools" ECLIPSE_RUN_REPO="https://download.eclipse.org/eclipse/updates/4.39-I-builds/" diff --git a/eclipse.platform.releng.tychoeclipsebuilder/eclipse/extras/produceChecksum.sh b/cje-production/scripts/produceChecksum.sh old mode 100755 new mode 100644 similarity index 95% rename from eclipse.platform.releng.tychoeclipsebuilder/eclipse/extras/produceChecksum.sh rename to cje-production/scripts/produceChecksum.sh index 4eda39dfa37..9ba9cd2da37 --- a/eclipse.platform.releng.tychoeclipsebuilder/eclipse/extras/produceChecksum.sh +++ b/cje-production/scripts/produceChecksum.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x #******************************************************************************* # Copyright (c) 2017, 2025 IBM Corporation and others. # @@ -12,10 +12,18 @@ # Contributors: # David Williams - initial API and implementation #******************************************************************************* -# + +if [ $# -ne 1 ]; then + echo USAGE: $0 client + exit 1 +fi +client=$1 + echo "[DEBUG] Producing checksums starting" echo "[DEBUG] current directory: ${PWD}" +mkdir checksum + allCheckSumsSHA512=checksum/${client}-${BUILD_ID}-SUMSSHA512 fileExtensionsToHash='zip dmg gz tar.xz jar' diff --git a/cje-production/scripts/publish.xml b/cje-production/scripts/publish.xml deleted file mode 100644 index dd9f70445d2..00000000000 --- a/cje-production/scripts/publish.xml +++ /dev/null @@ -1,97 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eclipse.platform.releng.tychoeclipsebuilder/eclipse/buildScripts/eclipse_compare.xml b/eclipse.platform.releng.tychoeclipsebuilder/eclipse/buildScripts/eclipse_compare.xml deleted file mode 100644 index ad8b6d9130b..00000000000 --- a/eclipse.platform.releng.tychoeclipsebuilder/eclipse/buildScripts/eclipse_compare.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eclipse.platform.releng.tychoeclipsebuilder/eclipse/helper.xml b/eclipse.platform.releng.tychoeclipsebuilder/eclipse/helper.xml deleted file mode 100644 index f1a7af56bab..00000000000 --- a/eclipse.platform.releng.tychoeclipsebuilder/eclipse/helper.xml +++ /dev/null @@ -1,136 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eclipse.platform.releng.tychoeclipsebuilder/equinox/helper.xml b/eclipse.platform.releng.tychoeclipsebuilder/equinox/helper.xml index 4a63256fb81..462f786b478 100644 --- a/eclipse.platform.releng.tychoeclipsebuilder/equinox/helper.xml +++ b/eclipse.platform.releng.tychoeclipsebuilder/equinox/helper.xml @@ -6,9 +6,6 @@ - - + - - - - - - - - @@ -44,121 +30,13 @@ name="publish" depends="init"> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/oomph/Platform.setup b/oomph/Platform.setup index 6857271e6f3..6dd5ec88376 100644 --- a/oomph/Platform.setup +++ b/oomph/Platform.setup @@ -815,72 +815,6 @@ The Platform Releng Agrregator components, including the Platform's products - - - - - github.remoteURIs - - - Platform Releng Build Tools Github Repository - - - remoteURI - - - - - - Platform Releng Build Tools - - - - - - - - - - - - - - - - - - - - The Platform Releng Build Tools - + + + + + diff --git a/scripts/.project b/scripts/.project new file mode 100644 index 00000000000..d791b34d1ae --- /dev/null +++ b/scripts/.project @@ -0,0 +1,17 @@ + + + eclipse.platform.releng.scripts + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/scripts/.settings/org.eclipse.jdt.core.prefs b/scripts/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000000..e58dd1940ae --- /dev/null +++ b/scripts/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,12 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.release=enabled +org.eclipse.jdt.core.compiler.source=25 diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000000..4b7c24df79e --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,6 @@ +# Eclipse build and RelEng scripts + +This folder contains scripts that are used in the Eclipse build, mainly for RelEng or Testing tasks. + +The contained `Java` classes are intended to be launched as single- or mutli-file _Source-Code Programs_ respectively as plain _Java scripts_. +This means the main source file is launched directly without being compiled and therefore this project must not have any external dependencies except the JDK. diff --git a/scripts/releng/ComparatorSummaryExtractor.java b/scripts/releng/ComparatorSummaryExtractor.java new file mode 100644 index 00000000000..c2ac5a8d2d8 --- /dev/null +++ b/scripts/releng/ComparatorSummaryExtractor.java @@ -0,0 +1,300 @@ + +/******************************************************************************* + * Copyright (c) 2013, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + * Hannes Wellmann - Convert to plain Java scripts + *******************************************************************************/ + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class is responsible for extracting the relevent "Debug" messages from + * the huge maven debug log. + * + * @author davidw + */ +public class ComparatorSummaryExtractor { + + private static final String EOL = System.lineSeparator(); + + public static void main(String[] args) throws IOException { + ComparatorSummaryExtractor extractor = new ComparatorSummaryExtractor(); + extractor.buildDirectory = Objects.requireNonNull(System.getProperty("buildDirectory"), + "Not set: buildDirectory"); + extractor.comparatorRepo = Objects.requireNonNull(System.getProperty("comparatorRepo"), + "Not set: comparatorRepo"); + extractor.processBuildfile(); + } + + private record LogEntry(String name, List reasons, List info) { + + static LogEntry create(String name) { + return new LogEntry(name, new ArrayList<>(), new ArrayList<>()); + } + + public void addInfo(final String infoline) { + info.add(infoline); + } + + public void addReason(final String reason) { + reasons.add(reason); + } + } + + private static final String BUILD_LOGS_DIRECTORY = "buildlogs"; + private static final String COMPARATOR_LOGS_DIRECTORY = "comparatorlogs"; + private String comparatorRepo = "comparatorRepo"; + private String buildDirectory; + private static final Pattern MAIN_PATTERN = Pattern.compile( + "^\\[WARNING\\].*eclipse.platform.releng.aggregator/(.*): baseline and build artifacts have same version but different contents"); + private static final Pattern NO_CLASSIFIER_PATTERN = Pattern.compile("^.*no-classifier:.*$"); + private static final Pattern CLASSIFIER_SOURCES_PATTERN = Pattern.compile("^.*classifier-sources:.*$"); + private static final Pattern CLASSIFIER_SOURCES_FEATURE_PATTERN = Pattern + .compile("^.*classifier-sources-feature:.*$"); + + private static final Pattern SIGN1_PATTERN = Pattern.compile("^.*META-INF/(ECLIPSE_|CODESIGN).RSA.*$"); + private static final Pattern SIGN2_PATTERN = Pattern.compile("^.*META-INF/(ECLIPSE_|CODESIGN).SF.*$"); + private static final Pattern DOC_NAME_PATTERN = Pattern.compile("^.*eclipse\\.platform\\.common.*\\.doc\\..*$"); + // jar pattern added for bug 416701 + private final Pattern JAR_PATTERN = Pattern.compile("^.*\\.jar.*$"); + private int count; + private int countSign; + private int countDoc; + private int countOther; + private int countSignPlusInnerJar; + private int countJDTCore; + + private boolean docItem(final LogEntry newEntry) { + boolean result = false; + final String name = newEntry.name(); + final Matcher matcher = DOC_NAME_PATTERN.matcher(name); + if (matcher.matches()) { + result = true; + } + return result; + } + + private String getInputFilename() { + return buildDirectory + "/" + BUILD_LOGS_DIRECTORY + "/" + "mb220_buildSdkPatch.sh.log"; + } + + private String getOutputFilenameDoc() { + return buildDirectory + "/" + BUILD_LOGS_DIRECTORY + "/" + COMPARATOR_LOGS_DIRECTORY + "/" + + "buildtimeComparatorDocBundle.log.txt"; + } + + private String getOutputFilenameFull() { + return buildDirectory + "/" + BUILD_LOGS_DIRECTORY + "/" + COMPARATOR_LOGS_DIRECTORY + "/" + + "buildtimeComparatorFull.log.txt"; + } + + private String getOutputFilenameOther() { + return buildDirectory + "/" + BUILD_LOGS_DIRECTORY + "/" + COMPARATOR_LOGS_DIRECTORY + "/" + + "buildtimeComparatorUnanticipated.log.txt"; + } + + private String getOutputFilenameSign() { + return buildDirectory + "/" + BUILD_LOGS_DIRECTORY + "/" + COMPARATOR_LOGS_DIRECTORY + "/" + + "buildtimeComparatorSignatureOnly.log.txt"; + } + + private String getOutputFilenameSignWithInnerJar() { + return buildDirectory + "/" + BUILD_LOGS_DIRECTORY + "/" + COMPARATOR_LOGS_DIRECTORY + "/" + + "buildtimeComparatorSignatureOnlyWithInnerJar.log.txt"; + } + + private String getOutputFilenameJDTCore() { + return buildDirectory + "/" + BUILD_LOGS_DIRECTORY + "/" + COMPARATOR_LOGS_DIRECTORY + "/" + + "buildtimeComparatorJDTCore.log.txt"; + } + + public void processBuildfile() throws IOException { + + // Make sure directory exists + File outputDir = new File(buildDirectory + "/" + BUILD_LOGS_DIRECTORY, COMPARATOR_LOGS_DIRECTORY); + if (!outputDir.exists()) { + outputDir.mkdirs(); + } + + final File infile = new File(getInputFilename()); + final File outfile = new File(getOutputFilenameFull()); + final File outfileSign = new File(getOutputFilenameSign()); + final File outfileDoc = new File(getOutputFilenameDoc()); + final File outfileOther = new File(getOutputFilenameOther()); + final File outfileSignWithInnerJar = new File(getOutputFilenameSignWithInnerJar()); + final File outfileJDTCore = new File(getOutputFilenameJDTCore()); + try (BufferedReader input = Files.newBufferedReader(infile.toPath()); + Writer out = new FileWriter(outfile); + BufferedWriter output = new BufferedWriter(out); + Writer outsign = new FileWriter(outfileSign); + BufferedWriter outputSign = new BufferedWriter(outsign); + Writer outdoc = new FileWriter(outfileDoc); + BufferedWriter outputDoc = new BufferedWriter(outdoc); + Writer outother = new FileWriter(outfileOther); + BufferedWriter outputOther = new BufferedWriter(outother); + Writer outsignWithJar = new FileWriter(outfileSignWithInnerJar); + BufferedWriter outputSignWithJar = new BufferedWriter(outsignWithJar); + Writer outJDTCore = new FileWriter(outfileJDTCore); + BufferedWriter outputJDTCore = new BufferedWriter(outJDTCore);) { + + writeHeader(output); + writeHeader(outputSign); + writeHeader(outputSignWithJar); + writeHeader(outputDoc); + writeHeader(outputOther); + writeHeader(outputJDTCore); + count = 0; + countSign = 0; + countSignPlusInnerJar = 0; + countDoc = 0; + countOther = 0; + countJDTCore = 0; + String inputLine = ""; + + while (inputLine != null) { + inputLine = input.readLine(); + if (inputLine != null) { + final Matcher matcher = MAIN_PATTERN.matcher(inputLine); + if (matcher.matches()) { + + final LogEntry newEntry = LogEntry.create(matcher.group(1)); + // read and write differences, until next blank line + do { + inputLine = input.readLine(); + if ((inputLine != null) && (inputLine.length() > 0)) { + newEntry.addReason(inputLine); + } + } while ((inputLine != null) && (inputLine.length() > 0)); + // //output.write(EOL); + // now, do one more, to get the "info" that says + // what was copied, or not. + do { + inputLine = input.readLine(); + if ((inputLine != null) && (inputLine.length() > 0)) { + // except leave out the first line, which is a + // long [INFO] line repeating what we already + // know. + if (!inputLine.startsWith("[INFO]")) { + newEntry.addInfo(inputLine); + } + } + } while ((inputLine != null) && (inputLine.length() > 0)); + // Write full log, for sanity check, if nothing else + writeEntry(++count, output, newEntry); + if (jdtCore(newEntry)) { + writeEntry(++countJDTCore, outputJDTCore, newEntry); + } else if (docItem(newEntry)) { + writeEntry(++countDoc, outputDoc, newEntry); + } else if (pureSignature(newEntry)) { + writeEntry(++countSign, outputSign, newEntry); + } else if (pureSignaturePlusInnerJar(newEntry)) { + writeEntry(++countSignPlusInnerJar, outputSignWithJar, newEntry); + } else { + writeEntry(++countOther, outputOther, newEntry); + } + } + } + } + } + } + + private void writeHeader(final BufferedWriter output) throws IOException { + output.write("Comparator differences from current build" + EOL); + output.write("\t" + buildDirectory + EOL); + output.write("compared to reference repo at " + EOL); + output.write("\t" + comparatorRepo + EOL + EOL); + } + + private boolean jdtCore(final LogEntry newEntry) { + boolean result = false; + final String name = newEntry.name(); + if (name.equals("eclipse.jdt.core/org.eclipse.jdt.core/pom.xml")) { + result = true; + } + return result; + } + + private boolean pureSignature(final LogEntry newEntry) { + // if all lines match one of these critical patterns, + // then assume "signature only" difference. If even + // one of them does not match, assume not. + boolean result = true; + final List reasons = newEntry.reasons(); + for (final String reason : reasons) { + final Matcher matcher1 = NO_CLASSIFIER_PATTERN.matcher(reason); + final Matcher matcher2 = CLASSIFIER_SOURCES_PATTERN.matcher(reason); + final Matcher matcher3 = CLASSIFIER_SOURCES_FEATURE_PATTERN.matcher(reason); + final Matcher matcher4 = SIGN1_PATTERN.matcher(reason); + final Matcher matcher5 = SIGN2_PATTERN.matcher(reason); + + if (matcher1.matches() || matcher2.matches() || matcher3.matches() || matcher4.matches() + || matcher5.matches()) { + } else { + result = false; + break; + } + } + + return result; + } + + private boolean pureSignaturePlusInnerJar(final LogEntry newEntry) { + // if all lines match one of these critical patterns, + // then assume "signature only plus inner jar" difference. If even + // one of them does not match, assume not. + // TODO: refactor so less copy/paste of pureSignature method. + boolean result = true; + final List reasons = newEntry.reasons(); + for (final String reason : reasons) { + final Matcher matcher1 = NO_CLASSIFIER_PATTERN.matcher(reason); + final Matcher matcher2 = CLASSIFIER_SOURCES_PATTERN.matcher(reason); + final Matcher matcher3 = CLASSIFIER_SOURCES_FEATURE_PATTERN.matcher(reason); + final Matcher matcher4 = SIGN1_PATTERN.matcher(reason); + final Matcher matcher5 = SIGN2_PATTERN.matcher(reason); + final Matcher matcher6 = JAR_PATTERN.matcher(reason); + + if (matcher1.matches() || matcher2.matches() || matcher3.matches() || matcher4.matches() + || matcher5.matches() || matcher6.matches()) { + } else { + result = false; + break; + } + } + + return result; + } + + private void writeEntry(int thistypeCount, final Writer output, final LogEntry newEntry) throws IOException { + + output.write(thistypeCount + ". " + newEntry.name() + EOL); + final List reasons = newEntry.reasons(); + for (final String reason : reasons) { + output.write(reason + EOL); + } + final List infolist = newEntry.info(); + for (final String info : infolist) { + output.write(info + EOL); + } + output.write(EOL); + } +} diff --git a/scripts/releng/CompileLogConverter.java b/scripts/releng/CompileLogConverter.java new file mode 100644 index 00000000000..3d20886d0e1 --- /dev/null +++ b/scripts/releng/CompileLogConverter.java @@ -0,0 +1,223 @@ + +/******************************************************************************* + * Copyright (c) 2000, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + * Hannes Wellmann - Convert to plain Java scripts + *******************************************************************************/ + + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Locale; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; +import org.xml.sax.helpers.DefaultHandler; + +import log.converter.DOMHtmlConverter; +import log.converter.LogDocumentNode; +import log.converter.LogDocumentNode.ProblemSummaryNode; +import log.converter.ProblemNode; +import log.converter.ProblemNode.SeverityType; +import log.converter.ProblemsNode; + +public class CompileLogConverter { + + private static final String HTML_EXTENSION = ".html"; //$NON-NLS-1$ + + private static final String XML_EXTENSION = ".xml"; //$NON-NLS-1$ + + private static List getAllFiles(Path root) throws IOException { + try (var files = Files.walk(root).filter(Files::isRegularFile)) { + return files.filter(f -> f.toString().toLowerCase(Locale.ROOT).endsWith(XML_EXTENSION)).toList(); + } + } + + public static void main(String[] args) throws IOException, ParserConfigurationException { + String input = System.getProperty("input"); + if (input == null) { + throw new IllegalArgumentException("An input file or directorty is required"); //$NON-NLS-1$ + } + CompileLogConverter converter = new CompileLogConverter(); + converter.parse2(Path.of(input)); + } + + private Path extractNameFrom(Path file) { + String inputFileName = file.getFileName().toString(); + final int index = inputFileName.lastIndexOf('.'); + return file.resolveSibling(inputFileName.substring(0, index) + HTML_EXTENSION); + } + + private void parse2(Path sourceDir) throws ParserConfigurationException, IOException { + DOMHtmlConverter converter = new DOMHtmlConverter(); + final DocumentBuilderFactory factory = createDocumentBuilderFactoryIgnoringDOCTYPE(); + factory.setValidating(true); + factory.setIgnoringElementContentWhitespace(true); + final DocumentBuilder builder = factory.newDocumentBuilder(); + // Commented due to + // https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/1943 + // builder.setEntityResolver((publicId, systemId) -> new InputSource(new + // ByteArrayInputStream(new byte[0]))); + if (true) { + // collect all xml files and iterate over them + if (!Files.exists(sourceDir)) { + throw new IllegalArgumentException("Directory " + sourceDir + " doesn't exist");//$NON-NLS-1$//$NON-NLS-2$ + } + if (!Files.isDirectory(sourceDir)) { + throw new IllegalArgumentException(sourceDir + " must be a directory in recursive mode");//$NON-NLS-1$ + } + List xmlFiles = getAllFiles(sourceDir); + for (Path xmlFile : xmlFiles) { + Path inputFile = xmlFile.toAbsolutePath(); + Path outputFile = extractNameFrom(inputFile); + try { + builder.setErrorHandler(new DefaultHandler() { + @Override + public void error(final SAXParseException e) throws SAXException { + reportError(inputFile, e); + throw e; + } + }); + Document document = builder.parse(inputFile.toFile()); + final LogDocumentNode documentNode = process(document); + converter.dump(inputFile, outputFile, documentNode); + } catch (final SAXException e) { + System.out.println(e); + } catch (final IOException e) { + e.printStackTrace(); + } + } + } + } + + private LogDocumentNode process(final Document document) { + final LogDocumentNode documentNode = new LogDocumentNode(); + NodeList nodeList = document.getElementsByTagName("problem_summary"); //$NON-NLS-1$ + if (nodeList.getLength() == 1) { + final Node problemSummaryNode = nodeList.item(0); + final NamedNodeMap problemSummaryMap = problemSummaryNode.getAttributes(); + final ProblemSummaryNode summaryNode = new ProblemSummaryNode( + Integer.parseInt(problemSummaryMap.getNamedItem("problems").getNodeValue()), + Integer.parseInt(problemSummaryMap.getNamedItem("errors").getNodeValue()), + Integer.parseInt(problemSummaryMap.getNamedItem("warnings").getNodeValue()), + Integer.parseInt(problemSummaryMap.getNamedItem("infos").getNodeValue())); + documentNode.setProblemSummary(summaryNode); + } + + nodeList = document.getElementsByTagName("problems"); //$NON-NLS-1$ + if (nodeList == null) { + return null; + } + + final int length = nodeList.getLength(); + int globalErrorNumber = 1; + for (int i = 0; i < length; i++) { + final Node problemsNode = nodeList.item(i); + final ProblemsNode node = new ProblemsNode(); + documentNode.addProblemsNode(node); + final Node sourceNode = problemsNode.getParentNode(); + final NamedNodeMap sourceNodeMap = sourceNode.getAttributes(); + final String sourceFileName = sourceNodeMap.getNamedItem("path").getNodeValue();//$NON-NLS-1$ + node.sourceFileName = sourceFileName; + final NamedNodeMap problemsNodeMap = problemsNode.getAttributes(); + node.numberOfErrors = Integer.parseInt(problemsNodeMap.getNamedItem("errors").getNodeValue());//$NON-NLS-1$ + node.numberOfWarnings = Integer.parseInt(problemsNodeMap.getNamedItem("warnings").getNodeValue());//$NON-NLS-1$ + node.numberOfProblems = Integer.parseInt(problemsNodeMap.getNamedItem("problems").getNodeValue());//$NON-NLS-1$ + node.numberOfInfos = Integer.parseInt(problemsNodeMap.getNamedItem("infos").getNodeValue());//$NON-NLS-1$ + + final NodeList children = problemsNode.getChildNodes(); + final int childrenLength = children.getLength(); + for (int j = 0; j < childrenLength; j++) { + final Node problemNode = children.item(j); + final NamedNodeMap problemNodeMap = problemNode.getAttributes(); + final String severity = problemNodeMap.getNamedItem("severity").getNodeValue();//$NON-NLS-1$ + final ProblemNode problem = new ProblemNode(); + problem.id = problemNodeMap.getNamedItem("id").getNodeValue();//$NON-NLS-1$ + switch (severity) { + case "ERROR": + problem.severityType = SeverityType.ERROR; + node.addError(problem); + break; + case "INFO": + problem.severityType = SeverityType.INFO; + node.addInfo(problem); + break; + case "WARNING": + problem.severityType = SeverityType.WARNING; + if (DOMHtmlConverter.FILTERED_WARNINGS_IDS.contains(problem.id)) { + if (DOMHtmlConverter.FORBIDDEN_REFERENCE.equals(problem.id)) { + node.addForbiddenWarning(problem); + } else { + node.addDiscouragedWarning(problem); + } + } else { + node.addOtherWarning(problem); + } + break; + } + problem.charStart = Integer.parseInt(problemNodeMap.getNamedItem("charStart").getNodeValue());//$NON-NLS-1$ + problem.charEnd = Integer.parseInt(problemNodeMap.getNamedItem("charEnd").getNodeValue());//$NON-NLS-1$ + problem.line = Integer.parseInt(problemNodeMap.getNamedItem("line").getNodeValue());//$NON-NLS-1$ + problem.globalProblemNumber = globalErrorNumber; + problem.problemNumber = j; + problem.sourceFileName = sourceFileName; + globalErrorNumber++; + final NodeList problemChildren = problemNode.getChildNodes(); + final int problemChildrenLength = problemChildren.getLength(); + for (int n = 0; n < problemChildrenLength; n++) { + final Node child = problemChildren.item(n); + final String nodeName = child.getNodeName(); + if ("message".equals(nodeName)) {//$NON-NLS-1$ + final NamedNodeMap childNodeMap = child.getAttributes(); + problem.message = childNodeMap.getNamedItem("value").getNodeValue();//$NON-NLS-1$ + } else if ("source_context".equals(nodeName)) {//$NON-NLS-1$ + final NamedNodeMap childNodeMap = child.getAttributes(); + problem.sourceStart = Integer.parseInt(childNodeMap.getNamedItem("sourceStart").getNodeValue());//$NON-NLS-1$ + problem.sourceEnd = Integer.parseInt(childNodeMap.getNamedItem("sourceEnd").getNodeValue());//$NON-NLS-1$ + problem.contextValue = childNodeMap.getNamedItem("value").getNodeValue();//$NON-NLS-1$ + } + } + } + } + return documentNode; + } + + private void reportError(final Path inputFileName, final SAXParseException e) { + System.err.println( + "Error in " + inputFileName + " at line " + e.getLineNumber() + " and column " + e.getColumnNumber()); //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ + System.err.println(e.getMessage()); + } + + // --- utility methods --- + + private static synchronized DocumentBuilderFactory createDocumentBuilderFactoryIgnoringDOCTYPE() { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + try { + // completely disable external entities declarations: + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); //$NON-NLS-1$ + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); //$NON-NLS-1$ + } catch (ParserConfigurationException e) { + throw new RuntimeException(e.getMessage(), e); + } + return factory; + } +} diff --git a/scripts/releng/ErrorTracker.java b/scripts/releng/ErrorTracker.java new file mode 100644 index 00000000000..de4e9e9bd36 --- /dev/null +++ b/scripts/releng/ErrorTracker.java @@ -0,0 +1,195 @@ + +/******************************************************************************* + * Copyright (c) 2000, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + * Hannes Wellmann - Convert to plain Java scripts + *******************************************************************************/ + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +public class ErrorTracker { + + private final Set testLogsSet = Collections.checkedSortedSet(new TreeSet<>(), String.class); + // Platforms keyed on + private final Map platforms = new HashMap<>(); + private final Map> logFiles = new HashMap<>(); + private final Map> typesMap = new HashMap<>(); + + private final List typesList = new ArrayList<>(); + + private String convertPathDelimiters(final String path) { + return new File(path).getPath(); + } + + public PlatformStatus[] getPlatforms(final String type) { + final List platformIDs = typesMap.get(type); + final PlatformStatus[] result = new PlatformStatus[platformIDs.size()]; + for (int i = 0; i < platformIDs.size(); i++) { + result[i] = platforms.get(platformIDs.get(i)); + } + return result; + } + + /** + * Returns the testLogs. + * + * @return Vector + */ + public List getTestLogs(List foundConfigs) { + // List of test logs expected at end of build + // We depend on both test logs and configs being sorted + ArrayList testLogs = new ArrayList<>(); + for (String initialLogName : testLogsSet) { + for (String config : foundConfigs) { + testLogs.add(initialLogName + "_" + config + ".xml"); + } + } + return testLogs; + } + + // Answer a string array of the zip type names in the order they appear in + // the .xml file. + public String[] getTypes() { + return typesList.toArray(new String[typesList.size()]); + } + + // Answer an array of PlatformStatus objects for a given type. + + public void loadFile(final String fileName) { + try { + DocumentBuilder parser = TestResultsGenerator.createDocumentBuilderWithErrorOnDOCTYPE(); + final Document document = parser.parse(fileName); + final NodeList elements = document.getElementsByTagName("platform"); + final int elementCount = elements.getLength(); + for (int i = 0; i < elementCount; i++) { + final PlatformStatus aPlatform = PlatformStatus.create((Element) elements.item(i)); + // System.out.println("ID: " + aPlatform.getId()); + platforms.put(aPlatform.id(), aPlatform); + + final Node zipType = elements.item(i).getParentNode(); + final String zipTypeName = zipType.getAttributes().getNamedItem("name").getNodeValue(); + + List aVector = typesMap.get(zipTypeName); + if (aVector == null) { + typesList.add(zipTypeName); + aVector = new ArrayList<>(); + typesMap.put(zipTypeName, aVector); + } + aVector.add(aPlatform.id()); + + } + + final NodeList effectedFiles = document.getElementsByTagName("effectedFile"); + final int effectedFilesCount = effectedFiles.getLength(); + for (int i = 0; i < effectedFilesCount; i++) { + final Node anEffectedFile = effectedFiles.item(i); + final Node logFile = anEffectedFile.getParentNode(); + String logFileName = logFile.getAttributes().getNamedItem("name").getNodeValue(); + logFileName = convertPathDelimiters(logFileName); + final String effectedFileID = anEffectedFile.getAttributes().getNamedItem("id").getNodeValue(); + // System.out.println(logFileName); + List aVector = logFiles.get(logFileName); + if (aVector == null) { + aVector = new ArrayList<>(); + logFiles.put(logFileName, aVector); + + } + final PlatformStatus ps = platforms.get(effectedFileID); + if (ps != null) { + aVector.add(ps); + } + } + + // store a list of the test logs expected after testing + final NodeList testLogList = document.getElementsByTagName("logFile"); + final int testLogCount = testLogList.getLength(); + for (int i = 0; i < testLogCount; i++) { + + final Node testLog = testLogList.item(i); + final String testLogName = testLog.getAttributes().getNamedItem("name").getNodeValue(); + final Node typeNode = testLog.getAttributes().getNamedItem("type"); + // String type = "test"; + // if (typeNode != null) { + // type = typeNode.getNodeValue(); + // } + // if (testLogName.endsWith(".xml") && type.equals("test")) { + // above is how it used to be checked, prior to 4/4/2016, but + // test logs are only log file in testManifest.xml without a "type" attribute + // -- I test for either/or, so that new versions of testManifest.xml + // can more correctly use "test" attribute, if desired. + if (typeNode == null || typeNode.getNodeValue().equals("test")) { + int firstUnderscore = testLogName.indexOf('_'); + String initialTestName = null; + if (firstUnderscore == -1) { + // no underscore found. Assume testManifest xml has been updated + // to mention minimal name. + initialTestName = testLogName; + } else { + initialTestName = testLogName.substring(0, firstUnderscore); + } + testLogsSet.add(initialTestName); + // System.out.println("Debug: initialTestName: " + initialTestName); + } + + } + + } catch (final IOException e) { + System.out.println("IOException: " + fileName); + // e.printStackTrace(); + + } catch (final SAXException e) { + System.out.println("SAXException: " + fileName); + e.printStackTrace(); + + } catch (final ParserConfigurationException e1) { + e1.printStackTrace(); + } + } + + public void registerError(final String fileName) { + // System.out.println("Found an error in: " + fileName); + if (logFiles.containsKey(fileName)) { + List aVector = logFiles.get(fileName); + for (PlatformStatus element : aVector) { + element.registerError(); + } + } else { + + // If a log file is not specified explicitly it effects + // all "platforms" except JDT + + for (PlatformStatus aValue : platforms.values()) { + if (!aValue.id().equals("JA") && !aValue.id().equals("EW") && !aValue.id().equals("EA")) { + aValue.registerError(); + } + } + } + } + +} diff --git a/scripts/releng/PlatformStatus.java b/scripts/releng/PlatformStatus.java new file mode 100644 index 00000000000..0f4d2af3dfa --- /dev/null +++ b/scripts/releng/PlatformStatus.java @@ -0,0 +1,40 @@ + +/******************************************************************************* + * Copyright (c) 2000, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ + +import java.util.List; + +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; + +record PlatformStatus(String id, String name, String fileName, String format, List images) { + + static PlatformStatus create(Element anElement) { + NamedNodeMap attributes = anElement.getAttributes(); + String id = attributes.getNamedItem("id").getNodeValue(); + Node node = attributes.getNamedItem("name"); + String name = node == null ? "" : node.getNodeValue(); + String fileName = attributes.getNamedItem("fileName").getNodeValue(); + node = attributes.getNamedItem("format"); + String format = node != null ? node.getNodeValue() : null; + node = attributes.getNamedItem("images"); + List images = node != null ? List.of(node.getNodeValue().split(",")) : null; + return new PlatformStatus(id, name, fileName, format, images); + } + + public void registerError() { + // Had no effect. Callers should be removed/clean-up. + } +} diff --git a/scripts/releng/TestResultsGenerator.java b/scripts/releng/TestResultsGenerator.java new file mode 100644 index 00000000000..34c1e3b77ec --- /dev/null +++ b/scripts/releng/TestResultsGenerator.java @@ -0,0 +1,1243 @@ + +/******************************************************************************* + * Copyright (c) 2000, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + * Hannes Wellmann - Convert to plain Java scripts + *******************************************************************************/ + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.Writer; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.TreeMap; +import java.util.TreeSet; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +/** + * @author Dean Roberts (circa 2000!) and David Williams (circa 2016) + */ +public class TestResultsGenerator { + + public class ResultsTable implements Iterable { + + private final Map rows = new TreeMap<>(); + private final List columns; + + public ResultsTable(List columns) { + this.columns = columns; + } + + public record Cell(int errorCount, File resultsFile) { + } + + private class Row { + + Map row = new TreeMap<>(); + + public Row(List columns) { + for (String column : columns) { + row.put(column, null); + } + } + + public Cell getCell(String column) { + return row.get(column); + } + + public void putCell(String columnName, Integer cellValue, File file) { + row.put(columnName, new Cell(cellValue, file)); + } + } + + private Row getRow(String rowname) { + Row row = rows.get(rowname); + if (row == null) { + row = new Row(columns); + rows.put(rowname, row); + } + return row; + } + + public Cell getCell(String rowName, String columnName) { + return getRow(rowName).getCell(columnName); + } + + public void putCell(String rowName, String columnName, Integer cellValue, File file) { + getRow(rowName).putCell(columnName, cellValue, file); + } + + @Override + public Iterator iterator() { + return rows.keySet().iterator(); + } + } + + private static final String HTML_EXTENSION = ".html"; + private static final String XML_EXTENSION = ".xml"; + private static final String WARNING_SEVERITY = "WARNING"; + private static final String ERROR_SEVERITY = "ERROR"; + private static final String INFO_SEVERITY = "INFO"; + private static final String ForbiddenReferenceID = "ForbiddenReference"; + private static final String DiscouragedReferenceID = "DiscouragedReference"; + + private static final String elementName = "testsuite"; + + private List expectedConfigs = null; + private static final String EOL = System.lineSeparator(); + private static boolean DEBUG = false; + + private final String expected_config_type = "expected"; + private final String expectedConfigFilename = "testConfigs.php"; + private final String foundConfigFilename = "testConfigsFound.php"; + + private ErrorTracker anErrorTracker; + + // Parameters + // build runs JUnit automated tests + private boolean isBuildTested; + + // buildType, I, N + private String buildType; + + // Comma separated list of drop tokens + private String dropTokenList; + + // Location of the xml files + private String xmlDirectoryName; + + // Location of the resulting index.php file. + private String dropDirectoryName; + + // Location and name of the template drop index.php file. + private String dropTemplateFileName; + + // Name of the HTML fragment file that any testResults.php file will + // "include". + // setting to common default. + private final String testResultsHtmlFileName = "testResultsTables.html"; + + // Name of the generated drop index php file; + private String dropHtmlFileName; + + // Arbitrary path used in the index.php page to href the + // generated .html files. + private final String hrefTestResultsTargetPath = "testresults"; + // Arbitrary path used in the index.php page to reference the compileLogs + private final String hrefCompileLogsTargetPath = "compilelogs/plugins/"; + // Location of compile logs base directory + private String compileLogsDirectoryName; + // Location and name of test manifest file + private String testManifestFileName; + // private static String testsConstant = ".tests"; + // private static int testsConstantLength = testsConstant.length(); + // temporary way to force "missing" list not to be printed (until complete + // solution found) + private final boolean doMissingList = true; + + private final Set missingManifestFiles = Collections.checkedSortedSet(new TreeSet<>(), String.class); + + class ExpectedConfigFiler implements FilenameFilter { + + String configEnding; + + public ExpectedConfigFiler(String expectedConfigEnding) { + configEnding = expectedConfigEnding; + } + + @Override + public boolean accept(File dir, String name) { + return (name.endsWith(configEnding)); + } + + } + + private void logException(final Throwable e) { + log(EOL + "ERROR: " + e.getMessage()); + StackTraceElement[] stackTrace = e.getStackTrace(); + for (StackTraceElement stackTraceElement : stackTrace) { + log(stackTraceElement.toString()); + } + } + + // Configuration of test machines. + // Add or change new configurations here + // and update titles in testResults.template.php. + // These are the suffixes used for JUnit's XML output files. + // On each invocation, all files in results directory are + // scanned, to see if they end with suffixes, and if so, + // are processed for summary row. The column order is determined by + // the order listed here. + // This suffix is determined, at test time, when the files junit files are + // generated, by the setting of a variable named "platform" in test.xml + // and associated property files. + + // no defaults set since adds to confusion or errors + // private String[] testsConfigDefaults = { "ep4" + getTestedBuildType() + + // "-unit-cen64-gtk2_linux.gtk.x86_64_8.0.xml", + // "ep4" + getTestedBuildType() + "-unit-mac64_macosx.cocoa.x86_64_8.0.xml", + // "ep4" + getTestedBuildType() + "-unit-win32_win32.win32.x86_8.0.xml", + // "ep4" + getTestedBuildType() + "-unit-cen64-gtk3_linux.gtk.x86_64_8.0.xml" }; + private String testsConfigExpected; + private final String compilerSummaryFilename = "compilerSummary.html"; + /* + * Default for "regenerate" is FALSE, but during development, is handy to set to + * TRUE. If TRUE, the "index.php" file and "compilerSummary.html" files are + * regenerated. In production that should seldom be required. The + * testResultsTables.html file, however, is regenerated each call (when + * 'isTested" is set) since the purpose is usually to include an additional + * tested platform. + */ + private final boolean regenerate = false; + + private int countCompileErrors(final String aString) { + return extractNumber(aString, "error"); + } + + private int countCompileWarnings(final String aString) { + return extractNumber(aString, "warning"); + } + + private int countDiscouragedWarnings(final String aString) { + return extractNumber(aString, "Discouraged access:"); + } + + private int countInfos(final String aString) { + return extractNumber(aString, "info"); + } + + /* + * returns number of errors plus number of failures. returns a negative number + * if the file is missing or something is wrong with the file (such as is + * incomplete). + */ + private int countErrors(final String fileName) { + int errorCount = -99; + // File should exists, since we are "driving" this based on file list + // ... but, just in case. + if (!new File(fileName).exists()) { + errorCount = -1; + } else { + + if (new File(fileName).length() == 0) { + errorCount = -2; + } else { + + try { + DocumentBuilder parser = createDocumentBuilderWithErrorOnDOCTYPE(); + final Document document = parser.parse(fileName); + final NodeList elements = document.getElementsByTagName(elementName); + + final int elementCount = elements.getLength(); + if (elementCount == 0) { + errorCount = -3; + } else { + // There can be multiple "testSuites" per file so we + // need to + // loop through each to count all errors and failures. + errorCount = 0; + for (int i = 0; i < elementCount; i++) { + final Element element = (Element) elements.item(i); + final NamedNodeMap attributes = element.getAttributes(); + Node aNode = attributes.getNamedItem("errors"); + if (aNode != null) { + errorCount = errorCount + Integer.parseInt(aNode.getNodeValue()); + } + aNode = attributes.getNamedItem("failures"); + errorCount = errorCount + Integer.parseInt(aNode.getNodeValue()); + } + } + + } catch (final IOException e) { + log(EOL + "ERROR: IOException: " + fileName); + logException(e); + errorCount = -4; + } catch (final SAXException e) { + log(EOL + "ERROR: SAXException: " + fileName); + logException(e); + errorCount = -5; + } catch (final ParserConfigurationException e) { + logException(e); + errorCount = -6; + } + } + } + return errorCount; + } + + private int countForbiddenWarnings(final String aString) { + return extractNumber(aString, "Access restriction:"); + } + + public static void main(String[] args) { + TestResultsGenerator generator = new TestResultsGenerator(); + generator.isBuildTested = Boolean.parseBoolean(System.getProperty("isBuildTested", "false")); + + generator.buildType = Objects.requireNonNull(System.getProperty("buildType"), "Not set: buildType"); + generator.compileLogsDirectoryName = System.getProperty("compileLogsDirectoryName"); + generator.dropDirectoryName = Objects.requireNonNull(System.getProperty("dropDirectoryName"), + "Not set: dropDirectoryName"); + generator.dropHtmlFileName = System.getProperty("dropHtmlFileName"); + generator.dropTemplateFileName = System.getProperty("dropTemplateFileName"); + generator.dropTokenList = System.getProperty("dropTokenList"); + generator.testManifestFileName = Objects.requireNonNull(System.getProperty("testManifestFileName"), + "Not set: testManifestFileName"); + + generator.xmlDirectoryName = System.getProperty("xmlDirectoryName"); + generator.testsConfigExpected = System.getProperty("testsConfigExpected"); + + generator.execute(); + } + + public void execute() { + + log(EOL + "INFO: Processing test and build results for "); + log("\t" + dropDirectoryName); + anErrorTracker = new ErrorTracker(); + anErrorTracker.loadFile(testManifestFileName); + + writeDropIndexFile(); + + try { + parseCompileLogs(); + } catch (IOException e) { + throw new IllegalStateException("Error while parsing Compiler Results File ", e); + } + + if (isBuildTested) { + + try { + parseJUnitTestsXml(); + + } catch (IOException e) { + throw new IllegalStateException("Error while parsing JUnit Tests Results Files", e); + } + + } else { + log(EOL + "INFO: isBuildTested value was not true, so did no processing for test files"); + } + log(EOL + "INFO: Completed processing test and build results"); + } + + private int extractNumber(final String aString, final String endToken) { + final int endIndex = aString.lastIndexOf(endToken); + if (endIndex == -1) { + return 0; + } + + int startIndex = endIndex; + while ((startIndex >= 0) && (aString.charAt(startIndex) != '(') && (aString.charAt(startIndex) != ',')) { + startIndex--; + } + + final String count = aString.substring(startIndex + 1, endIndex).trim(); + try { + return Integer.parseInt(count); + } catch (final NumberFormatException e) { + return 0; + } + + } + + private void formatAccessesErrorRow(final String fileName, final int forbiddenAccessesWarningsCount, + final int discouragedAccessesWarningsCount, final int infoCount, final StringBuilder buffer) { + + if ((forbiddenAccessesWarningsCount == 0) && (discouragedAccessesWarningsCount == 0) && (infoCount == 0)) { + return; + } + + String relativeName = computeRelativeName(fileName); + String shortName = computeShortName(relativeName); + + buffer.append("").append(EOL).append("").append(EOL).append("").append(shortName).append("").append("\n") + .append("").append("").append(forbiddenAccessesWarningsCount).append("") + .append("").append(EOL).append("").append("") + .append(discouragedAccessesWarningsCount).append("").append("").append(EOL) + .append("").append("").append(infoCount).append("").append("").append(EOL) + .append("").append(EOL); + } + + private String computeRelativeName(final String fileName) { + String relativeName; + final int i = fileName.indexOf(hrefCompileLogsTargetPath); + relativeName = fileName.substring(i); + return relativeName; + } + + private String computeShortName(final String relativeName) { + String shortName; + + int start = hrefCompileLogsTargetPath.length(); + int last = relativeName.lastIndexOf("/"); + // if there is no "last slash", that's a pretty weird case, but we'll + // just + // take the whole rest of string in that case. + if (last == -1) { + shortName = relativeName.substring(start); + } else { + shortName = relativeName.substring(start, last); + } + // further shortening (may need to "back out", so left as separate step) + // we always expect the name to start with "org.eclipse." .. but, just + // in case that changes, we'll check and handle if not. + String commonnamespace = "org.eclipse."; + if (shortName.startsWith(commonnamespace)) { + start = commonnamespace.length(); + } else { + start = 0; + } + // Similarly, we alwasy expect the name to end with '_version', but just + // in case not. + last = shortName.indexOf('_'); + if (last == -1) { + shortName = shortName.substring(start); + } else { + shortName = shortName.substring(start, last); + } + return shortName; + } + + private void formatCompileErrorRow(final String fileName, final int errorCount, final int warningCount, + final StringBuilder buffer) { + + if ((errorCount == 0) && (warningCount == 0)) { + return; + } + + String relativeName = computeRelativeName(fileName); + String shortName = computeShortName(relativeName); + + buffer.append("" + EOL + "" + EOL).append("").append(shortName).append("").append("\n").append("") + .append("").append(errorCount) + .append("").append("\n").append("").append("").append(warningCount).append("") + .append("\n").append("\n"); + } + + private void parseCompileLog(final String log, final StringBuilder compilerLog, final StringBuilder accessesLog) { + int errorCount = 0; + int warningCount = 0; + int forbiddenWarningCount = 0; + int discouragedWarningCount = 0; + int infoCount = 0; + + final File file = new File(log); + Document aDocument = null; + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + final InputSource inputSource = new InputSource(reader); + final DocumentBuilder builder = createDocumentBuilderIgnoringDOCTYPE(); + + aDocument = builder.parse(inputSource); + } catch (final ParserConfigurationException | IOException | SAXException e) { + logException(e); + } + + if (aDocument == null) { + return; + } + // Get summary of problems + final NodeList nodeList = aDocument.getElementsByTagName("problem"); + if ((nodeList == null) || (nodeList.getLength() == 0)) { + return; + } + + final int length = nodeList.getLength(); + for (int i = 0; i < length; i++) { + final Node problemNode = nodeList.item(i); + final NamedNodeMap aNamedNodeMap = problemNode.getAttributes(); + final Node severityNode = aNamedNodeMap.getNamedItem("severity"); + final Node idNode = aNamedNodeMap.getNamedItem("id"); + if (severityNode != null) { + final String severityNodeValue = severityNode.getNodeValue(); + if (WARNING_SEVERITY.equals(severityNodeValue)) { + // this is a warning + // need to check the id + final String nodeValue = idNode == null ? "" : idNode.getNodeValue(); + if (ForbiddenReferenceID.equals(nodeValue)) { + forbiddenWarningCount++; + } else if (DiscouragedReferenceID.equals(nodeValue)) { + discouragedWarningCount++; + } else { + warningCount++; + } + } else if (ERROR_SEVERITY.equals(severityNodeValue)) { + // this is an error + errorCount++; + } else if (INFO_SEVERITY.equals(severityNodeValue)) { + // this is an info warning + infoCount++; + } + } + } + if (errorCount != 0) { + // use wildcard in place of version number on directory names + // log(log + "/n"); + String logName = log.substring(compileLogsDirectoryName.length() + 1); + final StringBuilder buffer = new StringBuilder(logName); + buffer.replace(logName.indexOf("_") + 1, logName.indexOf(File.separator, logName.indexOf("_") + 1), "*"); + logName = new String(buffer); + + anErrorTracker.registerError(logName); + } + // make sure '.xml' extension is "last thing" in string. (bug 490320) + final String logName = log.replaceAll(XML_EXTENSION + "$", HTML_EXTENSION); + formatCompileErrorRow(logName, errorCount, warningCount, compilerLog); + formatAccessesErrorRow(logName, forbiddenWarningCount, discouragedWarningCount, infoCount, accessesLog); + } + + private void parseCompileLogs() throws IOException { + String compileLogsDirectory = compileLogsDirectoryName; + if (compileLogsDirectory == null || compileLogsDirectory.isBlank()) { + log(EOL + "INFO: Skip generating the compile logs summary page."); + return; + } + File sourceDirectory = new File(compileLogsDirectory); + File mainDir = new File(dropDirectoryName); + File compilerSummaryFile = new File(mainDir, compilerSummaryFilename); + // we do not recompute compiler summary each time, since it is + // fairly time consuming -- and no reason it would not be "complete", + // if it exists. + if (compilerSummaryFile.exists() && !regenerate) { + log(EOL + "INFO: Compile logs summary page, " + compilerSummaryFilename + + ", was found to exist already and not regenerated."); + } else { + if (compilerSummaryFile.exists()) { + log(EOL + "INFO: Compile logs summary page, " + compilerSummaryFilename + + ", was found to exist already and is being regenerated."); + } + log("DEBUG: BEGIN: Parsing compile logs and generating summary table."); + final StringBuilder compilerString = new StringBuilder(); + final StringBuilder accessesString = new StringBuilder(); + processCompileLogsDirectory(compileLogsDirectory, compilerString, accessesString); + if (compilerString.length() == 0) { + compilerString.append( + "None  " + + EOL); + } + if (accessesString.length() == 0) { + accessesString.append( + "None  " + + EOL); + } + String prefix = """ +

Plugins containing compile errors or warnings

+ +

The table below shows the plugins in which errors or warnings were encountered. Click on the jar file link to view its + detailed report.

+ + + + + + + + """; + String suffix = """ +
Compile Logs (Jar Files)ErrorsWarnings
+ +

Plugins containing access errors or warnings

+ + + + + + + + """; + StringBuilder compileLogResults = new StringBuilder(prefix).append(compilerString).append(suffix); + compileLogResults.append(accessesString.toString()); + compileLogResults.append("
Compile Logs (Jar Files)Forbidden AccessDiscouraged AccessInfo Warnings
").append(EOL); + // write the include file. The name of this file must match what is + // in testResults.template.php + writePhpIncludeCompilerResultsFile(sourceDirectory, compileLogResults.toString()); + log("DEBUG: End: Parsing compile logs and generating summary table."); + } + } + + private void parseJUnitTestsXml() throws IOException { + log("DEBUG: Begin: Parsing XML JUnit results files"); + List foundConfigs = new ArrayList<>(); + final File xmlResultsDirectory = new File(xmlDirectoryName); + ResultsTable resultsTable = new ResultsTable(getTestsConfig()); + if (xmlResultsDirectory.exists()) { + // reinitialize each time. + // We currently "re do" all of tests, but can improve in the future + // where the "found configs" are remembered, but then have to keep + // track of original order (not "found" order which has to with when + // tests completed). + foundConfigs.clear(); + + List allFileNames = new ArrayList<>(); + + for (String expectedConfig : getTestsConfig()) { + + FilenameFilter configfilter = new ExpectedConfigFiler("_" + expectedConfig + XML_EXTENSION); + // we end with "full" list of files, sorted by configfilter, and + // then alphabetical. + File[] xmlFileNamesForConfig = xmlResultsDirectory.listFiles(configfilter); + + if (xmlFileNamesForConfig.length > 0) { + // log("DEBUG: For " + expectedConfig + " found " + + // xmlFileNamesForConfig.length + " XML results files"); + foundConfigs.add(expectedConfig); + // sort by name, for each 'config' found. + Arrays.sort(xmlFileNamesForConfig); + Collections.addAll(allFileNames, xmlFileNamesForConfig); + } + } + File[] xmlFileNames = new File[allFileNames.size()]; + allFileNames.toArray(xmlFileNames); + // files MUST be alphabetical, for now? + Arrays.sort(xmlFileNames); + String sourceDirectoryCanonicalPath = dropDirectoryName; + for (File junitResultsFile : xmlFileNames) { + checkIfMissingFromTestManifestFile(junitResultsFile, foundConfigs); + String fullName = junitResultsFile.getPath(); + int errorCount = countErrors(fullName); + resultsTable.putCell(computeCoreName(junitResultsFile), computeConfig(junitResultsFile), errorCount, + junitResultsFile); + if (errorCount != 0) { + trackDataForMail(sourceDirectoryCanonicalPath, junitResultsFile, fullName); + } + } + } else { + // error? Or, just too early? + log(EOL + "WARNING: sourceDirectory did not exist at \n\t" + xmlResultsDirectory); + log(" either incorrect call to 'generate index' or called too early (tests not done yet)?"); + } + log("DEBUG: End: Parsing XML JUnit results files"); + // above is all "compute data". Now it is time to "display" it. + if (foundConfigs.size() > 0) { + log("DEBUG: Begin: Generating test results index tables in " + testResultsHtmlFileName); + writeHTMLResultsTable(foundConfigs, resultsTable); + log("DEBUG: End: Generating test results index tables"); + } else { + log(EOL + "WARNING: Test results not found in " + xmlResultsDirectory.getAbsolutePath()); + } + + } + + private void writeHTMLResultsTable(List foundConfigs, ResultsTable resultsTable) throws IOException { + // These first files reflect what we expected, and what we found. + String found_config_type = "found"; + writePhpConfigFile(found_config_type, foundConfigs, foundConfigFilename); + // write the table to main output directory in testResultsTables.html, + // which in turn is included by the testResults.php file. + + StringBuilder htmlString = new StringBuilder(); + // first we right a bit of "static" part. That comes before the table. + htmlString.append(EOL).append("

Unit Test Results

").append(EOL); + + if (buildType.equals("Y")) { + htmlString.append( + "

The unit tests are run on the releng ci instance.

"); + } else { + htmlString.append( + "

The unit tests are run on the releng ci instance.

"); + } + String tableDescription = """ +

The table shows the unit test results for this build on the platforms + tested. You may access the test results page specific to each + component on a specific platform by clicking the cell link. + Normally, the number of errors is indicated in the cell.

+

A negative number or \"DNF\" means the test \"Did Not Finish\" for unknown reasons + and hence no results page is available. In that case, + more information can sometimes be found in + the console logs.

+ + """; + htmlString.append(EOL).append(tableDescription); + + htmlString.append(startTableOfUnitResults()); + for (String row : resultsTable) { + htmlString.append(formatJUnitRow(row, resultsTable, foundConfigs)); + } + // Once we are done with the Unit tests rows, we must add end table + // tag, since the following methods may or may not add a table of + // their own. + htmlString.append(EOL).append("").append(EOL); + // check for missing test logs + // TODO put styling on these tables + htmlString.append(verifyAllTestsRan(xmlDirectoryName, foundConfigs)); + htmlString.append(listMissingManifestFiles()); + writeTestResultsFile(htmlString.toString()); + } + + private String startTableOfUnitResults() throws IOException { + StringBuilder result = new StringBuilder(); + int ncolumns = getTestsConfig().size(); + result.append("").append(EOL); + // table header + result.append("").append(EOL); + result.append("").append(EOL); + result.append("").append(EOL); + result.append("\n"); + + result.append("").append(EOL); + + for (String column : getTestsConfig()) { + result.append("\n"); + } + result.append("").append(EOL); + // end table header + return result.toString(); + } + + /* + * This function "breaks" the full config string at meaningful underscores, for + * improved display in tables and similar. Remember, some config values can have + * more than two underscores, such as ep46I-unit-lin64_linux.gtk.x86_64_8.0, + * which should be split as ep46I-unit-lin64 lin64_linux.gtk.x86_64 8.0 + */ + private String computeDisplayConfig(String config) { + int lastUnderscore = config.lastIndexOf("_"); + int firstUnderscore = config.indexOf('_', config.indexOf("x86_64") + 6); + // echo "
DEBUG: config: config firstUnderscore: firstUnderscore + // lastUnderscore: lastUnderscore lastMinusFirst: platformLength" + String jobname = config.substring(0, firstUnderscore); + String platformconfig = config.substring(firstUnderscore + 1, lastUnderscore); + String vmused = config.substring(lastUnderscore + 1); + // echo "DEBUG: jobname: ".jobname."
"; + // echo "DEBUG: platformconfig: ".platformconfig."
"; + // echo "DEBUG: vmused: ".vmused."
"; + return jobname + "
" + platformconfig + "
" + vmused; + + } + + /* + * As far as I know, this "work" was done to track data send out in an email. + */ + private void trackDataForMail(String sourceDirectoryCanonicalPath, File junitResultsFile, final String fullName) { + anErrorTracker.registerError(fullName.substring(xmlDirectoryName.length() + 1)); + } + + /* + * This is the "reverse" of checking for "missing test results". It is simple + * sanity check to see if all "known" test results are listed in in the + * testManifest.xml file. We only do this check if we also are checking for + * missing logs which depends on an accurate testManifest.xml file. + */ + private void checkIfMissingFromTestManifestFile(File junitResultsFile, List foundConfigs) { + if (doMissingList) { + if (!verifyLogInManifest(junitResultsFile.getName(), foundConfigs)) { + String corename = computeCoreName(junitResultsFile); + missingManifestFiles.add(corename); + } + } + } + + private String computeCoreName(File junitResultsFile) { + String fname = junitResultsFile.getName(); + // corename is all that needs to be listed in testManifest.xml + String corename = null; + int firstUnderscorepos = fname.indexOf('_'); + if (firstUnderscorepos == -1) { + // should not occur, but if it does, we will take whole name + corename = fname; + } else { + corename = fname.substring(0, firstUnderscorepos); + } + return corename; + } + + private String computeConfig(File junitResultsFile) { + String fname = junitResultsFile.getName(); + String configName = null; + int firstUnderscorepos = fname.indexOf('_'); + if (firstUnderscorepos == -1) { + // should not occur, but if it does, we will set to null + // and let calling program decide what to do. + configName = null; + } else { + int lastPos = fname.lastIndexOf(XML_EXTENSION); + if (lastPos == -1) { + configName = null; + } else { + configName = fname.substring(firstUnderscorepos + 1, lastPos); + } + } + return configName; + } + + private void writePhpConfigFile(String config_type, List configs, String phpfilename) throws IOException { + File mainDir = new File(dropDirectoryName); + File testConfigsFile = new File(mainDir, phpfilename); + try (Writer testconfigsPHP = new FileWriter(testConfigsFile)) { + testconfigsPHP.write("" + EOL); + compilerSummaryPHP.write(compilerSummary); + } + } + + private void processCompileLogsDirectory(final String directoryName, final StringBuilder compilerLog, + final StringBuilder accessesLog) { + final File sourceDirectory = new File(directoryName); + if (sourceDirectory.isFile()) { + if (sourceDirectory.getName().endsWith(".log")) { + readCompileLog(sourceDirectory.getAbsolutePath(), compilerLog, accessesLog); + } + if (sourceDirectory.getName().endsWith(XML_EXTENSION)) { + parseCompileLog(sourceDirectory.getAbsolutePath(), compilerLog, accessesLog); + } + } + if (sourceDirectory.isDirectory()) { + final File[] logFiles = sourceDirectory.listFiles(); + Arrays.sort(logFiles); + for (File logFile : logFiles) { + processCompileLogsDirectory(logFile.getAbsolutePath(), compilerLog, accessesLog); + } + } + } + + private String processDropRow(final PlatformStatus aPlatform) { + if ("equinox".equalsIgnoreCase(aPlatform.format())) { + return processEquinoxDropRow(aPlatform); + } else { + return processEclipseDropRow(aPlatform); + } + + } + + private String processEclipseDropRow(PlatformStatus aPlatform) { + StringBuilder result = new StringBuilder("\n\n"); + // generate file link, size and checksums in the php template + result.append("\n"); + result.append("\n"); + return result.toString(); + } + + private String processDropRows(final PlatformStatus[] platforms) { + StringBuilder result = new StringBuilder(); + for (PlatformStatus platform : platforms) { + result.append(processDropRow(platform)); + } + return result.toString(); + } + + /* + * Generate and return the HTML mark-up for a single row for an Equinox JAR on + * the downloads page. + */ + private String processEquinoxDropRow(final PlatformStatus aPlatform) { + StringBuilder result = new StringBuilder(""); + result.append("\n"); + result.append("{$generateDropSize(\"").append(filename).append("\")}\n"); + result.append("{$generateChecksumLinks(\"").append(filename).append("\", $buildlabel)}\n"); + result.append("\n"); + return result.toString(); + } + + private void readCompileLog(final String log, final StringBuilder compilerLog, final StringBuilder accessesLog) { + final String fileContents = readFile(log); + + final int errorCount = countCompileErrors(fileContents); + final int warningCount = countCompileWarnings(fileContents); + final int forbiddenWarningCount = countForbiddenWarnings(fileContents); + final int discouragedWarningCount = countDiscouragedWarnings(fileContents); + final int infoCount = countInfos(fileContents); + if (errorCount != 0) { + // use wildcard in place of version number on directory names + String logName = log.substring(compileLogsDirectoryName.length() + 1); + final StringBuilder stringBuilder = new StringBuilder(logName); + stringBuilder.replace(logName.indexOf("_") + 1, logName.indexOf(File.separator, logName.indexOf("_") + 1), + "*"); + logName = new String(stringBuilder); + + anErrorTracker.registerError(logName); + } + formatCompileErrorRow(log, errorCount, warningCount, compilerLog); + formatAccessesErrorRow(log, forbiddenWarningCount, discouragedWarningCount, infoCount, accessesLog); + } + + private String readFile(final String fileName) { + try { + return Files.readString(Path.of(fileName), Charset.defaultCharset()); + } catch (final IOException e) { + logException(e); + return ""; + } + } + + private String replace(final String source, final String original, final String replacement) { + + final int replaceIndex = source.indexOf(original); + if (replaceIndex > -1) { + StringBuilder resultString = new StringBuilder().append(source.substring(0, replaceIndex)); + resultString.append(replacement); + resultString.append(source.substring(replaceIndex + original.length())); + return resultString.toString(); + } else { + log(EOL + "WARNING: Could not find token: " + original); + return source; + } + + } + + private String verifyAllTestsRan(final String directory, List foundConfigs) { + StringBuilder replaceString = new StringBuilder(); + List missingFiles = new ArrayList<>(); + if (doMissingList) { + for (String testLogName : anErrorTracker.getTestLogs(foundConfigs)) { + + if (new File(directory + File.separator + testLogName).exists()) { + // log("DEBUG: found log existed: " + testLogName); + continue; + } + // log("DEBUG: found log DID NOT exist: " + testLogName); + anErrorTracker.registerError(testLogName); + // replaceString = replaceString + tmp; + missingFiles.add(testLogName); + } + } else { + // Note: we intentionally do not deal with missing file for perf. + // tests yet. + // (though, probably could, once fixed with "expected configs"). + replaceString + .append(""" + + + + """); + } + // TODO: we need lots more of separating "data" from "formating" + if (doMissingList && missingFiles.size() > 0) { + String ordinalWord = "File"; + if (missingFiles.size() > 1) { + ordinalWord = "Files"; + } + + replaceString.append("
org.eclipse
Test Bundles
Test Configurations (Hudson Job/os.ws.arch/VM)
").append(computeDisplayConfig(column)).append("
").append(aPlatform.name()).append("
"); + final String filename = aPlatform.fileName(); + // if there are images, put them in the same table column as the name of + // the file + final List images = aPlatform.images(); + if ((images != null) && !images.isEmpty()) { + for (String image : images) { + result.append(" "); + } + } + result.append("").append(filename) + .append("

NOTE: + Remember that for performance unit test tables, there are never any \"missing files\" listed, if there are any. + This is expected to be a temporary solution, until an exact fix can be implemented. For more details, see + bug 451890.

+
").append(EOL).append("").append(""); + for (String testLogName : missingFiles) { + replaceString.append(EOL).append(""); + } + replaceString.append(EOL).append("
Missing ") + .append(ordinalWord).append("
").append(testLogName).append("
"); + } + return replaceString.toString(); + } + + private void writeDropIndexFile() { + String dropHtmlFile = dropHtmlFileName; + if (dropHtmlFile == null || dropHtmlFile.isBlank()) { + log(EOL + "INFO: Skip generating the drop index file."); + return; + } + List dropTokens = List.of(dropTokenList.split(",")); + final String outputFileName = dropDirectoryName + File.separator + dropHtmlFile; + File outputIndexFile = new File(outputFileName); + // we assume if "eclipse" has been done, then "equinox" has been as + // well. + if (outputIndexFile.exists() && !regenerate) { + log(EOL + "INFO: The drop index file, " + dropHtmlFile + + ", was found to exist already and not regenerated."); + } else { + String dropTemplateString = readFile(dropTemplateFileName); + if (outputIndexFile.exists()) { + log(EOL + "INFO: The drop index file, " + dropHtmlFile + + ", was found to exist already and is being regenerated."); + } + log("DEBUG: Begin: Generating drop index page"); + final String[] types = anErrorTracker.getTypes(); + for (int i = 0; i < types.length; i++) { + final PlatformStatus[] platforms = anErrorTracker.getPlatforms(types[i]); + final String replaceString = processDropRows(platforms); + dropTemplateString = replace(dropTemplateString, dropTokens.get(i).toString(), replaceString); + } + writeFile(outputIndexFile, dropTemplateString); + log("DEBUG: End: Generating drop index page"); + } + } + + private void writeFile(File outputFile, final String contents) { + try { + Files.writeString(outputFile.toPath(), contents); + } catch (final FileNotFoundException e) { + log(EOL + "ERROR: File not found exception while writing: " + outputFile.getPath()); + } catch (final IOException e) { + log(EOL + "ERROR: IOException writing: " + outputFile.getPath()); + } + } + + /* + * This method writes the computed HTML to the file specified by caller in + * testResultsHtmlFileName. There must be an appropriate file on Download site + * that "includes" the file. + */ + private void writeTestResultsFile(String contents) { + final String outputFileName = dropDirectoryName + File.separator + testResultsHtmlFileName; + File outputFile = new File(outputFileName); + writeFile(outputFile, contents); + + } + + private List getTestsConfig() throws IOException { + if (expectedConfigs == null) { + expectedConfigs = new ArrayList<>(); + String expectedConfigParam = testsConfigExpected; + if (expectedConfigParam != null) { + StringTokenizer tokenizer = new StringTokenizer(expectedConfigParam, " ,\t"); + while (tokenizer.hasMoreTokens()) { + expectedConfigs.add(tokenizer.nextToken()); + } + } else { + throw new IllegalStateException("test configurations were not found. One or more must be set."); + } + if (DEBUG) { + // log("DEBUG: testsConfig array "); + for (String expected : expectedConfigs) { + log("\tDEBUG: expectedTestConfig: " + expected); + } + } + // write expected test config file here. This file is later used by + // the PHP file so the name passed in must match what was put in PHP + // file. + writePhpConfigFile(expected_config_type, expectedConfigs, expectedConfigFilename); + } + return expectedConfigs; + } + + /* + * This is the reverse of checking that all expected logs were found. If logs + * were found that are NOT in the test manifest, we write the list below missing + * files, so that they can be added to testManifest.xml. This allows them to be + * detected as missing, in future. We only do this check if "doMissingList" is + * true. + */ + private boolean verifyLogInManifest(String filename, List foundConfigs) { + boolean result = false; + if (doMissingList) { + for (String testLogName : anErrorTracker.getTestLogs(foundConfigs)) { + if (filename.equals(testLogName)) { + result = true; + break; + } + } + } + return result; + } + + private String listMissingManifestFiles() throws IOException { + StringBuilder results = new StringBuilder(); + if (doMissingList) { + StringBuilder xmlFragment = new StringBuilder(" ").append(EOL) + .append("").append(EOL); + + if (doMissingList && missingManifestFiles.size() > 0) { + String ordinalWord = "File"; + if (missingManifestFiles.size() > 1) { + ordinalWord = "Files"; + } + + results.append(EOL).append("").append( + ""); + for (String testLogName : missingManifestFiles) { + results.append(EOL).append(""); + xmlFragment.append("").append(EOL); + } + results.append(EOL).append("
Releng: Missing testManifest.xml ") + .append(ordinalWord).append("
").append(testLogName).append("
"); + xmlFragment.append("
"); + try (FileWriter xmlOutput = new FileWriter(dropDirectoryName + "/addToTestManifest.xml")) { + xmlOutput.write(xmlFragment.toString()); + } + } + } + return results.toString(); + } + + // Specific to the RelEng test results page + private String formatJUnitRow(String corename, ResultsTable resultsTable, List foundConfigs) + throws IOException { + + StringBuilder results = new StringBuilder(); + int orgEclipseLength = "org.eclipse.".length(); + // indexOf('_') assumes never part of file name? + final String displayName = corename.substring(orgEclipseLength); + + results.append(EOL).append("").append(displayName).append(""); + + for (String config : getTestsConfig()) { + TestResultsGenerator.ResultsTable.Cell cell = resultsTable.getCell(corename, config); + if (cell == null && foundConfigs.contains(config)) { + cell = new TestResultsGenerator.ResultsTable.Cell(-1, null); + } + results.append(printCell(cell)); + } + results.append("").append(EOL); + return results.toString(); + } + + private String printCell(TestResultsGenerator.ResultsTable.Cell cell) { + String result = null; + String displayName = null; + if (cell == null) { + displayName = " "; + result = displayName; + } else { + int cellErrorCount = cell.errorCount(); + File cellResultsFile = cell.resultsFile(); + String filename = null; + int beginFilename = 0; + String rawfilename = null; + if (cellResultsFile != null) { + filename = cellResultsFile.getName(); + beginFilename = filename.lastIndexOf(File.separatorChar); + rawfilename = filename.substring(beginFilename + 1, filename.length() - XML_EXTENSION.length()); + } + String startCell = null; + if (cellErrorCount == -999) { + displayName = " "; + result = displayName; + } else if (cellErrorCount == 0) { + startCell = ""; + displayName = "(0)"; + result = addLinks(startCell, displayName, rawfilename); + } else if (cellErrorCount < 0) { + startCell = ""; + displayName = "(" + Integer.toString(cellErrorCount) + ") DNF "; + result = startCell + displayName + ""; + } else if (cellErrorCount > 0) { + startCell = ""; + displayName = "(" + Integer.toString(cellErrorCount) + ")"; + result = addLinks(startCell, displayName, rawfilename); + } else { + // should never occur + displayName = "?" + Integer.toString(cellErrorCount) + "?"; + result = displayName; + } + } + return result; + + } + + private String addLinks(String startCell, String displayName, String rawfilename) { + return startCell + "" + displayName + "" + + " (XML)"; + } + + // --- utility methods --- + + private static void log(String msg) { + System.out.println(msg); + } + + private static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY_IGNORING_DOCTYPE = createDocumentBuilderFactoryIgnoringDOCTYPE(); + private static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY_ERROR_ON_DOCTYPE = createDocumentBuilderFactoryWithErrorOnDOCTYPE(); + + private static synchronized DocumentBuilderFactory createDocumentBuilderFactoryIgnoringDOCTYPE() { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + try { + // completely disable external entities declarations: + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); //$NON-NLS-1$ + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); //$NON-NLS-1$ + } catch (ParserConfigurationException e) { + throw new RuntimeException(e.getMessage(), e); + } + return factory; + } + + private static synchronized DocumentBuilderFactory createDocumentBuilderFactoryWithErrorOnDOCTYPE() { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + // completely disable DOCTYPE declaration: + try { + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); //$NON-NLS-1$ + } catch (ParserConfigurationException e) { + throw new RuntimeException(e.getMessage(), e); + } + return factory; + } + + static synchronized DocumentBuilder createDocumentBuilderIgnoringDOCTYPE() throws ParserConfigurationException { + DocumentBuilder builder = DOCUMENT_BUILDER_FACTORY_IGNORING_DOCTYPE.newDocumentBuilder(); + builder.setEntityResolver((__, ___) -> new InputSource(new ByteArrayInputStream(new byte[0]))); + return builder; + } + + static synchronized DocumentBuilder createDocumentBuilderWithErrorOnDOCTYPE() throws ParserConfigurationException { + return DOCUMENT_BUILDER_FACTORY_ERROR_ON_DOCTYPE.newDocumentBuilder(); + } + +} diff --git a/scripts/releng/log/converter/DOMHtmlConverter.java b/scripts/releng/log/converter/DOMHtmlConverter.java new file mode 100644 index 00000000000..5e825619b47 --- /dev/null +++ b/scripts/releng/log/converter/DOMHtmlConverter.java @@ -0,0 +1,154 @@ + +/******************************************************************************* + * Copyright (c) 2006, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + * Hannes Wellmann - Convert to plain Java scripts + *******************************************************************************/ + +package log.converter; + +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.ChoiceFormat; +import java.text.MessageFormat; +import java.util.List; +import java.util.ResourceBundle; +import java.util.Set; +import java.util.function.Function; +import java.util.function.ToIntFunction; + +import log.converter.LogDocumentNode.ProblemSummaryNode; + +public class DOMHtmlConverter { + + public static final String FORBIDDEN_REFERENCE = "ForbiddenReference"; //$NON-NLS-1$ + public static final String DISCOURAGED_REFERENCE = "DiscouragedReference"; //$NON-NLS-1$ + public static final Set FILTERED_WARNINGS_IDS = Set.of(FORBIDDEN_REFERENCE, DISCOURAGED_REFERENCE); + + private final ResourceBundle messages = ResourceBundle.getBundle("log.converter.html_messages"); //$NON-NLS-1$ + + private String convertToHTML(final String s) { + final StringBuilder buffer = new StringBuilder(); + for (int i = 0, max = s.length(); i < max; i++) { + final char c = s.charAt(i); + switch (c) { + case '<' -> buffer.append("<"); //$NON-NLS-1$ + case '>' -> buffer.append(">"); //$NON-NLS-1$ + case '\"' -> buffer.append("""); //$NON-NLS-1$ + case '&' -> buffer.append("&"); //$NON-NLS-1$ + case '^' -> buffer.append("∧"); //$NON-NLS-1$ + default -> buffer.append(c); + } + } + return buffer.toString(); + } + + private int globalErrorNumber; + + public void dump(Path inputFilename, Path outputFileName, LogDocumentNode documentNode) { + final ProblemSummaryNode summaryNode = documentNode.getSummaryNode(); + if ((summaryNode == null) || (summaryNode.numberOfProblems() == 0)) { + return; + } + try (Writer writer = Files.newBufferedWriter(outputFileName)) { + String pluginName = outputFileName.getParent().getFileName().toString(); + if (pluginName == null) { + writer.write(messages.getString("header")); //$NON-NLS-1$ + } else { + String pattern = messages.getString("dom_header"); //$NON-NLS-1$ + writer.write(MessageFormat.format(pattern, pluginName, inputFilename.getFileName().toString())); + } + writer.write(messages.getString("problem.summary.title_anchor"));//$NON-NLS-1$ + writer.write(MessageFormat.format(messages.getString("problem.summary"), // + summaryNode.numberOfProblems(), summaryNode.numberOfErrors(), summaryNode.numberOfWarnings(), + summaryNode.numberOfInfos())); + + writer.write(messages.getString("anchors.references.no_top"));//$NON-NLS-1$ + List problemsNodes = documentNode.getProblems(); + globalErrorNumber = 1; + // dump errors + writeIssueSection(writer, pluginName, problemsNodes, "error", "errors", ProblemsNode::getErrors, + n -> n.numberOfErrors); + // dump other warnings + writeIssueSection(writer, pluginName, problemsNodes, "warning", "other_warnings", + ProblemsNode::getOtherWarnings, n -> n.numberOfWarnings); + // dump infos + writeIssueSection(writer, pluginName, problemsNodes, "info", "infos", ProblemsNode::getInfos, + n -> n.numberOfInfos); + // dump forbidden accesses warnings + writeIssueSection(writer, pluginName, problemsNodes, "warning", "forbidden_warnings", + ProblemsNode::getForbiddenWarnings, n -> n.numberOfWarnings); + // dump discouraged accesses warnings + writeIssueSection(writer, pluginName, problemsNodes, "warning", "discouraged_warnings", + ProblemsNode::getDiscouragedWarnings, n -> n.numberOfWarnings); + + writer.write(messages.getString("footer")); //$NON-NLS-1$ + writer.flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private void writeIssueSection(Writer writer, String pluginName, List problemsNodes, String type, + String titleType, Function> getNodes, + ToIntFunction getNumberOfIssues) throws IOException { + writer.write(messages.getString(titleType + ".title_anchor"));//$NON-NLS-1$ + writer.write(messages.getString("anchors.references.no_" + titleType));//$NON-NLS-1$ + for (ProblemsNode problemsNode : problemsNodes) { + List problemNodes = getNodes.apply(problemsNode); + int numberOfIssues = getNumberOfIssues.applyAsInt(problemsNode); + if (problemNodes.isEmpty()) { + continue; + } + MessageFormat form = new MessageFormat(messages.getString(titleType + ".header")); + double[] warningsLimits = { 1, 2 }; + String[] warningParts = { messages.getString("one_" + type), //$NON-NLS-1$ + messages.getString("multiple_" + type + "s") //$NON-NLS-1$ + }; + ChoiceFormat warningForm = new ChoiceFormat(warningsLimits, warningParts); + String sourceFileName = extractRelativePath(problemsNode.sourceFileName, pluginName); + form.setFormatByArgumentIndex(1, warningForm); + Object[] arguments = new Object[] { sourceFileName, numberOfIssues }; + writer.write(form.format(arguments)); + for (int j = 0; j < problemNodes.size(); j++) { + ProblemNode problemNode = problemNodes.get(j); + String pattern = messages.getString(type + "s.entry." + ((j & 1) != 0 ? "odd" : "even")); + problemNode.setSources(); + writer.write(MessageFormat.format(pattern, sourceFileName, globalErrorNumber, j + 1, problemNode.id, + problemNode.line, convertToHTML(problemNode.message), + convertToHTML(problemNode.sourceCodeBefore), convertToHTML(problemNode.sourceCode), + convertToHTML(problemNode.sourceCodeAfter), "", problemNode.charStart, problemNode.charEnd)); + globalErrorNumber++; + } + writer.write(messages.getString(titleType + ".footer")); //$NON-NLS-1$ + } + } + + private String extractRelativePath(final String sourceFileName, final String pluginName) { + if (pluginName == null) { + return sourceFileName; + } + final int index = pluginName.indexOf('_'); + if (index == -1) { + return sourceFileName; + } + final String pluginShortName = pluginName.substring(0, index); + final int index2 = sourceFileName.indexOf(pluginShortName); + if (index2 == -1) { + return sourceFileName; + } + return sourceFileName.substring(index2 + pluginShortName.length(), sourceFileName.length()); + } + +} diff --git a/scripts/releng/log/converter/LogDocumentNode.java b/scripts/releng/log/converter/LogDocumentNode.java new file mode 100644 index 00000000000..3a810f27575 --- /dev/null +++ b/scripts/releng/log/converter/LogDocumentNode.java @@ -0,0 +1,52 @@ +/******************************************************************************* + * Copyright (c) 2006, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ + +package log.converter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class LogDocumentNode { + + public record ProblemSummaryNode(int numberOfProblems, int numberOfErrors, int numberOfWarnings, + int numberOfInfos) { + @Override + public String toString() { + return "problems : " + numberOfProblems // + + " errors : " + numberOfErrors // + + " warnings : " + numberOfWarnings // + + " infos : " + numberOfInfos; + } + } + + private final List problems = new ArrayList<>(); + private ProblemSummaryNode summaryNode; + + public void addProblemsNode(final ProblemsNode node) { + problems.add(node); + } + + public List getProblems() { + return Collections.unmodifiableList(problems); + } + + public ProblemSummaryNode getSummaryNode() { + return summaryNode; + } + + public void setProblemSummary(final ProblemSummaryNode node) { + summaryNode = node; + } +} diff --git a/scripts/releng/log/converter/ProblemNode.java b/scripts/releng/log/converter/ProblemNode.java new file mode 100644 index 00000000000..694ecd7eec9 --- /dev/null +++ b/scripts/releng/log/converter/ProblemNode.java @@ -0,0 +1,76 @@ +/******************************************************************************* + * Copyright (c) 2006, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ + +package log.converter; + +public class ProblemNode { + + public enum SeverityType { + ERROR, WARNING, INFO; + } + + public SeverityType severityType; + public int charStart; + public int charEnd; + public int line; + public String id; + public String message; + public int sourceStart; + public int sourceEnd; + public String contextValue; + public int globalProblemNumber; + public int problemNumber; + public String sourceFileName; + + public String sourceCodeBefore; + public String sourceCodeAfter; + public String sourceCode; + + public void setSources() { + if ((sourceStart == -1) || (sourceEnd == -1)) { + sourceCodeBefore = ""; + sourceCode = contextValue; + sourceCodeAfter = ""; + } else { + final int length = contextValue.length(); + if (sourceStart < length) { + sourceCodeBefore = contextValue.substring(0, sourceStart); + final int end = sourceEnd + 1; + if (end < length) { + sourceCode = contextValue.substring(sourceStart, end); + sourceCodeAfter = contextValue.substring(end, length); + } else { + sourceCode = contextValue.substring(sourceStart, length); + sourceCodeAfter = ""; + } + } else { + sourceCodeBefore = ""; + sourceCode = ""; + sourceCodeAfter = ""; + } + } + } + + @Override + public String toString() { + final StringBuilder buffer = new StringBuilder(); + switch (severityType) { + case ERROR -> buffer.append("ERROR ");//$NON-NLS-1$ + case WARNING -> buffer.append("WARNING ");//$NON-NLS-1$ + case INFO -> buffer.append("INFO ");//$NON-NLS-1$ + } + buffer.append("line : ").append(line).append(" message = ").append(message);//$NON-NLS-1$//$NON-NLS-2$ + return buffer.toString(); + } +} diff --git a/scripts/releng/log/converter/ProblemsNode.java b/scripts/releng/log/converter/ProblemsNode.java new file mode 100644 index 00000000000..92ef2765b87 --- /dev/null +++ b/scripts/releng/log/converter/ProblemsNode.java @@ -0,0 +1,74 @@ +/******************************************************************************* + * Copyright (c) 2006, 2025 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ + +package log.converter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ProblemsNode { + + public String sourceFileName; + public int numberOfProblems; + public int numberOfErrors; + public int numberOfWarnings; + public int numberOfInfos; + + private final List errorNodes = new ArrayList<>(); + private final List otherWarningNodes = new ArrayList<>(); + private final List discouragedWarningsNodes = new ArrayList<>(); + private final List forbiddenWarningsNodes = new ArrayList<>(); + private final List infoNodes = new ArrayList<>(); + + public void addDiscouragedWarning(final ProblemNode node) { + discouragedWarningsNodes.add(node); + } + + public void addError(final ProblemNode node) { + errorNodes.add(node); + } + + public void addForbiddenWarning(final ProblemNode node) { + forbiddenWarningsNodes.add(node); + } + + public void addOtherWarning(final ProblemNode node) { + otherWarningNodes.add(node); + } + + public void addInfo(final ProblemNode node) { + infoNodes.add(node); + } + + public List getDiscouragedWarnings() { + return Collections.unmodifiableList(discouragedWarningsNodes); + } + + public List getErrors() { + return Collections.unmodifiableList(errorNodes); + } + + public List getForbiddenWarnings() { + return Collections.unmodifiableList(forbiddenWarningsNodes); + } + + public List getOtherWarnings() { + return Collections.unmodifiableList(otherWarningNodes); + } + + public List getInfos() { + return Collections.unmodifiableList(infoNodes); + } +} diff --git a/scripts/releng/log/converter/html_messages.properties b/scripts/releng/log/converter/html_messages.properties new file mode 100644 index 00000000000..cc9cd77c342 --- /dev/null +++ b/scripts/releng/log/converter/html_messages.properties @@ -0,0 +1,271 @@ +############################################################################### +# Copyright (c) 2000, 2017 IBM Corporation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# Contributors: +# IBM Corporation - initial API and implementation +############################################################################### +# {0} plugin name +# {1} xml log file name +dom_header=\n\ +\n\ +\n\ +\ \ \ \n\ +\ \ \ Compiler log for {0}\n\ +\n\ +\n\ +

Compiler log for {0} : {1}

\n + +header=<\!doctype html public "-//w3c//dtd html 4.0 transitional//en">\n\ +\n\ +\n\ +\ \ \ \n\ +\ \ \ Compiler log\n\ +\n\ +\n + +footer=\n\n + +# {0] source file name +# {1} number of problems +# {2} number of errors +# {3} number of warnings +# {4} number of infos +problems.footer= + +errors.title_anchor=

ERRORS

\n + +# {0] source file name +# {1} number of errors +errors.header=

{0} : {1} :

\n\n +errors.footer=
\n + +other_warnings.title_anchor=

OTHER WARNINGS

\n + +# {0] source file name +# {1} number of warnings +other_warnings.header=

{0} : {1} :

\n\n\n +other_warnings.footer=
OTHER WARNINGS
\n + +forbidden_warnings.title_anchor=

FORBIDDEN ACCESS WARNINGS

\n +# {0] source file name +# {1} number of warnings +forbidden_warnings.header=

{0} : {1} :

\n\n\n +forbidden_warnings.footer=
FORBIDDEN ACCESS WARNINGS
\n + +discouraged_warnings.title_anchor=

DISCOURAGED ACCESS WARNINGS

\n +# {0] source file name +# {1} number of warnings +discouraged_warnings.header=

{0} : {1} :

\n\n\n +discouraged_warnings.footer=
DISCOURAGED ACCESS WARNINGS
\n + +infos.title_anchor=

INFO WARNINGS

\n +# {0] source file name +# {1} number of warnings +infos.header=

{0} : {1} :

\n\n\n +infos.footer=
INFO WARNINGS
\n + +# {0} source file name +# {1} global error number +# {2} error number +# {3} error id +# {4} line number of the error +# {5} error message +# {6} source code before this error +# {7} source code corresponding to this error +# {8} source code after this error +# {9} complement to underline the problem +# {10} starting position of the error +# {11} ending position of the error +errors.entry.even=\n\ +\ {2}. ERROR in {0}
\n\ +\  (at line {4})
\n\ +\ {6}{7}{8}
\n\ +\ {5}\n\ +\ \n\ +\\n + +# {0} source file name +# {1} global error number +# {2} error number +# {3} error id +# {4} line number of the error +# {5} error message +# {6} source code before this error +# {7} source code corresponding to this error +# {8} source code after this error +# {9} complement to underline the problem +# {10} starting position of the error +# {11} ending position of the error +errors.entry.odd=\n\ +\ {2}. ERROR in {0}
\n\ +\  (at line {4})
\n\ +\ {6}{7}{8}
\n\ +\ {5}\n\ +\ \n\ +\\n + +# {0} source file name +# {1} global error number +# {2} error number +# {3} error id +# {4} line number of the error +# {5} error message +# {6} source code before this error +# {7} source code corresponding to this error +# {8} source code after this error +# {9} complement to underline the problem +# {10} starting position of the error +# {11} ending position of the error +warnings.entry.even=\n\ +\ {2}. WARNING in {0}
\n\ +\  (at line {4})
\n\ +\ {6}{7}{8}
\n\ +\ {5}\n\ +\ \n\ +\\n + +# {0} source file name +# {1} global error number +# {2} error number +# {3} error id +# {4} line number of the error +# {5} error message +# {6} source code before this error +# {7} source code corresponding to this error +# {8} source code after this error +# {9} complement to underline the problem +# {10} starting position of the error +# {11} ending position of the error +warnings.entry.odd=\n\ +\ {2}. WARNING in {0}
\n\ +\  (at line {4})
\n\ +\ {6}{7}{8}
\n\ +\ {5}\n\ +\ \n\ +\\n + +# {0} source file name +# {1} global error number +# {2} error number +# {3} error id +# {4} line number of the error +# {5} error message +# {6} source code before this error +# {7} source code corresponding to this error +# {8} source code after this error +# {9} complement to underline the problem +# {10} starting position of the error +# {11} ending position of the error +infos.entry.even=\n\ +\ {2}. INFO in {0}
\n\ +\  (at line {4})
\n\ +\ {6}{7}{8}
\n\ +\ {5}\n\ +\ \n\ +\\n + +# {0} source file name +# {1} global error number +# {2} error number +# {3} error id +# {4} line number of the error +# {5} error message +# {6} source code before this error +# {7} source code corresponding to this error +# {8} source code after this error +# {9} complement to underline the problem +# {10} starting position of the error +# {11} ending position of the error +infos.entry.odd=\n\ +\ {2}. INFO in {0}
\n\ +\  (at line {4})
\n\ +\ {6}{7}{8}
\n\ +\ {5}\n\ +\ \n\ +\\n + +# {0} global number of problems +# {1} global number of errors +# {2} global number of warnings +# {3} global number of infos +problem.summary=

TOTAL : ERRORS: {1}, WARNINGS: {2}, INFOS: {3}

\n + +problem.summary.title_anchor=\n + +anchors.references.no_top=\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +
errorsothers warningsinfosforbidden warningsdiscouraged warnings
\n + +anchors.references.no_errors=\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +
topothers warningsinfosforbidden warningsdiscouraged warnings
\n + +anchors.references.no_other_warnings=\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +
toperrorsinfosforbidden warningsdiscouraged warnings
\n + +anchors.references.no_infos=\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +
errorsothers warningsforbidden warningsdiscouraged warnings
\n + +anchors.references.no_forbidden_warnings=\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +
toperrorsinfosothers warningsdiscouraged warnings
\n + +anchors.references.no_discouraged_warnings=\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +
toperrorsinfosothers warningsforbidden warnings
\n + +one_warning=1 warning +multiple_warnings={1,number} warnings + +one_error=1 error +multiple_errors={1,number} errors + +one_info=1 info +multiple_infos={1,number} infos \ No newline at end of file