diff --git a/zorg/jenkins/jobs/jobs/build-bisect b/zorg/jenkins/jobs/jobs/build-bisect new file mode 100644 index 000000000..5bfe79e90 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/build-bisect @@ -0,0 +1,70 @@ +#!/usr/bin/env groovy +@Library('llvm-jenkins-lib') _ + +// Bisection orchestrator that manages the binary search process +// Uses build-bisect-run jobs to do the actual testing work +def bisectionUtils = evaluate readTrusted('zorg/jenkins/lib/utils/BisectionUtils.groovy') + +pipeline { + options { + disableConcurrentBuilds() + } + + parameters { + string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on') + string(name: 'BISECT_GOOD', defaultValue: params.BISECT_GOOD ?: '', description: 'Known good commit') + string(name: 'BISECT_BAD', defaultValue: params.BISECT_BAD ?: '', description: 'Known bad commit') + + // Job template configuration to pass through to build-bisect-run + string(name: 'JOB_TEMPLATE', defaultValue: params.JOB_TEMPLATE ?: 'clang-stage2-Rthinlto', description: 'Job template to bisect') + string(name: 'BUILD_CONFIG', defaultValue: params.BUILD_CONFIG ?: '{}', description: 'Build configuration JSON') + string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Base artifact to use') + } + + agent { + node { + label params.LABEL + } + } + + stages { + stage('Setup') { + steps { + script { + if (!params.BISECT_GOOD || !params.BISECT_BAD || !params.JOB_TEMPLATE) { + error "BISECT_GOOD, BISECT_BAD, and JOB_TEMPLATE parameters are required" + } + + echo "Starting bisection of ${params.JOB_TEMPLATE}: ${params.BISECT_GOOD}...${params.BISECT_BAD}" + echo "Build Config: ${params.BUILD_CONFIG}" + + // Set build description + currentBuild.description = "🔍 BISECTING [${params.JOB_TEMPLATE}]: ${params.BISECT_GOOD.take(8)}..${params.BISECT_BAD.take(8)}" + + // Clone the repository to get commit information + def clangBuilder = evaluate readTrusted('zorg/jenkins/lib/builders/ClangBuilder.groovy') + clangBuilder.checkoutStage() + } + } + } + + stage('Perform Bisection') { + steps { + script { + def failingCommit = bisectionUtils.performBisectionWithRunner( + params.BISECT_GOOD, + params.BISECT_BAD, + params.JOB_TEMPLATE, + params.BUILD_CONFIG, + params.ARTIFACT + ) + + bisectionUtils.reportBisectionResult(failingCommit, params.JOB_TEMPLATE) + + // Update build description with result + currentBuild.description = "✅ BISECTED [${params.JOB_TEMPLATE}]: failing commit ${failingCommit.take(8)}" + } + } + } + } +} \ No newline at end of file diff --git a/zorg/jenkins/jobs/jobs/build-bisect-run b/zorg/jenkins/jobs/jobs/build-bisect-run new file mode 100644 index 000000000..70d0caccc --- /dev/null +++ b/zorg/jenkins/jobs/jobs/build-bisect-run @@ -0,0 +1,106 @@ +#!/usr/bin/env groovy + +// Generic build runner that can execute any job template configuration +// This job handles the actual building/testing work for bisection +def clangBuilder = evaluate readTrusted('zorg/jenkins/lib/builders/ClangBuilder.groovy') + +pipeline { + options { + disableConcurrentBuilds() + } + + parameters { + string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on') + string(name: 'GIT_SHA', defaultValue: params.GIT_SHA ?: '*/main', description: 'Git commit to build.') + string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Clang artifact to use') + + // Job template configuration + string(name: 'JOB_TEMPLATE', defaultValue: params.JOB_TEMPLATE ?: 'clang-stage2-Rthinlto', description: 'Job template to use') + string(name: 'BUILD_CONFIG', defaultValue: params.BUILD_CONFIG ?: '{}', description: 'Build configuration JSON') + + // Bisection context (for build description) + string(name: 'BISECT_GOOD', defaultValue: params.BISECT_GOOD ?: '', description: 'Good commit for bisection context') + string(name: 'BISECT_BAD', defaultValue: params.BISECT_BAD ?: '', description: 'Bad commit for bisection context') + } + + agent { + node { + label params.LABEL + } + } + + stages { + stage('Setup Build Description') { + steps { + script { + def commitInfo = params.GIT_SHA.take(8) + def template = params.JOB_TEMPLATE + + if (params.BISECT_GOOD && params.BISECT_BAD) { + def goodShort = params.BISECT_GOOD.take(8) + def badShort = params.BISECT_BAD.take(8) + currentBuild.description = "🔍 BISECT RUN [${template}]: ${commitInfo} (${goodShort}..${badShort})" + } else { + currentBuild.description = "🔧 BUILD RUN [${template}]: ${commitInfo}" + } + + echo "Job Template: ${template}" + echo "Testing commit: ${commitInfo}" + echo "Build Config: ${params.BUILD_CONFIG}" + } + } + } + + stage('Checkout') { + steps { + script { + clangBuilder.checkoutStage() + } + } + } + + stage('Setup Venv') { + steps { + script { + clangBuilder.setupVenvStage() + } + } + } + + stage('Fetch Artifact') { + steps { + script { + clangBuilder.fetchArtifactStage() + } + } + } + + stage('Build') { + steps { + script { + // Load the job template dynamically + def template = evaluate readTrusted("zorg/jenkins/lib/templates/${params.JOB_TEMPLATE}.groovy") + + // Parse user build configuration + def userBuildConfig = [:] + if (params.BUILD_CONFIG && params.BUILD_CONFIG != '{}') { + userBuildConfig = readJSON text: params.BUILD_CONFIG + } + + // Apply template defaults with user overrides + def buildConfig = template.getDefaultBuildConfig(userBuildConfig) + + clangBuilder.buildStage(buildConfig) + } + } + } + } + + post { + always { + script { + clangBuilder.cleanupStage() + } + } + } +} \ No newline at end of file diff --git a/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto-v2 b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto-v2 new file mode 120000 index 000000000..220028ad2 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto-v2 @@ -0,0 +1 @@ +templated-clang-job.groovy \ No newline at end of file diff --git a/zorg/jenkins/jobs/jobs/templated-clang-job.groovy b/zorg/jenkins/jobs/jobs/templated-clang-job.groovy new file mode 100644 index 000000000..d628126f9 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/templated-clang-job.groovy @@ -0,0 +1,49 @@ +#!/usr/bin/env groovy + +/* + * GENERIC TEMPLATED CLANG JOB + * + * This is a generic job script that automatically configures itself based on the Jenkins job name. + * It works by using symlinks - each job is just a symlink to this file with the appropriate name. + * + * NAMING CONVENTION: + * - Job names follow the pattern: clang-stage[N]-[CONFIG] + * - Template name is derived by stripping any version suffix (e.g., -v2) + * - Each template defines its own bisection policy in getJobConfig() + * + * EXAMPLES: + * clang-stage2-Rthinlto-v2 → template: clang-stage2-Rthinlto, bisection: per template + * clang-stage1-RA → template: clang-stage1-RA, bisection: per template + * clang-stage2-cmake-RgSan → template: clang-stage2-cmake-RgSan, bisection: per template + * + * TEMPLATE RESOLUTION: + * - Templates are loaded from zorg/jenkins/lib/templates/[JOB_TEMPLATE].groovy + * - Template defines build configuration (stage, cmake_type, projects, etc.) + * - Template defines job configuration (bisection policy, etc.) via getJobConfig() + * + * TO ADD A NEW JOB: + * 1. Create the template file: zorg/jenkins/lib/templates/your-job-pattern.groovy + * 2. Create symlink: ln -s templated-clang-job.groovy your-job-name + * 3. Done! The job will automatically use the correct template and settings. + * + * BISECTION: + * - Each template decides its own bisection policy in getJobConfig() + * - ThinLTO jobs enable bisection (useful for performance regressions) + * - Stage1 jobs disable bisection (failures often environmental) + * - Future templates can define custom bisection logic + */ + +def clangBuilder = evaluate readTrusted('zorg/jenkins/lib/builders/ClangBuilder.groovy') + +// Auto-configure based on Jenkins job name +def jobName = env.JOB_NAME ?: 'unknown' + +// Derive template name by stripping -v2 suffix if present +def templateName = jobName.replaceAll(/-v\d+$/, '') + +// Load the template and get its job configuration +def template = evaluate readTrusted("zorg/jenkins/lib/templates/${templateName}.groovy") +def jobConfig = template.getJobConfig(jobName) + +// Instantiate the templated pipeline +clangBuilder.createTemplatedPipeline(jobConfig).call() \ No newline at end of file diff --git a/zorg/jenkins/lib/builders/ClangBuilder.groovy b/zorg/jenkins/lib/builders/ClangBuilder.groovy new file mode 100644 index 000000000..7c81165ca --- /dev/null +++ b/zorg/jenkins/lib/builders/ClangBuilder.groovy @@ -0,0 +1,546 @@ +#!/usr/bin/env groovy + +class ClangBuilder { + + static def pipeline(config) { + def buildConfig = config.config ?: [:] + def stagesToRun = config.stages ?: ['checkout', 'build', 'test'] + def postFailureConfig = config.post_failure ?: [:] + + pipeline { + options { + disableConcurrentBuilds() + } + + parameters { + string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on') + string(name: 'GIT_SHA', defaultValue: params.GIT_REVISION ?: '*/main', description: 'Git commit to build.') + string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Clang artifact to use') + string(name: 'BISECT_GOOD', defaultValue: params.BISECT_GOOD ?: '', description: 'Good commit for bisection') + string(name: 'BISECT_BAD', defaultValue: params.BISECT_BAD ?: '', description: 'Bad commit for bisection') + booleanParam(name: 'IS_BISECT_JOB', defaultValue: params.IS_BISECT_JOB ?: false, description: 'Whether this is a bisection job') + } + + agent { + node { + label params.LABEL + } + } + + stages { + stage('Checkout') { + when { + expression { 'checkout' in stagesToRun } + } + steps { + script { + ClangBuilder.checkoutStage() + } + } + } + + stage('Setup Venv') { + when { + expression { 'checkout' in stagesToRun } + } + steps { + script { + ClangBuilder.setupVenvStage() + } + } + } + + stage('Fetch Artifact') { + when { + expression { 'build' in stagesToRun } + } + steps { + script { + ClangBuilder.fetchArtifactStage() + } + } + } + + stage('Build') { + when { + expression { 'build' in stagesToRun } + } + steps { + script { + ClangBuilder.buildStage(buildConfig) + } + } + } + + stage('Test') { + when { + expression { 'test' in stagesToRun } + } + steps { + script { + ClangBuilder.testStage() + } + } + post { + always { + script { + junit "clang-build/**/testresults.xunit.xml" + } + } + } + } + } + + post { + always { + script { + sh "rm -rf clang-build clang-install host-compiler *.tar.gz" + } + } + failure { + script { + // Only trigger bisection for main jobs, not bisection jobs themselves + if (!params.IS_BISECT_JOB && shouldTriggerBisection(postFailureConfig)) { + triggerBisection(config.name, postFailureConfig) + } + } + } + } + } + } + + static def checkoutStage() { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], extensions: [ + [$class: 'CloneOption', timeout: 30] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + + static def setupVenvStage() { + withEnv(["PATH=$PATH:/usr/bin:/usr/local/bin"]) { + sh ''' + # Non-incremental, so always delete. + rm -rf clang-build clang-install host-compiler *.tar.gz + rm -rf venv + python3 -m venv venv + set +u + source ./venv/bin/activate + pip install -r ./llvm-zorg/zorg/jenkins/jobs/requirements.txt + set -u + ''' + } + } + + static def fetchArtifactStage() { + withEnv(["PATH=$PATH:/usr/bin:/usr/local/bin"]) { + withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) { + sh """ + source ./venv/bin/activate + echo "ARTIFACT=${params.ARTIFACT}" + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + ls $WORKSPACE/host-compiler/lib/clang/ + VERSION=`ls $WORKSPACE/host-compiler/lib/clang/` + """ + } + } + } + + static def buildStage(config = [:]) { + def thinlto = config.thinlto ?: false + def cmakeType = config.cmake_type ?: "RelWithDebInfo" + def projects = config.projects ?: "clang;clang-tools-extra;compiler-rt" + def runtimes = config.runtimes ?: "" + def sanitizer = config.sanitizer ?: "" + def assertions = config.assertions ?: false + def timeout = config.timeout ?: 120 + def buildTarget = config.build_target ?: "" + def noinstall = config.noinstall ?: false + def extraCmakeFlags = config.cmake_flags ?: [] + def stage1Mode = config.stage1 ?: false + def extraEnvVars = config.env_vars ?: [:] + def testCommand = config.test_command ?: "cmake" + def testTargets = config.test_targets ?: [] + + // Build environment variables map + def envVars = [ + "PATH": "\$PATH:/usr/bin:/usr/local/bin", + "MACOSX_DEPLOYMENT_TARGET": stage1Mode ? "13.6" : null + ] + + // Add custom environment variables + extraEnvVars.each { key, value -> + envVars[key] = value + } + + // Filter out null values + envVars = envVars.findAll { k, v -> v != null } + + def envList = envVars.collect { k, v -> "${k}=${v}" } + + withEnv(envList) { + timeout(timeout) { + withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) { + // Build the command dynamically + def buildCmd = buildMonorepoBuildCommand(config) + + sh """ + set -u + ${stage1Mode ? 'rm -rf build.properties' : ''} + source ./venv/bin/activate + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=\$(git describe --match "first_commit") + export GIT_DISTANCE=\$(echo \${git_desc} | cut -f 2 -d "-") + + sha=\$(echo \${git_desc} | cut -f 3 -d "-") + export GIT_SHA=\${sha:1} + + ${stage1Mode ? 'export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ")' : ''} + + cd - + + ${stage1Mode ? 'echo "GIT_DISTANCE=\$GIT_DISTANCE" > build.properties' : ''} + ${stage1Mode ? 'echo "GIT_SHA=\$GIT_SHA" >> build.properties' : ''} + ${stage1Mode ? 'echo "ARTIFACT=\$JOB_NAME/clang-d\$GIT_DISTANCE-g\$GIT_SHA-t\$BUILD_ID-b\$BUILD_NUMBER.tar.gz" >> build.properties' : ''} + + ${stage1Mode ? 'rm -rf clang-build clang-install *.tar.gz' : ''} + ${buildCmd} + """ + } + } + } + } + + static def buildMonorepoBuildCommand(config) { + def testCommand = config.test_command ?: "cmake" + def projects = config.projects ?: "clang;clang-tools-extra;compiler-rt" + def runtimes = config.runtimes ?: "" + def cmakeType = config.cmake_type ?: "RelWithDebInfo" + def assertions = config.assertions ?: false + def timeout = config.timeout ?: 120 + def buildTarget = config.build_target ?: "" + def noinstall = config.noinstall ?: false + def thinlto = config.thinlto ?: false + def sanitizer = config.sanitizer ?: "" + def extraCmakeFlags = config.cmake_flags ?: [] + + // Start building command + def cmd = "python llvm-zorg/zorg/jenkins/monorepo_build.py ${testCommand} build" + + // Add cmake type if not default + if (cmakeType != "default") { + cmd += " --cmake-type=${cmakeType}" + } + + // Add projects + cmd += " --projects=\"${projects}\"" + + // Add runtimes if specified + if (runtimes) { + cmd += " --runtimes=\"${runtimes}\"" + } + + // Add assertions flag + if (assertions) { + cmd += " --assertions" + } + + // Add timeout if different from default + if (timeout != 2400) { + cmd += " --timeout=${timeout}" + } + + // Add build target if specified + if (buildTarget) { + cmd += " --cmake-build-target=${buildTarget}" + } + + // Add noinstall flag + if (noinstall) { + cmd += " --noinstall" + } + + // Build cmake flags + def cmakeFlags = [] + cmakeFlags.add("-DPython3_EXECUTABLE=\$(which python)") + + if (thinlto) { + cmakeFlags.add("-DLLVM_ENABLE_LTO=Thin") + } + + if (sanitizer) { + cmakeFlags.add("-DLLVM_USE_SANITIZER=${sanitizer}") + } + + // Add DYLD_LIBRARY_PATH for TSan + if (sanitizer == "Thread") { + cmakeFlags.add("-DDYLD_LIBRARY_PATH=\$DYLD_LIBRARY_PATH") + } + + // Add extra cmake flags from config + cmakeFlags.addAll(extraCmakeFlags) + + // Add all cmake flags to command + cmakeFlags.each { flag -> + cmd += " --cmake-flag=\"${flag}\"" + } + + return cmd + } + + static def testStage(config = [:]) { + def testCommand = config.test_command ?: "cmake" + def testType = config.test_type ?: "testlong" // testlong vs test + def testTargets = config.test_targets ?: [] + def timeout = config.test_timeout ?: 420 + def extraEnvVars = config.env_vars ?: [:] + + // Build environment variables map + def envVars = [ + "PATH": "\$PATH:/usr/bin:/usr/local/bin" + ] + + // Add custom environment variables (like ASAN_SYMBOLIZER_PATH) + extraEnvVars.each { key, value -> + envVars[key] = value + } + + def envList = envVars.collect { k, v -> "${k}=${v}" } + + withEnv(envList) { + timeout(timeout) { + // Build test command dynamically + def cmd = "python llvm-zorg/zorg/jenkins/monorepo_build.py ${testCommand} ${testType}" + + // Add specific test targets if provided + testTargets.each { target -> + cmd += " --cmake-test-target=${target}" + } + + sh """ + set -u + source ./venv/bin/activate + + rm -rf clang-build/testresults.xunit.xml + + ${cmd} + """ + } + } + } + } + + static def cleanupStage() { + sh "rm -rf clang-build clang-install host-compiler *.tar.gz" + } + + static def createTemplatedPipeline(config) { + def jobName = config.name + def jobTemplate = config.job_template ?: 'clang-stage2-Rthinlto' + def enableBisectionTrigger = config.enable_bisection_trigger ?: false + def bisectJobName = config.bisect_job_name ?: 'build-bisect' + def descriptionPrefix = config.description_prefix ?: "" + + def clangBuilder = evaluate readTrusted('zorg/jenkins/lib/builders/ClangBuilder.groovy') + + return { + pipeline { + options { + disableConcurrentBuilds() + } + + parameters { + string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on') + string(name: 'GIT_SHA', defaultValue: params.GIT_REVISION ?: '*/main', description: 'Git commit to build.') + string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'llvm.org/clang-stage1-RA/latest', description: 'Clang artifact to use') + booleanParam(name: 'IS_BISECT_JOB', defaultValue: params.IS_BISECT_JOB ?: false, description: 'Whether this is a bisection job') + string(name: 'BISECT_GOOD', defaultValue: params.BISECT_GOOD ?: '', description: 'Good commit for bisection') + string(name: 'BISECT_BAD', defaultValue: params.BISECT_BAD ?: '', description: 'Bad commit for bisection') + } + + agent { + node { + label params.LABEL + } + } + + stages { + stage('Setup Build Description') { + steps { + script { + // Set build description based on context + def buildType = params.IS_BISECT_JOB ? "🔍 BISECTION TEST" : "🔧 NORMAL BUILD" + def commitInfo = params.GIT_SHA.take(8) + + if (params.IS_BISECT_JOB && params.BISECT_GOOD && params.BISECT_BAD) { + def goodShort = params.BISECT_GOOD.take(8) + def badShort = params.BISECT_BAD.take(8) + currentBuild.description = "${buildType}: Testing ${commitInfo} (${goodShort}..${badShort})" + } else { + currentBuild.description = "${buildType}: ${commitInfo}" + } + + if (descriptionPrefix) { + currentBuild.description = "${descriptionPrefix}: ${currentBuild.description}" + } + + echo "Build Type: ${buildType}" + echo "Job Template: ${jobTemplate}" + if (params.IS_BISECT_JOB) { + echo "This is a bisection test run - results will be used by build-bisect job" + } else { + echo "This is a normal CI build" + } + } + } + } + stage('Checkout') { + steps { + script { + clangBuilder.checkoutStage() + } + } + } + + stage('Setup Venv') { + steps { + script { + clangBuilder.setupVenvStage() + } + } + } + + stage('Fetch Artifact') { + when { + expression { + // Load template to check if this is a stage2+ build + def template = evaluate readTrusted("zorg/jenkins/lib/templates/${jobTemplate}.groovy") + def buildConfig = template.getDefaultBuildConfig() + def stage = buildConfig.stage ?: 2 // Default to stage2 if not specified + return stage >= 2 + } + } + steps { + script { + clangBuilder.fetchArtifactStage() + } + } + } + + stage('Build') { + steps { + script { + // Load the shared template + def template = evaluate readTrusted("zorg/jenkins/lib/templates/${jobTemplate}.groovy") + def buildConfig = template.getDefaultBuildConfig() + + clangBuilder.buildStage(buildConfig) + } + } + } + + stage('Test') { + steps { + script { + // Load the shared template + def template = evaluate readTrusted("zorg/jenkins/lib/templates/${jobTemplate}.groovy") + def testConfig = template.getDefaultTestConfig() + + clangBuilder.testStage(testConfig) + } + } + post { + always { + script { + junit "clang-build/**/testresults.xunit.xml" + } + } + } + } + } + + post { + always { + script { + clangBuilder.cleanupStage() + } + } + failure { + script { + // Only trigger bisection if enabled and this is not already a bisection job + if (enableBisectionTrigger && !params.IS_BISECT_JOB && shouldTriggerBisection()) { + triggerBisection(jobName, bisectJobName, jobTemplate) + } + } + } + } + } + } + } + + static def shouldTriggerBisection() { + // Check if this is a new failure by looking at previous build result + def previousBuild = currentBuild.previousBuild + if (previousBuild == null) { + return false // First build, can't bisect + } + + // Only bisect if previous build was successful (new failure) + return previousBuild.result == 'SUCCESS' + } + + static def triggerBisection(currentJobName, bisectJobName, jobTemplate) { + // Get the commit range for bisection + def currentCommit = env.GIT_COMMIT + def goodCommit = getPreviousGoodCommit() + + if (goodCommit) { + echo "Triggering bisection: ${goodCommit}...${currentCommit}" + + // Launch the bisection orchestrator with template configuration + build job: bisectJobName, + parameters: [ + string(name: 'BISECT_GOOD', value: goodCommit), + string(name: 'BISECT_BAD', value: currentCommit), + string(name: 'JOB_TEMPLATE', value: jobTemplate), + string(name: 'BUILD_CONFIG', value: '{}'), // Use template defaults + string(name: 'ARTIFACT', value: params.ARTIFACT) + ], + wait: false + } else { + echo "Could not determine good commit for bisection" + } + } + + static def getPreviousGoodCommit() { + // Walk back through builds to find the last successful one + def build = currentBuild.previousBuild + while (build != null) { + if (build.result == 'SUCCESS') { + // Extract commit from the successful build + def buildEnv = build.getBuildVariables() + return buildEnv.GIT_COMMIT + } + build = build.previousBuild + } + return null + } +} + +return this \ No newline at end of file diff --git a/zorg/jenkins/lib/templates/clang-stage2-Rthinlto.groovy b/zorg/jenkins/lib/templates/clang-stage2-Rthinlto.groovy new file mode 100644 index 000000000..3bed98c4c --- /dev/null +++ b/zorg/jenkins/lib/templates/clang-stage2-Rthinlto.groovy @@ -0,0 +1,41 @@ +#!/usr/bin/env groovy + +// Template configuration for clang ThinLTO jobs (Release + ThinLTO) +class ClangThinLTOTemplate { + + static def getDefaultBuildConfig(userConfig = [:]) { + def defaults = [ + thinlto: true, + test_command: "clang", + projects: "clang;compiler-rt", + cmake_type: "Release" // R = Release + ] + + // User config overrides defaults + return defaults + userConfig + } + + static def getDefaultTestConfig(userConfig = [:]) { + def defaults = [ + test_command: "clang" + ] + + return defaults + userConfig + } + + static def getJobDescription() { + return "Clang ThinLTO build configuration (Release + ThinLTO)" + } + + // Template-specific job configuration + static def getJobConfig(jobName) { + return [ + name: jobName, + job_template: 'clang-stage2-Rthinlto', + enable_bisection_trigger: true, // All templated ThinLTO jobs enable bisection + bisect_job_name: 'build-bisect' + ] + } +} + +return ClangThinLTOTemplate \ No newline at end of file diff --git a/zorg/jenkins/lib/utils/BisectionUtils.groovy b/zorg/jenkins/lib/utils/BisectionUtils.groovy new file mode 100644 index 000000000..c558703b8 --- /dev/null +++ b/zorg/jenkins/lib/utils/BisectionUtils.groovy @@ -0,0 +1,200 @@ +#!/usr/bin/env groovy + +class BisectionUtils { + + // Static list to track bisection steps for reproduction + static def bisectionSteps = [] + static def bisectionStartTime = 0 + static def stepDurations = [] + + static def performBisectionWithRunner(goodCommit, badCommit, jobTemplate, buildConfig, artifact) { + // Initialize bisection tracking + bisectionSteps.clear() + bisectionSteps.add("git bisect start --first-parent") + bisectionSteps.add("git bisect bad ${badCommit}") + bisectionSteps.add("git bisect good ${goodCommit}") + + bisectionStartTime = System.currentTimeMillis() + stepDurations.clear() + + // Calculate and log estimated steps + def commits = getCommitRange(goodCommit, badCommit) + def estimatedSteps = Math.ceil(Math.log(commits.size()) / Math.log(2)) + echo "Starting bisection: ${commits.size()} commits to test, estimated ${estimatedSteps} steps" + + return performBisectionRecursive(goodCommit, badCommit, jobTemplate, buildConfig, artifact) + } + + static def performBisectionRecursive(goodCommit, badCommit, jobTemplate, buildConfig, artifact) { + def commits = getCommitRange(goodCommit, badCommit) + + if (commits.size() <= 2) { + echo "Bisection complete: failing commit is ${badCommit}" + return badCommit + } + + def midpoint = commits[commits.size() / 2] + + // Calculate progress and ETA + def remainingCommits = commits.size() + def remainingSteps = Math.ceil(Math.log(remainingCommits) / Math.log(2)) + def avgStepDuration = stepDurations.size() > 0 ? stepDurations.sum() / stepDurations.size() : 0 + + def etaText = "unknown" + if (avgStepDuration > 0) { + def etaMillis = remainingSteps * avgStepDuration + def etaDays = Math.floor(etaMillis / 86400000) // 24 * 60 * 60 * 1000 + def etaHours = Math.floor((etaMillis % 86400000) / 3600000) // 60 * 60 * 1000 + + if (etaDays > 0) { + etaText = "${etaDays}d ${etaHours}h" + } else { + etaText = "${etaHours}h" + } + } + + echo "Bisecting: testing commit ${midpoint} (${remainingCommits} commits remaining, ~${remainingSteps} steps left, ETA: ${etaText})" + + // Record step start time + def stepStartTime = System.currentTimeMillis() + + // Test the midpoint commit using the build-bisect-run job + def testResult = testCommitWithRunner(midpoint, jobTemplate, buildConfig, artifact, goodCommit, badCommit) + + // Record step duration + def stepDuration = System.currentTimeMillis() - stepStartTime + stepDurations.add(stepDuration) + + // Format step duration for logging + def stepHours = Math.floor(stepDuration / 3600000) + def stepMinutes = Math.ceil((stepDuration % 3600000) / 60000) + echo "Step completed in ${stepHours}h ${stepMinutes}m" + + if (testResult == 'SUCCESS') { + // Failure is in the second half + bisectionSteps.add("git bisect good ${midpoint}") + return performBisectionRecursive(midpoint, badCommit, jobTemplate, buildConfig, artifact) + } else { + // Failure is in the first half + bisectionSteps.add("git bisect bad ${midpoint}") + return performBisectionRecursive(goodCommit, midpoint, jobTemplate, buildConfig, artifact) + } + } + + static def testCommitWithRunner(commit, jobTemplate, buildConfig, artifact, goodCommit, badCommit) { + echo "Testing commit ${commit} using job template ${jobTemplate}" + + def result = build job: 'build-bisect-run', + parameters: [ + string(name: 'GIT_SHA', value: commit), + string(name: 'JOB_TEMPLATE', value: jobTemplate), + string(name: 'BUILD_CONFIG', value: buildConfig), + string(name: 'ARTIFACT', value: artifact), + string(name: 'BISECT_GOOD', value: goodCommit), + string(name: 'BISECT_BAD', value: badCommit) + ], + propagate: false + + echo "Test result for ${commit}: ${result.result}" + return result.result + } + + static def getCommitRange(goodCommit, badCommit) { + def commits = sh( + script: "cd llvm-project && git rev-list --reverse ${goodCommit}..${badCommit}", + returnStdout: true + ).trim().split('\n') + + // Add the boundary commits + return [goodCommit] + commits + [badCommit] + } + + static def reportBisectionResult(failingCommit, originalJobName) { + def commitInfo = getCommitInfo(failingCommit) + + // Add final bisect step + bisectionSteps.add("git bisect reset") + + // Calculate final timing statistics + def totalDuration = System.currentTimeMillis() - bisectionStartTime + def totalDays = Math.floor(totalDuration / 86400000) + def totalHours = Math.floor((totalDuration % 86400000) / 3600000) + def totalMinutes = Math.ceil((totalDuration % 3600000) / 60000) + + def avgStepDuration = stepDurations.size() > 0 ? stepDurations.sum() / stepDurations.size() : 0 + def avgStepHours = Math.floor(avgStepDuration / 3600000) + def avgStepMinutes = Math.ceil((avgStepDuration % 3600000) / 60000) + + // Format total time + def totalTimeText = "" + if (totalDays > 0) { + totalTimeText = "${totalDays}d ${totalHours}h ${totalMinutes}m" + } else if (totalHours > 0) { + totalTimeText = "${totalHours}h ${totalMinutes}m" + } else { + totalTimeText = "${totalMinutes}m" + } + + // Format average step time + def avgStepText = "${avgStepHours}h ${avgStepMinutes}m" + + // Format reproduction steps + def reproductionSteps = bisectionSteps.join('\n') + + def report = """ +=== BISECTION COMPLETE === +Original Job: ${originalJobName} +Failing commit: ${failingCommit} +Author: ${commitInfo.author} +Date: ${commitInfo.date} +Message: ${commitInfo.message} + +Bisection Statistics: +- Total steps: ${stepDurations.size()} +- Total time: ${totalTimeText} +- Average step time: ${avgStepText} + +This commit appears to be the first one that introduced the failure. + +To reproduce this bisection locally: +1. Clone the repository and navigate to llvm-project/ +2. Run the following git bisect commands in sequence: + +${reproductionSteps} + +3. At each bisect step, test the commit using your build configuration +4. Mark commits as 'git bisect good' or 'git bisect bad' based on build results +5. The bisection will converge on commit ${failingCommit} + +To reproduce the specific failure: +1. Check out commit ${failingCommit} +2. Run the ${originalJobName} job configuration +3. The failure should reproduce consistently + +=========================== + """.trim() + + echo report + + // Write the report to the workspace for archival + writeFile file: 'bisection-result.txt', text: report + archiveArtifacts artifacts: 'bisection-result.txt', allowEmptyArchive: false + + return failingCommit + } + + static def getCommitInfo(commit) { + def authorInfo = sh( + script: "cd llvm-project && git show -s --format='%an|%ad|%s' ${commit}", + returnStdout: true + ).trim().split('\\|') + + return [ + author: authorInfo[0], + date: authorInfo[1], + message: authorInfo[2] + ] + } +} + +return this \ No newline at end of file