|
| 1 | +/** |
| 2 | + * This script defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize |
| 3 | + * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes |
| 4 | + * with -PgitBaseRef. |
| 5 | + */ |
| 6 | +import java.nio.file.Paths |
| 7 | + |
| 8 | +allprojects { project -> |
| 9 | + project.ext { |
| 10 | + activePartition = true |
| 11 | + } |
| 12 | + final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") |
| 13 | + if (shouldUseTaskPartitions) { |
| 14 | + final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int |
| 15 | + final int taskPartition = project.rootProject.property("taskPartition") as int |
| 16 | + final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) |
| 17 | + project.setProperty("activePartition", currentTaskPartition == taskPartition) |
| 18 | + } |
| 19 | +} |
| 20 | + |
| 21 | +File relativeToGitRoot(File f) { |
| 22 | + return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() |
| 23 | +} |
| 24 | + |
| 25 | +String isAffectedBy(Task baseTask, Map<Project, Set<String>> affectedProjects) { |
| 26 | + HashSet<Task> visited = [] |
| 27 | + LinkedList<Task> queue = [baseTask] |
| 28 | + while (!queue.isEmpty()) { |
| 29 | + Task t = queue.poll() |
| 30 | + if (visited.contains(t)) { |
| 31 | + continue |
| 32 | + } |
| 33 | + visited.add(t) |
| 34 | + |
| 35 | + final Set<String> affectedTasks = affectedProjects.get(t.project) |
| 36 | + if (affectedTasks != null) { |
| 37 | + if (affectedTasks.contains("all")) { |
| 38 | + return "${t.project.path}:${t.name}" |
| 39 | + } |
| 40 | + if (affectedTasks.contains(t.name)) { |
| 41 | + return "${t.project.path}:${t.name}" |
| 42 | + } |
| 43 | + } |
| 44 | + |
| 45 | + t.taskDependencies.each { queue.addAll(it.getDependencies(t)) } |
| 46 | + } |
| 47 | + return null |
| 48 | +} |
| 49 | + |
| 50 | +List<File> getChangedFiles(String baseRef, String newRef) { |
| 51 | + final stdout = new StringBuilder() |
| 52 | + final stderr = new StringBuilder() |
| 53 | + final proc = "git diff --name-only ${baseRef}..${newRef}".execute() |
| 54 | + proc.consumeProcessOutput(stdout, stderr) |
| 55 | + proc.waitForOrKill(1000) |
| 56 | + assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" |
| 57 | + def out = stdout.toString().trim() |
| 58 | + if (out.isEmpty()) { |
| 59 | + return [] |
| 60 | + } |
| 61 | + logger.debug("git diff output: ${out}") |
| 62 | + return out.split("\n").collect { |
| 63 | + new File(rootProject.projectDir, it.trim()) |
| 64 | + } |
| 65 | +} |
| 66 | + |
| 67 | +rootProject.ext { |
| 68 | + useGitChanges = false |
| 69 | +} |
| 70 | + |
| 71 | +if (rootProject.hasProperty("gitBaseRef")) { |
| 72 | + final String baseRef = rootProject.property("gitBaseRef") |
| 73 | + final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" |
| 74 | + |
| 75 | + rootProject.ext { |
| 76 | + it.changedFiles = getChangedFiles(baseRef, newRef) |
| 77 | + useGitChanges = true |
| 78 | + } |
| 79 | + |
| 80 | + final ignoredFiles = fileTree(rootProject.projectDir) { |
| 81 | + include '.gitignore', '.editorconfig' |
| 82 | + include '*.md', '**/*.md' |
| 83 | + include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' |
| 84 | + include 'NOTICE' |
| 85 | + include 'static-analysis.datadog.yml' |
| 86 | + } |
| 87 | + rootProject.changedFiles.each { File f -> |
| 88 | + if (ignoredFiles.contains(f)) { |
| 89 | + logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") |
| 90 | + } |
| 91 | + } |
| 92 | + rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } |
| 93 | + |
| 94 | + final globalEffectFiles = fileTree(rootProject.projectDir) { |
| 95 | + include '.circleci/**' |
| 96 | + include 'build.gradle' |
| 97 | + include 'gradle/**' |
| 98 | + } |
| 99 | + |
| 100 | + for (File f in rootProject.changedFiles) { |
| 101 | + if (globalEffectFiles.contains(f)) { |
| 102 | + logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") |
| 103 | + rootProject.useGitChanges = false |
| 104 | + break |
| 105 | + } |
| 106 | + } |
| 107 | + |
| 108 | + if (rootProject.useGitChanges) { |
| 109 | + logger.warn("Git change tracking is enabled: ${baseRef}..${newRef}") |
| 110 | + |
| 111 | + final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } |
| 112 | + Map<Project, Set<String>> _affectedProjects = [:] |
| 113 | + // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in |
| 114 | + // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. |
| 115 | + final List<Map<String, String>> matchers = [ |
| 116 | + [prefix: 'src/testFixtures/', task: 'testFixturesClasses'], |
| 117 | + [prefix: 'src/test/', task: 'testClasses'], |
| 118 | + [prefix: 'src/jmh/', task: 'jmhCompileGeneratedClasses'] |
| 119 | + ] |
| 120 | + for (File f in rootProject.changedFiles) { |
| 121 | + Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } |
| 122 | + if (p == null) { |
| 123 | + logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") |
| 124 | + rootProject.useGitChanges = false |
| 125 | + break |
| 126 | + } |
| 127 | + // Make sure path separator is / |
| 128 | + final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()).collect { it.toString() }.join('/') |
| 129 | + final String task = matchers.find { relPath.startsWith(it.prefix) }?.task ?: "all" |
| 130 | + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (${task})") |
| 131 | + _affectedProjects.computeIfAbsent(p, { new HashSet<String>() }).add(task) |
| 132 | + } |
| 133 | + rootProject.ext { |
| 134 | + it.affectedProjects = _affectedProjects |
| 135 | + } |
| 136 | + } |
| 137 | +} |
| 138 | + |
| 139 | +def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { |
| 140 | + def createRootTask = { String rootTaskName, String subProjTaskName -> |
| 141 | + def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") |
| 142 | + tasks.register(rootTaskName) { aggTest -> |
| 143 | + subprojects { subproject -> |
| 144 | + if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { |
| 145 | + Task testTask = subproject.tasks.findByName(subProjTaskName) |
| 146 | + boolean isAffected = true |
| 147 | + if (testTask != null) { |
| 148 | + if (rootProject.useGitChanges) { |
| 149 | + final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) |
| 150 | + if (fileTrigger != null) { |
| 151 | + logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") |
| 152 | + } else { |
| 153 | + logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") |
| 154 | + isAffected = false |
| 155 | + } |
| 156 | + } |
| 157 | + if (isAffected) { |
| 158 | + aggTest.dependsOn(testTask) |
| 159 | + } |
| 160 | + } |
| 161 | + if (isAffected && coverage) { |
| 162 | + def coverageTask = subproject.tasks.findByName("jacocoTestReport") |
| 163 | + if (coverageTask != null) { |
| 164 | + aggTest.dependsOn(coverageTask) |
| 165 | + } |
| 166 | + coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") |
| 167 | + if (coverageTask != null) { |
| 168 | + aggTest.dependsOn(coverageTask) |
| 169 | + } |
| 170 | + } |
| 171 | + } |
| 172 | + } |
| 173 | + } |
| 174 | + } |
| 175 | + |
| 176 | + createRootTask "${baseTaskName}Test", 'allTests' |
| 177 | + createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' |
| 178 | + createRootTask "${baseTaskName}Check", 'check' |
| 179 | +} |
| 180 | + |
| 181 | +testAggregate("smoke", [":dd-smoke-tests"], []) |
| 182 | +testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) |
| 183 | +testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) |
| 184 | +testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) |
| 185 | +testAggregate("base", [":"], [ |
| 186 | + ":dd-java-agent:instrumentation", |
| 187 | + ":dd-smoke-tests", |
| 188 | + ":dd-java-agent:agent-profiling", |
| 189 | + ":dd-java-agent:agent-debugger" |
| 190 | +]) |
0 commit comments