Skip to content

Commit c175937

Browse files
sarahchen6sarahchen6
committed
Turn ci_jobs into a convention plugin and extension
1 parent 4c89f79 commit c175937

File tree

4 files changed

+440
-198
lines changed

4 files changed

+440
-198
lines changed

‎build.gradle.kts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ plugins {
55
id("datadog.dependency-locking")
66
id("datadog.tracer-version")
77
id("datadog.dump-hanged-test")
8+
id("datadog.ci-jobs")
89

910
id("com.diffplug.spotless") version "6.13.0"
1011
id("com.github.spotbugs") version "5.0.14"
@@ -137,4 +138,16 @@ allprojects {
137138
}
138139
}
139140

140-
apply(from = "$rootDir/gradle/ci_jobs.gradle")
141+
testAggregate("smoke", listOf(":dd-smoke-tests"), emptyList())
142+
testAggregate("instrumentation", listOf(":dd-java-agent:instrumentation"), emptyList())
143+
testAggregate("profiling", listOf(":dd-java-agent:agent-profiling"), emptyList())
144+
testAggregate("debugger", listOf(":dd-java-agent:agent-debugger"), emptyList(), true)
145+
testAggregate(
146+
"base", listOf(":"),
147+
listOf(
148+
":dd-java-agent:instrumentation",
149+
":dd-smoke-tests",
150+
":dd-java-agent:agent-profiling",
151+
":dd-java-agent:agent-debugger"
152+
)
153+
)

‎buildSrc/src/main/kotlin/CIJobsExtensions.kt‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import org.gradle.api.Project
2+
import org.gradle.api.Task
3+
import org.gradle.kotlin.dsl.extra
4+
5+
/**
6+
* Checks if a task is affected by git changes
7+
*/
8+
internal fun isAffectedBy(baseTask: Task, affectedProjects: Map<Project, Set<String>>): String? {
9+
val visited = mutableSetOf<Task>()
10+
val queue = mutableListOf(baseTask)
11+
12+
while (queue.isNotEmpty()) {
13+
val t = queue.removeAt(0)
14+
if (visited.contains(t)) {
15+
continue
16+
}
17+
visited.add(t)
18+
19+
val affectedTasks = affectedProjects[t.project]
20+
if (affectedTasks != null) {
21+
if (affectedTasks.contains("all")) {
22+
return "${t.project.path}:${t.name}"
23+
}
24+
if (affectedTasks.contains(t.name)) {
25+
return "${t.project.path}:${t.name}"
26+
}
27+
}
28+
29+
t.taskDependencies.getDependencies(t).forEach { queue.add(it) }
30+
}
31+
return null
32+
}
33+
34+
/**
35+
* Creates aggregate test tasks for CI
36+
*
37+
* Creates three tasks for the given base name:
38+
* - ${baseTaskName}Test - runs allTests
39+
* - ${baseTaskName}LatestDepTest - runs allLatestDepTests
40+
* - ${baseTaskName}Check - runs check
41+
*/
42+
fun Project.testAggregate(
43+
baseTaskName: String,
44+
includePrefixes: List<String>,
45+
excludePrefixes: List<String>,
46+
forceCoverage: Boolean = false
47+
) {
48+
fun createRootTask(rootTaskName: String, subProjTaskName: String) {
49+
val coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
50+
val proj = this@testAggregate
51+
tasks.register(rootTaskName) {
52+
proj.subprojects.forEach { subproject ->
53+
val activePartition = subproject.extra.get("activePartition") as Boolean
54+
if (activePartition &&
55+
includePrefixes.any { subproject.path.startsWith(it) } &&
56+
!excludePrefixes.any { subproject.path.startsWith(it) }) {
57+
58+
val testTask = subproject.tasks.findByName(subProjTaskName)
59+
var isAffected = true
60+
61+
if (testTask != null) {
62+
val useGitChanges = proj.rootProject.extra.get("useGitChanges") as Boolean
63+
if (useGitChanges) {
64+
@Suppress("UNCHECKED_CAST")
65+
val affectedProjects = proj.rootProject.extra.get("affectedProjects") as Map<Project, Set<String>>
66+
val fileTrigger = isAffectedBy(testTask, affectedProjects)
67+
if (fileTrigger != null) {
68+
proj.logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)")
69+
} else {
70+
proj.logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)")
71+
isAffected = false
72+
}
73+
}
74+
if (isAffected) {
75+
dependsOn(testTask)
76+
}
77+
}
78+
79+
if (isAffected && coverage) {
80+
val coverageTask = subproject.tasks.findByName("jacocoTestReport")
81+
if (coverageTask != null) {
82+
dependsOn(coverageTask)
83+
}
84+
val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
85+
if (verificationTask != null) {
86+
dependsOn(verificationTask)
87+
}
88+
}
89+
}
90+
}
91+
}
92+
}
93+
94+
createRootTask("${baseTaskName}Test", "allTests")
95+
createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests")
96+
createRootTask("${baseTaskName}Check", "check")
97+
}
98+

‎buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/*
2+
* This plugin defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize
3+
* jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes
4+
* with -PgitBaseRef.
5+
*/
6+
7+
import java.io.File
8+
import kotlin.math.abs
9+
10+
// Set up activePartition property on all projects
11+
allprojects {
12+
extra.set("activePartition", true)
13+
14+
val shouldUseTaskPartitions = rootProject.hasProperty("taskPartitionCount") && rootProject.hasProperty("taskPartition")
15+
if (shouldUseTaskPartitions) {
16+
val taskPartitionCount = rootProject.property("taskPartitionCount") as String
17+
val taskPartition = rootProject.property("taskPartition") as String
18+
val currentTaskPartition = abs(project.path.hashCode() % taskPartitionCount.toInt())
19+
extra.set("activePartition", currentTaskPartition == taskPartition.toInt())
20+
}
21+
}
22+
23+
fun relativeToGitRoot(f: File): File {
24+
return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile()
25+
}
26+
27+
fun getChangedFiles(baseRef: String, newRef: String): List<File> {
28+
val stdout = StringBuilder()
29+
val stderr = StringBuilder()
30+
31+
val proc = Runtime.getRuntime().exec(arrayOf("git", "diff", "--name-only", "$baseRef..$newRef"))
32+
proc.inputStream.bufferedReader().use { stdout.append(it.readText()) }
33+
proc.errorStream.bufferedReader().use { stderr.append(it.readText()) }
34+
proc.waitFor()
35+
require(proc.exitValue() == 0) { "git diff command failed, stderr: $stderr" }
36+
37+
val out = stdout.toString().trim()
38+
if (out.isEmpty()) {
39+
return emptyList()
40+
}
41+
42+
logger.debug("git diff output: $out")
43+
return out.split("\n").map { File(rootProject.projectDir, it.trim()) }
44+
}
45+
46+
// Initialize git change tracking
47+
rootProject.extra.set("useGitChanges", false)
48+
49+
if (rootProject.hasProperty("gitBaseRef")) {
50+
val baseRef = rootProject.property("gitBaseRef") as String
51+
val newRef = if (rootProject.hasProperty("gitNewRef")) {
52+
rootProject.property("gitNewRef") as String
53+
} else {
54+
"HEAD"
55+
}
56+
57+
val changedFiles = getChangedFiles(baseRef, newRef)
58+
rootProject.extra.set("changedFiles", changedFiles)
59+
rootProject.extra.set("useGitChanges", true)
60+
61+
val ignoredFiles = fileTree(rootProject.projectDir) {
62+
include(".gitignore", ".editorconfig")
63+
include("*.md", "**/*.md")
64+
include("gradlew", "gradlew.bat", "mvnw", "mvnw.cmd")
65+
include("NOTICE")
66+
include("static-analysis.datadog.yml")
67+
}
68+
69+
changedFiles.forEach { f ->
70+
if (ignoredFiles.contains(f)) {
71+
logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}")
72+
}
73+
}
74+
75+
val filteredChangedFiles = changedFiles.filter { !ignoredFiles.contains(it) }
76+
rootProject.extra.set("changedFiles", filteredChangedFiles)
77+
78+
val globalEffectFiles = fileTree(rootProject.projectDir) {
79+
include(".gitlab/**")
80+
include("build.gradle")
81+
include("gradle/**")
82+
}
83+
84+
for (f in filteredChangedFiles) {
85+
if (globalEffectFiles.contains(f)) {
86+
logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)")
87+
rootProject.extra.set("useGitChanges", false)
88+
break
89+
}
90+
}
91+
92+
if (rootProject.extra.get("useGitChanges") as Boolean) {
93+
logger.warn("Git change tracking is enabled: $baseRef..$newRef")
94+
95+
val projects = subprojects.sortedByDescending { it.projectDir.path.length }
96+
val affectedProjects = mutableMapOf<Project, MutableSet<String>>()
97+
98+
// Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in
99+
// the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used.
100+
val matchers = listOf(
101+
mapOf("prefix" to "src/testFixtures/", "task" to "testFixturesClasses"),
102+
mapOf("prefix" to "src/test/", "task" to "testClasses"),
103+
mapOf("prefix" to "src/jmh/", "task" to "jmhCompileGeneratedClasses")
104+
)
105+
106+
for (f in filteredChangedFiles) {
107+
val p = projects.find { f.toString().startsWith(it.projectDir.path + "/") }
108+
if (p == null) {
109+
logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)")
110+
rootProject.extra.set("useGitChanges", false)
111+
break
112+
}
113+
114+
// Make sure path separator is /
115+
val relPath = p.projectDir.toPath().relativize(f.toPath()).joinToString("/")
116+
val task = matchers.find { relPath.startsWith(it["prefix"]!!) }?.get("task") ?: "all"
117+
logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} ($task)")
118+
affectedProjects.computeIfAbsent(p) { mutableSetOf() }.add(task)
119+
}
120+
121+
rootProject.extra.set("affectedProjects", affectedProjects)
122+
}
123+
}
124+
125+
tasks.register("runMuzzle") {
126+
val muzzleSubprojects = subprojects.filter { p ->
127+
val activePartition = p.extra.get("activePartition") as Boolean
128+
activePartition && p.plugins.hasPlugin("java") && p.plugins.hasPlugin("muzzle")
129+
}
130+
dependsOn(muzzleSubprojects.map { p -> "${p.path}:muzzle" })
131+
}

0 commit comments

Comments
 (0)