Skip to content

Commit 1f652f6

Browse files
dweissdweiss
authored
SOLR-15436: Gradle's RAT task has missing inputs, so it can't figure out when to run (#272)
1 parent 0a81be3 commit 1f652f6

File tree

1 file changed

+148
-93
lines changed

1 file changed

+148
-93
lines changed

‎gradle/validation/rat-sources.gradle‎

Lines changed: 148 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import groovy.xml.NamespaceBuilder
1919

20+
// Configure rat dependencies for use in the custom task.
2021
configure(rootProject) {
2122
configurations {
2223
ratDeps
@@ -27,111 +28,164 @@ configure(rootProject) {
2728
}
2829
}
2930

31+
// Configure the rat validation task and all scanned directories.
3032
allprojects {
3133
task("rat", type: RatTask) {
3234
group = 'Verification'
3335
description = 'Runs Apache Rat checks.'
34-
}
35-
}
3636

37-
configure(rootProject) {
38-
rat {
39-
includes += [
40-
"buildSrc/**/*.java",
41-
"gradle/**/*.gradle",
42-
"lucene/tools/forbiddenApis/**",
43-
"lucene/tools/prettify/**",
44-
]
45-
excludes += [
46-
// Unclear if this needs ASF header, depends on how much was copied from ElasticSearch
47-
"**/ErrorReportingTestListener.java"
48-
]
49-
}
50-
}
37+
def defaultScanFileTree = project.fileTree(projectDir, {
38+
// Don't check under the project's build folder.
39+
exclude project.buildDir.name
5140

52-
configure(project(":solr:core")) {
53-
rat {
54-
srcExcludes += [
55-
"**/htmlStripReaderTest.html"
56-
]
57-
}
58-
}
41+
// Exclude any generated stuff.
42+
exclude "src/generated"
43+
44+
// Don't check any of the subprojects - they have their own rat tasks.
45+
exclude subprojects.collect { it.projectDir.name }
46+
47+
// At the module scope we only check selected file patterns as folks have various .gitignore-d resources
48+
// generated by IDEs, etc.
49+
include "**/*.gradle"
50+
include "**/*.xml"
51+
include "**/*.md"
52+
include "**/*.py"
53+
include "**/*.sh"
54+
include "**/*.bat"
55+
56+
// Include selected patterns from any source folders. We could make this
57+
// relative to source sets but it seems to be of little value - all our source sets
58+
// live under 'src' anyway.
59+
include "src/**"
60+
exclude "src/**/*.png"
61+
exclude "src/**/*.txt"
62+
exclude "src/**/*.zip"
63+
exclude "src/**/*.properties"
64+
exclude "src/**/*.utf8"
65+
exclude "src/**/*.svg"
66+
exclude "src/**/*.csv"
67+
68+
// TODO: SOLR-15601: Some of these should carry the license, perhaps?
69+
exclude "**/README.md"
70+
exclude "**/*.html"
71+
exclude "**/*.xml"
72+
exclude "**/*.json"
73+
exclude "**/*.js"
74+
exclude "**/*.py"
75+
76+
// Conditionally apply module-specific patterns. We do it here instead
77+
// of reconfiguring each project because the provider can be made lazy
78+
// and it's easier to manage this way.
79+
switch (project.path) {
80+
case ":":
81+
include "gradlew"
82+
include "gradlew.bat"
83+
exclude ".gradle"
84+
exclude ".idea"
85+
exclude ".muse"
86+
exclude ".git"
87+
88+
// Exclude github stuff (templates, workflows).
89+
exclude ".github"
90+
91+
exclude "dev-tools/scripts/cloud.sh"
92+
exclude "dev-tools/scripts/README.md"
93+
exclude "dev-tools/scripts/create_line_file_docs.py"
5994

60-
configure(project(":solr:webapp")) {
61-
rat {
62-
includes = ["**"]
63-
excludes += [
64-
"web/img/**",
65-
"*.iml",
66-
"build.gradle",
67-
"build/**",
68-
]
95+
// The root project also includes patterns for the boostrap (buildSrc) and composite
96+
// projects. Include their sources in the scan.
97+
include "buildSrc/src/**"
98+
include "dev-tools/missing-doclet/src/**"
99+
break
100+
101+
case ":solr:contrib:clustering":
102+
exclude "src/test-files/META-INF/services/*"
103+
break
104+
105+
case ":solr:contrib:langid":
106+
exclude "**/langdetect-profiles/*"
107+
break
108+
109+
case ":solr:documentation":
110+
exclude "src/markdown/*.md"
111+
break
112+
113+
case ":solr:core":
114+
exclude "**/htmlStripReaderTest.html"
115+
exclude "src/resources/*.json"
116+
exclude "src/resources/*.xml"
117+
exclude "src/test-files/**/*.csv"
118+
exclude "src/test-files/**/*.json"
119+
exclude "src/test-files/**/*.aff"
120+
exclude "src/test-files/**/*.dic"
121+
exclude "src/test-files/**/*.conf"
122+
exclude "src/test-files/**/external_eff"
123+
exclude "src/test-files/**/*.incl"
124+
break
125+
126+
case ":solr:server":
127+
exclude "**/*.xml"
128+
exclude "**/*.sh"
129+
exclude "**/*.bat"
130+
break
131+
132+
case ":solr:webapp":
133+
exclude "web/img/**"
134+
break
135+
136+
case ":solr:solr-ref-guide":
137+
exclude "src/**"
138+
break
139+
140+
case ":solr:docker":
141+
exclude "tests/**/*.xml"
142+
break
143+
144+
case ":solr:example":
145+
exclude "**/*.xml"
146+
exclude "films/README.md"
147+
break
148+
149+
case ":solr:solrj":
150+
exclude "src/**/*.json"
151+
exclude "src/test-files/**/*.cfg"
152+
exclude "src/test-files/**/*.xml"
153+
break
154+
}
155+
})
156+
inputFileTrees.add(defaultScanFileTree)
69157
}
70158
}
71159

72-
// Structure inspired by existing task from Apache Kafka, heavily modified since then.
160+
/**
161+
* An Apache RAT adapter that validates whether files contain acceptable licenses.
162+
*/
73163
class RatTask extends DefaultTask {
74-
@Input
75-
List<String> includes = [
76-
"*.gradle",
77-
"*.xml",
78-
"src/tools/**"
79-
]
80-
81-
@Input
82-
List<String> excludes = []
83-
84-
@Input
85-
List<String> srcExcludes = [
86-
"**/TODO",
87-
"**/*.txt",
88-
"**/*.md",
89-
"**/*.iml",
90-
"build/**"
91-
]
164+
@InputFiles
165+
final ListProperty<ConfigurableFileTree> inputFileTrees = project.objects.listProperty(ConfigurableFileTree)
92166

93167
@OutputFile
94-
def xmlReport = new File(new File(project.buildDir, 'rat'), 'rat-report.xml')
168+
final RegularFileProperty xmlReport = project.objects.fileProperty().convention(
169+
project.layout.buildDirectory.file("rat/rat-report.xml"))
95170

96-
def generateXmlReport() {
97-
def uri = 'antlib:org.apache.rat.anttasks'
171+
def generateReport(File reportFile) {
172+
// Set up ant rat task.
98173
def ratClasspath = project.rootProject.configurations.ratDeps.asPath
99-
ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml', uri: uri, classpath: ratClasspath)
174+
ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml', classpath: ratClasspath)
100175

101-
def rat = NamespaceBuilder.newInstance(ant, uri)
102-
rat.report(format: 'xml', reportFile: xmlReport, addDefaultLicenseMatchers: true) {
103-
ant.fileset(dir: "${project.projectDir}") {
104-
includes.each { pattern -> ant.include(name: pattern) }
105-
excludes.each { pattern -> ant.exclude(name: pattern) }
106-
}
176+
// Collect all output files for debugging.
177+
String inputFileList = inputFileTrees.get().collectMany { fileTree ->
178+
fileTree.asList()
179+
}.sort().join("\n")
180+
project.file(reportFile.path.replaceAll('.xml$', '-filelist.txt')).setText(inputFileList, "UTF-8")
107181

108-
if (project.plugins.findPlugin(JavaPlugin)) {
109-
def checkSets = [
110-
project.sourceSets.main.java.srcDirs,
111-
project.sourceSets.test.java.srcDirs,
112-
]
113-
114-
project.sourceSets.matching { it.name == 'tools' }.all {
115-
checkSets += project.sourceSets.tools.java.srcDirs
116-
}
117-
118-
checkSets.flatten().each { srcLocation ->
119-
ant.fileset(dir: srcLocation, erroronmissingdir: false) {
120-
srcExcludes.each { pattern -> ant.exclude(name: pattern) }
121-
}
122-
}
123-
124-
[
125-
project.sourceSets.main.resources.srcDirs
126-
].flatten().each { srcLocation ->
127-
ant.fileset(dir: srcLocation, erroronmissingdir: false) {
128-
ant.include(name: "META-INF/**")
129-
}
130-
}
182+
// Run rat via ant.
183+
ant.report(format: 'xml', reportFile: reportFile, addDefaultLicenseMatchers: true) {
184+
// Pass all gradle file trees to the ant task (Gradle's internal adapters are used).
185+
inputFileTrees.get().each { fileTree ->
186+
fileTree.addToAntBuilder(ant, 'resources', FileCollection.AntType.ResourceCollection)
131187
}
132188

133-
// The license rules below were manually copied from lucene/common-build.xml, there is currently no mechanism to sync them
134-
135189
// BSD 4-clause stuff (is disallowed below)
136190
substringMatcher(licenseFamilyCategory: "BSD4 ", licenseFamilyName: "Original BSD License (with advertising clause)") {
137191
pattern(substring: "All advertising materials")
@@ -160,7 +214,7 @@ class RatTask extends DefaultTask {
160214
// ICU license
161215
pattern(substring: "Permission is hereby granted, free of charge, to any person obtaining a copy")
162216
// ui-grid
163-
pattern(substring: " ; License: MIT")
217+
pattern(substring: " ; License: MIT")
164218
}
165219

166220
// Apache
@@ -186,8 +240,8 @@ class RatTask extends DefaultTask {
186240
}
187241
}
188242

189-
def printUnknownFiles() {
190-
def ratXml = new XmlParser().parse(xmlReport)
243+
def printUnknownFiles(File reportFile) {
244+
def ratXml = new XmlParser().parse(reportFile)
191245
def errors = []
192246
ratXml.resource.each { resource ->
193247
if (resource.'license-approval'.@name[0] == "false") {
@@ -201,14 +255,15 @@ class RatTask extends DefaultTask {
201255
}
202256

203257
@TaskAction
204-
def rat() {
258+
def execute() {
205259
def origEncoding = System.getProperty("file.encoding")
206260
try {
207-
generateXmlReport()
208-
printUnknownFiles()
261+
File reportFile = xmlReport.get().asFile
262+
generateReport(reportFile)
263+
printUnknownFiles(reportFile)
209264
} finally {
210265
if (System.getProperty("file.encoding") != origEncoding) {
211-
throw new GradleException("Insane: rat changed file.encoding to ${System.getProperty('file.encoding')}?")
266+
throw new GradleException("Something is wrong: Apache RAT changed file.encoding to ${System.getProperty('file.encoding')}?")
212267
}
213268
}
214269
}

0 commit comments

Comments
 (0)