diff --git a/test1 b/test1 new file mode 100644 index 0000000..61b68ed --- /dev/null +++ b/test1 @@ -0,0 +1,3855 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import org.ajoberstar.grgit.Grgit +import org.gradle.api.JavaVersion + +import java.nio.charset.StandardCharsets + +buildscript { + repositories { + mavenCentral() + } + apply from: "$rootDir/gradle/dependencies.gradle" + + dependencies { + // For Apache Rat plugin to ignore non-Git files + classpath "org.ajoberstar.grgit:grgit-core:$versions.grgit" + } +} + +plugins { + id 'com.github.ben-manes.versions' version '0.48.0' + id 'idea' + id 'jacoco' + id 'java-library' + id 'org.owasp.dependencycheck' version '8.2.1' + id 'org.nosphere.apache.rat' version "0.8.1" + id "io.swagger.core.v3.swagger-gradle-plugin" version "${swaggerVersion}" + + id "com.github.spotbugs" version '6.0.25' apply false + id 'org.scoverage' version '8.0.3' apply false + id 'com.gradleup.shadow' version '8.3.6' apply false + id 'com.diffplug.spotless' version "6.25.0" +} + +ext { + gradleVersion = versions.gradle + minClientJavaVersion = 11 + minNonClientJavaVersion = 17 + modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams-scala", ":test-common:test-common-util"] + + buildVersionFileName = "kafka-version.properties" + + defaultMaxHeapSize = "2g" + defaultJvmArgs = ["-Xss4m", "-XX:+UseParallelGC"] + + // "JEP 403: Strongly Encapsulate JDK Internals" causes some tests to fail when they try + // to access internals (often via mocking libraries). We use `--add-opens` as a workaround + // for now and we'll fix it properly (where possible) via KAFKA-13275. + if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_16)) + defaultJvmArgs.addAll( + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.nio.file=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.regex=ALL-UNNAMED", + "--add-opens=java.base/java.util.stream=ALL-UNNAMED", + "--add-opens=java.base/java.text=ALL-UNNAMED", + "--add-opens=java.base/java.time=ALL-UNNAMED", + "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED" + ) + + maxTestForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : Runtime.runtime.availableProcessors() + maxScalacThreads = project.hasProperty('maxScalacThreads') ? maxScalacThreads.toInteger() : + Math.min(Runtime.runtime.availableProcessors(), 8) + userIgnoreFailures = project.hasProperty('ignoreFailures') ? ignoreFailures.toBoolean() : false + + userMaxTestRetries = project.hasProperty('maxTestRetries') ? maxTestRetries.toInteger() : 0 + userMaxTestRetryFailures = project.hasProperty('maxTestRetryFailures') ? maxTestRetryFailures.toInteger() : 0 + + skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean() + shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") + + mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : '' + mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : '' + mavenPassword = project.hasProperty('mavenPassword') ? project.mavenPassword : '' + + userShowStandardStreams = project.hasProperty("showStandardStreams") ? showStandardStreams : null + + userTestLoggingEvents = project.hasProperty("testLoggingEvents") ? Arrays.asList(testLoggingEvents.split(",")) : null + + userEnableTestCoverage = project.hasProperty("enableTestCoverage") ? enableTestCoverage : false + + userKeepAliveModeString = project.hasProperty("keepAliveMode") ? keepAliveMode : "daemon" + userKeepAliveMode = KeepAliveMode.values().find(m -> m.name().toLowerCase().equals(userKeepAliveModeString)) + if (userKeepAliveMode == null) { + def keepAliveValues = KeepAliveMode.values().collect(m -> m.name.toLowerCase()) + throw new GradleException("Unexpected value for keepAliveMode property. Expected one of $keepAliveValues, but received: $userKeepAliveModeString") + } + + // Used by :test task + isGithubActions = System.getenv('GITHUB_ACTIONS') != null + + // See README.md for details on this option and the reasoning for the default + userScalaOptimizerMode = project.hasProperty("scalaOptimizerMode") ? scalaOptimizerMode : "inline-kafka" + def scalaOptimizerValues = ["none", "method", "inline-kafka", "inline-scala"] + if (!scalaOptimizerValues.contains(userScalaOptimizerMode)) + throw new GradleException("Unexpected value for scalaOptimizerMode property. Expected one of $scalaOptimizerValues, but received: $userScalaOptimizerMode") + + generatedDocsDir = new File("${project.rootDir}/docs/generated") + repo = file("$rootDir/.git").isDirectory() ? Grgit.open(currentDir: project.getRootDir()) : null + + commitId = determineCommitId() + + configureJavaCompiler = { name, options, projectPath -> + // -parameters generates arguments with parameter names in TestInfo#getDisplayName. + // ref: https://github.com/junit-team/junit5/blob/4c0dddad1b96d4a20e92a2cd583954643ac56ac0/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java#L161-L164 + + def releaseVersion = modulesNeedingJava11.any { projectPath == it } ? minClientJavaVersion : minNonClientJavaVersion + + options.compilerArgs << "-encoding" << "UTF-8" + options.release = releaseVersion + + if (name in ["compileTestJava", "compileTestScala"]) { + options.compilerArgs << "-parameters" + } else if (name in ["compileJava", "compileScala"]) { + options.compilerArgs << "-Xlint:all" + options.compilerArgs << "-Xlint:-serial" + options.compilerArgs << "-Xlint:-try" + options.compilerArgs << "-Werror" + } + } + + runtimeTestLibs = [ + libs.slf4jLog4j2, + libs.junitPlatformLanucher, + libs.jacksonDatabindYaml, + project(":test-common:test-common-util") + ] + + log4jReleaseLibs = [ + libs.slf4jLog4j2, + libs.log4j1Bridge2Api, + libs.jacksonDatabindYaml + ] + + log4j2Libs = [ + libs.log4j2Api, + libs.log4j2Core + ] + + testLog4j2Libs = [ + libs.slf4jApi, + libs.slf4jLog4j2, + libs.log4j2Api, + libs.log4j2Core + ] + +} + +allprojects { + + repositories { + mavenCentral() + } + + dependencyUpdates { + revision="release" + resolutionStrategy { + componentSelection { rules -> + rules.all { ComponentSelection selection -> + boolean rejected = ['snap', 'alpha', 'beta', 'rc', 'cr', 'm'].any { qualifier -> + selection.candidate.version ==~ /(?i).*[.-]${qualifier}[.\d-]*/ + } + if (rejected) { + selection.reject('Release candidate') + } + } + } + } + } + + configurations.all { + // zinc is the Scala incremental compiler, it has a configuration for its own dependencies + // that are unrelated to the project dependencies, we should not change them + if (name != "zinc") { + resolutionStrategy { + force( + // be explicit about the javassist dependency version instead of relying on the transitive version + libs.javassist, + // ensure we have a single version in the classpath despite transitive dependencies + libs.scalaLibrary, + libs.scalaReflect, + libs.jacksonAnnotations + ) + } + } + } + task printAllDependencies(type: DependencyReportTask) {} + + tasks.withType(Javadoc) { + options.charSet = 'UTF-8' + options.docEncoding = 'UTF-8' + options.encoding = 'UTF-8' + options.memberLevel = JavadocMemberLevel.PUBLIC // Document only public members/API + // Turn off doclint for now, see https://blog.joda.org/2014/02/turning-off-doclint-in-jdk-8-javadoc.html for rationale + options.addStringOption('Xdoclint:none', '-quiet') + // Javadoc warnings should fail the build in JDK 15+ https://bugs.openjdk.org/browse/JDK-8200363 + options.addBooleanOption('Werror', JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_15)) + options.links "https://docs.oracle.com/en/java/javase/${JavaVersion.current().majorVersion}/docs/api/" + } + + tasks.withType(Checkstyle) { + minHeapSize = "200m" + maxHeapSize = "1g" + } + + clean { + delete "${projectDir}/src/generated" + delete "${projectDir}/src/generated-test" + } +} + +def determineCommitId() { + def takeFromHash = 16 + if (project.hasProperty('commitId')) { + commitId.take(takeFromHash) + } else if (repo != null) { + repo.head().id.take(takeFromHash) + } else { + "unknown" + } +} + +/** + * For a given Project, compute a nice dash separated directory name + * to store the JUnit XML files in. E.g., Project ":connect:api" -> "connect-api" + */ +static def projectToJUnitXmlPath(project) { + var p = project + var projectNames = [] + while (p != null) { + projectNames.push(p.name) + p = p.parent + if (p.name == "kafka") { + break + } + } + return projectNames.join("/") +} + + +apply from: file('wrapper.gradle') + +if (repo != null) { + rat { + dependsOn subprojects.collect { + it.tasks.matching { + it.name == "processMessages" || it.name == "processTestMessages" + } + } + + verbose.set(true) + reportDir.set(project.file('build/rat')) + stylesheet.set(file('gradle/resources/rat-output-to-html.xsl')) + + // Exclude everything under the directory that git should be ignoring via .gitignore or that isn't checked in. These + // restrict us only to files that are checked in or are staged. + excludes = new ArrayList(repo.clean(ignore: false, directories: true, dryRun: true)) + // And some of the files that we have checked in should also be excluded from this check + excludes.addAll([ + '**/.git/**', + '**/build/**', + '.github/pull_request_template.md', + 'CONTRIBUTING.md', + 'gradlew', + 'gradlew.bat', + 'gradle/wrapper/gradle-wrapper.properties', + 'trogdor/README.md', + '**/README.md', + '**/id_rsa', + '**/id_rsa.pub', + 'checkstyle/suppressions.xml', + 'streams/quickstart/java/src/test/resources/projects/basic/goal.txt', + 'streams/streams-scala/logs/*', + 'licenses/*', + '**/generated/**', + 'clients/src/test/resources/serializedData/*', + 'docker/test/fixtures/secrets/*', + 'docker/examples/fixtures/secrets/*', + 'docker/docker_official_images/.gitkeep' + ]) + } +} else { + rat.enabled = false +} +println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}") +println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries") + +subprojects { + + // enable running :dependencies task recursively on all subprojects + // eg: ./gradlew allDeps + task allDeps(type: DependencyReportTask) {} + // enable running :dependencyInsight task recursively on all subprojects + // eg: ./gradlew allDepInsight --configuration runtime --dependency com.fasterxml.jackson.core:jackson-databind + task allDepInsight(type: DependencyInsightReportTask) {showingAllVariants = false} doLast {} + + apply plugin: 'java-library' + apply plugin: 'checkstyle' + apply plugin: "com.github.spotbugs" + + // We use the shadow plugin for the jmh-benchmarks module and the `-all` jar can get pretty large, so + // don't publish it + def shouldPublish = !project.name.equals('jmh-benchmarks') + def shouldPublishWithShadow = (['clients'].contains(project.name)) + + if (shouldPublish) { + apply plugin: 'maven-publish' + apply plugin: 'signing' + + // Add aliases for the task names used by the maven plugin for backwards compatibility + // The maven plugin was replaced by the maven-publish plugin in Gradle 7.0 + tasks.register('install').configure { dependsOn(publishToMavenLocal) } + tasks.register('uploadArchives').configure { dependsOn(publish) } + } + + // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. + if (!project.name.equals('connect')) { + apply plugin: 'eclipse' + fineTuneEclipseClasspathFile(eclipse, project) + } + + java { + consistentResolution { + // resolve the compileClasspath and then "inject" the result of resolution as strict constraints into the runtimeClasspath + useCompileClasspathVersions() + } + } + + tasks.withType(JavaCompile) { + configureJavaCompiler(name, options, project.path) + } + + if (shouldPublish) { + + publishing { + repositories { + // To test locally, invoke gradlew with `-PmavenUrl=file:///some/local/path` + maven { + url = mavenUrl + credentials { + username = mavenUsername + password = mavenPassword + } + } + } + publications { + mavenJava(MavenPublication) { + if (!shouldPublishWithShadow) { + from components.java + } else { + apply plugin: 'com.gradleup.shadow' + from components.shadow + + // Fix for avoiding inclusion of runtime dependencies marked as 'shadow' in MANIFEST Class-Path. + // https://github.com/GradleUp/shadow/issues/324 + afterEvaluate { + pom.withXml { xml -> + def dependenciesNode = xml.asNode().get('dependencies') ?: xml.asNode().appendNode('dependencies') + project.configurations.shadowed.allDependencies.each { + def dependencyNode = dependenciesNode.appendNode('dependency') + dependencyNode.appendNode('groupId', it.group) + dependencyNode.appendNode('artifactId', it.name) + dependencyNode.appendNode('version', it.version) + dependencyNode.appendNode('scope', 'runtime') + } + } + } + } + + afterEvaluate { + ["srcJar", "javadocJar", "scaladocJar", "testJar", "testSrcJar"].forEach { taskName -> + def task = tasks.findByName(taskName) + if (task != null) + artifact task + } + + artifactId = base.archivesName.get() + pom { + name = 'Apache Kafka' + url = 'https://kafka.apache.org' + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + distribution = 'repo' + } + } + } + } + } + } + } + + if (shouldSign) { + signing { + sign publishing.publications.mavenJava + } + } + } + + def testLoggingEvents = ["passed", "skipped", "failed"] + def testShowStandardStreams = false + def testExceptionFormat = 'full' + // Gradle built-in logging only supports sending test output to stdout, which generates a lot + // of noise, especially for passing tests. We really only want output for failed tests. This + // hooks into the output and logs it (so we don't have to buffer it all in memory) and only + // saves the output for failing tests. Directory and filenames are such that you can, e.g., + // create a Jenkins rule to collect failed test output. + def logTestStdout = { + def testId = { TestDescriptor descriptor -> + "${descriptor.className}.${descriptor.name}".toString() + } + + def logFiles = new HashMap() + def logStreams = new HashMap() + beforeTest { TestDescriptor td -> + def tid = testId(td) + // truncate the file name if it's too long + def logFile = new File( + "${projectDir}/build/reports/testOutput/${tid.substring(0, Math.min(tid.size(),240))}.test.stdout" + ) + logFile.parentFile.mkdirs() + logFiles.put(tid, logFile) + logStreams.put(tid, new FileOutputStream(logFile)) + } + onOutput { TestDescriptor td, TestOutputEvent toe -> + def tid = testId(td) + // Some output can happen outside the context of a specific test (e.g. at the class level) + // and beforeTest/afterTest seems to not be invoked for these cases (and similarly, there's + // a TestDescriptor hierarchy that includes the thread executing the test, Gradle tasks, + // etc). We see some of these in practice and it seems like something buggy in the Gradle + // test runner since we see it *before* any tests and it is frequently not related to any + // code in the test (best guess is that it is tail output from last test). We won't have + // an output file for these, so simply ignore them. If they become critical for debugging, + // they can be seen with showStandardStreams. + if (td.name == td.className || td.className == null) { + // silently ignore output unrelated to specific test methods + return + } else if (logStreams.get(tid) == null) { + println "WARNING: unexpectedly got output for a test [${tid}]" + + " that we didn't previously see in the beforeTest hook." + + " Message for debugging: [" + toe.message + "]." + return + } + try { + logStreams.get(tid).write(toe.message.getBytes(StandardCharsets.UTF_8)) + } catch (Exception e) { + println "ERROR: Failed to write output for test ${tid}" + e.printStackTrace() + } + } + afterTest { TestDescriptor td, TestResult tr -> + def tid = testId(td) + try { + logStreams.get(tid).close() + if (tr.resultType != TestResult.ResultType.FAILURE) { + logFiles.get(tid).delete() + } else { + def file = logFiles.get(tid) + println "${tid} failed, log available in ${file}" + } + } catch (Exception e) { + println "ERROR: Failed to close stdout file for ${tid}" + e.printStackTrace() + } finally { + logFiles.remove(tid) + logStreams.remove(tid) + } + } + } + + // The suites are for running sets of tests in IDEs. + // Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice. + def testsToExclude = ['**/*Suite.class'] + + + // This task will copy JUnit XML files out of the sub-project's build directory and into + // a top-level build/junit-xml directory. This is necessary to avoid reporting on tests which + // were not run, but instead were restored via FROM-CACHE. See KAFKA-17479 for more details. + def copyTestXml = tasks.register('copyTestXml') { + onlyIf("Environment GITHUB_ACTIONS is set") { isGithubActions } + onlyIf("Project '${project.name}:test' has sources") { ! test.state.noSource } + onlyIf("Task '${project.name}:test' did work") { test.state.didWork } + + ext { + output = project.findProperty("kafka.test.xml.output.dir") + } + + // Never cache this task + outputs.cacheIf { false } + outputs.upToDateWhen { false } + + doLast { + def moduleDirPath = projectToJUnitXmlPath(project) + def dest = rootProject.layout.buildDirectory.dir("junit-xml/${moduleDirPath}/${output}").get().asFile + println "Copy JUnit XML for ${project.name} to $dest" + ant.copy(todir: "$dest") { + ant.fileset(dir: "${test.reports.junitXml.entryPoint}") { + ant.include(name: "**/*.xml") + } + } + } + } + + test { + maxParallelForks = maxTestForks + ignoreFailures = userIgnoreFailures + + maxHeapSize = "3g" + jvmArgs = defaultJvmArgs + + // KAFKA-17433 Used by deflake.yml github action to repeat individual tests + systemProperty("kafka.cluster.test.repeat", project.findProperty("kafka.cluster.test.repeat")) + systemProperty("kafka.test.catalog.file", project.findProperty("kafka.test.catalog.file")) + systemProperty("kafka.test.run.new", project.findProperty("kafka.test.run.new")) + systemProperty("kafka.test.run.flaky", project.findProperty("kafka.test.run.flaky")) + systemProperty("kafka.test.verbose", project.findProperty("kafka.test.verbose")) + + testLogging { + events = userTestLoggingEvents ?: testLoggingEvents + showStandardStreams = userShowStandardStreams ?: testShowStandardStreams + exceptionFormat = testExceptionFormat + displayGranularity = 0 + } + logTestStdout.rehydrate(delegate, owner, this)() + + exclude testsToExclude + + useJUnitPlatform { + includeEngines 'junit-jupiter' + } + + develocity { + testRetry { + maxRetries = userMaxTestRetries + maxFailures = userMaxTestRetryFailures + } + } + + finalizedBy("copyTestXml") + } + + task integrationTest(type: Test, dependsOn: compileJava) { + maxParallelForks = maxTestForks + ignoreFailures = userIgnoreFailures + + // Increase heap size for integration tests + maxHeapSize = "2560m" + jvmArgs = defaultJvmArgs + + + testLogging { + events = userTestLoggingEvents ?: testLoggingEvents + showStandardStreams = userShowStandardStreams ?: testShowStandardStreams + exceptionFormat = testExceptionFormat + displayGranularity = 0 + } + logTestStdout.rehydrate(delegate, owner, this)() + + exclude testsToExclude + + useJUnitPlatform { + includeTags "integration" + includeEngines 'junit-jupiter' + } + + develocity { + testRetry { + maxRetries = userMaxTestRetries + maxFailures = userMaxTestRetryFailures + } + } + } + + task unitTest(type: Test, dependsOn: compileJava) { + maxParallelForks = maxTestForks + ignoreFailures = userIgnoreFailures + + maxHeapSize = defaultMaxHeapSize + jvmArgs = defaultJvmArgs + + testLogging { + events = userTestLoggingEvents ?: testLoggingEvents + showStandardStreams = userShowStandardStreams ?: testShowStandardStreams + exceptionFormat = testExceptionFormat + displayGranularity = 0 + } + logTestStdout.rehydrate(delegate, owner, this)() + + exclude testsToExclude + + useJUnitPlatform { + excludeTags "integration" + includeEngines 'junit-jupiter' + } + + develocity { + testRetry { + maxRetries = userMaxTestRetries + maxFailures = userMaxTestRetryFailures + } + } + } + + // remove test output from all test types + tasks.withType(Test).all { t -> + cleanTest { + delete t.reports.junitXml.outputLocation + delete t.reports.html.outputLocation + } + } + + jar { + from "$rootDir/LICENSE" + from "$rootDir/NOTICE" + } + + task srcJar(type: Jar) { + archiveClassifier = 'sources' + from "$rootDir/LICENSE" + from "$rootDir/NOTICE" + from sourceSets.main.allSource + } + + task javadocJar(type: Jar, dependsOn: javadoc) { + archiveClassifier = 'javadoc' + from "$rootDir/LICENSE" + from "$rootDir/NOTICE" + from javadoc.destinationDir + } + + task docsJar(dependsOn: javadocJar) + + check.dependsOn('javadoc') + + task systemTestLibs(dependsOn: jar) + + if (!sourceSets.test.allSource.isEmpty()) { + task testJar(type: Jar) { + archiveClassifier = 'test' + from "$rootDir/LICENSE" + from "$rootDir/NOTICE" + from sourceSets.test.output + // The junit-platform.properties file is used for configuring and customizing the behavior of the JUnit platform. + // It should only apply to Kafka's own JUnit tests, and should not exist in the test JAR. + // If we include it in the test JAR, it could lead to conflicts with user configurations. + exclude 'junit-platform.properties' + } + + task testSrcJar(type: Jar, dependsOn: testJar) { + archiveClassifier = 'test-sources' + from "$rootDir/LICENSE" + from "$rootDir/NOTICE" + from sourceSets.test.allSource + } + + } + + plugins.withType(ScalaPlugin) { + + scala { + zincVersion = versions.zinc + } + + task scaladocJar(type:Jar, dependsOn: scaladoc) { + archiveClassifier = 'scaladoc' + from "$rootDir/LICENSE" + from "$rootDir/NOTICE" + from scaladoc.destinationDir + } + + //documentation task should also trigger building scala doc jar + docsJar.dependsOn scaladocJar + + } + + tasks.withType(ScalaCompile) { + def releaseVersion = modulesNeedingJava11.any { project.path == it } ? minClientJavaVersion : minNonClientJavaVersion + scalaCompileOptions.keepAliveMode = userKeepAliveMode + + scalaCompileOptions.additionalParameters = [ + "-deprecation:false", + "-unchecked", + "-encoding", "utf8", + "-Xlog-reflective-calls", + "-feature", + "-language:postfixOps", + "-language:implicitConversions", + "-language:existentials", + "-Ybackend-parallelism", maxScalacThreads.toString(), + "-Xlint:constant", + "-Xlint:delayedinit-select", + "-Xlint:doc-detached", + "-Xlint:missing-interpolator", + "-Xlint:nullary-unit", + "-Xlint:option-implicit", + "-Xlint:package-object-classes", + "-Xlint:poly-implicit-overload", + "-Xlint:private-shadow", + "-Xlint:stars-align", + "-Xlint:type-parameter-shadow", + "-Xlint:unused" + ] + + // See README.md for details on this option and the meaning of each value + if (userScalaOptimizerMode.equals("method")) + scalaCompileOptions.additionalParameters += ["-opt:l:method"] + else if (userScalaOptimizerMode.startsWith("inline-")) { + List inlineFrom = ["-opt-inline-from:org.apache.kafka.**"] + if (project.name.equals('core')) + inlineFrom.add("-opt-inline-from:kafka.**") + if (userScalaOptimizerMode.equals("inline-scala")) + inlineFrom.add("-opt-inline-from:scala.**") + + scalaCompileOptions.additionalParameters += ["-opt:l:inline"] + scalaCompileOptions.additionalParameters += inlineFrom + } + + scalaCompileOptions.additionalParameters += ["-opt-warnings", "-Xlint:strict-unsealed-patmat"] + // Scala 2.13.2 introduces compiler warnings suppression, which is a pre-requisite for -Xfatal-warnings + scalaCompileOptions.additionalParameters += ["-Xfatal-warnings"] + scalaCompileOptions.additionalParameters += ["--release", String.valueOf(releaseVersion)] + + // Gradle does not support the `release` configuration when performing joint Java-Scala compilation. + // For more details, refer to https://github.com/gradle/gradle/issues/13762. + // As a result, we need to explicitly configure the Scala compiler with this setting. + options.compilerArgs += ["--release", String.valueOf(releaseVersion)] + + configureJavaCompiler(name, options, project.path) + + configure(scalaCompileOptions.forkOptions) { + memoryMaximumSize = defaultMaxHeapSize + jvmArgs = defaultJvmArgs + } + } + + checkstyle { + configDirectory = rootProject.layout.projectDirectory.dir("checkstyle") + configProperties = checkstyleConfigProperties("import-control.xml") + toolVersion = versions.checkstyle + } + + configure(checkstyleMain) { + group = 'Verification' + description = 'Run checkstyle on all main Java sources' + } + + configure(checkstyleTest) { + group = 'Verification' + description = 'Run checkstyle on all test Java sources' + } + + test.dependsOn('checkstyleMain', 'checkstyleTest') + + spotbugs { + toolVersion = versions.spotbugs + excludeFilter = file("$rootDir/gradle/spotbugs-exclude.xml") + ignoreFailures = false + } + test.dependsOn('spotbugsMain') + + tasks.withType(com.github.spotbugs.snom.SpotBugsTask).configureEach { + reports.configure { + // Continue supporting `xmlFindBugsReport` for compatibility + xml.enabled(project.hasProperty('xmlSpotBugsReport') || project.hasProperty('xmlFindBugsReport')) + html.enabled(!project.hasProperty('xmlSpotBugsReport') && !project.hasProperty('xmlFindBugsReport')) + } + maxHeapSize = defaultMaxHeapSize + jvmArgs = defaultJvmArgs + } + + // Ignore core since its a scala project + if (it.path != ':core') { + if (userEnableTestCoverage) { + apply plugin: "jacoco" + + jacoco { + toolVersion = versions.jacoco + } + + jacocoTestReport { + dependsOn tasks.test + sourceSets sourceSets.main + reports { + html.required = true + xml.required = true + csv.required = false + } + } + + } + } + + if (userEnableTestCoverage) { + def coverageGen = it.path == ':core' ? 'reportTestScoverage' : 'jacocoTestReport' + tasks.register('reportCoverage').configure { dependsOn(coverageGen) } + } + + dependencyCheck { + suppressionFile = "$rootDir/gradle/resources/dependencycheck-suppressions.xml" + skipProjects = [ ":jmh-benchmarks", ":trogdor" ] + skipConfigurations = [ "zinc" ] + } + apply plugin: 'com.diffplug.spotless' + spotless { + java { + targetExclude('**/generated/**/*.java','**/generated-test/**/*.java') + importOrder('kafka', 'org.apache.kafka', 'com', 'net', 'org', 'java', 'javax', '', '\\#') + removeUnusedImports() + } + } +} + +gradle.taskGraph.whenReady { taskGraph -> + taskGraph.getAllTasks().findAll { it.name.contains('spotbugsScoverage') || it.name.contains('spotbugsTest') }.each { task -> + task.enabled = false + } +} + +def fineTuneEclipseClasspathFile(eclipse, project) { + eclipse.classpath.file { + beforeMerged { cp -> + cp.entries.clear() + // for the core project add the directories defined under test/scala as separate source directories + if (project.name.equals('core')) { + cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/integration", null)) + cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/other", null)) + cp.entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("src/test/scala/unit", null)) + } + } + whenMerged { cp -> + // for the core project exclude the separate sub-directories defined under test/scala. These are added as source dirs above + if (project.name.equals('core')) { + cp.entries.findAll { it.kind == "src" && it.path.equals("src/test/scala") }*.excludes = ["integration/", "other/", "unit/"] + } + /* + * Set all eclipse build output to go to 'build_eclipse' directory. This is to ensure that gradle and eclipse use different + * build output directories, and also avoid using the eclipse default of 'bin' which clashes with some of our script directories. + * https://discuss.gradle.org/t/eclipse-generated-files-should-be-put-in-the-same-place-as-the-gradle-generated-files/6986/2 + */ + cp.entries.findAll { it.kind == "output" }*.path = "build_eclipse" + /* + * Some projects have explicitly added test output dependencies. These are required for the gradle build but not required + * in Eclipse since the dependent projects are added as dependencies. So clean up these from the generated classpath. + */ + cp.entries.removeAll { it.kind == "lib" && it.path.matches(".*/build/(classes|resources)/test") } + } + } +} + +def checkstyleConfigProperties(configFileName) { + [importControlFile: "$configFileName"] +} + +if (userEnableTestCoverage) { + tasks.register('reportCoverage').configure { dependsOn(subprojects.reportCoverage) } +} + +def connectPkgs = [ + 'connect:api', + 'connect:basic-auth-extension', + 'connect:file', + 'connect:json', + 'connect:runtime', + 'connect:test-plugins', + 'connect:transforms', + 'connect:mirror', + 'connect:mirror-client' +] + +tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" }) {} + +tasks.create(name: "testConnect", dependsOn: connectPkgs.collect { it + ":test" }) {} + +project(':server') { + base { + archivesName = "kafka-server" + } + + dependencies { + implementation project(':clients') + implementation project(':metadata') + implementation project(':server-common') + implementation project(':storage') + implementation project(':group-coordinator') + implementation project(':transaction-coordinator') + implementation project(':raft') + implementation project(':share-coordinator') + implementation project(':storage:storage-api') + implementation libs.jacksonDatabind + implementation libs.metrics + implementation libs.slf4jApi + + testImplementation project(':clients').sourceSets.test.output + + testImplementation libs.mockitoCore + testImplementation libs.junitJupiter + testImplementation testLog4j2Libs + testImplementation project(':test-common:test-common-internal-api') + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':storage:storage-api').sourceSets.test.output + + testRuntimeOnly runtimeTestLibs + } + + task createVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + outputs.cacheIf { true } + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + jar { + dependsOn createVersionFile + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "${layout.buildDirectory.get().asFile.path}/kafka/" + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-server.xml") + } + + javadoc { + enabled = false + } +} + +project(':core') { + apply plugin: 'scala' + + // scaladoc generation is configured at the sub-module level with an artifacts + // block (cf. see streams-scala). If scaladoc generation is invoked explicitly + // for the `core` module, this ensures the generated jar doesn't include scaladoc + // files since the `core` module doesn't include public APIs. + scaladoc { + enabled = false + } + if (userEnableTestCoverage) + apply plugin: "org.scoverage" + + base { + archivesName = "kafka_${versions.baseScala}" + } + + configurations { + // manually excludes some unnecessary dependencies + implementation.exclude module: 'javax' + implementation.exclude module: 'jline' + implementation.exclude module: 'jms' + implementation.exclude module: 'jmxri' + implementation.exclude module: 'jmxtools' + implementation.exclude module: 'mail' + // To prevent a UniqueResourceException due the same resource existing in both + // org.apache.directory.api/api-all and org.apache.directory.api/api-ldap-schema-data + testImplementation.exclude module: 'api-ldap-schema-data' + releaseOnly + } + + dependencies { + releaseOnly log4jReleaseLibs + // `core` is often used in users' tests, define the following dependencies as `api` for backwards compatibility + // even though the `core` module doesn't expose any public API + api project(':clients') + api libs.scalaLibrary + + compileOnly log4j2Libs + + implementation project(':server-common') + implementation project(':group-coordinator:group-coordinator-api') + implementation project(':group-coordinator') + implementation project(':transaction-coordinator') + implementation project(':metadata') + implementation project(':storage:storage-api') + implementation project(':tools:tools-api') + implementation project(':raft') + implementation project(':storage') + implementation project(':server') + implementation project(':coordinator-common') + implementation project(':share-coordinator') + + implementation libs.argparse4j + implementation libs.commonsValidator + implementation libs.jacksonDatabind + implementation libs.jacksonDataformatCsv + implementation libs.jacksonJDK8Datatypes + implementation libs.jacksonDatabindYaml + implementation libs.joptSimple + implementation libs.jose4j + implementation libs.metrics + // only needed transitively, but set it explicitly to ensure it has the same version as scala-library + implementation libs.scalaReflect + implementation libs.scalaLogging + implementation libs.slf4jApi + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':group-coordinator').sourceSets.test.output + testImplementation project(':share-coordinator').sourceSets.test.output + testImplementation project(':metadata').sourceSets.test.output + testImplementation project(':raft').sourceSets.test.output + testImplementation project(':server-common').sourceSets.test.output + testImplementation project(':storage:storage-api').sourceSets.test.output + testImplementation project(':server').sourceSets.test.output + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':test-common:test-common-internal-api') + testImplementation project(':test-common:test-common-util') + testImplementation libs.bcpkix + testImplementation libs.mockitoCore + testImplementation libs.jqwik + testImplementation(libs.apacheda) { + exclude group: 'xml-apis', module: 'xml-apis' + // `mina-core` is a transitive dependency for `apacheds` and `apacheda`. + // It is safer to use from `apacheds` since that is the implementation. + exclude module: 'mina-core' + } + testImplementation libs.apachedsCoreApi + testImplementation libs.apachedsInterceptorKerberos + testImplementation libs.apachedsProtocolShared + testImplementation libs.apachedsProtocolKerberos + testImplementation libs.apachedsProtocolLdap + testImplementation libs.apachedsLdifPartition + testImplementation libs.apachedsMavibotPartition + testImplementation libs.apachedsJdbmPartition + testImplementation libs.junitJupiter + testImplementation libs.caffeine + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + if (userEnableTestCoverage) { + scoverage { + scoverageVersion = versions.scoverage + if (versions.baseScala == '2.13') { + scoverageScalaVersion = '2.13.9' // there's no newer 2.13 artifact, org.scoverage:scalac-scoverage-plugin_2.13.9:2.0.11 is the latest as of now + } + reportDir = file("${layout.buildDirectory.get().asFile.path}/scoverage") + highlighting = false + minimumRate = 0.0 + } + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + task genProtocolErrorDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.common.protocol.Errors' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "protocol_errors.html").newOutputStream() + } + + task genProtocolTypesDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.common.protocol.types.Type' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "protocol_types.html").newOutputStream() + } + + task genProtocolApiKeyDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.common.protocol.ApiKeys' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "protocol_api_keys.html").newOutputStream() + } + + task genProtocolMessageDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.common.protocol.Protocol' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "protocol_messages.html").newOutputStream() + } + + task genAdminClientConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.clients.admin.AdminClientConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "admin_client_config.html").newOutputStream() + } + + task genProducerConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.clients.producer.ProducerConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "producer_config.html").newOutputStream() + } + + task genConsumerConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.clients.consumer.ConsumerConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "consumer_config.html").newOutputStream() + } + + task genKafkaConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'kafka.server.KafkaConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "kafka_config.html").newOutputStream() + } + + task genTopicConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.storage.internals.log.LogConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "topic_config.html").newOutputStream() + } + + task genConsumerMetricsDocs(type: JavaExec) { + classpath = sourceSets.test.runtimeClasspath + mainClass = 'org.apache.kafka.clients.consumer.internals.ConsumerMetrics' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "consumer_metrics.html").newOutputStream() + } + + task genProducerMetricsDocs(type: JavaExec) { + classpath = sourceSets.test.runtimeClasspath + mainClass = 'org.apache.kafka.clients.producer.internals.ProducerMetrics' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream() + } + + task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs', + 'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs', + 'genKafkaConfigDocs', 'genTopicConfigDocs', + ':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs', + ':connect:runtime:genConnectPredicateDocs', + ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs', + ':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs', + ':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs', + ':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs', + ':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs', + ':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) { + archiveClassifier = 'site-docs' + compression = Compression.GZIP + from project.file("$rootDir/docs") + into 'site-docs' + duplicatesStrategy 'exclude' + } + + tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) { + into "kafka_${versions.baseScala}-${archiveVersion.get()}" + compression = Compression.GZIP + from(project.file("$rootDir/bin")) { into "bin/" } + from(project.file("$rootDir/config")) { into "config/" } + from(project.file("$rootDir/licenses")) { into "licenses/" } + from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")} + from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")} + from(configurations.runtimeClasspath) { into("libs/") } + from(configurations.archives.artifacts.files) { into("libs/") } + from(configurations.releaseOnly) { into("libs/") } + from(project.siteDocsTar) { into("site-docs/") } + from(project(':tools').jar) { into("libs/") } + from(project(':tools').configurations.runtimeClasspath) { into("libs/") } + from(project(':trogdor').jar) { into("libs/") } + from(project(':trogdor').configurations.runtimeClasspath) { into("libs/") } + from(project(':shell').jar) { into("libs/") } + from(project(':shell').configurations.runtimeClasspath) { into("libs/") } + from(project(':connect:api').jar) { into("libs/") } + from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") } + from(project(':connect:runtime').jar) { into("libs/") } + from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") } + from(project(':connect:transforms').jar) { into("libs/") } + from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") } + from(project(':connect:json').jar) { into("libs/") } + from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") } + from(project(':connect:file').jar) { into("libs/") } + from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") } + from(project(':connect:basic-auth-extension').jar) { into("libs/") } + from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") } + from(project(':connect:mirror').jar) { into("libs/") } + from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") } + from(project(':connect:mirror-client').jar) { into("libs/") } + from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") } + from(project(':streams').jar) { into("libs/") } + from(project(':streams').configurations.runtimeClasspath) { into("libs/") } + from(project(':streams:streams-scala').jar) { into("libs/") } + from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") } + from(project(':streams:test-utils').jar) { into("libs/") } + from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") } + from(project(':streams:examples').jar) { into("libs/") } + from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") } + from(project(':tools:tools-api').jar) { into("libs/") } + from(project(':tools:tools-api').configurations.runtimeClasspath) { into("libs/") } + duplicatesStrategy 'exclude' + } + + jar { + dependsOn copyDependantLibs + } + + jar.manifest { + attributes( + 'Version': "${version}" + ) + } + + test { + useJUnitPlatform { + includeEngines 'jqwik', 'junit-jupiter' + } + } + + tasks.create(name: "copyDependantTestLibs", type: Copy) { + from (configurations.testRuntimeClasspath) { + include('*.jar') + } + from (configurations.releaseOnly) + into "${layout.buildDirectory.get().asFile.path}/dependant-testlibs" + //By default gradle does not handle test dependencies between the sub-projects + //This line is to include clients project test jar to dependant-testlibs + from (project(':clients').testJar ) { "${layout.buildDirectory.get().asFile.path}/dependant-testlibs" } + duplicatesStrategy 'exclude' + } + + systemTestLibs.dependsOn('jar', 'testJar', 'copyDependantTestLibs') + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-core.xml") + } + + sourceSets { + // Set java/scala source folders in the `scala` block to enable joint compilation + main { + java { + srcDirs = [] + } + scala { + srcDirs = ["src/main/java", "src/main/scala"] + } + } + test { + java { + srcDirs = [] + } + scala { + srcDirs = ["src/test/java", "src/test/scala"] + } + } + } +} + +project(':metadata') { + base { + archivesName = "kafka-metadata" + } + + configurations { + generator + } + + dependencies { + implementation project(':server-common') + implementation project(':clients') + implementation project(':raft') + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.metrics + implementation libs.slf4jApi + + testImplementation testLog4j2Libs + testImplementation libs.junitJupiter + testImplementation libs.jqwik + testImplementation libs.mockitoCore + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':raft').sourceSets.test.output + testImplementation project(':server-common').sourceSets.test.output + testImplementation project(':test-common:test-common-util') + + testRuntimeOnly runtimeTestLibs + + generator project(':generator') + } + + task processMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.common.metadata", + "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/common/metadata", + "-i", "src/main/resources/common/metadata", + "-m", "MessageDataGenerator", "JsonConverterGenerator", + "-t", "MetadataRecordTypeGenerator", "MetadataJsonConvertersGenerator" + ] + inputs.dir("src/main/resources/common/metadata") + .withPropertyName("messages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/common/metadata") + } + + compileJava.dependsOn 'processMessages' + srcJar.dependsOn 'processMessages' + + sourceSets { + main { + java { + srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + javadoc { + enabled = false + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-metadata.xml") + } +} + +project(':group-coordinator:group-coordinator-api') { + base { + archivesName = "kafka-group-coordinator-api" + } + + dependencies { + implementation project(':clients') + } + + task createVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + outputs.cacheIf { true } + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + jar { + dependsOn createVersionFile + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "${layout.buildDirectory.get().asFile.path}/kafka/" + } + + javadoc { + include "**/org/apache/kafka/coordinator/group/api/**" + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-group-coordinator.xml") + } +} + +project(':group-coordinator') { + base { + archivesName = "kafka-group-coordinator" + } + + configurations { + generator + } + + dependencies { + implementation project(':server-common') + implementation project(':clients') + implementation project(':metadata') + implementation project(':group-coordinator:group-coordinator-api') + implementation project(':storage') + implementation project(':coordinator-common') + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.metrics + implementation libs.hdrHistogram + implementation libs.re2j + implementation libs.slf4jApi + implementation libs.guava + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':server-common').sourceSets.test.output + testImplementation project(':coordinator-common').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + + generator project(':generator') + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + javadoc { + enabled = false + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-group-coordinator.xml") + } + + task processMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.coordinator.group.generated", + "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/group/generated", + "-i", "src/main/resources/common/message", + "-m", "MessageDataGenerator", "JsonConverterGenerator", + "-t", "CoordinatorRecordTypeGenerator", "CoordinatorRecordJsonConvertersGenerator" + ] + inputs.dir("src/main/resources/common/message") + .withPropertyName("messages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/group/generated") + } + + compileJava.dependsOn 'processMessages' + srcJar.dependsOn 'processMessages' +} + + +project(':test-common:test-common-internal-api') { + // Interfaces, config classes, and other test APIs. Java 17 is the minimum Java version. + base { + archivesName = "kafka-test-common-internal-api" + } + + dependencies { + implementation project(':server-common') // Only project dependency allowed + + implementation libs.junitJupiterApi + + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-test-common-internal-api.xml") + } + + javadoc { + enabled = false + } +} + +project(':test-common:test-common-util') { + // Runtime-only JUnit extensions for entire project. Java 11 is the minimum Java version required. + base { + archivesName = "kafka-test-common-util" + } + + dependencies { + implementation libs.junitPlatformLanucher + implementation libs.junitJupiterApi + implementation libs.junitJupiter + implementation libs.slf4jApi + testImplementation testLog4j2Libs + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-test-common-util.xml") + } + + javadoc { + enabled = false + } +} + +project(':test-common:test-common-runtime') { + // Runtime-only JUnit extensions for integration tests. Java 17 is the minimum Java version. + base { + archivesName = "kafka-test-common-runtime" + } + + dependencies { + api project(':core') + api project(':clients') + + implementation project(':server') + implementation project(':server-common') + implementation project(':group-coordinator') + implementation project(':test-common:test-common-internal-api') + implementation project(':metadata') + implementation project(':raft') + implementation project(':storage') + + implementation libs.junitPlatformLanucher + implementation libs.junitJupiter + implementation libs.jacksonDatabindYaml + implementation libs.slf4jApi + + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-test-common-runtime.xml") + } + + javadoc { + enabled = false + } +} + +project(':transaction-coordinator') { + base { + archivesName = "kafka-transaction-coordinator" + } + + configurations { + generator + } + + dependencies { + implementation libs.jacksonDatabind + implementation project(':clients') + implementation project(':server-common') + implementation project(':coordinator-common') + implementation libs.slf4jApi + + testImplementation testLog4j2Libs + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':test-common:test-common-internal-api') + + testRuntimeOnly runtimeTestLibs + + generator project(':generator') + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-transaction-coordinator.xml") + } + + task processMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.coordinator.transaction.generated", + "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/transaction/generated", + "-i", "src/main/resources/common/message", + "-m", "MessageDataGenerator", "JsonConverterGenerator", + "-t", "CoordinatorRecordTypeGenerator", "CoordinatorRecordJsonConvertersGenerator" + ] + inputs.dir("src/main/resources/common/message") + .withPropertyName("messages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/transaction/generated") + } + + compileJava.dependsOn 'processMessages' + srcJar.dependsOn 'processMessages' + + javadoc { + enabled = false + } +} + +project(':coordinator-common') { + base { + archivesName = "kafka-coordinator-common" + } + + dependencies { + implementation project(':clients') + implementation project(':server-common') + implementation project(':metadata') + implementation project(':storage') + implementation libs.slf4jApi + implementation libs.metrics + implementation libs.hdrHistogram + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':server-common').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-coordinator-common.xml") + } + + javadoc { + enabled = false + } +} + +project(':share-coordinator') { + base { + archivesName = "kafka-share-coordinator" + } + + configurations { + generator + } + + dependencies { + implementation libs.jacksonDatabind + implementation project(':clients') + implementation project(':coordinator-common') + implementation project(':metadata') + implementation project(':server-common') + implementation libs.metrics + implementation libs.slf4jApi + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':server-common').sourceSets.test.output + testImplementation project(':coordinator-common').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + + generator project(':generator') + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-share-coordinator.xml") + } + + task processMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.coordinator.share.generated", + "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/share/generated", + "-i", "src/main/resources/common/message", + "-m", "MessageDataGenerator", "JsonConverterGenerator", + "-t", "CoordinatorRecordTypeGenerator", "CoordinatorRecordJsonConvertersGenerator" + ] + inputs.dir("src/main/resources/common/message") + .withPropertyName("messages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/share/generated") + } + + compileJava.dependsOn 'processMessages' + srcJar.dependsOn 'processMessages' + + javadoc { + enabled = false + } +} + +project(':examples') { + base { + archivesName = "kafka-examples" + } + + dependencies { + implementation project(':clients') + } + + javadoc { + enabled = false + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-examples.xml") + } +} + +project(':generator') { + dependencies { + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.jacksonJakartarsJsonProvider + + implementation 'org.eclipse.jgit:org.eclipse.jgit:6.4.0.202211300538-r' + // SSH support for JGit based on Apache MINA sshd + implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.apache:6.4.0.202211300538-r' + // GPG support for JGit based on BouncyCastle (commit signing) + implementation 'org.eclipse.jgit:org.eclipse.jgit.gpg.bc:6.4.0.202211300538-r' + + testImplementation libs.junitJupiter + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } +} + +project(':clients') { + base { + archivesName = "kafka-clients" + } + + configurations { + generator + shadowed + } + + dependencies { + implementation libs.zstd + implementation libs.lz4 + implementation libs.snappy + implementation libs.opentelemetryProto + implementation libs.protobuf + implementation libs.slf4jApi + + // libraries which should be added as runtime dependencies in generated pom.xml should be defined here: + shadowed libs.zstd + shadowed libs.lz4 + shadowed libs.snappy + shadowed libs.slf4jApi + + compileOnly libs.jacksonDatabind // for SASL/OAUTHBEARER bearer token parsing + compileOnly libs.jacksonJDK8Datatypes + compileOnly libs.jose4j // for SASL/OAUTHBEARER JWT validation; only used by broker + + + testImplementation project(':test-common:test-common-util') + testImplementation libs.bcpkix + testImplementation libs.jacksonJakartarsJsonProvider + testImplementation libs.jose4j + testImplementation libs.junitJupiter + testImplementation libs.jqwik + testImplementation libs.spotbugs + testImplementation libs.mockitoCore + testImplementation libs.mockitoJunitJupiter // supports MockitoExtension + testImplementation testLog4j2Libs + + testCompileOnly libs.bndlib + + testRuntimeOnly libs.jacksonDatabind + testRuntimeOnly libs.jacksonJDK8Datatypes + testRuntimeOnly runtimeTestLibs + + generator project(':generator') + } + + tasks.withType(GenerateModuleMetadata) { + enabled = false + } + + task createVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + outputs.cacheIf { true } + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + shadowJar { + dependsOn createVersionFile + // archiveClassifier defines the classifier for the shadow jar, the default is 'all'. + // We don't want to use the default classifier because it will cause the shadow jar to + // overwrite the original jar. We also don't want to use the 'shadow' classifier because + // it will cause the shadow jar to be named kafka-clients-shadow.jar. We want to use the + // same name as the original jar, kafka-clients.jar. + archiveClassifier = null + // KIP-714: move shaded dependencies to a shaded location + relocate('io.opentelemetry.proto', 'org.apache.kafka.shaded.io.opentelemetry.proto') + relocate('com.google.protobuf', 'org.apache.kafka.shaded.com.google.protobuf') + + // dependencies excluded from the final jar, since they are declared as runtime dependencies + dependencies { + project.configurations.shadowed.allDependencies.each { + exclude(dependency(it)) + } + // exclude proto files from the jar + exclude "**/opentelemetry/proto/**/*.proto" + exclude "**/google/protobuf/*.proto" + } + + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildVersionFileName" + } + + from "$rootDir/LICENSE" + from "$rootDir/NOTICE" + } + + jar { + enabled false + dependsOn 'shadowJar' + } + + clean.doFirst { + delete "${layout.buildDirectory.get().asFile.path}/kafka/" + } + + task processMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.common.message", + "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/common/message", + "-i", "src/main/resources/common/message", + "-t", "ApiMessageTypeGenerator", + "-m", "MessageDataGenerator", "JsonConverterGenerator" + ] + inputs.dir("src/main/resources/common/message") + .withPropertyName("messages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/common/message") + } + + task processTestMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.common.message", + "-o", "${projectDir}/build/generated/test/java/org/apache/kafka/common/message", + "-i", "src/test/resources/common/message", + "-m", "MessageDataGenerator", "JsonConverterGenerator" + ] + inputs.dir("src/test/resources/common/message") + .withPropertyName("testMessages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/test/java/org/apache/kafka/common/message") + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java", "${projectDir}/build/generated/test/java"] + } + } + } + + compileJava.dependsOn 'processMessages' + srcJar.dependsOn 'processMessages' + + compileTestJava.dependsOn 'processTestMessages' + + javadoc { + include "**/org/apache/kafka/clients/admin/*" + include "**/org/apache/kafka/clients/consumer/*" + include "**/org/apache/kafka/clients/producer/*" + include "**/org/apache/kafka/common/*" + include "**/org/apache/kafka/common/acl/*" + include "**/org/apache/kafka/common/annotation/*" + include "**/org/apache/kafka/common/errors/*" + include "**/org/apache/kafka/common/header/*" + include "**/org/apache/kafka/common/metrics/*" + include "**/org/apache/kafka/common/metrics/stats/*" + include "**/org/apache/kafka/common/quota/*" + include "**/org/apache/kafka/common/resource/*" + include "**/org/apache/kafka/common/serialization/*" + include "**/org/apache/kafka/common/config/*" + include "**/org/apache/kafka/common/config/provider/*" + include "**/org/apache/kafka/common/security/auth/*" + include "**/org/apache/kafka/common/security/plain/*" + include "**/org/apache/kafka/common/security/scram/*" + include "**/org/apache/kafka/common/security/token/delegation/*" + include "**/org/apache/kafka/common/security/oauthbearer/*" + include "**/org/apache/kafka/common/security/oauthbearer/secured/*" + include "**/org/apache/kafka/server/authorizer/*" + include "**/org/apache/kafka/server/policy/*" + include "**/org/apache/kafka/server/quota/*" + include "**/org/apache/kafka/server/telemetry/*" + } +} + +project(':clients:clients-integration-tests') { + base { + archivesName = "kafka-clients-integration-tests" + } + + dependencies { + testImplementation libs.slf4jApi + testImplementation project(':test-common:test-common-internal-api') + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':metadata') + testImplementation project(':server') + testImplementation project(':storage') + testImplementation project(':core').sourceSets.test.output + testImplementation project(':clients').sourceSets.test.output + implementation project(':server-common') + testImplementation project(':metadata') + implementation project(':group-coordinator') + implementation project(':transaction-coordinator') + + testImplementation libs.junitJupiter + testImplementation libs.junitPlatformSuiteEngine + + testRuntimeOnly runtimeTestLibs + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-clients-integration-tests.xml") + } +} + + +project(':raft') { + base { + archivesName = "kafka-raft" + } + + configurations { + generator + } + + dependencies { + implementation project(':server-common') + implementation project(':clients') + implementation libs.jacksonDatabind + implementation libs.slf4jApi + + testImplementation project(':server-common') + testImplementation project(':server-common').sourceSets.test.output + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation libs.jqwik + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + + generator project(':generator') + } + + task createVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + outputs.cacheIf { true } + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + task processMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.raft.generated", + "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/raft/generated", + "-i", "src/main/resources/common/message", + "-m", "MessageDataGenerator", "JsonConverterGenerator"] + inputs.dir("src/main/resources/common/message") + .withPropertyName("messages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/raft/generated") + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + compileJava.dependsOn 'processMessages' + srcJar.dependsOn 'processMessages' + + jar { + dependsOn createVersionFile + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildVersionFileName" + } + } + + test { + useJUnitPlatform { + includeEngines 'jqwik', 'junit-jupiter' + } + } + + clean.doFirst { + delete "${layout.buildDirectory.get().asFile.path}/kafka/" + } + + javadoc { + enabled = false + } +} + +project(':server-common') { + base { + archivesName = "kafka-server-common" + } + + dependencies { + api project(':clients') + implementation libs.metrics + implementation libs.joptSimple + implementation libs.jacksonDatabind + implementation libs.pcollections + implementation libs.slf4jApi + + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + task createVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + outputs.cacheIf { true } + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + jar { + dependsOn createVersionFile + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "${layout.buildDirectory.get().asFile.path}/kafka/" + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-server-common.xml") + } + + javadoc { + enabled = false + } +} + +project(':storage:storage-api') { + base { + archivesName = "kafka-storage-api" + } + + dependencies { + implementation project(':clients') + implementation project(':server-common') + implementation libs.metrics + implementation libs.slf4jApi + + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + task createVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + outputs.cacheIf { true } + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + jar { + dependsOn createVersionFile + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "${layout.buildDirectory.get().asFile.path}/kafka/" + } + + javadoc { + include "**/org/apache/kafka/server/log/remote/storage/*" + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-storage.xml") + } +} + +project(':storage') { + base { + archivesName = "kafka-storage" + } + + configurations { + generator + } + + dependencies { + implementation project(':storage:storage-api') + implementation project(':server-common') + implementation project(':clients') + implementation(libs.caffeine) { + exclude group: 'org.checkerframework', module: 'checker-qual' + } + implementation libs.slf4jApi + implementation libs.jacksonDatabind + implementation libs.metrics + + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':core').sourceSets.test.output + testImplementation project(':storage:storage-api').sourceSets.test.output + testImplementation project(':test-common:test-common-internal-api') + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':test-common:test-common-util') + testImplementation project(':server') + testImplementation project(':server-common') + testImplementation project(':server-common').sourceSets.test.output + testImplementation project(':transaction-coordinator') + testImplementation libs.hamcrest + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation libs.bcpkix + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + + generator project(':generator') + } + + task createVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + outputs.cacheIf { true } + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + task processMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.server.log.remote.metadata.storage.generated", + "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/server/log/remote/metadata/storage/generated", + "-i", "src/main/resources/message", + "-m", "MessageDataGenerator", "JsonConverterGenerator", + "-t", "MetadataRecordTypeGenerator", "MetadataJsonConvertersGenerator" ] + inputs.dir("src/main/resources/message") + .withPropertyName("messages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/server/log/remote/metadata/storage/generated") + } + + task genRemoteLogManagerConfigDoc(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "remote_log_manager_config.html").newOutputStream() + } + + task genRemoteLogMetadataManagerConfigDoc(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "remote_log_metadata_manager_config.html").newOutputStream() + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + compileJava.dependsOn 'processMessages' + srcJar.dependsOn 'processMessages' + + jar { + dependsOn createVersionFile + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "${layout.buildDirectory.get().asFile.path}/kafka/" + } + + javadoc { + enabled = false + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-storage.xml") + } +} + +project(':tools:tools-api') { + base { + archivesName = "kafka-tools-api" + } + + dependencies { + implementation project(':clients') + testImplementation libs.junitJupiter + testImplementation testLog4j2Libs + testRuntimeOnly runtimeTestLibs + } + + task createVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + outputs.cacheIf { true } + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + jar { + dependsOn createVersionFile + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "${layout.buildDirectory.get().asFile.path}/kafka/" + } + + javadoc { + include "**/org/apache/kafka/tools/api/*" + } +} + +project(':tools') { + base { + archivesName = "kafka-tools" + } + + configurations { + releaseOnly + } + + dependencies { + releaseOnly log4jReleaseLibs + + implementation project(':clients') + implementation project(':metadata') + implementation project(':storage') + implementation project(':server') + implementation project(':server-common') + implementation project(':connect:runtime') + implementation project(':tools:tools-api') + implementation project(':transaction-coordinator') + implementation project(':group-coordinator') + implementation project(':coordinator-common') + implementation project(':share-coordinator') + implementation project(':raft') + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonDataformatCsv + implementation libs.jacksonJDK8Datatypes + implementation libs.joptSimple + implementation libs.slf4jApi + implementation libs.re2j + + implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation + implementation libs.jacksonJakartarsJsonProvider + + compileOnly libs.spotbugs + + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':server') + testImplementation project(':server').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':core').sourceSets.test.output + testImplementation project(':test-common:test-common-internal-api') + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':server-common') + testImplementation project(':server-common').sourceSets.test.output + testImplementation project(':connect:api') + testImplementation project(':connect:runtime') + testImplementation project(':connect:runtime').sourceSets.test.output + testImplementation project(':storage:storage-api').sourceSets.main.output + testImplementation project(':storage').sourceSets.test.output + testImplementation project(':streams') + testImplementation project(':streams').sourceSets.test.output + testImplementation project(':streams:integration-tests').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation libs.mockitoJunitJupiter // supports MockitoExtension + testImplementation libs.bcpkix // required by the clients test module, but we have to specify it explicitly as gradle does not include the transitive test dependency automatically + testImplementation(libs.jfreechart) { + exclude group: 'junit', module: 'junit' + } + testImplementation libs.apachedsCoreApi + testImplementation libs.apachedsInterceptorKerberos + testImplementation libs.apachedsProtocolShared + testImplementation libs.apachedsProtocolKerberos + testImplementation libs.apachedsProtocolLdap + testImplementation libs.apachedsLdifPartition + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + testRuntimeOnly libs.hamcrest + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + } + from (configurations.releaseOnly) + into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn 'copyDependantLibs' + } +} + +project(':trogdor') { + base { + archivesName = "trogdor" + } + + dependencies { + implementation project(':clients') + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.slf4jApi + + implementation libs.jacksonJakartarsJsonProvider + implementation libs.jerseyContainerServlet + implementation libs.jerseyHk2 + implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 + implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 + implementation (libs.jettyServer) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation (libs.jettyServlet) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation (libs.jettyServlets) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + + implementation project(':group-coordinator') + implementation project(':group-coordinator:group-coordinator-api') + + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':group-coordinator') + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + testRuntimeOnly libs.junitPlatformLanucher + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn 'copyDependantLibs' + } +} + +project(':shell') { + base { + archivesName = "kafka-shell" + } + + dependencies { + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.jline + implementation libs.slf4jApi + implementation project(':server-common') + implementation project(':clients') + implementation project(':core') + implementation project(':metadata') + implementation project(':raft') + + implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation + implementation libs.jacksonJakartarsJsonProvider + + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':server-common') + testImplementation project(':server-common').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + include('jline-*jar') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn 'copyDependantLibs' + } +} + +project(':streams') { + base { + archivesName = "kafka-streams" + } + + ext.buildStreamsVersionFileName = "kafka-streams-version.properties" + + configurations { + generator + } + + dependencies { + api project(path: ':clients', configuration: 'shadow') + // `org.rocksdb.Options` is part of Kafka Streams public api via `RocksDBConfigSetter` + api libs.rocksDBJni + + implementation libs.jacksonAnnotations + implementation libs.jacksonDatabind + implementation libs.slf4jApi + + // testCompileOnly prevents streams from exporting a dependency on test-utils, which would cause a dependency cycle + testCompileOnly project(':streams:test-utils') + testCompileOnly libs.bndlib + + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation libs.bcpkix + testImplementation libs.hamcrest + testImplementation libs.mockitoCore + testImplementation libs.mockitoJunitJupiter // supports MockitoExtension + testImplementation libs.junitPlatformSuiteEngine // supports suite test + testImplementation testLog4j2Libs + + testRuntimeOnly project(':streams:test-utils') + testRuntimeOnly runtimeTestLibs + + generator project(':generator') + } + + task processMessages(type:JavaExec) { + mainClass = "org.apache.kafka.message.MessageGenerator" + classpath = configurations.generator + args = [ "-p", "org.apache.kafka.streams.internals.generated", + "-o", "${projectDir}/build/generated/main/java/org/apache/kafka/streams/internals/generated", + "-i", "src/main/resources/common/message", + "-m", "MessageDataGenerator" + ] + inputs.dir("src/main/resources/common/message") + .withPropertyName("messages") + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.cacheIf { true } + outputs.dir("${projectDir}/build/generated/main/java/org/apache/kafka/streams/internals/generated") + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java", "${projectDir}/build/generated/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + compileJava.dependsOn 'processMessages' + srcJar.dependsOn 'processMessages' + + javadoc { + include "**/org/apache/kafka/streams/**" + exclude "**/org/apache/kafka/streams/internals/**", "**/org/apache/kafka/streams/**/internals/**" + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + task createStreamsVersionFile() { + def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildStreamsVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + jar { + dependsOn 'createStreamsVersionFile' + from("${layout.buildDirectory.get().asFile.path}") { + include "kafka/$buildStreamsVersionFileName" + } + dependsOn 'copyDependantLibs' + } + + systemTestLibs { + dependsOn testJar + } + + task genStreamsConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.streams.StreamsConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "streams_config.html").newOutputStream() + } + + task testAll( + dependsOn: [ + ':streams:test', + ':streams:integration-tests:test', + ':streams:test-utils:test', + ':streams:streams-scala:test', + ':streams:upgrade-system-tests-0110:test', + ':streams:upgrade-system-tests-10:test', + ':streams:upgrade-system-tests-11:test', + ':streams:upgrade-system-tests-20:test', + ':streams:upgrade-system-tests-21:test', + ':streams:upgrade-system-tests-22:test', + ':streams:upgrade-system-tests-23:test', + ':streams:upgrade-system-tests-24:test', + ':streams:upgrade-system-tests-25:test', + ':streams:upgrade-system-tests-26:test', + ':streams:upgrade-system-tests-27:test', + ':streams:upgrade-system-tests-28:test', + ':streams:upgrade-system-tests-30:test', + ':streams:upgrade-system-tests-31:test', + ':streams:upgrade-system-tests-32:test', + ':streams:upgrade-system-tests-33:test', + ':streams:upgrade-system-tests-34:test', + ':streams:upgrade-system-tests-35:test', + ':streams:upgrade-system-tests-36:test', + ':streams:upgrade-system-tests-37:test', + ':streams:upgrade-system-tests-38:test', + ':streams:upgrade-system-tests-39:test', + ':streams:upgrade-system-tests-40:test', + ':streams:examples:test' + ] + ) +} + +project(':streams:streams-scala') { + apply plugin: 'scala' + + base { + archivesName = "kafka-streams-scala_${versions.baseScala}" + } + + dependencies { + api project(':streams') + + api libs.scalaLibrary + testImplementation project(':streams').sourceSets.test.output + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':streams:test-utils') + + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation libs.mockitoJunitJupiter // supports MockitoExtension + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + include "**/org/apache/kafka/streams/scala/**" + } + + scaladoc { + scalaDocOptions.additionalParameters = ["-no-link-warnings"] + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-streams*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn 'copyDependantLibs' + } + + apply plugin: 'com.diffplug.spotless' + spotless { + scala { + target '**/*.scala' + scalafmt("$versions.scalafmt").configFile('../../checkstyle/.scalafmt.conf').scalaMajorVersion(versions.baseScala) + licenseHeaderFile '../../checkstyle/java.header', 'package' + } + } +} + +project(':streams:integration-tests') { + apply plugin: 'scala' + + base { + archivesName = "kafka-streams-integration-tests" + } + + dependencies { + implementation libs.slf4jApi + implementation libs.scalaLibrary + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':group-coordinator') + testImplementation project(':server') + testImplementation project(':server-common') + testImplementation project(':server-common').sourceSets.test.output + testImplementation project(':storage') + testImplementation project(':streams').sourceSets.test.output + testImplementation project(':streams:streams-scala') + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':tools') + testImplementation project(':transaction-coordinator') + testImplementation libs.bcpkix + testImplementation libs.hamcrest + testImplementation libs.junitJupiter + testImplementation libs.junitPlatformSuiteEngine // supports suite test + testImplementation libs.mockitoCore + testImplementation testLog4j2Libs + testImplementation project(':streams:test-utils') + + testRuntimeOnly runtimeTestLibs + } + + sourceSets { + // Set java/scala source folders in the `scala` block to enable joint compilation + main { + java { + srcDirs = [] + } + scala { + srcDirs = [] + } + } + test { + java { + srcDirs = [] + } + scala { + srcDirs = ["src/test/java", "src/test/scala"] + } + } + } +} + +project(':streams:test-utils') { + base { + archivesName = "kafka-streams-test-utils" + } + + dependencies { + api project(':streams') + api project(':clients') + + implementation libs.slf4jApi + + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation libs.hamcrest + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-streams*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn 'copyDependantLibs' + } + +} + +project(':streams:examples') { + base { + archivesName = "kafka-streams-examples" + } + + dependencies { + // this dependency should be removed after we unify data API + implementation(project(':connect:json')) + implementation project(':streams') + implementation libs.slf4jApi + + testImplementation project(':streams:test-utils') + testImplementation project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest + testImplementation libs.junitJupiter + testImplementation libs.hamcrest + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-streams*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn 'copyDependantLibs' + } +} + +project(':streams:upgrade-system-tests-0110') { + base{ + archivesName = "kafka-streams-upgrade-system-tests-0110" + } + + dependencies { + testImplementation libs.kafkaStreams_0110 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-10') { + base { + archivesName = "kafka-streams-upgrade-system-tests-10" + } + + dependencies { + testImplementation libs.kafkaStreams_10 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-11') { + base { + archivesName = "kafka-streams-upgrade-system-tests-11" + } + + dependencies { + testImplementation libs.kafkaStreams_11 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-20') { + base { + archivesName = "kafka-streams-upgrade-system-tests-20" + } + + dependencies { + testImplementation libs.kafkaStreams_20 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-21') { + base { + archivesName = "kafka-streams-upgrade-system-tests-21" + } + + dependencies { + testImplementation libs.kafkaStreams_21 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-22') { + base { + archivesName = "kafka-streams-upgrade-system-tests-22" + } + + dependencies { + testImplementation libs.kafkaStreams_22 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-23') { + base { + archivesName = "kafka-streams-upgrade-system-tests-23" + } + + dependencies { + testImplementation libs.kafkaStreams_23 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-24') { + base { + archivesName = "kafka-streams-upgrade-system-tests-24" + } + + dependencies { + testImplementation libs.kafkaStreams_24 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-25') { + base { + archivesName = "kafka-streams-upgrade-system-tests-25" + } + + dependencies { + testImplementation libs.kafkaStreams_25 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-26') { + base { + archivesName = "kafka-streams-upgrade-system-tests-26" + } + + dependencies { + testImplementation libs.kafkaStreams_26 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-27') { + base { + archivesName = "kafka-streams-upgrade-system-tests-27" + } + + dependencies { + testImplementation libs.kafkaStreams_27 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-28') { + base { + archivesName = "kafka-streams-upgrade-system-tests-28" + } + + dependencies { + testImplementation libs.kafkaStreams_28 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-30') { + base { + archivesName = "kafka-streams-upgrade-system-tests-30" + } + + dependencies { + testImplementation libs.kafkaStreams_30 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-31') { + base { + archivesName = "kafka-streams-upgrade-system-tests-31" + } + + dependencies { + testImplementation libs.kafkaStreams_31 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-32') { + base { + archivesName = "kafka-streams-upgrade-system-tests-32" + } + + dependencies { + testImplementation libs.kafkaStreams_32 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-33') { + base { + archivesName = "kafka-streams-upgrade-system-tests-33" + } + + dependencies { + testImplementation libs.kafkaStreams_33 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-34') { + base { + archivesName = "kafka-streams-upgrade-system-tests-34" + } + + dependencies { + testImplementation libs.kafkaStreams_34 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-35') { + base { + archivesName = "kafka-streams-upgrade-system-tests-35" + } + + dependencies { + testImplementation libs.kafkaStreams_35 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-36') { + base { + archivesName = "kafka-streams-upgrade-system-tests-36" + } + + dependencies { + testImplementation libs.kafkaStreams_36 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-37') { + base { + archivesName = "kafka-streams-upgrade-system-tests-37" + } + + dependencies { + testImplementation libs.kafkaStreams_37 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-38') { + base { + archivesName = "kafka-streams-upgrade-system-tests-38" + } + + dependencies { + testImplementation libs.kafkaStreams_38 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-39') { + base { + archivesName = "kafka-streams-upgrade-system-tests-39" + } + + dependencies { + testImplementation libs.kafkaStreams_39 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-40') { + base { + archivesName = "kafka-streams-upgrade-system-tests-40" + } + + dependencies { + testImplementation libs.kafkaStreams_40 + testRuntimeOnly libs.junitJupiter + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':jmh-benchmarks') { + + apply plugin: 'com.gradleup.shadow' + + shadowJar { + archiveBaseName = 'kafka-jmh-benchmarks' + } + + dependencies { + implementation(project(':core')) { + // jmh requires jopt 4.x while `core` depends on 5.0, they are not binary compatible + exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' + } + implementation project(':server-common') + implementation project(':server') + implementation project(':raft') + implementation project(':clients') + implementation project(':coordinator-common') + implementation project(':group-coordinator') + implementation project(':group-coordinator:group-coordinator-api') + implementation project(':metadata') + implementation project(':storage') + implementation project(':streams') + implementation project(':transaction-coordinator') + implementation project(':core') + implementation project(':connect:api') + implementation project(':connect:transforms') + implementation project(':connect:json') + implementation project(':clients').sourceSets.test.output + implementation project(':core').sourceSets.test.output + implementation project(':server-common').sourceSets.test.output + implementation project(':metadata').sourceSets.test.output + + implementation libs.jmhCore + annotationProcessor libs.jmhGeneratorAnnProcess + implementation libs.jmhCoreBenchmarks + implementation libs.jacksonDatabind + implementation libs.metrics + implementation libs.mockitoCore + implementation libs.scalaLibrary + implementation libs.slf4jApi + } + + tasks.withType(JavaCompile) { + // Suppress warning caused by code generated by jmh: `warning: [cast] redundant cast to long` + options.compilerArgs << "-Xlint:-cast" + } + + jar { + manifest { + attributes "Main-Class": "org.openjdk.jmh.Main" + } + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-jmh-benchmarks.xml") + } + + task jmh(type: JavaExec, dependsOn: [':jmh-benchmarks:clean', ':jmh-benchmarks:shadowJar']) { + + mainClass = "-jar" + + doFirst { + if (System.getProperty("jmhArgs")) { + args System.getProperty("jmhArgs").split(' ') + } + args = [shadowJar.archivePath, *args] + } + } + + javadoc { + enabled = false + } +} + +project(':connect:api') { + base { + archivesName = "connect-api" + } + + dependencies { + api project(':clients') + implementation libs.jakartaRsApi + implementation libs.slf4jApi + + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + include "**/org/apache/kafka/connect/**" // needed for the `aggregatedJavadoc` task + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + exclude('connect-*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn copyDependantLibs + } +} + +project(':connect:transforms') { + base { + archivesName = "connect-transforms" + } + + dependencies { + api project(':connect:api') + + implementation libs.slf4jApi + + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output + testImplementation testLog4j2Libs + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + exclude('connect-*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn copyDependantLibs + } +} + +project(':connect:json') { + base { + archivesName = "connect-json" + } + + dependencies { + api project(':connect:api') + + api libs.jacksonDatabind + api libs.jacksonJDK8Datatypes + api libs.jacksonBlackbird + + implementation libs.slf4jApi + + testImplementation libs.junitJupiter + testImplementation testLog4j2Libs + testImplementation project(':clients').sourceSets.test.output + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + exclude('connect-*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn copyDependantLibs + } +} + +project(':connect:runtime') { + configurations { + swagger + } + + base { + archivesName = "connect-runtime" + } + + dependencies { + // connect-runtime is used in tests, use `api` for modules below for backwards compatibility even though + // applications should generally not depend on `connect-runtime` + api project(':connect:api') + api project(':clients') + api project(':connect:json') + api project(':connect:transforms') + + compileOnly log4j2Libs + + implementation libs.slf4jApi + implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation + implementation libs.jacksonAnnotations + implementation libs.jacksonJakartarsJsonProvider + implementation libs.jerseyContainerServlet + implementation libs.jerseyHk2 + implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 + implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 + implementation (libs.jettyServer) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation (libs.jettyServlet) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation (libs.jettyServlets) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation (libs.jettyClient) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation libs.classgraph + implementation libs.mavenArtifact + implementation libs.swaggerAnnotations + + compileOnly libs.bndlib + compileOnly libs.spotbugs + + // We use this library to generate OpenAPI docs for the REST API, but we don't want or need it at compile + // or run time. So, we add it to a separate configuration, which we use later on during docs generation + swagger libs.jakartaServletApi + swagger libs.jaxrs2Jakarta + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':server') + testImplementation project(':metadata') + testImplementation project(':server-common') + testImplementation project(':test-common:test-common-internal-api') + testImplementation project(':test-common:test-common-util') + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':server-common') + testImplementation project(':server') + testImplementation project(':group-coordinator') + testImplementation project(':storage') + testImplementation project(':connect:test-plugins') + testImplementation project(':server-common').sourceSets.test.output + + testImplementation libs.jacksonDatabindYaml + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation libs.mockitoJunitJupiter + testImplementation libs.httpclient + testImplementation testLog4j2Libs + + testCompileOnly libs.bndlib + + testRuntimeOnly libs.bcpkix + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + exclude('connect-*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn copyDependantLibs + } + + task genConnectConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.runtime.distributed.DistributedConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "connect_config.html").newOutputStream() + } + + task genSinkConnectorConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.runtime.SinkConnectorConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "sink_connector_config.html").newOutputStream() + } + + task genSourceConnectorConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.runtime.SourceConnectorConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "source_connector_config.html").newOutputStream() + } + + task genConnectTransformationDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.tools.TransformationDoc' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "connect_transforms.html").newOutputStream() + } + + task genConnectPredicateDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.tools.PredicateDoc' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "connect_predicates.html").newOutputStream() + } + + task genConnectMetricsDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.runtime.ConnectMetrics' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "connect_metrics.html").newOutputStream() + } + + task setVersionInOpenAPISpec(type: Copy) { + from "$rootDir/gradle/openapi.template" + into "${layout.buildDirectory.get().asFile.path}/resources/docs" + rename ('openapi.template', 'openapi.yaml') + expand(kafkaVersion: "$rootProject.version") + } + + task genConnectOpenAPIDocs(type: io.swagger.v3.plugins.gradle.tasks.ResolveTask, dependsOn: setVersionInOpenAPISpec) { + classpath = sourceSets.main.runtimeClasspath + + buildClasspath = classpath + configurations.swagger + outputFileName = 'connect_rest' + outputFormat = 'YAML' + prettyPrint = 'TRUE' + sortOutput = 'TRUE' + openApiFile = file("${layout.buildDirectory.get().asFile.path}/resources/docs/openapi.yaml") + resourcePackages = ['org.apache.kafka.connect.runtime.rest.resources'] + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + outputDir = file(generatedDocsDir) + } + +} + +project(':connect:file') { + base { + archivesName = "connect-file" + } + + dependencies { + implementation project(':connect:api') + implementation libs.slf4jApi + + testImplementation testLog4j2Libs + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':connect:runtime') + testImplementation project(':connect:runtime').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':server-common').sourceSets.test.output + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + exclude('connect-*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn copyDependantLibs + } +} + +project(':connect:basic-auth-extension') { + base { + archivesName = "connect-basic-auth-extension" + } + + dependencies { + implementation project(':connect:api') + + implementation libs.slf4jApi + implementation libs.jakartaRsApi + implementation libs.jaxAnnotationApi + + testImplementation libs.bcpkix + testImplementation libs.mockitoCore + testImplementation libs.junitJupiter + testImplementation testLog4j2Libs + testImplementation project(':clients').sourceSets.test.output + + testRuntimeOnly libs.jerseyContainerServlet + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + exclude('connect-*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn copyDependantLibs + } +} + +project(':connect:mirror') { + base { + archivesName = "connect-mirror" + } + + dependencies { + implementation project(':connect:api') + implementation project(':connect:runtime') + implementation project(':connect:mirror-client') + implementation project(':clients') + + implementation libs.argparse4j + implementation libs.slf4jApi + implementation libs.jacksonAnnotations + implementation libs.jacksonJakartarsJsonProvider + implementation libs.jerseyContainerServlet + implementation libs.jerseyHk2 + implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 + implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 + implementation (libs.jettyServer) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation (libs.jettyServlet) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation (libs.jettyServlets) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation (libs.jettyClient) { + exclude group: 'org.slf4j', module: 'slf4j-api' + } + implementation libs.swaggerAnnotations + + testImplementation testLog4j2Libs + testImplementation libs.junitJupiter + testImplementation libs.bndlib + testImplementation libs.mockitoCore + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':connect:runtime').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':test-common:test-common-runtime') + testImplementation project(':server') + testImplementation project(':server-common') + + + testRuntimeOnly project(':connect:runtime') + testRuntimeOnly libs.bcpkix + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + exclude('connect-*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs" + duplicatesStrategy 'exclude' + } + + task genMirrorConnectorConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.mirror.MirrorConnectorConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "mirror_connector_config.html").newOutputStream() + } + + task genMirrorSourceConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.mirror.MirrorSourceConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "mirror_source_config.html").newOutputStream() + } + + task genMirrorCheckpointConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.mirror.MirrorCheckpointConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "mirror_checkpoint_config.html").newOutputStream() + } + + task genMirrorHeartbeatConfigDocs(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.apache.kafka.connect.mirror.MirrorHeartbeatConfig' + if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() } + standardOutput = new File(generatedDocsDir, "mirror_heartbeat_config.html").newOutputStream() + } + + jar { + dependsOn copyDependantLibs + } +} + +project(':connect:mirror-client') { + base { + archivesName = "connect-mirror-client" + } + + dependencies { + implementation project(':clients') + implementation libs.slf4jApi + + testImplementation testLog4j2Libs + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output + + testRuntimeOnly runtimeTestLibs + } + + javadoc { + enabled = true + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.runtimeClasspath) { + exclude('kafka-clients*') + exclude('connect-*') + } + into "${layout.buildDirectory.get().asFile.path}/dependant-libs" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn copyDependantLibs + } +} + +project(':connect:test-plugins') { + base { + archivesName = "connect-test-plugins" + } + + dependencies { + api project(':connect:api') + + implementation project(':server-common') + implementation libs.slf4jApi + implementation libs.jacksonDatabind + + testImplementation testLog4j2Libs + } +} + +task aggregatedJavadoc(type: Javadoc, dependsOn: compileJava) { + def projectsWithJavadoc = subprojects.findAll { it.javadoc.enabled } + source = projectsWithJavadoc.collect { it.sourceSets.main.allJava } + classpath = files(projectsWithJavadoc.collect { it.sourceSets.main.compileClasspath }) + includes = projectsWithJavadoc.collectMany { it.javadoc.getIncludes() } + excludes = projectsWithJavadoc.collectMany { it.javadoc.getExcludes() } +}