Skip to content

Commit 27cab9f

Browse files
nicoweidnernicoweidner
authored
DE-8810: Pipeline SDK shouldn't mark kafka as a dependency (#81)
* DE-8810: Set Kafka connector dependency to compileOnly so we can bundle our own fork * DE-8810: Add testImplementation dependency on Kafka connector * DE-8810: Add explicit kafka-clients dependency to example project * DE-8810: Change kafka-clients dependency in example project to flink-connector-kafka * DE-8810: Add explicit matrix for Flink and Kafka connector versions in workflow * DE-8810: Fix project dir variable * DE-8810: More workflow fixes * DE-8810: Revert 1.20 dependency changes to see if that fixes the build * DE-8810: Don't include aws-msk-iam-auth in SDK * DE-8810: Instead of automagically excluding compileOnly dependencies from shadow, use explicit configuration excludeFromShadow for that purpose; make flink-connector-kafka a testImplementation dependency of the example project as it is now bundled with our Flink images * DE-8810: Do not exclude transitive dependencies of flink-connector-kafka from example (that would exclude Jackson) * DE-8810: Fix Kafka connector version * DE-8810: Fix Maven build dependencies * DE-8810: Move flink-connector-kafka to compileOnly and testImplementation for 1.20 * DE-8810: Remove unnecessary dependencies from tests
1 parent 04c53e7 commit 27cab9f

File tree

7 files changed

+82
-30
lines changed

7 files changed

+82
-30
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616

1717
strategy:
1818
matrix:
19-
value: [flink-1.16,flink-1.18,flink-1.19,flink-1.20]
19+
value: [{flink: 1.16.3, kafka-connector: 1.16.3, project-dir: flink-1.16},{flink: 1.18.1, kafka-connector: 3.2.0-1.18, project-dir: flink-1.18},{flink: 1.19.2, kafka-connector: 3.3.0-1.19, project-dir: flink-1.19},{flink: 1.20.1, kafka-connector: 3.4.0-1.20, project-dir: flink-1.20}]
2020

2121
steps:
2222
- name: 'Check out repository'
@@ -45,17 +45,21 @@ jobs:
4545
id: version
4646
run: |
4747
RELEASE_VERSION="1.0.0.ci"
48-
FLINK_VERSION=`cd sdk/${{ matrix.value }} && grep "def flinkVersion" build.gradle | sed "s/.*'\([0-9.]*\)'.*/\1/"`
48+
FLINK_VERSION=${{ matrix.value.flink }}
49+
KAFKA_CONNECTOR_VERSION=${{ matrix.value.kafka-connector }}
4950

50-
echo "version=$FLINK_VERSION-$RELEASE_VERSION" > sdk/${{ matrix.value }}/gradle.properties
51+
echo "version=$FLINK_VERSION-$RELEASE_VERSION" > sdk/${{ matrix.value.project-dir }}/gradle.properties
5152
sed -i 's/<decodable.pipeline.sdk.version>.*<\/decodable.pipeline.sdk.version>/<decodable.pipeline.sdk.version>'"$FLINK_VERSION-$RELEASE_VERSION"'<\/decodable.pipeline.sdk.version>/g' examples/custom-pipelines-hello-world/pom.xml
5253
sed -i 's/sdkVersion = \x27.*\x27/sdkVersion = \x27'"$FLINK_VERSION-$RELEASE_VERSION"'\x27/g' examples/custom-pipelines-hello-world/build.gradle
5354

5455
sed -i 's/<flink.version>.*<\/flink.version>/<flink.version>'"$FLINK_VERSION"'<\/flink.version>/g' examples/custom-pipelines-hello-world/pom.xml
5556
sed -i 's/flinkVersion = \x27.*\x27/flinkVersion = \x27'"$FLINK_VERSION"'\x27/g' examples/custom-pipelines-hello-world/build.gradle
5657

58+
sed -i 's/<flink.kafka.connector.version>.*<\/flink.kafka.connector.version>/<flink.kafka.connector.version>'"$KAFKA_CONNECTOR_VERSION"'<\/flink.kafka.connector.version>/g' examples/custom-pipelines-hello-world/pom.xml
59+
sed -i 's/kafkaConnectorVersion = \x27.*\x27/kafkaConnectorVersion = \x27'"$KAFKA_CONNECTOR_VERSION"'\x27/g' examples/custom-pipelines-hello-world/build.gradle
60+
5761
- name: Build SDK
58-
run: cd sdk/${{ matrix.value }} && ../gradlew build publishToMavenLocal
62+
run: cd sdk/${{ matrix.value.project-dir }} && ../gradlew build publishToMavenLocal
5963

6064
- name: Build example project with Maven
6165
run: cd examples/custom-pipelines-hello-world && ./mvnw -B clean verify

‎examples/custom-pipelines-hello-world/build.gradle‎

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ version = '0.2'
1717

1818
ext {
1919
flinkVersion = '1.20.1'
20+
kafkaConnectorVersion = '3.4.0-1.20'
2021
log4jVersion = '2.17.1'
2122
sdkVersion = '1.20.1-1.0.0.Beta8'
2223
}
@@ -30,20 +31,43 @@ application {
3031
mainClass = 'co.decodable.examples.cpdemo.DataStreamJob'
3132
}
3233

34+
configurations {
35+
// Dependencies which are provided by Flink at runtime should be excluded from the uber JAR.
36+
// We use a custom configuration for this, which is similar to compileOnly except that we exclude these AND THEIR
37+
// TRANSITIVE DEPENDENCIES from shadowJar further down
38+
excludeFromShadow
39+
}
40+
41+
tasks.withType(JavaCompile).configureEach {
42+
// Include the excludeFromShadow configuration in the compile classpath
43+
classpath += configurations.excludeFromShadow
44+
}
45+
46+
tasks.withType(Javadoc).configureEach {
47+
// Include the excludeFromShadow configuration in the compile classpath
48+
classpath += configurations.excludeFromShadow
49+
}
50+
3351
dependencies {
3452
annotationProcessor "co.decodable:decodable-pipeline-sdk:$sdkVersion"
3553

3654
implementation "co.decodable:decodable-pipeline-sdk:$sdkVersion"
3755
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
3856
implementation "org.apache.flink:flink-json:$flinkVersion"
3957

40-
compileOnly "org.apache.flink:flink-streaming-java:$flinkVersion"
41-
compileOnly "org.apache.flink:flink-table-api-java:$flinkVersion"
42-
compileOnly "org.apache.flink:flink-table-api-java-bridge:$flinkVersion"
43-
compileOnly "org.apache.flink:flink-table-planner_2.12:$flinkVersion"
44-
compileOnly "org.apache.flink:flink-table-common:$flinkVersion"
45-
compileOnly "org.apache.flink:flink-table-runtime:$flinkVersion"
58+
// Kafka connector is bundled with the Flink image used by Decodable
59+
// Transitive dependencies of this should NOT be excluded (they contain Jackson, for instance)
60+
compileOnly "org.apache.flink:flink-connector-kafka:$kafkaConnectorVersion"
61+
62+
excludeFromShadow "org.apache.flink:flink-streaming-java:$flinkVersion"
63+
excludeFromShadow "org.apache.flink:flink-table-api-java:$flinkVersion"
64+
excludeFromShadow "org.apache.flink:flink-table-api-java-bridge:$flinkVersion"
65+
excludeFromShadow "org.apache.flink:flink-table-planner_2.12:$flinkVersion"
66+
excludeFromShadow "org.apache.flink:flink-table-common:$flinkVersion"
67+
excludeFromShadow "org.apache.flink:flink-table-runtime:$flinkVersion"
4668

69+
testImplementation "org.apache.flink:flink-connector-kafka:$kafkaConnectorVersion"
70+
testImplementation "commons-codec:commons-codec:1.18.0"
4771
testImplementation "org.apache.flink:flink-clients:$flinkVersion"
4872
testImplementation "org.apache.flink:flink-table-api-java:$flinkVersion"
4973
testImplementation "org.apache.flink:flink-table-api-java-bridge:$flinkVersion"
@@ -71,18 +95,10 @@ tasks.named('test') {
7195
useJUnitPlatform()
7296
}
7397

74-
configurations {
75-
// compileOnly-scoped dependencies should be excluded from the uber JAR,
76-
// but the compileOnly scope is not allowed to be resolved by Gradle.
77-
// so we're deriving a configuration from it which we actually can resolve.
78-
provided
79-
provided.extendsFrom compileOnly
80-
}
81-
8298
shadowJar {
8399
def exclude_modules = project
84100
.configurations
85-
.provided
101+
.excludeFromShadow
86102
.resolvedConfiguration
87103
.getLenientConfiguration()
88104
.getAllModuleDependencies()

‎examples/custom-pipelines-hello-world/pom.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
<maven.compiler.source>${target.java.version}</maven.compiler.source>
2727
<maven.compiler.target>${target.java.version}</maven.compiler.target>
2828
<decodable.pipeline.sdk.version>1.20.1-1.0.0.Beta8</decodable.pipeline.sdk.version>
29+
<flink.kafka.connector.version>3.4.0-1.20</flink.kafka.connector.version>
30+
<aws.msk.iam.auth.version>1.1.6</aws.msk.iam.auth.version>
2931
<flink.version>1.20.1</flink.version>
3032
<log4j.version>2.17.1</log4j.version>
3133
</properties>
@@ -47,6 +49,12 @@
4749
<version>${flink.version}</version>
4850
</dependency>
4951

52+
<dependency>
53+
<groupId>org.apache.flink</groupId>
54+
<artifactId>flink-connector-kafka</artifactId>
55+
<version>${flink.kafka.connector.version}</version>
56+
<scope>provided</scope>
57+
</dependency>
5058
<dependency>
5159
<groupId>org.apache.flink</groupId>
5260
<artifactId>flink-streaming-java</artifactId>
@@ -84,6 +92,18 @@
8492
<scope>provided</scope>
8593
</dependency>
8694

95+
<dependency>
96+
<groupId>org.apache.flink</groupId>
97+
<artifactId>flink-connector-kafka</artifactId>
98+
<version>${flink.kafka.connector.version}</version>
99+
<scope>test</scope>
100+
</dependency>
101+
<dependency>
102+
<groupId>commons-codec</groupId>
103+
<artifactId>commons-codec</artifactId>
104+
<version>1.18.0</version>
105+
<scope>test</scope>
106+
</dependency>
87107
<dependency>
88108
<groupId>org.apache.flink</groupId>
89109
<artifactId>flink-clients</artifactId>

‎sdk/flink-1.16/build.gradle‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@ dependencies {
1010
def flinkVersion = '1.16.3'
1111

1212
api "org.apache.flink:flink-core:$flinkVersion"
13-
implementation "org.apache.flink:flink-connector-kafka:$flinkVersion"
1413
implementation "org.apache.flink:flink-streaming-java:$flinkVersion"
1514
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
16-
implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.6'
1715

16+
// A Kafka connector with custom patches is supplied by the Decodable platform at runtime
17+
compileOnly "org.apache.flink:flink-connector-kafka:$flinkVersion"
18+
19+
testImplementation "org.apache.flink:flink-connector-kafka:$flinkVersion"
1820
testImplementation "org.apache.flink:flink-clients:$flinkVersion"
1921
testImplementation "org.apache.flink:flink-table-api-java:$flinkVersion"
2022
testImplementation "org.apache.flink:flink-table-api-java-bridge:$flinkVersion"

‎sdk/flink-1.18/build.gradle‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,14 @@ dependencies {
1212
api "org.apache.flink:flink-core:$flinkVersion"
1313
//NOTE: flink-connector-base is needed as of Flink 1.18
1414
implementation "org.apache.flink:flink-connector-base:$flinkVersion"
15-
//NOTE: flink-connector-kafka released with different versioning schema starting with Flink 1.18
16-
implementation "org.apache.flink:flink-connector-kafka:3.2.0-1.18"
1715
implementation "org.apache.flink:flink-streaming-java:$flinkVersion"
1816
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
19-
implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.6'
2017

18+
// NOTE: flink-connector-kafka released with different versioning schema starting with Flink 1.18
19+
// A Kafka connector with custom patches is supplied by the Decodable platform at runtime
20+
compileOnly "org.apache.flink:flink-connector-kafka:3.2.0-1.18"
21+
22+
testImplementation "org.apache.flink:flink-connector-kafka:3.2.0-1.18"
2123
testImplementation "org.apache.flink:flink-clients:$flinkVersion"
2224
testImplementation "org.apache.flink:flink-table-api-java:$flinkVersion"
2325
testImplementation "org.apache.flink:flink-table-api-java-bridge:$flinkVersion"

‎sdk/flink-1.19/build.gradle‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,16 @@ dependencies {
1212
api "org.apache.flink:flink-core:$flinkVersion"
1313
//NOTE: flink-connector-base is needed as of Flink 1.18
1414
implementation "org.apache.flink:flink-connector-base:$flinkVersion"
15-
//NOTE: flink-connector-kafka released with different versioning schema starting with Flink 1.18
16-
implementation "org.apache.flink:flink-connector-kafka:3.3.0-1.19"
1715
implementation "org.apache.flink:flink-streaming-java:$flinkVersion"
1816
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
19-
implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.6'
2017

18+
// NOTE: flink-connector-kafka released with different versioning schema starting with Flink 1.18
19+
// A Kafka connector with custom patches is supplied by the Decodable platform at runtime
20+
compileOnly "org.apache.flink:flink-connector-kafka:3.3.0-1.19"
21+
22+
// For some reason, with Flink 1.18+, the Redpanda testcontainer requires commons-codec to be added explicitly
23+
testImplementation "commons-codec:commons-codec:1.18.0"
24+
testImplementation "org.apache.flink:flink-connector-kafka:3.3.0-1.19"
2125
testImplementation "org.apache.flink:flink-clients:$flinkVersion"
2226
testImplementation "org.apache.flink:flink-table-api-java:$flinkVersion"
2327
testImplementation "org.apache.flink:flink-table-api-java-bridge:$flinkVersion"

‎sdk/flink-1.20/build.gradle‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,16 @@ dependencies {
1212
api "org.apache.flink:flink-core:$flinkVersion"
1313
//NOTE: flink-connector-base is needed as of Flink 1.18
1414
implementation "org.apache.flink:flink-connector-base:$flinkVersion"
15-
//NOTE: flink-connector-kafka released with different versioning schema starting with Flink 1.18
16-
implementation "org.apache.flink:flink-connector-kafka:3.4.0-1.20"
1715
implementation "org.apache.flink:flink-streaming-java:$flinkVersion"
1816
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
19-
implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.6'
2017

18+
//NOTE: flink-connector-kafka released with different versioning schema starting with Flink 1.18
19+
// A Kafka connector with custom patches is supplied by the Decodable platform at runtime
20+
compileOnly "org.apache.flink:flink-connector-kafka:3.4.0-1.20"
21+
22+
// For some reason, with Flink 1.18+, the Redpanda testcontainer requires commons-codec to be added explicitly
23+
testImplementation "commons-codec:commons-codec:1.18.0"
24+
testImplementation "org.apache.flink:flink-connector-kafka:3.4.0-1.20"
2125
testImplementation "org.apache.flink:flink-clients:$flinkVersion"
2226
testImplementation "org.apache.flink:flink-table-api-java:$flinkVersion"
2327
testImplementation "org.apache.flink:flink-table-api-java-bridge:$flinkVersion"

0 commit comments

Comments
 (0)