Skip to content

Commit fd7ff11

Browse files
mzitnikmzitnik
committed
Split into two modules 2.0.0 and 1.17+
1 parent 72cefb5 commit fd7ff11

File tree

56 files changed

+2377
-6
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+2377
-6
lines changed

‎.github/workflows/tests.yaml‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@ name: Apache Flink ClickHouse Connector Tests CI (Java)
33
on: [push]
44

55
jobs:
6-
build:
6+
test-flink-17:
77
runs-on: ubuntu-latest
88
strategy:
99
fail-fast: false
1010
matrix:
1111
clickhouse: [ "23.7", "24.3", "latest", "cloud" ]
12-
name: Apache Flink ClickHouse Connector tests with ClickHouse ${{ matrix.clickhouse }}
12+
flink: [ "1.17.2", "1.18.1", "1.19.3", "1.20.2"]
13+
name: Apache Flink 1.17+ ClickHouse Connector tests with ClickHouse ${{ matrix.clickhouse }}
1314
steps:
1415
- name: Check for Cloud Credentials
1516
id: check-cloud-credentials
@@ -36,5 +37,41 @@ jobs:
3637
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
3738
CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }}
3839
CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }}
40+
FLINK_VERSION: ${{ matrix.flink }}
3941
with:
40-
arguments: test
42+
arguments: :flink-connector-clickhouse-1.17:test
43+
test-flink-2:
44+
runs-on: ubuntu-latest
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
clickhouse: [ "23.7", "24.3", "latest", "cloud" ]
49+
name: Apache Flink 2.0.0 ClickHouse Connector tests with ClickHouse ${{ matrix.clickhouse }}
50+
steps:
51+
- name: Check for Cloud Credentials
52+
id: check-cloud-credentials
53+
run: |
54+
if [[ "${{ matrix.clickhouse }}" == "cloud" && (-z "${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }}" || -z "${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }}") ]]; then
55+
echo "SKIP_STEP=true" >> $GITHUB_ENV
56+
else
57+
echo "SKIP_STEP=false" >> $GITHUB_ENV
58+
fi
59+
shell: bash
60+
- uses: actions/checkout@v3
61+
if: env.SKIP_STEP != 'true'
62+
- name: Set up JDK 21
63+
if: env.SKIP_STEP != 'true'
64+
uses: actions/setup-java@v3
65+
with:
66+
java-version: '21'
67+
distribution: 'adopt'
68+
architecture: x64
69+
- name: Setup and execute Gradle 'test' task
70+
if: env.SKIP_STEP != 'true'
71+
uses: gradle/gradle-build-action@v2
72+
env:
73+
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
74+
CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }}
75+
CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }}
76+
with:
77+
arguments: :flink-connector-clickhouse-2.0.0:test

‎build.gradle.kts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
plugins {
2+
`maven-publish`
3+
scala
4+
java
5+
signing
6+
id("com.gradleup.nmcp") version "0.0.8"
7+
id("com.github.johnrengelman.shadow") version "8.1.1"
8+
9+
}
10+
11+
val sinkVersion by extra("0.0.1")
12+
val flinkVersion by extra("1.18.0")
13+
val clickhouseVersion by extra("0.4.6")
14+
val junitVersion by extra("5.8.2")
15+
16+
allprojects {
17+
group = "org.apache.flink"
18+
version = "1.0.0"
19+
20+
repositories {
21+
mavenCentral()
22+
}
23+
}
24+
25+
subprojects {
26+
apply(plugin = "java-library")
27+
apply(plugin = "maven-publish")
28+
29+
dependencies {
30+
testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion")
31+
}
32+
33+
java {
34+
toolchain {
35+
languageVersion.set(JavaLanguageVersion.of(11))
36+
}
37+
}
38+
}
39+
40+
sourceSets {
41+
main {
42+
scala {
43+
srcDirs("src/main/scala")
44+
}
45+
java {
46+
srcDirs("src/main/java")
47+
}
48+
}
49+
test {
50+
scala {
51+
srcDirs("src/test/scala")
52+
}
53+
java {
54+
srcDirs("src/test/java")
55+
}
56+
}
57+
}

‎flink-connector-clickhouse-1.17/build.gradle.kts‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
/*
2+
* This file is the build file of flink-connector-clickhouse-base submodule
3+
*
4+
*/
5+
6+
plugins {
7+
`maven-publish`
8+
scala
9+
java
10+
signing
11+
id("com.gradleup.nmcp") version "0.0.8"
12+
id("com.github.johnrengelman.shadow") version "8.1.1"
13+
}
14+
15+
val scalaVersion = "2.13.12"
16+
val sinkVersion: String by rootProject.extra
17+
18+
repositories {
19+
// Use Maven Central for resolving dependencies.
20+
// mavenLocal()
21+
maven("https://s01.oss.sonatype.org/content/groups/staging/") // Temporary until we have a Java Client release
22+
mavenCentral()
23+
}
24+
25+
val flinkVersion = System.getenv("FLINK_VERSION") ?: "1.17.2"
26+
27+
extra.apply {
28+
set("clickHouseDriverVersion", "0.9.0-SNAPSHOT") // Temporary until we have a Java Client release
29+
set("flinkVersion", flinkVersion)
30+
set("log4jVersion","2.17.2")
31+
set("testContainersVersion", "1.21.0")
32+
set("byteBuddyVersion", "1.17.5")
33+
}
34+
35+
dependencies {
36+
// Use JUnit Jupiter for testing.
37+
testImplementation(libs.junit.jupiter)
38+
39+
testRuntimeOnly("org.junit.platform:junit-platform-launcher")
40+
41+
implementation("net.bytebuddy:byte-buddy:${project.extra["byteBuddyVersion"]}")
42+
implementation("net.bytebuddy:byte-buddy-agent:${project.extra["byteBuddyVersion"]}")
43+
// This dependency is used by the application.
44+
implementation(libs.guava)
45+
implementation("org.scala-lang:scala-library:$scalaVersion")
46+
implementation("org.scala-lang:scala-compiler:$scalaVersion")
47+
// logger
48+
implementation("org.apache.logging.log4j:log4j-slf4j-impl:${project.extra["log4jVersion"]}")
49+
implementation("org.apache.logging.log4j:log4j-api:${project.extra["log4jVersion"]}")
50+
implementation("org.apache.logging.log4j:log4j-1.2-api:${project.extra["log4jVersion"]}")
51+
implementation("org.apache.logging.log4j:log4j-core:${project.extra["log4jVersion"]}")
52+
53+
// ClickHouse Client Libraries
54+
implementation("com.clickhouse:client-v2:${project.extra["clickHouseDriverVersion"]}:all")
55+
// Apache Flink Libraries
56+
implementation("org.apache.flink:flink-connector-base:${project.extra["flinkVersion"]}")
57+
implementation("org.apache.flink:flink-streaming-java:${project.extra["flinkVersion"]}")
58+
59+
60+
testImplementation("org.apache.flink:flink-connector-files:${project.extra["flinkVersion"]}")
61+
testImplementation("org.apache.flink:flink-connector-base:${project.extra["flinkVersion"]}")
62+
testImplementation("org.apache.flink:flink-streaming-java:${project.extra["flinkVersion"]}")
63+
testImplementation("org.apache.flink:flink-clients:${project.extra["flinkVersion"]}")
64+
testImplementation("org.apache.flink:flink-runtime:${project.extra["flinkVersion"]}")
65+
// logger
66+
testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:${project.extra["log4jVersion"]}")
67+
testImplementation("org.apache.logging.log4j:log4j-api:${project.extra["log4jVersion"]}")
68+
testImplementation("org.apache.logging.log4j:log4j-1.2-api:${project.extra["log4jVersion"]}")
69+
testImplementation("org.apache.logging.log4j:log4j-core:${project.extra["log4jVersion"]}")
70+
// flink tests
71+
testImplementation("org.apache.flink:flink-test-utils:${project.extra["flinkVersion"]}")
72+
//
73+
testImplementation("org.testcontainers:testcontainers:${project.extra["testContainersVersion"]}")
74+
testImplementation("org.testcontainers:clickhouse:${project.extra["testContainersVersion"]}")
75+
testImplementation("org.scalatest:scalatest_2.13:3.2.19")
76+
testRuntimeOnly("org.scalatestplus:junit-4-13_2.13:3.2.18.0")
77+
// testRuntimeOnly("org.pegdown:pegdown:1.6.0") // sometimes required by ScalaTest
78+
}
79+
80+
sourceSets {
81+
main {
82+
scala {
83+
srcDirs("src/main/scala")
84+
}
85+
java {
86+
srcDirs("src/main/java")
87+
}
88+
}
89+
test {
90+
scala {
91+
srcDirs("src/test/scala")
92+
}
93+
java {
94+
srcDirs("src/test/java")
95+
}
96+
}
97+
}
98+
99+
// Apply a specific Java toolchain to ease working on different environments.
100+
java {
101+
toolchain {
102+
languageVersion = JavaLanguageVersion.of(11)
103+
}
104+
}
105+
106+
tasks.test {
107+
useJUnitPlatform()
108+
109+
include("**/*Test.class", "**/*Tests.class", "**/*Spec.class")
110+
testLogging {
111+
events("passed", "failed", "skipped")
112+
//showStandardStreams = true - , "standardOut", "standardError"
113+
}
114+
}
115+
116+
tasks.withType<ScalaCompile> {
117+
scalaCompileOptions.apply {
118+
encoding = "UTF-8"
119+
isDeprecation = true
120+
additionalParameters = listOf("-feature", "-unchecked")
121+
}
122+
}
123+
124+
tasks.named<Test>("test") {
125+
// Use JUnit Platform for unit tests.
126+
useJUnitPlatform()
127+
}
128+
129+
tasks.register<JavaExec>("runScalaTests") {
130+
group = "verification"
131+
mainClass.set("org.scalatest.tools.Runner")
132+
classpath = sourceSets["test"].runtimeClasspath
133+
args = listOf(
134+
"-R", "build/classes/scala/test",
135+
"-oD", // show durations
136+
"-s", "org.apache.flink.connector.clickhouse.test.scala.ClickHouseSinkTests"
137+
)
138+
}
139+
140+
tasks.shadowJar {
141+
archiveClassifier.set("all")
142+
143+
dependencies {
144+
exclude(dependency("org.apache.flink:.*"))
145+
}
146+
mergeServiceFiles()
147+
}
148+
149+
val shadowSourcesJar by tasks.registering(Jar::class) {
150+
archiveClassifier.set("all-sources")
151+
from(sourceSets.main.get().allSource)
152+
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
153+
}
154+
155+
tasks.jar {
156+
enabled = false
157+
}
158+
159+
publishing {
160+
publications {
161+
create<MavenPublication>("maven") {
162+
artifact(tasks.shadowJar)
163+
groupId = "com.clickhouse.flink"
164+
artifactId = "flink-connector-clickhouse"
165+
version = sinkVersion
166+
167+
artifact(shadowSourcesJar)
168+
169+
pom {
170+
name.set("ClickHouse Flink Connector")
171+
description.set("Official Apache Flink connector for ClickHouse")
172+
url.set("https://github.com/ClickHouse/flink-connector-clickhouse")
173+
174+
licenses {
175+
license {
176+
name.set("The Apache License, Version 2.0")
177+
url.set("https://github.com/ClickHouse/flink-connector-clickhouse/blob/main/LICENSE")
178+
}
179+
}
180+
181+
developers {
182+
developer {
183+
id.set("mzitnik")
184+
name.set("Mark Zitnik")
185+
email.set("[email protected]")
186+
}
187+
developer {
188+
id.set("BentsiLeviav")
189+
name.set("Bentsi Leviav")
190+
email.set("[email protected]")
191+
}
192+
}
193+
194+
scm {
195+
connection.set("[email protected]:ClickHouse/flink-connector-clickhouse.git")
196+
url.set("https://github.com/ClickHouse/flink-connector-clickhouse")
197+
}
198+
199+
organization {
200+
name.set("ClickHouse")
201+
url.set("https://clickhouse.com")
202+
}
203+
204+
issueManagement {
205+
system.set("GitHub Issues")
206+
url.set("https://github.com/ClickHouse/flink-connector-clickhouse/issues")
207+
}
208+
}
209+
}
210+
}
211+
}
212+
213+
signing {
214+
val signingKey = System.getenv("SIGNING_KEY")
215+
val signingPassword = System.getenv("SIGNING_PASSWORD")
216+
if (signingKey != null && signingPassword != null) {
217+
useInMemoryPgpKeys(signingKey, signingPassword)
218+
sign(publishing.publications["maven"])
219+
}
220+
}
221+
222+
nmcp {
223+
publish("maven") {
224+
username = System.getenv("NMCP_USERNAME")
225+
password = System.getenv("NMCP_PASSWORD")
226+
publicationType = "AUTOMATIC"
227+
}
228+
}

‎flink-connector-clickhouse-base/src/main/java/com/clickhouse/utils/Serialize.java‎ renamed to ‎flink-connector-clickhouse-1.17/src/main/java/com/clickhouse/utils/Serialize.java‎

File renamed without changes.

‎flink-connector-clickhouse-base/src/main/java/com/clickhouse/utils/Utils.java‎ renamed to ‎flink-connector-clickhouse-1.17/src/main/java/com/clickhouse/utils/Utils.java‎

File renamed without changes.

0 commit comments

Comments
 (0)