Skip to content

Commit 7b0b53b

Browse files
mzitnikmzitnik
committed
Merge branch 'main' into integration-tests
# Conflicts: # flink-connector-clickhouse-1.17/build.gradle.kts # flink-connector-clickhouse-2.0.0/build.gradle.kts # flink-connector-clickhouse-base/build.gradle.kts
2 parents 22b65fa + ea762f0 commit 7b0b53b

File tree

17 files changed

+625
-14
lines changed

17 files changed

+625
-14
lines changed

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@ build/
2020
gradle-app.setting
2121
!gradle-wrapper.jar
2222

23+
##############################
24+
## Sbt
25+
##############################
26+
project/project/
27+
project/target/
28+
target/
29+
2330
##############################
2431
## IntelliJ
2532
##############################

‎CONTRIBUTING.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,9 @@ Does not require a running ClickHouse server.
3434
./gradlew test
3535
```
3636

37+
if you want to run scala unit tests
38+
39+
```bash
40+
cd flink-connector-clickhouse
41+
./gradlew clean runScalaTests
42+
```

‎examples/README.md‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
<div align="center">
2+
<p><img src="https://github.com/ClickHouse/clickhouse-js/blob/a332672bfb70d54dfd27ae1f8f5169a6ffeea780/.static/logo.svg" width="200px" align="center"></p>
3+
<h1>ClickHouse Flink Connector</h1>
4+
</div>
5+
6+
Table of Contents
7+
* [Covid Flink Example](#covid-flink-application-example)
8+
* [Build Application](#build-covid-application)
9+
* [Build Connector](#build-clickhouse-flink-connector)
10+
* [Java Application](#java-covid-app)
11+
* [Scala Application](#scala-covid-app)
12+
* [Running Example](#running-the-example)
13+
* [Download Data](#download-covid-data)
14+
* [Create table](#create-a-destination-covid-table)
15+
* [Submit Flink](#submit-flink-job)
16+
17+
# Covid Flink Application example
18+
19+
Read covid data from a file and insert into ClickHouse
20+
21+
### Build Covid Application
22+
23+
#### Build ClickHouse Flink Connector
24+
If you wish to build the connector locally run before building the example
25+
```bash
26+
./gradlew publishToMavenLocal
27+
```
28+
29+
#### Java Covid App
30+
31+
From the project directory, run the following command, which will create a `covid-1.0-SNAPSHOT.jar` artifact that can be found in your target folder
32+
33+
```bash
34+
mvn clean package -DskipTests
35+
```
36+
37+
#### Scala Covid App
38+
39+
From project directory run this will create a `covid.jar` can be found in `target/scala-2.12` folder
40+
41+
Build Covid Scala App
42+
43+
```bash
44+
sbt clean assembly
45+
```
46+
47+
## Running the example
48+
49+
- Prepare ClickHouse OSS or [ClickHouse Cloud](https://clickhouse.com/)
50+
- Flink Cluster or Standalone running
51+
- Download covid data
52+
53+
### Download covid data
54+
55+
Download covid data set and save it in a location that is accessible to Flink
56+
57+
```bash
58+
curl -L -# -o epidemiology.csv https://storage.googleapis.com/covid19-open-data/v3/epidemiology.csv
59+
```
60+
61+
### Create a destination covid table
62+
63+
```sql
64+
65+
CREATE TABLE IF NOT EXISTS `default`.`covid` (
66+
date Date,
67+
location_key LowCardinality(String),
68+
new_confirmed Int32,
69+
new_deceased Int32,
70+
new_recovered Int32,
71+
new_tested Int32,
72+
cumulative_confirmed Int32,
73+
cumulative_deceased Int32,
74+
cumulative_recovered Int32,
75+
cumulative_tested Int32
76+
) ENGINE = MergeTree
77+
ORDER BY (location_key, date);
78+
```
79+
80+
### Submit Flink Job
81+
82+
With the Java `covid-1.0-SNAPSHOT.jar` or Scala `covid.jar` built, you can now submit the job to your Flink cluster (or standalone instance)
83+
84+
```bash
85+
# Run the application
86+
./bin/flink run \
87+
/path/to/your/generated/jar \
88+
-input "/path/to/epidemiology.csv" \
89+
-url "/url/clickhouse" \
90+
-username "default" \
91+
-password "" \
92+
-database "default" \
93+
-table "covid"
94+
```

‎examples/maven/covid/pom.xml‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
23+
<groupId>com.clickhouse.example.covid</groupId>
24+
<artifactId>covid</artifactId>
25+
<version>1.0-SNAPSHOT</version>
26+
<packaging>jar</packaging>
27+
28+
<name>Flink Quickstart Job</name>
29+
30+
<properties>
31+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
32+
<flink.version>2.0.0</flink.version>
33+
<target.java.version>11</target.java.version>
34+
<scala.binary.version>2.12</scala.binary.version>
35+
<maven.compiler.source>${target.java.version}</maven.compiler.source>
36+
<maven.compiler.target>${target.java.version}</maven.compiler.target>
37+
<log4j.version>2.24.1</log4j.version>
38+
</properties>
39+
40+
<repositories>
41+
42+
<repository>
43+
<id>apache.snapshots</id>
44+
<name>Apache Development Snapshot Repository</name>
45+
<url>https://repository.apache.org/content/repositories/snapshots/</url>
46+
<releases>
47+
<enabled>false</enabled>
48+
</releases>
49+
<snapshots>
50+
<enabled>true</enabled>
51+
</snapshots>
52+
</repository>
53+
</repositories>
54+
55+
<dependencies>
56+
<!-- Apache Flink dependencies -->
57+
<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
58+
<dependency>
59+
<groupId>org.apache.flink</groupId>
60+
<artifactId>flink-streaming-java</artifactId>
61+
<version>${flink.version}</version>
62+
<scope>provided</scope>
63+
</dependency>
64+
<dependency>
65+
<groupId>org.apache.flink</groupId>
66+
<artifactId>flink-clients</artifactId>
67+
<version>${flink.version}</version>
68+
<scope>provided</scope>
69+
</dependency>
70+
<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-files -->
71+
<dependency>
72+
<groupId>org.apache.flink</groupId>
73+
<artifactId>flink-connector-files</artifactId>
74+
<version>2.0.0</version>
75+
<scope>provided</scope>
76+
</dependency>
77+
78+
<dependency>
79+
<groupId>com.clickhouse.flink</groupId>
80+
<artifactId>flink-connector-clickhouse-2.0.0</artifactId>
81+
<version>0.0.1</version>
82+
<classifier>all</classifier>
83+
</dependency>
84+
85+
86+
<!-- Add connector dependencies here. They must be in the default scope (compile). -->
87+
88+
<!-- Example:
89+
90+
<dependency>
91+
<groupId>org.apache.flink</groupId>
92+
<artifactId>flink-connector-kafka</artifactId>
93+
<version>3.0.0-1.17</version>
94+
</dependency>
95+
-->
96+
97+
<!-- Add logging framework, to produce console output when running in the IDE. -->
98+
<!-- These dependencies are excluded from the application JAR by default. -->
99+
<dependency>
100+
<groupId>org.apache.logging.log4j</groupId>
101+
<artifactId>log4j-slf4j-impl</artifactId>
102+
<version>${log4j.version}</version>
103+
<scope>runtime</scope>
104+
</dependency>
105+
<dependency>
106+
<groupId>org.apache.logging.log4j</groupId>
107+
<artifactId>log4j-api</artifactId>
108+
<version>${log4j.version}</version>
109+
<scope>runtime</scope>
110+
</dependency>
111+
<dependency>
112+
<groupId>org.apache.logging.log4j</groupId>
113+
<artifactId>log4j-core</artifactId>
114+
<version>${log4j.version}</version>
115+
<scope>runtime</scope>
116+
</dependency>
117+
</dependencies>
118+
119+
<build>
120+
<plugins>
121+
122+
<!-- Java Compiler -->
123+
<plugin>
124+
<groupId>org.apache.maven.plugins</groupId>
125+
<artifactId>maven-compiler-plugin</artifactId>
126+
<version>3.1</version>
127+
<configuration>
128+
<source>${target.java.version}</source>
129+
<target>${target.java.version}</target>
130+
</configuration>
131+
</plugin>
132+
133+
<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
134+
<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
135+
<plugin>
136+
<groupId>org.apache.maven.plugins</groupId>
137+
<artifactId>maven-shade-plugin</artifactId>
138+
<version>3.1.1</version>
139+
<executions>
140+
<!-- Run shade goal on package phase -->
141+
<execution>
142+
<phase>package</phase>
143+
<goals>
144+
<goal>shade</goal>
145+
</goals>
146+
<configuration>
147+
<createDependencyReducedPom>false</createDependencyReducedPom>
148+
<artifactSet>
149+
<excludes>
150+
<exclude>org.apache.flink:flink-shaded-force-shading</exclude>
151+
<exclude>com.google.code.findbugs:jsr305</exclude>
152+
<exclude>org.slf4j:*</exclude>
153+
<exclude>org.apache.logging.log4j:*</exclude>
154+
</excludes>
155+
</artifactSet>
156+
<filters>
157+
<filter>
158+
<!-- Do not copy the signatures in the META-INF folder.
159+
Otherwise, this might cause SecurityExceptions when using the JAR. -->
160+
<artifact>*:*</artifact>
161+
<excludes>
162+
<exclude>META-INF/*.SF</exclude>
163+
<exclude>META-INF/*.DSA</exclude>
164+
<exclude>META-INF/*.RSA</exclude>
165+
</excludes>
166+
</filter>
167+
</filters>
168+
<transformers>
169+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
170+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
171+
<mainClass>com.clickhouse.example.covid.DataStreamJob</mainClass>
172+
</transformer>
173+
</transformers>
174+
</configuration>
175+
</execution>
176+
</executions>
177+
</plugin>
178+
</plugins>
179+
180+
<pluginManagement>
181+
<plugins>
182+
183+
<!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->
184+
<plugin>
185+
<groupId>org.eclipse.m2e</groupId>
186+
<artifactId>lifecycle-mapping</artifactId>
187+
<version>1.0.0</version>
188+
<configuration>
189+
<lifecycleMappingMetadata>
190+
<pluginExecutions>
191+
<pluginExecution>
192+
<pluginExecutionFilter>
193+
<groupId>org.apache.maven.plugins</groupId>
194+
<artifactId>maven-shade-plugin</artifactId>
195+
<versionRange>[3.1.1,)</versionRange>
196+
<goals>
197+
<goal>shade</goal>
198+
</goals>
199+
</pluginExecutionFilter>
200+
<action>
201+
<ignore/>
202+
</action>
203+
</pluginExecution>
204+
<pluginExecution>
205+
<pluginExecutionFilter>
206+
<groupId>org.apache.maven.plugins</groupId>
207+
<artifactId>maven-compiler-plugin</artifactId>
208+
<versionRange>[3.1,)</versionRange>
209+
<goals>
210+
<goal>testCompile</goal>
211+
<goal>compile</goal>
212+
</goals>
213+
</pluginExecutionFilter>
214+
<action>
215+
<ignore/>
216+
</action>
217+
</pluginExecution>
218+
</pluginExecutions>
219+
</lifecycleMappingMetadata>
220+
</configuration>
221+
</plugin>
222+
</plugins>
223+
</pluginManagement>
224+
</build>
225+
</project>

0 commit comments

Comments
 (0)