Skip to content

Commit d0210f9

Browse files
Paddy0523FlechazoW
Paddy0523
authored and
FlechazoW
committed
[feat-861][protobuf]add protobuf format
1 parent b5e6b08 commit d0210f9

File tree

26 files changed

+5160
-83
lines changed

26 files changed

+5160
-83
lines changed

‎chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,10 @@
131131
<pattern>org.apache.http</pattern>
132132
<shadedPattern>com.dtstack.chunjun.connector.hbase14.org.apache.http</shadedPattern>
133133
</relocation>
134+
<relocation>
135+
<pattern>com.google.protobuf</pattern>
136+
<shadedPattern>com.dtstack.chunjun.connector.hbase14.com.google.protobuf</shadedPattern>
137+
</relocation>
134138
</relocations>
135139
</configuration>
136140
</execution>

‎chunjun-connectors/chunjun-connector-hive/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,12 @@
282282
</excludes>
283283
</filter>
284284
</filters>
285+
<relocations>
286+
<relocation>
287+
<pattern>com.google.protobuf</pattern>
288+
<shadedPattern>com.dtstack.chunjun.connector.hive.com.google.protobuf</shadedPattern>
289+
</relocation>
290+
</relocations>
285291
<transformers>
286292
<transformer
287293
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">

‎chunjun-connectors/chunjun-connector-kafka/pom.xml‎

Lines changed: 92 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,47 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<project xmlns="http://maven.apache.org/POM/4.0.0"
3-
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5-
<parent>
6-
<artifactId>chunjun-connectors</artifactId>
7-
<groupId>com.dtstack.chunjun</groupId>
8-
<version>1.12-SNAPSHOT</version>
9-
</parent>
10-
<modelVersion>4.0.0</modelVersion>
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>chunjun-connectors</artifactId>
7+
<groupId>com.dtstack.chunjun</groupId>
8+
<version>1.12-SNAPSHOT</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
1111

12-
<artifactId>chunjun-connector-kafka</artifactId>
12+
<artifactId>chunjun-connector-kafka</artifactId>
1313
<name>ChunJun : Connectors : Kafka</name>
1414

15-
<dependencies>
16-
<dependency>
17-
<groupId>org.apache.flink</groupId>
18-
<artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
19-
<version>${flink.version}</version>
20-
<exclusions>
21-
<exclusion>
22-
<artifactId>flink-core</artifactId>
23-
<groupId>org.apache.flink</groupId>
24-
</exclusion>
25-
</exclusions>
26-
</dependency>
15+
<dependencies>
16+
<dependency>
17+
<groupId>org.apache.flink</groupId>
18+
<artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
19+
<version>${flink.version}</version>
20+
<exclusions>
21+
<exclusion>
22+
<artifactId>flink-core</artifactId>
23+
<groupId>org.apache.flink</groupId>
24+
</exclusion>
25+
</exclusions>
26+
</dependency>
2727

2828
<dependency>
2929
<groupId>com.pingcap.ticdc.cdc</groupId>
3030
<artifactId>ticdc-decoder</artifactId>
3131
<version>5.2.0-SNAPSHOT</version>
32-
<exclusions>
33-
<exclusion>
34-
<groupId>com.alibaba</groupId>
35-
<artifactId>fastjson</artifactId>
36-
</exclusion>
37-
</exclusions>
32+
<exclusions>
33+
<exclusion>
34+
<groupId>com.alibaba</groupId>
35+
<artifactId>fastjson</artifactId>
36+
</exclusion>
37+
</exclusions>
3838
</dependency>
3939

40-
<dependency>
41-
<groupId>com.alibaba</groupId>
42-
<artifactId>fastjson</artifactId>
43-
<version>1.2.79</version>
44-
</dependency>
40+
<dependency>
41+
<groupId>com.alibaba</groupId>
42+
<artifactId>fastjson</artifactId>
43+
<version>1.2.79</version>
44+
</dependency>
4545

4646
<dependency>
4747
<groupId>org.apache.flink</groupId>
@@ -54,13 +54,13 @@
5454
<artifactId>avro</artifactId>
5555
<version>1.10.0</version>
5656
</dependency>
57-
</dependencies>
57+
</dependencies>
5858

59-
<build>
60-
<plugins>
61-
<plugin>
62-
<groupId>org.apache.maven.plugins</groupId>
63-
<artifactId>maven-shade-plugin</artifactId>
59+
<build>
60+
<plugins>
61+
<plugin>
62+
<groupId>org.apache.maven.plugins</groupId>
63+
<artifactId>maven-shade-plugin</artifactId>
6464
<version>3.2.4</version>
6565
<executions>
6666
<execution>
@@ -70,11 +70,11 @@
7070
</goals>
7171
<configuration>
7272
<createDependencyReducedPom>false</createDependencyReducedPom>
73-
<relocations>
74-
<relocation>
75-
<pattern>com.alibaba.fastjson</pattern>
76-
<shadedPattern>com.shade.alibaba.fastjson</shadedPattern>
77-
</relocation>
73+
<relocations>
74+
<relocation>
75+
<pattern>com.alibaba.fastjson</pattern>
76+
<shadedPattern>com.shade.alibaba.fastjson</shadedPattern>
77+
</relocation>
7878
<!--Solution conflicts with flink-sql-avro.all shades are shaded in the way of flink-sql-avro-->
7979
<relocation>
8080
<pattern>org.apache.avro</pattern>
@@ -88,7 +88,11 @@
8888
<pattern>org.apache.commons.compress</pattern>
8989
<shadedPattern>org.apache.flink.avro.shaded.org.apache.commons.compress</shadedPattern>
9090
</relocation>
91-
</relocations>
91+
<relocation>
92+
<pattern>com.google.protobuf</pattern>
93+
<shadedPattern>com.dtstack.chunjun.connector.kafka.shaded.com.google.protobuf</shadedPattern>
94+
</relocation>
95+
</relocations>
9296
<artifactSet>
9397
<excludes>
9498
<exclude>org.slf4j</exclude>
@@ -116,28 +120,50 @@
116120
<exclude>META-INF/*.RSA</exclude>
117121
</excludes>
118122
</filter>
123+
124+
<filter>
125+
<artifact>org.apache.flink:flink-avro</artifact>
126+
<excludes>
127+
<exclude>META-INF/services/*</exclude>
128+
</excludes>
129+
</filter>
130+
<filter>
131+
<artifact>org.apache.flink:flink-connector-kafka</artifact>
132+
<excludes>
133+
<exclude>META-INF/services/*</exclude>
134+
</excludes>
135+
</filter>
136+
<filter>
137+
<artifact>org.glassfish.jersey.core:jersey-common</artifact>
138+
<excludes>
139+
<exclude>META-INF/services/*</exclude>
140+
</excludes>
141+
</filter>
119142
</filters>
143+
<transformers>
144+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
145+
</transformers>
120146
</configuration>
121147
</execution>
122148
</executions>
123149
</plugin>
124150

125-
<plugin>
126-
<artifactId>maven-antrun-plugin</artifactId>
127-
<executions>
128-
<execution>
129-
<id>copy-resources</id>
130-
<!-- here the phase you need -->
131-
<phase>package</phase>
132-
<goals>
133-
<goal>run</goal>
134-
</goals>
135-
<configuration>
136-
<tasks>
137-
<copy todir="${basedir}/../../${dist.dir}/connector/kafka"
151+
<plugin>
152+
<artifactId>maven-antrun-plugin</artifactId>
153+
<executions>
154+
<execution>
155+
<id>copy-resources</id>
156+
<!-- here the phase you need -->
157+
<phase>package</phase>
158+
<goals>
159+
<goal>run</goal>
160+
</goals>
161+
<configuration>
162+
<tasks>
163+
<copy todir="${basedir}/../../${dist.dir}/connector/kafka"
138164
file="${basedir}/target/${project.artifactId}-${project.version}.jar"/>
139-
<move file="${basedir}/../../${dist.dir}/connector/kafka/${project.artifactId}-${project.version}.jar"
140-
tofile="${basedir}/../../${dist.dir}/connector/kafka/${project.artifactId}-${git.branch}.jar"/>
165+
<move file="${basedir}/../../${dist.dir}/connector/kafka/${project.artifactId}-${project.version}.jar"
166+
tofile="${basedir}/../../${dist.dir}/connector/kafka/${project.artifactId}-${git.branch}.jar"/>
141167
<delete>
142168
<fileset dir="${basedir}/../../${dist.dir}/connector/kafka/"
143169
includes="${project.artifactId}-*.jar"
@@ -153,11 +179,11 @@
153179
includes="${project.artifactId}-*.jar"
154180
excludes="${project.artifactId}-${git.branch}.jar"/>
155181
</delete>
156-
</tasks>
157-
</configuration>
158-
</execution>
159-
</executions>
160-
</plugin>
161-
</plugins>
162-
</build>
182+
</tasks>
183+
</configuration>
184+
</execution>
185+
</executions>
186+
</plugin>
187+
</plugins>
188+
</build>
163189
</project>

‎chunjun-connectors/chunjun-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,3 @@
1515

1616
com.dtstack.chunjun.connector.kafka.table.KafkaDynamicTableFactory
1717
com.dtstack.chunjun.connector.upsertkafka.table.UpsertKafkaDynamicTableFactory
18-
org.apache.flink.formats.avro.registry.confluent.RegistryAvroFormatFactory
19-
org.apache.flink.formats.avro.registry.confluent.debezium.DebeziumAvroFormatFactory

‎chunjun-core/src/main/java/com/dtstack/chunjun/constants/ConstantValue.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ public class ConstantValue {
7272

7373
public static final String CONNECTOR_DIR_NAME = "connector";
7474

75+
public static final String FORMAT_DIR_NAME = "formats";
76+
7577
public static final String DIRTY_DATA_DIR_NAME = "dirty-data-collector";
7678

7779
public static final String RESTORE_DIR_NAME = "restore-plugins";

‎chunjun-core/src/main/java/com/dtstack/chunjun/util/ConnectorNameConvertUtil.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public class ConnectorNameConvertUtil {
4040
connectorNameMap.put("adbpostgresql", new Tuple2<>("postgresql", "postgresql"));
4141
connectorNameMap.put("dorisbatch", new Tuple2<>("doris", "doris"));
4242
connectorNameMap.put("gbase", new Tuple2<>("gBase", "gBase"));
43+
connectorNameMap.put("protobuf", new Tuple2<>("pbformat", "pbformat"));
4344
}
4445

4546
public static String convertClassPrefix(String originName) {

‎chunjun-core/src/main/java/org/apache/flink/table/factories/FactoryUtil.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import java.util.HashSet;
5050
import java.util.LinkedList;
5151
import java.util.List;
52+
import java.util.Locale;
5253
import java.util.Optional;
5354
import java.util.ServiceConfigurationError;
5455
import java.util.ServiceLoader;
@@ -654,6 +655,14 @@ private <F extends Factory> Optional<F> discoverOptionalFormatFactory(
654655
if (identifier == null) {
655656
return Optional.empty();
656657
}
658+
if (identifier.toLowerCase(Locale.ROOT).endsWith("-x")) {
659+
String s = identifier.substring(0, identifier.length() - 2);
660+
FactoryHelper factoryHelper = factoryHelperThreadLocal.get();
661+
if (factoryHelper != null) {
662+
factoryHelper.registerCachedFile(
663+
s, context.getClassLoader(), ConstantValue.FORMAT_DIR_NAME);
664+
}
665+
}
657666
final F factory =
658667
discoverFactory(context.getClassLoader(), formatFactoryClass, identifier);
659668
String formatPrefix = formatPrefix(factory, formatOption);

0 commit comments

Comments
 (0)