Skip to content

Commit cb97ae3

Browse files
author
wangjin18
committed
Merge remote-tracking branch 'upstream/4.1.0-SNAPSHOT' into pre_release
merge
2 parents 12aaefe + 7151ea9 commit cb97ae3

File tree

57 files changed

+1094
-1334
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+1094
-1334
lines changed

‎joyqueue-client/joyqueue-client-all-shaded/pom.xml‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
16+
-->
17+
<project xmlns="http://maven.apache.org/POM/4.0.0"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<parent>
21+
<artifactId>joyqueue-client</artifactId>
22+
<groupId>com.jd.joyqueue</groupId>
23+
<version>4.1.0-SNAPSHOT</version>
24+
</parent>
25+
<modelVersion>4.0.0</modelVersion>
26+
27+
<artifactId>joyqueue-client-all-shaded</artifactId>
28+
29+
<dependencies>
30+
<dependency>
31+
<groupId>com.jd.joyqueue</groupId>
32+
<artifactId>joyqueue-client-core</artifactId>
33+
</dependency>
34+
<dependency>
35+
<groupId>com.jd.joyqueue</groupId>
36+
<artifactId>joyqueue-client-kafka</artifactId>
37+
</dependency>
38+
<dependency>
39+
<groupId>org.slf4j</groupId>
40+
<artifactId>slf4j-api</artifactId>
41+
</dependency>
42+
</dependencies>
43+
44+
<build>
45+
<resources>
46+
<resource>
47+
<directory>src/main/resources</directory>
48+
<filtering>true</filtering>
49+
</resource>
50+
</resources>
51+
<plugins>
52+
<plugin>
53+
<groupId>org.apache.maven.plugins</groupId>
54+
<artifactId>maven-shade-plugin</artifactId>
55+
<version>3.2.1</version>
56+
<executions>
57+
<execution>
58+
<phase>package</phase>
59+
<goals>
60+
<goal>shade</goal>
61+
</goals>
62+
<configuration>
63+
<createDependencyReducedPom>false</createDependencyReducedPom>
64+
<minimizeJar>false</minimizeJar>
65+
<artifactSet>
66+
<excludes>
67+
<exclude>org.slf4j:slf4j-api:*</exclude>
68+
</excludes>
69+
</artifactSet>
70+
<relocations>
71+
<relocation>
72+
<pattern>com.codahale</pattern>
73+
<shadedPattern>com.jd.joyqueue.shaded.com.codahale</shadedPattern>
74+
</relocation>
75+
<relocation>
76+
<pattern>org.objectweb.asm</pattern>
77+
<shadedPattern>com.jd.joyqueue.shaded.org.objectweb.asm</shadedPattern>
78+
</relocation>
79+
<relocation>
80+
<pattern>com.google</pattern>
81+
<shadedPattern>com.jd.joyqueue.shaded.com.google</shadedPattern>
82+
</relocation>
83+
<relocation>
84+
<pattern>io.netty</pattern>
85+
<shadedPattern>com.jd.joyqueue.shaded.io.netty</shadedPattern>
86+
</relocation>
87+
<relocation>
88+
<pattern>javax.inject</pattern>
89+
<shadedPattern>com.jd.joyqueue.shaded.javax.inject</shadedPattern>
90+
</relocation>
91+
<relocation>
92+
<pattern>javax.validation</pattern>
93+
<shadedPattern>com.jd.joyqueue.shaded.javax.validation</shadedPattern>
94+
</relocation>
95+
<relocation>
96+
<pattern>org.aopalliance</pattern>
97+
<shadedPattern>com.jd.joyqueue.shaded.org.aopalliance</shadedPattern>
98+
</relocation>
99+
<relocation>
100+
<pattern>org.apache</pattern>
101+
<shadedPattern>com.jd.joyqueue.shaded.org.apache</shadedPattern>
102+
</relocation>
103+
<relocation>
104+
<pattern>org.xerial.snappy</pattern>
105+
<shadedPattern>com.jd.joyqueue.shaded.org.xerial.snappy</shadedPattern>
106+
</relocation>
107+
<relocation>
108+
<pattern>net.jpountz.lz4</pattern>
109+
<shadedPattern>com.jd.joyqueue.shaded.net.jpountz.lz4</shadedPattern>
110+
</relocation>
111+
<relocation>
112+
<pattern>com.jd.laf.extension</pattern>
113+
<shadedPattern>com.jd.joyqueue.shaded.com.jd.laf.extension</shadedPattern>
114+
</relocation>
115+
</relocations>
116+
</configuration>
117+
</execution>
118+
</executions>
119+
</plugin>
120+
</plugins>
121+
</build>
122+
</project>

‎joyqueue-client/joyqueue-client-all-shaded/src/main/java/com/jd/joyqueue/package-info.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/**
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.jd.joyqueue;

‎joyqueue-client/joyqueue-client-all/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
<dependencies>
3232
<dependency>
3333
<groupId>com.jd.joyqueue</groupId>
34-
<artifactId>joyqueue-client-core-shaded</artifactId>
34+
<artifactId>joyqueue-client-core</artifactId>
3535
</dependency>
3636
<dependency>
3737
<groupId>com.jd.joyqueue</groupId>

‎joyqueue-client/joyqueue-client-core-shaded/pom.xml‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@
3131
<groupId>com.jd.joyqueue</groupId>
3232
<artifactId>joyqueue-client-core</artifactId>
3333
</dependency>
34+
<dependency>
35+
<groupId>org.slf4j</groupId>
36+
<artifactId>slf4j-api</artifactId>
37+
</dependency>
3438
</dependencies>
3539

3640
<build>
@@ -52,38 +56,22 @@
5256
<goal>shade</goal>
5357
</goals>
5458
<configuration>
59+
<createDependencyReducedPom>false</createDependencyReducedPom>
5560
<minimizeJar>false</minimizeJar>
5661
<artifactSet>
57-
<includes>
58-
<include>org.apache.commons:commons-lang3:*</include>
59-
<include>commons-collections:commons-collections:*</include>
60-
<include>com.google.inject:guice:*</include>
61-
<include>javax.inject:javax.inject:*</include>
62-
<include>aopalliance:aopalliance:*</include>
63-
<include>com.google.guava:guava:*</include>
64-
<include>javax.validation:validation-api:*</include>
65-
<include>io.dropwizard.metrics:*</include>
66-
<include>io.netty:netty-all:*</include>
67-
<include>com.alibaba:fastjson:*</include>
68-
<include>commons-io:commons-io:*</include>
69-
<include>com.jd.joyqueue:joyqueue-network:*</include>
70-
<include>com.jd.joyqueue:joyqueue-model:*</include>
71-
<include>com.jd.joyqueue:joyqueue-toolkit:*</include>
72-
<include>com.jd.laf:laf-extension-core:*</include>
73-
</includes>
7462
<excludes>
7563
<exclude>org.slf4j:slf4j-api:*</exclude>
7664
</excludes>
7765
</artifactSet>
7866
<relocations>
79-
<relocation>
80-
<pattern>com.alibaba</pattern>
81-
<shadedPattern>com.jd.joyqueue.shaded.com.alibaba</shadedPattern>
82-
</relocation>
8367
<relocation>
8468
<pattern>com.codahale</pattern>
8569
<shadedPattern>com.jd.joyqueue.shaded.com.codahale</shadedPattern>
8670
</relocation>
71+
<relocation>
72+
<pattern>org.objectweb.asm</pattern>
73+
<shadedPattern>com.jd.joyqueue.shaded.org.objectweb.asm</shadedPattern>
74+
</relocation>
8775
<relocation>
8876
<pattern>com.google</pattern>
8977
<shadedPattern>com.jd.joyqueue.shaded.com.google</shadedPattern>

‎joyqueue-client/joyqueue-client-core/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,5 +48,9 @@
4848
<groupId>io.openmessaging</groupId>
4949
<artifactId>openmessaging-api</artifactId>
5050
</dependency>
51+
<dependency>
52+
<groupId>com.google.code.gson</groupId>
53+
<artifactId>gson</artifactId>
54+
</dependency>
5155
</dependencies>
5256
</project>

‎joyqueue-client/joyqueue-client-core/src/main/java/com/jd/joyqueue/client/internal/cluster/ClusterClientManager.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,11 @@ protected void validate() throws Exception {
7676
transportConfig.setConnections(1);
7777

7878
URL url = URL.valueOf(String.format("joyqueue://%s", nameServerConfig.getAddress()));
79-
bootstrapNode = new BrokerNode(url.getHost(), url.getPort());
79+
if (url.getPort() == 0) {
80+
bootstrapNode = new BrokerNode(url.getHost(), -1);
81+
} else {
82+
bootstrapNode = new BrokerNode(url.getHost(), url.getPort());
83+
}
8084
clientManager = new ClientManager(transportConfig, nameServerConfig);
8185
}
8286

‎joyqueue-client/joyqueue-client-core/src/main/java/com/jd/joyqueue/client/internal/consumer/support/ConsumerLocalIndexStore.java‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,11 @@
1313
*/
1414
package com.jd.joyqueue.client.internal.consumer.support;
1515

16-
import com.alibaba.fastjson.JSON;
17-
import com.alibaba.fastjson.JSONObject;
18-
import com.alibaba.fastjson.serializer.SerializerFeature;
1916
import com.google.common.collect.HashBasedTable;
2017
import com.google.common.collect.Maps;
2118
import com.google.common.collect.Table;
19+
import com.google.gson.Gson;
20+
import com.google.gson.GsonBuilder;
2221
import com.jd.joyqueue.client.internal.consumer.domain.LocalIndexData;
2322
import com.jd.joyqueue.toolkit.concurrent.NamedThreadFactory;
2423
import com.jd.joyqueue.toolkit.service.Service;
@@ -132,19 +131,24 @@ protected Table<String, String, Map<Short, LocalIndexData>> doRead(File persistF
132131
return result;
133132
}
134133

135-
JSONObject jsonObject = JSON.parseObject(json);
136-
for (String app : jsonObject.keySet()) {
137-
JSONObject appJsonObject = jsonObject.getJSONObject(app);
138-
for (String topic : appJsonObject.keySet()) {
139-
JSONObject topicJsonObject = appJsonObject.getJSONObject(topic);
134+
Gson gson = new GsonBuilder().create();
135+
Map<String, Map<String, Map<String, Map<String, Object>>>> map = gson.fromJson(json, Map.class);
140136

137+
for (Map.Entry<String, Map<String, Map<String, Map<String, Object>>>> appEntry : map.entrySet()) {
138+
String app = appEntry.getKey();
139+
140+
for (Map.Entry<String, Map<String, Map<String, Object>>> topicEntry : appEntry.getValue().entrySet()) {
141+
String topic = topicEntry.getKey();
141142
Map<Short, LocalIndexData> partitions = Maps.newHashMap();
142-
for (String partition : topicJsonObject.keySet()) {
143-
JSONObject partitionJsonObject = topicJsonObject.getJSONObject(partition);
144-
long index = partitionJsonObject.getLong("index");
145-
long updateTime = partitionJsonObject.getLong("updateTime");
146-
long createTime = partitionJsonObject.getLong("createTime");
147-
partitions.put(Short.valueOf(partition), new LocalIndexData(index, updateTime, createTime));
143+
144+
for (Map.Entry<String, Map<String, Object>> partitionEntry : topicEntry.getValue().entrySet()) {
145+
short partition = Short.valueOf(partitionEntry.getKey());
146+
Map<String, Object> values = partitionEntry.getValue();
147+
148+
long index = Double.valueOf(String.valueOf(values.get("index"))).longValue();
149+
long updateTime = Double.valueOf(String.valueOf(values.get("updateTime"))).longValue();
150+
long createTime = Double.valueOf(String.valueOf(values.get("createTime"))).longValue();
151+
partitions.put(partition, new LocalIndexData(index, updateTime, createTime));
148152
}
149153

150154
result.put(app, topic, partitions);
@@ -162,7 +166,7 @@ protected void persist() {
162166
}
163167

164168
protected void doPersist(File persistFile) {
165-
String json = JSON.toJSONString(indexTable.rowMap(), SerializerFeature.DisableCircularReferenceDetect);
169+
String json = new GsonBuilder().create().toJson(indexTable.rowMap());
166170

167171
try {
168172
FileUtils.writeStringToFile(persistFile, json);

‎joyqueue-client/joyqueue-client-core/src/main/java/com/jd/joyqueue/client/internal/metadata/MetadataManager.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
*/
1414
package com.jd.joyqueue.client.internal.metadata;
1515

16-
import com.alibaba.fastjson.JSON;
1716
import com.google.common.collect.Lists;
1817
import com.jd.joyqueue.client.internal.cluster.ClusterClientManager;
1918
import com.jd.joyqueue.client.internal.metadata.converter.ClusterMetadataConverter;
@@ -49,7 +48,7 @@ public TopicMetadata fetchMetadata(String topic, String app) {
4948
public ClusterMetadata fetchMetadata(List<String> topics, String app) {
5049
FetchClusterResponse fetchClusterResponse = clusterClientManager.getOrCreateClient().fetchCluster(topics, app);
5150
if (logger.isDebugEnabled()) {
52-
logger.debug("fetch metadata, topics: {}, app: {}, metadata: {}", topics, app, JSON.toJSONString(fetchClusterResponse));
51+
logger.debug("fetch metadata, topics: {}, app: {}, metadata: {}", topics, app, fetchClusterResponse);
5352
}
5453
return ClusterMetadataConverter.convert(fetchClusterResponse);
5554
}

‎joyqueue-client/joyqueue-client-core/src/main/java/com/jd/joyqueue/client/internal/support/DefaultMessageAccessPoint.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
package com.jd.joyqueue.client.internal.support;
1515

16+
import com.google.common.base.Preconditions;
1617
import com.google.common.collect.Maps;
1718
import com.jd.joyqueue.client.internal.MessageAccessPoint;
1819
import com.jd.joyqueue.client.internal.cluster.ClusterClientManager;
@@ -40,7 +41,6 @@
4041
import com.jd.joyqueue.client.internal.producer.transport.ProducerClientManagerFactory;
4142
import com.jd.joyqueue.client.internal.transport.config.TransportConfig;
4243
import com.jd.joyqueue.client.internal.transport.config.TransportConfigChecker;
43-
import com.google.common.base.Preconditions;
4444
import com.jd.joyqueue.toolkit.service.Service;
4545
import org.apache.commons.lang3.StringUtils;
4646

@@ -179,55 +179,55 @@ protected ClusterManager getClusterManager() {
179179
if (clusterManager != null) {
180180
return clusterManager;
181181
}
182-
clusterManager = ClusterManagerFactory.create(nameServerConfig, getClusterClientManager());
182+
ClusterManager clusterManager = ClusterManagerFactory.create(nameServerConfig, getClusterClientManager());
183183
try {
184184
clusterManager.start();
185185
} catch (Exception e) {
186-
clusterManager = null;
187186
throw new ClientException(e);
188187
}
188+
this.clusterManager = clusterManager;
189189
return clusterManager;
190190
}
191191

192192
protected ClusterClientManager getClusterClientManager() {
193193
if (clusterClientManager != null) {
194194
return clusterClientManager;
195195
}
196-
clusterClientManager = ClusterClientManagerFactory.create(nameServerConfig, transportConfig);
196+
ClusterClientManager clusterClientManager = ClusterClientManagerFactory.create(nameServerConfig, transportConfig);
197197
try {
198198
clusterClientManager.start();
199199
} catch (Exception e) {
200-
clusterClientManager = null;
201200
throw new ClientException(e);
202201
}
202+
this.clusterClientManager = clusterClientManager;
203203
return clusterClientManager;
204204
}
205205

206206
protected ProducerClientManager getProducerClientManager() {
207207
if (producerClientManager != null) {
208208
return producerClientManager;
209209
}
210-
producerClientManager = ProducerClientManagerFactory.create(nameServerConfig, transportConfig);
210+
ProducerClientManager producerClientManager = ProducerClientManagerFactory.create(nameServerConfig, transportConfig);
211211
try {
212212
producerClientManager.start();
213213
} catch (Exception e) {
214-
producerClientManager = null;
215214
throw new ClientException(e);
216215
}
216+
this.producerClientManager = producerClientManager;
217217
return producerClientManager;
218218
}
219219

220220
protected ConsumerClientManager getConsumerClientManager() {
221221
if (consumerClientManager != null) {
222222
return consumerClientManager;
223223
}
224-
consumerClientManager = ConsumerClientManagerFactory.create(nameServerConfig, transportConfig);
224+
ConsumerClientManager consumerClientManager = ConsumerClientManagerFactory.create(nameServerConfig, transportConfig);
225225
try {
226226
consumerClientManager.start();
227227
} catch (Exception e) {
228-
consumerClientManager = null;
229228
throw new ClientException(e);
230229
}
230+
this.consumerClientManager = consumerClientManager;
231231
return consumerClientManager;
232232
}
233233

‎joyqueue-client/joyqueue-client-core/src/main/java/com/jd/joyqueue/client/internal/transport/Client.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,11 @@ public long getLastUseTime() {
168168

169169
@Override
170170
protected void doStart() throws Exception {
171-
transport = transportClient.createTransport(new InetSocketAddress(node.getHost(), node.getPort()), transportConfig.getSendTimeout());
171+
if (node.getPort() <= 0) {
172+
transport = transportClient.createTransport(node.getHost(), transportConfig.getSendTimeout());
173+
} else {
174+
transport = transportClient.createTransport(new InetSocketAddress(node.getHost(), node.getPort()), transportConfig.getSendTimeout());
175+
}
172176
}
173177

174178
@Override

0 commit comments

Comments
 (0)