Skip to content

Commit 67eb866

Browse files
dyx1234dyx1234
authored
[OSPP]Support Kubernetes ConfigMap for Apollo java, golang client (#79)
Solve the problem of Apollo client configuration information files being lost due to server downtime or Pod restart in the Kubernetes environment. By using Kubernetes ConfigMap as a new persistent storage solution, the reliability and fault tolerance of configuration information are improved. discussion apolloconfig/apollo#5210
1 parent e43ddf5 commit 67eb866

File tree

15 files changed

+1053
-5
lines changed

15 files changed

+1053
-5
lines changed

‎CHANGES.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,12 @@ Release Notes.
55
Apollo Java 2.4.0
66

77
------------------
8+
89
* [Fix the Cannot enhance @Configuration bean definition issue](https://github.com/apolloconfig/apollo-java/pull/82)
910
* [Feature openapi query namespace support not fill item](https://github.com/apolloconfig/apollo-java/pull/83)
1011
* [Add more observability in apollo config client](https://github.com/apolloconfig/apollo-java/pull/74)
12+
* [Feature Support Kubernetes ConfigMap cache for Apollo java, golang client](https://github.com/apolloconfig/apollo-java/pull/79)
13+
1114

1215
------------------
1316
All issues and pull requests are [here](https://github.com/apolloconfig/apollo-java/milestone/4?closed=1)

‎apollo-client/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@
7070
<artifactId>spring-boot-configuration-processor</artifactId>
7171
<optional>true</optional>
7272
</dependency>
73+
<dependency>
74+
<groupId>io.kubernetes</groupId>
75+
<artifactId>client-java</artifactId>
76+
<optional>true</optional>
77+
</dependency>
7378
<!-- test -->
7479
<dependency>
7580
<groupId>org.eclipse.jetty</groupId>

‎apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@
2222
* @since 1.1.0
2323
*/
2424
public enum ConfigSourceType {
25-
REMOTE("Loaded from remote config service"), LOCAL("Loaded from local cache"), NONE("Load failed");
25+
REMOTE("Loaded from remote config service"),
26+
LOCAL("Loaded from local cache"),
27+
CONFIGMAP("Loaded from k8s config map"),
28+
NONE("Load failed");
2629

2730
private final String description;
2831

‎apollo-client/src/main/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepository.java‎

Lines changed: 263 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,263 @@
1+
/*
2+
* Copyright 2022 Apollo Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
package com.ctrip.framework.apollo.internals;
18+
19+
import com.ctrip.framework.apollo.kubernetes.KubernetesManager;
20+
import com.ctrip.framework.apollo.build.ApolloInjector;
21+
import com.ctrip.framework.apollo.core.ConfigConsts;
22+
import com.ctrip.framework.apollo.core.utils.DeferredLoggerFactory;
23+
import com.ctrip.framework.apollo.core.utils.StringUtils;
24+
import com.ctrip.framework.apollo.enums.ConfigSourceType;
25+
import com.ctrip.framework.apollo.exceptions.ApolloConfigException;
26+
import com.ctrip.framework.apollo.tracer.Tracer;
27+
import com.ctrip.framework.apollo.tracer.spi.Transaction;
28+
import com.ctrip.framework.apollo.util.ConfigUtil;
29+
import com.ctrip.framework.apollo.util.ExceptionUtil;
30+
import com.ctrip.framework.apollo.util.escape.EscapeUtil;
31+
import com.google.common.base.Preconditions;
32+
import com.google.gson.Gson;
33+
import com.google.gson.reflect.TypeToken;
34+
import org.slf4j.Logger;
35+
36+
import java.lang.reflect.Type;
37+
import java.util.HashMap;
38+
import java.util.Map;
39+
import java.util.Properties;
40+
41+
/**
42+
* @author dyx1234
43+
*/
44+
public class K8sConfigMapConfigRepository extends AbstractConfigRepository
45+
implements RepositoryChangeListener {
46+
private static final Logger logger = DeferredLoggerFactory.getLogger(K8sConfigMapConfigRepository.class);
47+
private final String namespace;
48+
private String configMapName;
49+
private String configMapKey;
50+
private final String k8sNamespace;
51+
private final ConfigUtil configUtil;
52+
private final KubernetesManager kubernetesManager;
53+
private volatile Properties configMapProperties;
54+
private volatile ConfigRepository upstream;
55+
private volatile ConfigSourceType sourceType = ConfigSourceType.CONFIGMAP;
56+
private static final Gson GSON = new Gson();
57+
58+
59+
public K8sConfigMapConfigRepository(String namespace, ConfigRepository upstream) {
60+
this.namespace = namespace;
61+
configUtil = ApolloInjector.getInstance(ConfigUtil.class);
62+
kubernetesManager = ApolloInjector.getInstance(KubernetesManager.class);
63+
k8sNamespace = configUtil.getK8sNamespace();
64+
65+
this.setConfigMapKey(configUtil.getCluster(), namespace);
66+
this.setConfigMapName(configUtil.getAppId(), false);
67+
this.setUpstreamRepository(upstream);
68+
}
69+
70+
private void setConfigMapKey(String cluster, String namespace) {
71+
// cluster: User Definition >idc>default
72+
if (StringUtils.isBlank(cluster)) {
73+
configMapKey = EscapeUtil.createConfigMapKey("default", namespace);
74+
return;
75+
}
76+
configMapKey = EscapeUtil.createConfigMapKey(cluster, namespace);
77+
}
78+
79+
private void setConfigMapName(String appId, boolean syncImmediately) {
80+
Preconditions.checkNotNull(appId, "AppId cannot be null");
81+
configMapName = ConfigConsts.APOLLO_CONFIG_CACHE + appId;
82+
this.checkConfigMapName(configMapName);
83+
if (syncImmediately) {
84+
this.sync();
85+
}
86+
}
87+
88+
private void checkConfigMapName(String configMapName) {
89+
if (StringUtils.isBlank(configMapName)) {
90+
throw new IllegalArgumentException("ConfigMap name cannot be null");
91+
}
92+
if (kubernetesManager.checkConfigMapExist(k8sNamespace, configMapName)) {
93+
return;
94+
}
95+
// Create an empty configmap, write the new value in the update event
96+
Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "createK8sConfigMap");
97+
transaction.addData("configMapName", configMapName);
98+
try {
99+
kubernetesManager.createConfigMap(k8sNamespace, configMapName, null);
100+
transaction.setStatus(Transaction.SUCCESS);
101+
} catch (Throwable ex) {
102+
Tracer.logEvent("ApolloConfigException", ExceptionUtil.getDetailMessage(ex));
103+
transaction.setStatus(ex);
104+
throw new ApolloConfigException("Create configmap failed!", ex);
105+
} finally {
106+
transaction.complete();
107+
}
108+
}
109+
110+
@Override
111+
public Properties getConfig() {
112+
if (configMapProperties == null) {
113+
sync();
114+
}
115+
Properties result = propertiesFactory.getPropertiesInstance();
116+
result.putAll(configMapProperties);
117+
return result;
118+
}
119+
120+
/**
121+
* Update the memory when the configuration center changes
122+
*
123+
* @param upstreamConfigRepository the upstream repo
124+
*/
125+
@Override
126+
public void setUpstreamRepository(ConfigRepository upstreamConfigRepository) {
127+
if (upstreamConfigRepository == null) {
128+
return;
129+
}
130+
//clear previous listener
131+
if (upstream != null) {
132+
upstream.removeChangeListener(this);
133+
}
134+
upstream = upstreamConfigRepository;
135+
upstreamConfigRepository.addChangeListener(this);
136+
}
137+
138+
@Override
139+
public ConfigSourceType getSourceType() {
140+
return sourceType;
141+
}
142+
143+
/**
144+
* Sync the configmap
145+
*/
146+
@Override
147+
protected void sync() {
148+
// Chain recovery, first read from upstream data source
149+
boolean syncFromUpstreamResultSuccess = trySyncFromUpstream();
150+
151+
if (syncFromUpstreamResultSuccess) {
152+
return;
153+
}
154+
155+
Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "syncK8sConfigMap");
156+
Throwable exception = null;
157+
try {
158+
configMapProperties = loadFromK8sConfigMap();
159+
sourceType = ConfigSourceType.CONFIGMAP;
160+
transaction.setStatus(Transaction.SUCCESS);
161+
} catch (Throwable ex) {
162+
Tracer.logEvent("ApolloConfigException", ExceptionUtil.getDetailMessage(ex));
163+
transaction.setStatus(ex);
164+
exception = ex;
165+
} finally {
166+
transaction.complete();
167+
}
168+
169+
if (configMapProperties == null) {
170+
sourceType = ConfigSourceType.NONE;
171+
throw new ApolloConfigException(
172+
"Load config from Kubernetes ConfigMap failed!", exception);
173+
}
174+
}
175+
176+
Properties loadFromK8sConfigMap() {
177+
Preconditions.checkNotNull(configMapName, "ConfigMap name cannot be null");
178+
179+
try {
180+
String jsonConfig = kubernetesManager.getValueFromConfigMap(k8sNamespace, configMapName, configMapKey);
181+
182+
// Convert jsonConfig to properties
183+
Properties properties = propertiesFactory.getPropertiesInstance();
184+
if (jsonConfig != null && !jsonConfig.isEmpty()) {
185+
Type type = new TypeToken<Map<String, String>>() {}.getType();
186+
Map<String, String> configMap = GSON.fromJson(jsonConfig, type);
187+
configMap.forEach(properties::setProperty);
188+
}
189+
return properties;
190+
} catch (Exception ex) {
191+
Tracer.logError(ex);
192+
throw new ApolloConfigException(String
193+
.format("Load config from Kubernetes ConfigMap %s failed!", configMapName), ex);
194+
}
195+
}
196+
197+
private boolean trySyncFromUpstream() {
198+
if (upstream == null) {
199+
return false;
200+
}
201+
try {
202+
updateConfigMapProperties(upstream.getConfig(), upstream.getSourceType());
203+
return true;
204+
} catch (Throwable ex) {
205+
Tracer.logError(ex);
206+
logger.warn("Sync config from upstream repository {} failed, reason: {}", upstream.getClass(),
207+
ExceptionUtil.getDetailMessage(ex));
208+
}
209+
return false;
210+
}
211+
212+
private synchronized void updateConfigMapProperties(Properties newProperties, ConfigSourceType sourceType) {
213+
this.sourceType = sourceType;
214+
if (newProperties == null || newProperties.equals(configMapProperties)) {
215+
return;
216+
}
217+
this.configMapProperties = newProperties;
218+
persistConfigMap(configMapProperties);
219+
}
220+
221+
/**
222+
* Update the memory
223+
*
224+
* @param namespace the namespace of this repository change
225+
* @param newProperties the properties after change
226+
*/
227+
@Override
228+
public void onRepositoryChange(String namespace, Properties newProperties) {
229+
if (newProperties == null || newProperties.equals(configMapProperties)) {
230+
return;
231+
}
232+
Properties newFileProperties = propertiesFactory.getPropertiesInstance();
233+
newFileProperties.putAll(newProperties);
234+
updateConfigMapProperties(newFileProperties, upstream.getSourceType());
235+
this.fireRepositoryChange(namespace, newProperties);
236+
}
237+
238+
void persistConfigMap(Properties properties) {
239+
Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "persistK8sConfigMap");
240+
transaction.addData("configMapName", configMapName);
241+
transaction.addData("k8sNamespace", k8sNamespace);
242+
try {
243+
// Convert properties to a JSON string using Gson
244+
String jsonConfig = GSON.toJson(properties);
245+
Map<String, String> data = new HashMap<>();
246+
data.put(configMapKey, jsonConfig);
247+
248+
// update configmap
249+
kubernetesManager.updateConfigMap(k8sNamespace, configMapName, data);
250+
transaction.setStatus(Transaction.SUCCESS);
251+
} catch (Exception ex) {
252+
ApolloConfigException exception =
253+
new ApolloConfigException(
254+
String.format("Persist config to Kubernetes ConfigMap %s failed!", configMapName), ex);
255+
Tracer.logError(exception);
256+
transaction.setStatus(exception);
257+
logger.error("Persist config to Kubernetes ConfigMap failed!", exception);
258+
} finally {
259+
transaction.complete();
260+
}
261+
}
262+
263+
}

0 commit comments

Comments
 (0)