Skip to content

Commit 1881bf9

Browse files
ryanjbaxterryanjbaxter
authored
Merge pull request #2107 from wind57/k8s-native-leader-election
K8s client leader election
2 parents 6b51391 + 1819a50 commit 1881bf9

File tree

32 files changed

+2216
-8
lines changed

32 files changed

+2216
-8
lines changed

‎docs/modules/ROOT/pages/leader-election.adoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ management.info.leader.enabled=false
5353
5454
'''
5555
56-
There is another way you can configure leader election, and it comes with native support in the fabric8 library (k8s native client support is not yet implemented). In the long run, this will be the default way to configure leader election, while the previous one will be dropped. You can treat this one much like the JDK's "preview" features.
56+
There is another way you can configure leader election, and it comes with native support in both fabric8 and kubernetes client. In the long run, this will be the default way to configure leader election, while the previous one will be dropped. You can treat this one much like the JDK's "preview" features.
5757
5858
To be able to use it, you need to set the property:
5959

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@
131131
<module>spring-cloud-starter-kubernetes-fabric8-loadbalancer</module>
132132
<module>spring-cloud-kubernetes-discovery</module>
133133
<module>spring-cloud-starter-kubernetes-discoveryclient</module>
134+
<module>spring-cloud-kubernetes-client-leader</module>
134135
</modules>
135136

136137
<dependencyManagement>

‎spring-cloud-kubernetes-client-leader/pom.xml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
<parent>
7+
<groupId>org.springframework.cloud</groupId>
8+
<artifactId>spring-cloud-kubernetes</artifactId>
9+
<version>5.0.1-SNAPSHOT</version>
10+
</parent>
11+
12+
<artifactId>spring-cloud-kubernetes-client-leader</artifactId>
13+
<name>K8s Client Spring Cloud Kubernetes :: Leader</name>
14+
15+
<dependencies>
16+
<dependency>
17+
<groupId>org.springframework.cloud</groupId>
18+
<artifactId>spring-cloud-kubernetes-client-autoconfig</artifactId>
19+
</dependency>
20+
<dependency>
21+
<groupId>org.springframework.boot</groupId>
22+
<artifactId>spring-boot-starter-actuator</artifactId>
23+
<optional>true</optional>
24+
</dependency>
25+
<dependency>
26+
<groupId>org.springframework.boot</groupId>
27+
<artifactId>spring-boot-configuration-processor</artifactId>
28+
<optional>true</optional>
29+
</dependency>
30+
<dependency>
31+
<groupId>org.springframework.boot</groupId>
32+
<artifactId>spring-boot-starter-web</artifactId>
33+
<scope>test</scope>
34+
</dependency>
35+
<dependency>
36+
<groupId>org.springframework.boot</groupId>
37+
<artifactId>spring-boot-starter-webflux</artifactId>
38+
<scope>test</scope>
39+
</dependency>
40+
<dependency>
41+
<groupId>org.springframework.boot</groupId>
42+
<artifactId>spring-boot-starter-test</artifactId>
43+
<scope>test</scope>
44+
</dependency>
45+
<dependency>
46+
<groupId>org.springframework.cloud</groupId>
47+
<artifactId>spring-cloud-kubernetes-test-support</artifactId>
48+
<scope>test</scope>
49+
</dependency>
50+
<dependency>
51+
<groupId>org.springframework.boot</groupId>
52+
<artifactId>spring-boot-webtestclient</artifactId>
53+
<scope>test</scope>
54+
</dependency>
55+
<dependency>
56+
<groupId>org.wiremock</groupId>
57+
<artifactId>wiremock-standalone</artifactId>
58+
<scope>test</scope>
59+
</dependency>
60+
</dependencies>
61+
</project>

‎spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionAutoConfiguration.java‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/*
2+
* Copyright 2013-present the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.cloud.kubernetes.client.leader.election;
18+
19+
import java.util.List;
20+
import java.util.Objects;
21+
import java.util.function.BooleanSupplier;
22+
23+
import io.kubernetes.client.extended.leaderelection.LeaderElectionConfig;
24+
import io.kubernetes.client.extended.leaderelection.Lock;
25+
import io.kubernetes.client.extended.leaderelection.resourcelock.ConfigMapLock;
26+
import io.kubernetes.client.extended.leaderelection.resourcelock.LeaseLock;
27+
import io.kubernetes.client.openapi.ApiClient;
28+
import io.kubernetes.client.openapi.ApiException;
29+
import io.kubernetes.client.openapi.apis.CoreV1Api;
30+
import io.kubernetes.client.openapi.apis.CustomObjectsApi;
31+
import io.kubernetes.client.openapi.models.V1APIResource;
32+
import io.kubernetes.client.openapi.models.V1Pod;
33+
import io.kubernetes.client.openapi.models.V1PodCondition;
34+
35+
import org.springframework.boot.actuate.autoconfigure.info.ConditionalOnEnabledInfoContributor;
36+
import org.springframework.boot.actuate.info.InfoContributor;
37+
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
38+
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
39+
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
40+
import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform;
41+
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
42+
import org.springframework.boot.cloud.CloudPlatform;
43+
import org.springframework.boot.context.properties.EnableConfigurationProperties;
44+
import org.springframework.cloud.kubernetes.commons.leader.election.ConditionalOnLeaderElectionEnabled;
45+
import org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionProperties;
46+
import org.springframework.context.annotation.Bean;
47+
import org.springframework.context.annotation.Configuration;
48+
import org.springframework.core.log.LogAccessor;
49+
50+
import static org.springframework.cloud.kubernetes.commons.leader.LeaderUtils.COORDINATION_GROUP;
51+
import static org.springframework.cloud.kubernetes.commons.leader.LeaderUtils.COORDINATION_VERSION;
52+
53+
/**
54+
* @author wind57
55+
*/
56+
@Configuration(proxyBeanMethods = false)
57+
@EnableConfigurationProperties(LeaderElectionProperties.class)
58+
@ConditionalOnBean(ApiClient.class)
59+
@ConditionalOnLeaderElectionEnabled
60+
@ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES)
61+
@AutoConfigureAfter(KubernetesClientLeaderElectionCallbacksAutoConfiguration.class)
62+
class KubernetesClientLeaderElectionAutoConfiguration {
63+
64+
private static final LogAccessor LOG = new LogAccessor(KubernetesClientLeaderElectionAutoConfiguration.class);
65+
66+
@Bean
67+
@ConditionalOnClass(InfoContributor.class)
68+
@ConditionalOnEnabledInfoContributor("leader.election")
69+
KubernetesClientLeaderElectionInfoContributor kubernetesClientLeaderElectionInfoContributor(
70+
String candidateIdentity, LeaderElectionConfig leaderElectionConfig) {
71+
return new KubernetesClientLeaderElectionInfoContributor(candidateIdentity, leaderElectionConfig);
72+
}
73+
74+
@Bean
75+
@ConditionalOnMissingBean
76+
KubernetesClientLeaderElectionInitiator kubernetesClientLeaderElectionInitiator(String candidateIdentity,
77+
String podNamespace, LeaderElectionConfig leaderElectionConfig,
78+
LeaderElectionProperties leaderElectionProperties, BooleanSupplier podReadySupplier,
79+
KubernetesClientLeaderElectionCallbacks callbacks) {
80+
return new KubernetesClientLeaderElectionInitiator(candidateIdentity, podNamespace, leaderElectionConfig,
81+
leaderElectionProperties, podReadySupplier, callbacks);
82+
}
83+
84+
@Bean
85+
@ConditionalOnMissingBean
86+
BooleanSupplier kubernetesClientPodReadySupplier(CoreV1Api coreV1Api, String candidateIdentity,
87+
String podNamespace) {
88+
return () -> {
89+
try {
90+
V1Pod pod = coreV1Api.readNamespacedPod(candidateIdentity, podNamespace).execute();
91+
return isPodReady(pod);
92+
}
93+
catch (ApiException e) {
94+
throw new RuntimeException(e);
95+
}
96+
};
97+
}
98+
99+
@Bean
100+
@ConditionalOnMissingBean
101+
LeaderElectionConfig kubernetesClientLeaderElectionConfig(LeaderElectionProperties properties, Lock lock) {
102+
return new LeaderElectionConfig(lock, properties.leaseDuration(), properties.renewDeadline(),
103+
properties.retryPeriod());
104+
}
105+
106+
@Bean
107+
@ConditionalOnMissingBean
108+
Lock kubernetesClientLeaderElectionLock(ApiClient apiClient, LeaderElectionProperties properties,
109+
String candidateIdentity) {
110+
111+
CustomObjectsApi customObjectsApi = new CustomObjectsApi(apiClient);
112+
boolean leaseSupported;
113+
try {
114+
List<V1APIResource> resources = customObjectsApi.getAPIResources(COORDINATION_GROUP, COORDINATION_VERSION)
115+
.execute()
116+
.getResources();
117+
118+
leaseSupported = resources.stream().map(V1APIResource::getKind).anyMatch("Lease"::equals);
119+
}
120+
catch (ApiException e) {
121+
throw new RuntimeException(e);
122+
}
123+
124+
if (leaseSupported) {
125+
if (properties.useConfigMapAsLock()) {
126+
LOG.info(() -> "leases are supported on the cluster, but config map will be used "
127+
+ "(because 'spring.cloud.kubernetes.leader.election.use-config-map-as-lock=true')");
128+
return new ConfigMapLock(properties.lockNamespace(), properties.lockName(), candidateIdentity);
129+
}
130+
else {
131+
LOG.info(() -> "will use lease as the lock for leader election");
132+
return new LeaseLock(properties.lockNamespace(), properties.lockName(), candidateIdentity, apiClient);
133+
}
134+
}
135+
else {
136+
LOG.info(() -> "will use configmap as the lock for leader election");
137+
return new ConfigMapLock(properties.lockNamespace(), properties.lockName(), candidateIdentity, apiClient);
138+
}
139+
}
140+
141+
// following two methods are a verbatim copy of the fabric8 implementation
142+
private static boolean isPodReady(V1Pod pod) {
143+
Objects.requireNonNull(pod, "Pod can't be null.");
144+
V1PodCondition condition = getPodReadyCondition(pod);
145+
146+
if (condition == null) {
147+
return false;
148+
}
149+
return condition.getStatus().equalsIgnoreCase("True");
150+
}
151+
152+
private static V1PodCondition getPodReadyCondition(V1Pod pod) {
153+
if (pod.getStatus() == null || pod.getStatus().getConditions() == null) {
154+
return null;
155+
}
156+
157+
for (V1PodCondition condition : pod.getStatus().getConditions()) {
158+
if ("Ready".equals(condition.getType())) {
159+
return condition;
160+
}
161+
}
162+
return null;
163+
}
164+
165+
}

‎spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionCallbacks.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright 2013-present the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.cloud.kubernetes.client.leader.election;
18+
19+
import java.util.function.Consumer;
20+
21+
/**
22+
* @author wind57
23+
*/
24+
record KubernetesClientLeaderElectionCallbacks(Runnable onStartLeadingCallback, Runnable onStopLeadingCallback,
25+
Consumer<String> onNewLeaderCallback) {
26+
}

‎spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionCallbacksAutoConfiguration.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright 2013-present the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.cloud.kubernetes.client.leader.election;
18+
19+
import java.util.function.Consumer;
20+
21+
import io.kubernetes.client.openapi.ApiClient;
22+
23+
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
24+
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
25+
import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform;
26+
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
27+
import org.springframework.boot.cloud.CloudPlatform;
28+
import org.springframework.boot.context.properties.EnableConfigurationProperties;
29+
import org.springframework.cloud.kubernetes.client.KubernetesClientAutoConfiguration;
30+
import org.springframework.cloud.kubernetes.commons.leader.election.ConditionalOnLeaderElectionEnabled;
31+
import org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionCallbacks;
32+
import org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionProperties;
33+
import org.springframework.context.annotation.Bean;
34+
import org.springframework.context.annotation.Configuration;
35+
36+
/**
37+
* @author wind57
38+
*/
39+
@Configuration(proxyBeanMethods = false)
40+
@EnableConfigurationProperties(LeaderElectionProperties.class)
41+
@ConditionalOnBean(ApiClient.class)
42+
@ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES)
43+
@ConditionalOnLeaderElectionEnabled
44+
@AutoConfigureAfter({ KubernetesClientAutoConfiguration.class })
45+
public class KubernetesClientLeaderElectionCallbacksAutoConfiguration extends LeaderElectionCallbacks {
46+
47+
@Bean
48+
@ConditionalOnMissingBean
49+
KubernetesClientLeaderElectionCallbacks kubernetesClientLeaderElectionCallbacks(Runnable onStartLeadingCallback,
50+
Runnable onStopLeadingCallback, Consumer<String> onNewLeaderCallback) {
51+
return new KubernetesClientLeaderElectionCallbacks(onStartLeadingCallback, onStopLeadingCallback,
52+
onNewLeaderCallback);
53+
}
54+
55+
}

‎spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributor.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright 2013-present the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.cloud.kubernetes.client.leader.election;
18+
19+
import java.util.HashMap;
20+
import java.util.Map;
21+
import java.util.Optional;
22+
23+
import io.kubernetes.client.extended.leaderelection.LeaderElectionConfig;
24+
import io.kubernetes.client.openapi.ApiException;
25+
26+
import org.springframework.boot.actuate.info.Info;
27+
import org.springframework.boot.actuate.info.InfoContributor;
28+
import org.springframework.core.log.LogAccessor;
29+
30+
/**
31+
* @author wind57
32+
*/
33+
final class KubernetesClientLeaderElectionInfoContributor implements InfoContributor {
34+
35+
private static final LogAccessor LOG = new LogAccessor(KubernetesClientLeaderElectionInfoContributor.class);
36+
37+
private final String candidateIdentity;
38+
39+
private final LeaderElectionConfig leaderElectionConfig;
40+
41+
KubernetesClientLeaderElectionInfoContributor(String candidateIdentity, LeaderElectionConfig leaderElectionConfig) {
42+
this.candidateIdentity = candidateIdentity;
43+
this.leaderElectionConfig = leaderElectionConfig;
44+
}
45+
46+
@Override
47+
public void contribute(Info.Builder builder) {
48+
Map<String, Object> details = new HashMap<>();
49+
try {
50+
Optional.ofNullable(leaderElectionConfig.getLock().get()).ifPresentOrElse(leaderRecord -> {
51+
boolean isLeader = candidateIdentity.equals(leaderRecord.getHolderIdentity());
52+
details.put("leaderId", candidateIdentity);
53+
details.put("isLeader", isLeader);
54+
}, () -> details.put("leaderId", "Unknown"));
55+
}
56+
catch (ApiException e) {
57+
LOG.error(e, "error in leader election info contributor");
58+
}
59+
60+
builder.withDetail("leaderElection", details);
61+
}
62+
63+
}

0 commit comments

Comments
 (0)