Skip to content

Commit 0256d80

Browse files
committed
Terminate is not delete
1 parent faec58f commit 0256d80

File tree

7 files changed

+129
-103
lines changed

7 files changed

+129
-103
lines changed

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,14 @@ public void dispatchPodWatch(Watch.Response<V1Pod> item) {
147147
if (info != null) {
148148
switch (item.type) {
149149
case "ADDED":
150+
info.setServerPodBeingDeleted(serverName, Boolean.FALSE);
151+
// fall through
150152
case "MODIFIED":
151153
info.setServerPodFromEvent(serverName, pod);
152154
break;
153155
case "DELETED":
154156
boolean removed = info.deleteServerPodFromEvent(serverName, pod);
155-
if (removed && info.isNotDeleting()) {
157+
if (removed && info.isNotDeleting() && !info.isServerPodBeingDeleted(serverName)) {
156158
LOGGER.info(
157159
MessageKeys.POD_DELETED, domainUID, metadata.getNamespace(), serverName);
158160
makeRightDomainPresence(info, true, false, true);

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,18 @@ private V1Pod getPod(ServerKubernetesObjects sko) {
122122
return sko.getPod().get();
123123
}
124124

125+
private Boolean getPodIsBeingDeleted(ServerKubernetesObjects sko) {
126+
return sko.isPodBeingDeleted().get();
127+
}
128+
129+
public Boolean isServerPodBeingDeleted(String serverName) {
130+
return getPodIsBeingDeleted(getSko(serverName));
131+
}
132+
133+
public void setServerPodBeingDeleted(String serverName, Boolean isBeingDeleted) {
134+
getSko(serverName).isPodBeingDeleted().set(isBeingDeleted);
135+
}
136+
125137
/**
126138
* Returns a collection of all servers defined.
127139
*

operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java

Lines changed: 44 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,21 @@ private Step getConflictStep() {
103103

104104
abstract Map<String, String> getPodAnnotations();
105105

106-
private class ConflictStep extends Step {
106+
private abstract class BaseStep extends Step {
107+
BaseStep() {
108+
this(null);
109+
}
110+
111+
BaseStep(Step next) {
112+
super(next);
113+
}
114+
115+
protected String getDetail() {
116+
return getServerName();
117+
}
118+
}
119+
120+
private class ConflictStep extends BaseStep {
107121

108122
@Override
109123
public NextAction apply(Packet packet) {
@@ -225,8 +239,12 @@ private List<V1ContainerPort> getContainerPorts() {
225239
// ----------------------- step methods ------------------------------
226240

227241
// Prevent the watcher from recreating pod with old spec
228-
private void clearRecord() {
229-
setRecordedPod(null);
242+
private void markBeingDeleted() {
243+
info.setServerPodBeingDeleted(getServerName(), Boolean.TRUE);
244+
}
245+
246+
private void clearBeingDeleted() {
247+
info.setServerPodBeingDeleted(getServerName(), Boolean.FALSE);
230248
}
231249

232250
private void setRecordedPod(V1Pod pod) {
@@ -269,6 +287,7 @@ private Step deletePod(Step next) {
269287
* @return a step to be scheduled.
270288
*/
271289
Step createPod(Step next) {
290+
clearBeingDeleted();
272291
return createPodAsync(createResponse(next));
273292
}
274293

@@ -337,15 +356,15 @@ Step createCyclePodStep(Step next) {
337356
return new CyclePodStep(next);
338357
}
339358

340-
private class CyclePodStep extends Step {
359+
private class CyclePodStep extends BaseStep {
341360

342361
CyclePodStep(Step next) {
343362
super(next);
344363
}
345364

346365
@Override
347366
public NextAction apply(Packet packet) {
348-
clearRecord();
367+
markBeingDeleted();
349368
return doNext(deletePod(getNext()), packet);
350369
}
351370
}
@@ -373,7 +392,7 @@ private String getReasonToRecycle(V1Pod currentPod) {
373392
return compatibility.getIncompatibility();
374393
}
375394

376-
private class VerifyPodStep extends Step {
395+
private class VerifyPodStep extends BaseStep {
377396

378397
VerifyPodStep(Step next) {
379398
super(next);
@@ -399,11 +418,21 @@ public NextAction apply(Packet packet) {
399418
}
400419
}
401420

421+
private abstract class BaseResponseStep extends ResponseStep<V1Pod> {
422+
BaseResponseStep(Step next) {
423+
super(next);
424+
}
425+
426+
protected String getDetail() {
427+
return getServerName();
428+
}
429+
}
430+
402431
private ResponseStep<V1Pod> createResponse(Step next) {
403432
return new CreateResponseStep(next);
404433
}
405434

406-
private class CreateResponseStep extends ResponseStep<V1Pod> {
435+
private class CreateResponseStep extends BaseResponseStep {
407436
CreateResponseStep(Step next) {
408437
super(next);
409438
}
@@ -433,6 +462,10 @@ private class DeleteResponseStep extends ResponseStep<V1Status> {
433462
super(next);
434463
}
435464

465+
protected String getDetail() {
466+
return getServerName();
467+
}
468+
436469
@Override
437470
public NextAction onFailure(Packet packet, CallResponse<V1Status> callResponses) {
438471
if (callResponses.getStatusCode() == CallBuilder.NOT_FOUND) {
@@ -451,12 +484,10 @@ private ResponseStep<V1Pod> replaceResponse(Step next) {
451484
return new ReplacePodResponseStep(next);
452485
}
453486

454-
private class ReplacePodResponseStep extends ResponseStep<V1Pod> {
455-
private final Step next;
487+
private class ReplacePodResponseStep extends BaseResponseStep {
456488

457489
ReplacePodResponseStep(Step next) {
458490
super(next);
459-
this.next = next;
460491
}
461492

462493
@Override
@@ -473,15 +504,16 @@ public NextAction onSuccess(Packet packet, CallResponse<V1Pod> callResponse) {
473504
setRecordedPod(newPod);
474505
}
475506

476-
return doNext(next, packet);
507+
PodAwaiterStepFactory pw = packet.getSPI(PodAwaiterStepFactory.class);
508+
return doNext(pw.waitForReady(newPod, getNext()), packet);
477509
}
478510
}
479511

480512
private ResponseStep<V1Pod> patchResponse(Step next) {
481513
return new PatchPodResponseStep(next);
482514
}
483515

484-
private class PatchPodResponseStep extends ResponseStep<V1Pod> {
516+
private class PatchPodResponseStep extends BaseResponseStep {
485517
private final Step next;
486518

487519
PatchPodResponseStep(Step next) {

operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java

Lines changed: 53 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,8 @@
55
package oracle.kubernetes.operator.helpers;
66

77
import io.kubernetes.client.models.V1Pod;
8-
import java.util.ArrayList;
9-
import java.util.Collection;
10-
import java.util.HashMap;
11-
import java.util.Iterator;
12-
import java.util.List;
13-
import java.util.Map;
14-
import oracle.kubernetes.operator.PodAwaiterStepFactory;
8+
import java.util.*;
9+
import java.util.concurrent.ConcurrentLinkedQueue;
1510
import oracle.kubernetes.operator.ProcessingConstants;
1611
import oracle.kubernetes.operator.logging.LoggingFacade;
1712
import oracle.kubernetes.operator.logging.LoggingFactory;
@@ -74,7 +69,7 @@ public NextAction apply(Packet packet) {
7469
List<String> availableServers = getReadyServers(info);
7570

7671
Collection<StepAndPacket> serversThatCanRestartNow = new ArrayList<>();
77-
Map<String, Collection<StepAndPacket>> clusteredRestarts = new HashMap<>();
72+
Map<String, Queue<StepAndPacket>> clusteredRestarts = new HashMap<>();
7873

7974
List<String> servers = new ArrayList<>();
8075
for (Map.Entry<String, StepAndPacket> entry : rolling.entrySet()) {
@@ -96,9 +91,9 @@ public NextAction apply(Packet packet) {
9691
}
9792

9893
// clustered server
99-
Collection<StepAndPacket> cr = clusteredRestarts.get(clusterName);
94+
Queue<StepAndPacket> cr = clusteredRestarts.get(clusterName);
10095
if (cr == null) {
101-
cr = new ArrayList<>();
96+
cr = new ConcurrentLinkedQueue<>();
10297
clusteredRestarts.put(clusterName, cr);
10398
}
10499
cr.add(entry.getValue());
@@ -116,7 +111,7 @@ public NextAction apply(Packet packet) {
116111
}
117112

118113
if (!clusteredRestarts.isEmpty()) {
119-
for (Map.Entry<String, Collection<StepAndPacket>> entry : clusteredRestarts.entrySet()) {
114+
for (Map.Entry<String, Queue<StepAndPacket>> entry : clusteredRestarts.entrySet()) {
120115
work.add(
121116
new StepAndPacket(
122117
new RollSpecificClusterStep(entry.getKey(), entry.getValue(), null), packet));
@@ -160,13 +155,13 @@ private static List<String> getReadyServers(DomainPresenceInfo info) {
160155

161156
private static class RollSpecificClusterStep extends Step {
162157
private final String clusterName;
163-
private final Iterator<StepAndPacket> it;
158+
private final Queue<StepAndPacket> servers;
164159

165160
public RollSpecificClusterStep(
166-
String clusterName, Collection<StepAndPacket> clusteredServerRestarts, Step next) {
161+
String clusterName, Queue<StepAndPacket> clusteredServerRestarts, Step next) {
167162
super(next);
168163
this.clusterName = clusterName;
169-
it = clusteredServerRestarts.iterator();
164+
servers = clusteredServerRestarts;
170165
}
171166

172167
@Override
@@ -176,89 +171,58 @@ public String getDetail() {
176171

177172
@Override
178173
public NextAction apply(Packet packet) {
179-
synchronized (it) {
180-
if (it.hasNext()) {
181-
DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class);
182-
WlsDomainConfig config =
183-
(WlsDomainConfig) packet.get(ProcessingConstants.DOMAIN_TOPOLOGY);
184-
185-
// Refresh as this is constantly changing
186-
Domain dom = info.getDomain();
187-
// These are presently Ready servers
188-
List<String> availableServers = getReadyServers(info);
189-
190-
List<String> servers = new ArrayList<>();
191-
List<String> readyServers = new ArrayList<>();
192-
List<V1Pod> notReadyServers = new ArrayList<>();
193-
194-
Collection<StepAndPacket> serversThatCanRestartNow = new ArrayList<>();
195-
196-
int countReady = 0;
197-
WlsClusterConfig cluster = config != null ? config.getClusterConfig(clusterName) : null;
198-
if (cluster != null) {
199-
List<WlsServerConfig> serversConfigs = cluster.getServerConfigs();
200-
if (serversConfigs != null) {
201-
for (WlsServerConfig s : serversConfigs) {
202-
// figure out how many servers are currently ready
203-
String name = s.getName();
204-
if (availableServers.contains(name)) {
205-
readyServers.add(s.getName());
206-
countReady++;
207-
} else {
208-
V1Pod pod = info.getServerPod(name);
209-
if (pod != null) {
210-
notReadyServers.add(pod);
211-
}
212-
}
213-
}
214-
}
215-
}
174+
DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class);
175+
WlsDomainConfig config = (WlsDomainConfig) packet.get(ProcessingConstants.DOMAIN_TOPOLOGY);
216176

217-
// then add as many as possible next() entries leaving at least minimum cluster
218-
// availability
219-
while (countReady-- > dom.getMinAvailable(clusterName)) {
220-
StepAndPacket current = it.next();
221-
WlsServerConfig serverConfig =
222-
(WlsServerConfig) current.packet.get(ProcessingConstants.SERVER_SCAN);
223-
String serverName = null;
224-
if (serverConfig != null) {
225-
serverName = serverConfig.getName();
226-
} else if (config != null) {
227-
serverName = config.getAdminServerName();
228-
}
229-
if (serverName != null) {
230-
servers.add(serverName);
231-
}
232-
serversThatCanRestartNow.add(current);
233-
if (!it.hasNext()) {
234-
break;
235-
}
236-
}
177+
// Refresh as this is constantly changing
178+
Domain dom = info.getDomain();
179+
// These are presently Ready servers
180+
List<String> availableServers = getReadyServers(info);
237181

238-
if (serversThatCanRestartNow.isEmpty()) {
239-
// Not enough servers are ready to let us restart a server now
240-
if (!notReadyServers.isEmpty()) {
241-
PodAwaiterStepFactory pw = PodHelper.getPodAwaiterStepFactory(packet);
242-
Collection<StepAndPacket> waitForUnreadyServers = new ArrayList<>();
243-
for (V1Pod pod : notReadyServers) {
244-
waitForUnreadyServers.add(
245-
new StepAndPacket(pw.waitForReady(pod, null), packet.clone()));
246-
}
247-
248-
// Wait for at least one of the not-yet-ready servers to become ready
249-
return doForkAtLeastOne(this, packet, waitForUnreadyServers);
250-
} else {
251-
throw new IllegalStateException();
182+
List<String> readyServers = new ArrayList<>();
183+
184+
int countReady = 0;
185+
WlsClusterConfig cluster = config != null ? config.getClusterConfig(clusterName) : null;
186+
if (cluster != null) {
187+
List<WlsServerConfig> serversConfigs = cluster.getServerConfigs();
188+
if (serversConfigs != null) {
189+
for (WlsServerConfig s : serversConfigs) {
190+
// figure out how many servers are currently ready
191+
String name = s.getName();
192+
if (availableServers.contains(name)) {
193+
readyServers.add(s.getName());
194+
countReady++;
252195
}
253196
}
197+
}
198+
}
254199

255-
readyServers.removeAll(servers);
256-
LOGGER.info(MessageKeys.ROLLING_SERVERS, dom.getDomainUID(), servers, readyServers);
200+
LOGGER.info(MessageKeys.ROLLING_SERVERS, dom.getDomainUID(), servers, readyServers);
257201

258-
return doNext(new ServersThatCanRestartNowStep(serversThatCanRestartNow, this), packet);
259-
}
202+
int countToRestartNow = Math.max(1, countReady - dom.getMinAvailable(clusterName));
203+
Collection<StepAndPacket> restarts = new ArrayList<>();
204+
for (int i = 0; i < countToRestartNow; i++) {
205+
restarts.add(new StepAndPacket(new RestartOneClusteredServerStep(servers, null), packet));
260206
}
207+
return doForkJoin(getNext(), packet, restarts);
208+
}
209+
}
210+
211+
private static class RestartOneClusteredServerStep extends Step {
212+
private final Queue<StepAndPacket> servers;
261213

214+
public RestartOneClusteredServerStep(Queue<StepAndPacket> servers, Step next) {
215+
super(next);
216+
this.servers = servers;
217+
}
218+
219+
@Override
220+
public NextAction apply(Packet packet) {
221+
StepAndPacket serverToRestart = servers.poll();
222+
if (serverToRestart != null) {
223+
Collection<StepAndPacket> col = Collections.singleton(serverToRestart);
224+
return doForkJoin(this, packet, col);
225+
}
262226
return doNext(packet);
263227
}
264228
}

0 commit comments

Comments
 (0)