Skip to content

Commit 5ff6c6c

Browse files
rjeberhardrjeberhard
authored
Merge pull request #522 from oracle/watch-fix-old
Protect against outdated watch events and close possible intermittent timing windows
2 parents 697b401 + 96e4777 commit 5ff6c6c

15 files changed

+787
-994
lines changed

‎operator/src/main/java/oracle/kubernetes/operator/DomainPresenceControl.java

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
package oracle.kubernetes.operator;
66

77
import com.google.common.base.Strings;
8-
import java.util.concurrent.ScheduledFuture;
9-
import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
108
import oracle.kubernetes.weblogic.domain.v2.DomainSpec;
119

1210
class DomainPresenceControl {
@@ -30,11 +28,4 @@ private static void normalizeImagePullPolicy(DomainSpec spec) {
3028
: KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY);
3129
}
3230
}
33-
34-
static void cancelDomainStatusUpdating(DomainPresenceInfo info) {
35-
ScheduledFuture<?> statusUpdater = info.getStatusUpdater().getAndSet(null);
36-
if (statusUpdater != null) {
37-
statusUpdater.cancel(true);
38-
}
39-
}
4031
}

‎operator/src/main/java/oracle/kubernetes/operator/DomainProcessor.java

Lines changed: 667 additions & 0 deletions
Large diffs are not rendered by default.

‎operator/src/main/java/oracle/kubernetes/operator/Main.java

Lines changed: 35 additions & 702 deletions
Large diffs are not rendered by default.

‎operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java

Lines changed: 11 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,7 @@
1515
import java.util.Set;
1616
import java.util.concurrent.ConcurrentHashMap;
1717
import java.util.concurrent.ConcurrentMap;
18-
import java.util.concurrent.CopyOnWriteArraySet;
1918
import java.util.concurrent.ScheduledFuture;
20-
import java.util.concurrent.atomic.AtomicBoolean;
2119
import java.util.concurrent.atomic.AtomicReference;
2220
import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig;
2321
import oracle.kubernetes.operator.wlsconfig.WlsServerConfig;
@@ -40,10 +38,6 @@ public class DomainPresenceInfo {
4038
private final ConcurrentMap<String, V1Service> clusters = new ConcurrentHashMap<>();
4139
private final ConcurrentMap<String, V1beta1Ingress> ingresses = new ConcurrentHashMap<>();
4240

43-
private final AtomicBoolean explicitRestartAdmin = new AtomicBoolean(false);
44-
private final Set<String> explicitRestartServers = new CopyOnWriteArraySet<>();
45-
private final Set<String> explicitRestartClusters = new CopyOnWriteArraySet<>();
46-
4741
private V1PersistentVolumeClaimList claims = null;
4842

4943
private WlsDomainConfig domainConfig;
@@ -202,33 +196,6 @@ public ConcurrentMap<String, V1beta1Ingress> getIngresses() {
202196
return ingresses;
203197
}
204198

205-
/**
206-
* Control for if domain has outstanding restart admin server pending
207-
*
208-
* @return Control for pending admin server restart
209-
*/
210-
public AtomicBoolean getExplicitRestartAdmin() {
211-
return explicitRestartAdmin;
212-
}
213-
214-
/**
215-
* Control list for outstanding server restarts
216-
*
217-
* @return Control list for outstanding server restarts
218-
*/
219-
public Set<String> getExplicitRestartServers() {
220-
return explicitRestartServers;
221-
}
222-
223-
/**
224-
* Control list for outstanding cluster restarts
225-
*
226-
* @return Control list for outstanding cluster restarts
227-
*/
228-
public Set<String> getExplicitRestartClusters() {
229-
return explicitRestartClusters;
230-
}
231-
232199
/**
233200
* Server objects (Pods and Services) for admin server
234201
*
@@ -260,13 +227,18 @@ public void setServerStartupInfo(Collection<ServerStartupInfo> serverStartupInfo
260227

261228
@Override
262229
public String toString() {
263-
StringBuilder sb =
264-
new StringBuilder(
265-
String.format(
266-
"DomainPresenceInfo{uid=%s, namespace=%s",
267-
getDomain().getSpec().getDomainUID(), getDomain().getMetadata().getNamespace()));
230+
StringBuilder sb = new StringBuilder("DomainPresenceInfo{");
231+
Domain d = getDomain();
232+
if (d != null) {
233+
sb.append(
234+
String.format(
235+
"uid=%s, namespace=%s",
236+
getDomain().getSpec().getDomainUID(), getDomain().getMetadata().getNamespace()));
237+
} else {
238+
sb.append(", namespace=").append(namespace);
239+
}
268240
if (!ingresses.isEmpty()) {
269-
sb.append(", ingresses ").append(String.join(",", ingresses.keySet()));
241+
sb.append(", ingresses=").append(String.join(",", ingresses.keySet()));
270242
}
271243
sb.append("}");
272244

‎operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -83,23 +83,6 @@ String getPodReplacedMessageKey() {
8383
return MessageKeys.ADMIN_POD_REPLACED;
8484
}
8585

86-
@Override
87-
void updateRestartForNewPod() {
88-
getExplicitRestartAdmin().set(false);
89-
getExplicitRestartServers().remove(getServerName());
90-
}
91-
92-
@Override
93-
void updateRestartForReplacedPod() {
94-
getExplicitRestartAdmin().set(false);
95-
}
96-
97-
@Override
98-
protected boolean isExplicitRestartThisServer() {
99-
return getExplicitRestartAdmin().get()
100-
|| getExplicitRestartServers().contains(getServerName());
101-
}
102-
10386
@Override
10487
protected V1PodSpec createSpec(TuningParameters tuningParameters) {
10588
return super.createSpec(tuningParameters).hostname(getPodName());
@@ -252,14 +235,6 @@ Step createNewPod(Step next) {
252235
DomainStatusUpdater.MANAGED_SERVERS_STARTING_PROGRESS_REASON, false, createPod(next));
253236
}
254237

255-
@Override
256-
void updateRestartForNewPod() {
257-
getExplicitRestartServers().remove(getServerName());
258-
}
259-
260-
@Override
261-
void updateRestartForReplacedPod() {}
262-
263238
@Override
264239
String getPodCreatedMessageKey() {
265240
return MessageKeys.MANAGED_POD_CREATED;
@@ -275,12 +250,6 @@ protected String getPodReplacedMessageKey() {
275250
return MessageKeys.MANAGED_POD_REPLACED;
276251
}
277252

278-
@Override
279-
protected boolean isExplicitRestartThisServer() {
280-
return getExplicitRestartServers().contains(getServerName())
281-
|| (getClusterName() != null && getExplicitRestartClusters().contains(getClusterName()));
282-
}
283-
284253
@Override
285254
protected V1ObjectMeta createMetadata() {
286255
V1ObjectMeta metadata = super.createMetadata();

‎operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@
3333
import java.util.List;
3434
import java.util.Map;
3535
import java.util.Optional;
36-
import java.util.Set;
37-
import java.util.concurrent.atomic.AtomicBoolean;
3836
import oracle.kubernetes.operator.KubernetesConstants;
3937
import oracle.kubernetes.operator.LabelConstants;
4038
import oracle.kubernetes.operator.PodAwaiterStepFactory;
@@ -287,20 +285,6 @@ private void logPodReplaced() {
287285

288286
abstract String getPodReplacedMessageKey();
289287

290-
AtomicBoolean getExplicitRestartAdmin() {
291-
return info.getExplicitRestartAdmin();
292-
}
293-
294-
abstract void updateRestartForNewPod();
295-
296-
abstract void updateRestartForReplacedPod();
297-
298-
Set<String> getExplicitRestartClusters() {
299-
return info.getExplicitRestartClusters();
300-
}
301-
302-
protected abstract boolean isExplicitRestartThisServer();
303-
304288
Step createCyclePodStep(Step next) {
305289
return new CyclePodStep(next);
306290
}
@@ -319,7 +303,7 @@ public NextAction apply(Packet packet) {
319303
}
320304

321305
private boolean canUseCurrentPod(V1Pod currentPod) {
322-
return !isExplicitRestartThisServer() && isCurrentPodValid(getPodModel(), currentPod);
306+
return isCurrentPodValid(getPodModel(), currentPod);
323307
}
324308

325309
// We want to detect changes that would require replacing an existing Pod
@@ -393,10 +377,6 @@ private static <T> boolean areUnequal(List<T> a, List<T> b) {
393377
return false;
394378
}
395379

396-
Set<String> getExplicitRestartServers() {
397-
return info.getExplicitRestartServers();
398-
}
399-
400380
private class VerifyPodStep extends Step {
401381

402382
VerifyPodStep(Step next) {
@@ -407,7 +387,6 @@ private class VerifyPodStep extends Step {
407387
public NextAction apply(Packet packet) {
408388
V1Pod currentPod = getSko().getPod().get();
409389
if (currentPod == null) {
410-
updateRestartForNewPod();
411390
return doNext(createNewPod(getNext()), packet);
412391
} else if (canUseCurrentPod(currentPod)) {
413392
logPodExists();
@@ -461,7 +440,6 @@ public NextAction onFailure(Packet packet, CallResponse<V1Status> callResponses)
461440

462441
@Override
463442
public NextAction onSuccess(Packet packet, CallResponse<V1Status> callResponses) {
464-
updateRestartForReplacedPod();
465443
return doNext(replacePod(getNext()), packet);
466444
}
467445
}

‎operator/src/main/java/oracle/kubernetes/operator/helpers/StorageHelper.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,12 @@
88
import oracle.kubernetes.operator.work.NextAction;
99
import oracle.kubernetes.operator.work.Packet;
1010
import oracle.kubernetes.operator.work.Step;
11+
import oracle.kubernetes.weblogic.domain.v2.Domain;
1112

1213
public class StorageHelper {
13-
public static Step insertStorageSteps(DomainPresenceInfo info, Step next) {
14-
if (info.getDomain().getRequiredPersistentVolumeClaim() != null)
15-
next = new PersistentVolumeClaimStep(next);
16-
if (info.getDomain().getRequiredPersistentVolume() != null)
17-
next = new PersistentVolumeStep(next);
14+
public static Step insertStorageSteps(Domain dom, Step next) {
15+
if (dom.getRequiredPersistentVolumeClaim() != null) next = new PersistentVolumeClaimStep(next);
16+
if (dom.getRequiredPersistentVolume() != null) next = new PersistentVolumeStep(next);
1817
return next;
1918
}
2019

‎operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,11 @@
1111
import io.kubernetes.client.models.V1PersistentVolumeList;
1212
import io.kubernetes.client.models.V1ServiceList;
1313
import io.kubernetes.client.models.V1beta1IngressList;
14+
import java.util.concurrent.ScheduledFuture;
1415
import oracle.kubernetes.operator.calls.CallResponse;
1516
import oracle.kubernetes.operator.helpers.CallBuilder;
17+
import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
18+
import oracle.kubernetes.operator.helpers.DomainPresenceInfoManager;
1619
import oracle.kubernetes.operator.logging.LoggingFacade;
1720
import oracle.kubernetes.operator.logging.LoggingFactory;
1821
import oracle.kubernetes.operator.logging.MessageKeys;
@@ -23,27 +26,44 @@
2326
public class DeleteDomainStep extends Step {
2427
private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator");
2528

29+
private final DomainPresenceInfo info;
2630
private final String namespace;
2731
private final String domainUID;
2832

29-
public DeleteDomainStep(String namespace, String domainUID) {
33+
public DeleteDomainStep(DomainPresenceInfo info, String namespace, String domainUID) {
3034
super(null);
35+
this.info = info;
3136
this.namespace = namespace;
3237
this.domainUID = domainUID;
3338
}
3439

3540
@Override
3641
public NextAction apply(Packet packet) {
42+
if (info != null) {
43+
cancelDomainStatusUpdating(info);
44+
}
45+
3746
return doNext(
3847
Step.chain(
3948
deleteIngresses(),
4049
deleteServices(),
4150
deletePods(),
4251
deletePersistentVolumes(),
43-
deletePersistentVolumeClaims()),
52+
deletePersistentVolumeClaims(),
53+
removeDomainPresenceInfo()),
4454
packet);
4555
}
4656

57+
private Step removeDomainPresenceInfo() {
58+
return new Step() {
59+
@Override
60+
public NextAction apply(Packet packet) {
61+
DomainPresenceInfoManager.remove(domainUID);
62+
return doNext(packet);
63+
}
64+
};
65+
}
66+
4767
private Step deleteIngresses() {
4868
LOGGER.finer(MessageKeys.LIST_INGRESS_FOR_DOMAIN, this.domainUID, namespace);
4969
return new CallBuilder()
@@ -58,6 +78,13 @@ Step createSuccessStep(V1beta1IngressList result, Step next) {
5878
});
5979
}
6080

81+
static void cancelDomainStatusUpdating(DomainPresenceInfo info) {
82+
ScheduledFuture<?> statusUpdater = info.getStatusUpdater().getAndSet(null);
83+
if (statusUpdater != null) {
84+
statusUpdater.cancel(true);
85+
}
86+
}
87+
6188
private Step deleteServices() {
6289
return new CallBuilder()
6390
.withLabelSelectors(forDomainUid(domainUID), CREATEDBYOPERATOR_LABEL)

‎operator/src/main/java/oracle/kubernetes/operator/steps/DomainPresenceStep.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@
44

55
package oracle.kubernetes.operator.steps;
66

7-
import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
87
import oracle.kubernetes.operator.work.NextAction;
98
import oracle.kubernetes.operator.work.Packet;
109
import oracle.kubernetes.operator.work.Step;
10+
import oracle.kubernetes.weblogic.domain.v2.Domain;
1111

1212
public class DomainPresenceStep extends Step {
1313

@@ -16,9 +16,8 @@ private DomainPresenceStep(Step adminStep) {
1616
}
1717

1818
public static DomainPresenceStep createDomainPresenceStep(
19-
DomainPresenceInfo info, Step adminStep, Step managedServerStep) {
20-
return new DomainPresenceStep(
21-
info.getDomain().isShuttingDown() ? managedServerStep : adminStep);
19+
Domain dom, Step adminStep, Step managedServerStep) {
20+
return new DomainPresenceStep(dom.isShuttingDown() ? managedServerStep : adminStep);
2221
}
2322

2423
@Override

‎operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,6 @@ public NextAction apply(Packet packet) {
9999
LOGGER.fine(SERVERS_UP_MSG, factory.domain.getDomainUID(), getRunningServers(info));
100100
}
101101

102-
updateExplicitRestart(info);
103-
104102
for (WlsServerConfig serverConfig : info.getScan().getServerConfigs().values()) {
105103
factory.addServerIfNeeded(serverConfig, null);
106104
}
@@ -119,18 +117,6 @@ public NextAction apply(Packet packet) {
119117
packet);
120118
}
121119

122-
private void updateExplicitRestart(DomainPresenceInfo info) {
123-
for (String clusterName : info.getExplicitRestartClusters()) {
124-
WlsClusterConfig cluster = info.getScan().getClusterConfig(clusterName);
125-
if (cluster != null) {
126-
for (WlsServerConfig server : cluster.getServerConfigs()) {
127-
info.getExplicitRestartServers().add(server.getName());
128-
}
129-
}
130-
}
131-
info.getExplicitRestartClusters().clear();
132-
}
133-
134120
private Collection<String> getRunningServers(DomainPresenceInfo info) {
135121
Collection<String> runningList = new ArrayList<>();
136122
for (Map.Entry<String, ServerKubernetesObjects> entry : info.getServers().entrySet()) {

0 commit comments

Comments
 (0)