Skip to content

Commit 835f1d6

Browse files
rjeberhardrjeberhard
committed
Tuning parameters
1 parent 5e8bdc1 commit 835f1d6

File tree

6 files changed

+80
-46
lines changed

6 files changed

+80
-46
lines changed

‎src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ public NextAction apply(Packet packet) {
126126
if (scan != null) {
127127
for (Map.Entry<String, WlsServerConfig> entry : scan.getServerConfigs().entrySet()) {
128128
String serverName = entry.getKey();
129-
ServerHealth health = new ServerHealth().withState(serverState.getOrDefault(serverName, "UNKNOWN"));
129+
ServerHealth health = new ServerHealth().withState(serverState.getOrDefault(serverName, "SHUTDOWN"));
130130
ServerStatus ss = new ServerStatus()
131131
.withServerName(serverName)
132132
.withHealth(health);
@@ -160,7 +160,7 @@ public NextAction apply(Packet packet) {
160160
V1Pod pod = entry.getValue().getPod().get();
161161
if (pod != null) {
162162
ServerHealth health = new ServerHealth()
163-
.withState(serverState.getOrDefault(serverName, "UNKNOWN"))
163+
.withState(serverState.getOrDefault(serverName, "SHUTDOWN"))
164164
.withClusterName(pod.getMetadata().getLabels().get(LabelConstants.CLUSTERNAME_LABEL))
165165
.withNodeName(pod.getSpec().getNodeName())
166166
.withStartTime(pod.getMetadata().getCreationTimestamp());
@@ -187,16 +187,6 @@ public NextAction apply(Packet packet) {
187187
madeChange = true;
188188
}
189189

190-
// This will control if we need to re-check states soon or if we can slow down checks
191-
Boolean areServerStatesClean = Boolean.TRUE;
192-
for (ServerStatus ss : status.getStatus()) {
193-
if (!"RUNNING".equals(ss.getHealth().getState()) && !"SHUTDOWN".equals(ss.getHealth().getState())) {
194-
areServerStatesClean = Boolean.FALSE;
195-
break;
196-
}
197-
}
198-
packet.put(ProcessingConstants.CLEAN_STATUS, areServerStatesClean);
199-
200190
// Now, we'll build the conditions.
201191
// Possible condition types are Progressing, Available, and Failed
202192
// Each condition is either True, False, or Unknown
@@ -302,8 +292,13 @@ public NextAction apply(Packet packet) {
302292
conditions.add(dc);
303293
madeChange = true;
304294
}
295+
296+
// This will control if we need to re-check states soon or if we can slow down checks
297+
packet.put(ProcessingConstants.STATUS_UNCHANGED, Boolean.valueOf(!madeChange));
305298

306-
LOGGER.info(MessageKeys.DOMAIN_STATUS, spec.getDomainUID(), status);
299+
if (madeChange) {
300+
LOGGER.info(MessageKeys.DOMAIN_STATUS, spec.getDomainUID(), status);
301+
}
307302
LOGGER.exiting();
308303

309304
return madeChange == true ? doDomainUpdate(dom, info, packet, StatusUpdateStep.this, next) : doNext(packet);

‎src/main/java/oracle/kubernetes/operator/Main.java

Lines changed: 43 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import java.util.concurrent.ScheduledFuture;
1717
import java.util.concurrent.TimeUnit;
1818
import java.util.concurrent.atomic.AtomicBoolean;
19+
import java.util.concurrent.atomic.AtomicInteger;
1920
import java.util.concurrent.atomic.AtomicReference;
2021

2122
import io.kubernetes.client.ApiException;
@@ -89,6 +90,14 @@ public class Main {
8990
private static final FiberGate domainUpdaters = new FiberGate(engine);
9091
private static final ConcurrentMap<String, DomainPresenceInfo> domains = new ConcurrentHashMap<String, DomainPresenceInfo>();
9192

93+
private static final ConfigMapConsumer config = new ConfigMapConsumer("/operator/config");
94+
95+
// tuning parameters
96+
private static final int statusUpdateTimeoutSeconds = (int) readTuningParameter("statusUpdateTimeoutSeconds", 10);
97+
private static final int unchangedCountToDelayStatusRecheck = (int) readTuningParameter("unchangedCountToDelayStatusRecheck", 10);
98+
private static final long initialShortDelay = readTuningParameter("initialShortDelay", 3);
99+
private static final long eventualLongDelay = readTuningParameter("eventualLongDelay", 30);
100+
92101
private static final ConcurrentMap<String, Boolean> initialized = new ConcurrentHashMap<>();
93102
private static final AtomicBoolean stopping = new AtomicBoolean(false);
94103

@@ -122,8 +131,7 @@ public static void main(String[] args) {
122131

123132
Collection<String> targetNamespaces = getTargetNamespaces(namespace);
124133

125-
ConfigMapConsumer cmc = new ConfigMapConsumer("/operator/config");
126-
String serviceAccountName = cmc.get("serviceaccount");
134+
String serviceAccountName = config.get("serviceaccount");
127135
if (serviceAccountName == null) {
128136
serviceAccountName = "default";
129137
}
@@ -409,6 +417,19 @@ private static void normalizeDomainSpec(DomainSpec spec) {
409417
}
410418
}
411419

420+
private static long readTuningParameter(String parameter, long defaultValue) {
421+
String val = config.get(parameter);
422+
if (val != null) {
423+
try {
424+
return Long.parseLong(val);
425+
} catch (NumberFormatException nfe) {
426+
LOGGER.warning(MessageKeys.EXCEPTION, nfe);
427+
}
428+
}
429+
430+
return defaultValue;
431+
}
432+
412433
/**
413434
* Restarts the admin server, if already running
414435
* @param principal Service principal
@@ -458,43 +479,51 @@ public static void doRollingRestartClusters(String principal, String domainUID,
458479
}
459480
}
460481

461-
private static void scheduleDomainStatusUpdating(DomainPresenceInfo info, long initialShortDelay, long eventualLongDelay, TimeUnit timeUnit) {
482+
private static void scheduleDomainStatusUpdating(DomainPresenceInfo info) {
462483
String domainUID = info.getDomain().getSpec().getDomainUID();
484+
AtomicInteger unchangedCount = new AtomicInteger(0);
463485
AtomicReference<ScheduledFuture<?>> statusUpdater = info.getStatusUpdater();
464486
ScheduledFuture<?> existing = statusUpdater.get();
465-
if (existing == null || !validateExisting(initialShortDelay, timeUnit, existing)) {
487+
if (existing == null || !validateExisting(initialShortDelay, existing)) {
466488
Runnable command = new Runnable() {
467489
public void run() {
468-
Runnable r = this; // resolve visibility later
490+
Runnable r = this; // resolve visibility
469491
Packet packet = new Packet();
470492
packet.getComponents().put(ProcessingConstants.DOMAIN_COMPONENT_NAME, Component.createFor(info, version));
471-
Step strategy = DomainStatusUpdater.createStatusStep(10, null); // FIXME: configure
493+
Step strategy = DomainStatusUpdater.createStatusStep(statusUpdateTimeoutSeconds, null);
472494
domainUpdaters.startFiberIfNoCurrentFiber(domainUID, strategy, packet, new CompletionCallback() {
473495
@Override
474496
public void onCompletion(Packet packet) {
475-
Boolean isStatusClean = (Boolean) packet.get(ProcessingConstants.CLEAN_STATUS);
476-
long delay = Boolean.TRUE.equals(isStatusClean) ? eventualLongDelay : initialShortDelay;
497+
Boolean isStatusUnchanged = (Boolean) packet.get(ProcessingConstants.STATUS_UNCHANGED);
498+
long delay = initialShortDelay;
499+
if (Boolean.TRUE.equals(isStatusUnchanged)) {
500+
if (unchangedCount.incrementAndGet() > unchangedCountToDelayStatusRecheck) {
501+
delay = eventualLongDelay;
502+
}
503+
} else {
504+
unchangedCount.set(0);
505+
}
477506
// retry after delay
478-
statusUpdater.set(engine.getExecutor().schedule(r, delay, timeUnit));
507+
statusUpdater.set(engine.getExecutor().schedule(r, delay, TimeUnit.SECONDS));
479508
}
480509

481510
@Override
482511
public void onThrowable(Packet packet, Throwable throwable) {
483512
LOGGER.severe(MessageKeys.EXCEPTION, throwable);
484513
// retry after delay
485-
statusUpdater.set(engine.getExecutor().schedule(r, initialShortDelay, timeUnit));
514+
statusUpdater.set(engine.getExecutor().schedule(r, initialShortDelay, TimeUnit.SECONDS));
486515
}
487516
});
488517
}
489518
};
490-
statusUpdater.set(engine.getExecutor().schedule(command, initialShortDelay, timeUnit));
519+
statusUpdater.set(engine.getExecutor().schedule(command, initialShortDelay, TimeUnit.SECONDS));
491520
}
492521
}
493522

494-
private static boolean validateExisting(long initialShortDelay, TimeUnit timeUnit, ScheduledFuture<?> existing) {
523+
private static boolean validateExisting(long initialShortDelay, ScheduledFuture<?> existing) {
495524
if (existing.isCancelled())
496525
return false;
497-
return existing.getDelay(timeUnit) <= initialShortDelay;
526+
return existing.getDelay(TimeUnit.SECONDS) <= initialShortDelay;
498527
}
499528

500529
private static void cancelDomainStatusUpdating(DomainPresenceInfo info) {
@@ -542,7 +571,7 @@ private static void doCheckAndCreateDomainPresence(
542571
}
543572
info.setDomain(dom);
544573
}
545-
scheduleDomainStatusUpdating(info, 3, 30, TimeUnit.SECONDS); // FIXME: configure
574+
scheduleDomainStatusUpdating(info);
546575

547576
LOGGER.info(MessageKeys.PROCESSING_DOMAIN, domainUID);
548577

‎src/main/java/oracle/kubernetes/operator/PodWatcher.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,12 @@ private void processEventCallback(Watch.Response<V1Pod> item) {
144144

145145
LOGGER.exiting();
146146
}
147-
147+
148148
static boolean isReady(V1Pod pod) {
149+
return isReady(pod, false);
150+
}
151+
152+
static boolean isReady(V1Pod pod, boolean isStatusCheck) {
149153
V1PodStatus status = pod.getStatus();
150154
if (status != null) {
151155
if ("Running".equals(status.getPhase())) {
@@ -155,7 +159,9 @@ static boolean isReady(V1Pod pod) {
155159
if ("Ready".equals(cond.getType())) {
156160
if ("True".equals(cond.getStatus())) {
157161
// Pod is Ready!
158-
LOGGER.info(MessageKeys.POD_IS_READY, pod.getMetadata().getName());
162+
if (!isStatusCheck) {
163+
LOGGER.info(MessageKeys.POD_IS_READY, pod.getMetadata().getName());
164+
}
159165
return true;
160166
}
161167
}

‎src/main/java/oracle/kubernetes/operator/ProcessingConstants.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,6 @@ public interface ProcessingConstants {
3131
public static final String SCRIPT_CONFIG_MAP = "scriptConfigMap";
3232
public static final String SERVER_STATE_MAP = "serverStateMap";
3333

34-
public static final String CLEAN_STATUS = "cleanStatus";
34+
public static final String STATUS_UNCHANGED = "statusUnchanged";
3535

3636
}

‎src/main/java/oracle/kubernetes/operator/ServerStatusReader.java

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import io.kubernetes.client.ApiException;
2121
import io.kubernetes.client.Exec;
2222
import io.kubernetes.client.models.V1ObjectMeta;
23+
import io.kubernetes.client.models.V1Pod;
2324
import oracle.kubernetes.operator.helpers.CallBuilder;
2425
import oracle.kubernetes.operator.helpers.ClientHelper;
2526
import oracle.kubernetes.operator.helpers.ClientHolder;
@@ -60,7 +61,8 @@ public DomainStatusReaderStep(DomainPresenceInfo info, long timeoutSeconds, Step
6061

6162
@Override
6263
public NextAction apply(Packet packet) {
63-
packet.put(ProcessingConstants.SERVER_STATE_MAP, new ConcurrentHashMap<String, String>());
64+
ConcurrentMap<String, String> serverStateMap = new ConcurrentHashMap<>();
65+
packet.put(ProcessingConstants.SERVER_STATE_MAP, serverStateMap);
6466

6567
Domain domain = info.getDomain();
6668
V1ObjectMeta meta = domain.getMetadata();
@@ -73,10 +75,17 @@ public NextAction apply(Packet packet) {
7375
for (Map.Entry<String, ServerKubernetesObjects> entry : info.getServers().entrySet()) {
7476
String serverName = entry.getKey();
7577
ServerKubernetesObjects sko = entry.getValue();
76-
if (sko != null && sko.getPod().get() != null) {
77-
Packet p = packet.clone();
78-
startDetails.add(new StepAndPacket(
79-
createServerStatusReaderStep(namespace, domainUID, serverName, timeoutSeconds, null), p));
78+
if (sko != null) {
79+
V1Pod pod = sko.getPod().get();
80+
if (pod != null) {
81+
if (PodWatcher.isReady(pod, true)) {
82+
serverStateMap.put(serverName, "RUNNING");
83+
} else {
84+
Packet p = packet.clone();
85+
startDetails.add(new StepAndPacket(
86+
createServerStatusReaderStep(namespace, domainUID, serverName, timeoutSeconds, null), p));
87+
}
88+
}
8089
}
8190
}
8291

@@ -130,26 +139,17 @@ public NextAction apply(Packet packet) {
130139
ClientHolder holder = helper.take();
131140
Exec exec = new Exec(holder.getApiClient());
132141
Process proc = null;
142+
String state = null;
133143
try {
134144
proc = exec.exec(namespace, podName,
135145
new String[] { "/weblogic-operator/scripts/readState.sh" },
136146
KubernetesConstants.CONTAINER_NAME, stdin, tty);
137147

138148
InputStream in = proc.getInputStream();
139149
if (proc.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
140-
String state = null;
141150
try (final Reader reader = new InputStreamReader(in, Charsets.UTF_8)) {
142151
state = CharStreams.toString(reader);
143152
}
144-
145-
@SuppressWarnings("unchecked")
146-
ConcurrentMap<String, String> serverStateMap = (ConcurrentMap<String, String>) packet
147-
.get(ProcessingConstants.SERVER_STATE_MAP);
148-
if (state != null) {
149-
serverStateMap.put(serverName, parseState(state));
150-
} else {
151-
serverStateMap.remove(serverName);
152-
}
153153
}
154154
} catch (IOException | ApiException | InterruptedException e) {
155155
LOGGER.warning(MessageKeys.EXCEPTION, e);
@@ -160,6 +160,10 @@ public NextAction apply(Packet packet) {
160160
}
161161
}
162162

163+
@SuppressWarnings("unchecked")
164+
ConcurrentMap<String, String> serverStateMap = (ConcurrentMap<String, String>) packet
165+
.get(ProcessingConstants.SERVER_STATE_MAP);
166+
serverStateMap.put(serverName, parseState(state));
163167
fiber.resume(packet);
164168
});
165169
}

‎src/main/java/oracle/kubernetes/operator/helpers/ConfigMapConsumer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public String get(Object key) {
6161
}
6262
return null;
6363
}
64-
64+
6565
@Override
6666
public String put(String key, String value) {
6767
throw new UnsupportedOperationException();

0 commit comments

Comments
 (0)