Skip to content

Commit d67b5c2

Browse files
rjeberhardrjeberhard
authored
Merge pull request #118 from oracle/simplify-ingress
Simplify ingress processing flow by having per-cluster service
2 parents 20ff764 + 0c90b6b commit d67b5c2

File tree

9 files changed

+592
-683
lines changed

9 files changed

+592
-683
lines changed

‎src/integration-tests/bash/run.sh

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1852,7 +1852,7 @@ function verify_domain_created {
18521852
fi
18531853
}
18541854

1855-
function verify_service_and_pod_deleted {
1855+
function verify_pod_deleted {
18561856
if [ "$#" != 2 ] ; then
18571857
fail "requires 2 parameters: domainkey serverNum, set serverNum to 0 to indicate the admin server"
18581858
fi
@@ -1869,27 +1869,9 @@ function verify_service_and_pod_deleted {
18691869
local POD_NAME="${DOMAIN_UID}-${MS_BASE_NAME}${SERVER_NUM}"
18701870
fi
18711871

1872-
local SERVICE_NAME="${POD_NAME}"
1873-
18741872
local max_count_srv=50
18751873
local max_count_pod=50
18761874
local wait_time=10
1877-
local count=1
1878-
local srv_count=1
1879-
1880-
trace "checking if service $SERVICE_NAME is deleted"
1881-
while [ "${srv_count:=Error}" != "0" -a $count -lt $max_count_srv ] ; do
1882-
local count=`expr $count + 1`
1883-
local srv_count=`kubectl -n $NAMESPACE get services | grep "^$SERVICE_NAME " | wc -l`
1884-
if [ "${srv_count:=Error}" != "0" ]; then
1885-
trace "service $SERVICE_NAME still exists, iteration $count of $max_count_srv"
1886-
sleep $wait_time
1887-
fi
1888-
done
1889-
1890-
if [ "${srv_count:=Error}" != "0" ]; then
1891-
fail "ERROR: the service $SERVICE_NAME is not deleted, exiting!"
1892-
fi
18931875

18941876
trace "checking if pod $POD_NAME is deleted"
18951877
local pod_count=1
@@ -1936,15 +1918,15 @@ function verify_domain_deleted {
19361918
fail "ERROR: domain still exists, exiting!"
19371919
fi
19381920

1939-
trace "verify the service and pod of admin server is deleted"
1940-
verify_service_and_pod_deleted $DOM_KEY 0
1921+
trace "verify the pod of admin server is deleted"
1922+
verify_pod_deleted $DOM_KEY 0
19411923

19421924
trace "verify $MS_NUM number of managed servers for deletion"
19431925
local i
19441926
for i in $(seq 1 $MS_NUM);
19451927
do
1946-
trace "verify service and pod of managed server $i is deleted"
1947-
verify_service_and_pod_deleted $DOM_KEY $i
1928+
trace "verify pod of managed server $i is deleted"
1929+
verify_pod_deleted $DOM_KEY $i
19481930
done
19491931
trace Done. Verified.
19501932
}
@@ -2168,7 +2150,7 @@ function test_cluster_scale {
21682150
sed -i -e "0,/replicas:/s/replicas:.*/replicas: 2/" $domainCR
21692151
kubectl apply -f $domainCR
21702152

2171-
verify_service_and_pod_deleted $DOM_KEY 3
2153+
verify_pod_deleted $DOM_KEY 3
21722154
verify_webapp_load_balancing $DOM_KEY 2
21732155

21742156
# verify that scaling $DOM_KEY had no effect on another domain

‎src/main/java/oracle/kubernetes/operator/Main.java

Lines changed: 154 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -890,14 +890,18 @@ else if (StartupControlConstants.AUTO_STARTUPCONTROL.equals(sc)) {
890890

891891
info.setServerStartupInfo(ssic);
892892
LOGGER.exiting();
893-
return doNext(scaleDownIfNecessary(info, servers, new ManagedServerUpIteratorStep(ssic, next)), packet);
893+
return doNext(scaleDownIfNecessary(info, servers,
894+
checkServerServices(info, new ManagedServerUpIteratorStep(ssic, next))),
895+
packet);
894896
case StartupControlConstants.ADMIN_STARTUPCONTROL:
895897
case StartupControlConstants.NONE_STARTUPCONTROL:
896898
default:
897899

898900
info.setServerStartupInfo(null);
899901
LOGGER.exiting();
900-
return doNext(scaleDownIfNecessary(info, servers, next), packet);
902+
return doNext(scaleDownIfNecessary(info, servers,
903+
checkServerServices(info, next)),
904+
packet);
901905
}
902906
}
903907
}
@@ -933,6 +937,113 @@ private static List<V1EnvVar> startInAdminMode(List<V1EnvVar> env) {
933937
return env;
934938
}
935939

940+
private static Step checkServerServices(DomainPresenceInfo info, Step next) {
941+
Collection<String> allServers = new ArrayList<>();
942+
Collection<ServerStartupInfo> ssic = new ArrayList<>();
943+
944+
WlsDomainConfig scan = info.getScan();
945+
946+
// Iterate all servers
947+
for (WlsClusterConfig wlsClusterConfig : scan.getClusterConfigs().values()) {
948+
for (WlsServerConfig wlsServerConfig : wlsClusterConfig.getServerConfigs()) {
949+
String serverName = wlsServerConfig.getListenAddress();
950+
if (!allServers.contains(serverName)) {
951+
allServers.add(serverName);
952+
ssic.add(new ServerStartupInfo(wlsServerConfig, wlsClusterConfig, null, null));
953+
}
954+
}
955+
}
956+
for (Map.Entry<String, WlsServerConfig> wlsServerConfig : scan.getServerConfigs().entrySet()) {
957+
String serverName = wlsServerConfig.getKey();
958+
if (!allServers.contains(serverName)) {
959+
allServers.add(serverName);
960+
ssic.add(new ServerStartupInfo(wlsServerConfig.getValue(), null, null, null));
961+
}
962+
}
963+
964+
return new ManagedServerServicesStep(info, ssic, next);
965+
}
966+
967+
private static class ManagedServerServicesStep extends Step {
968+
private final DomainPresenceInfo info;
969+
private final Collection<ServerStartupInfo> ssic;
970+
971+
public ManagedServerServicesStep(DomainPresenceInfo info, Collection<ServerStartupInfo> ssic, Step next) {
972+
super(next);
973+
this.info = info;
974+
this.ssic = ssic;
975+
}
976+
977+
@Override
978+
public NextAction apply(Packet packet) {
979+
Collection<StepAndPacket> startDetails = new ArrayList<>();
980+
981+
for (ServerStartupInfo ssi : ssic) {
982+
Packet p = packet.clone();
983+
WlsServerConfig serverConfig = ssi.serverConfig;
984+
ServerStartup serverStartup = ssi.serverStartup;
985+
String serverName = serverConfig.getName();
986+
p.put(ProcessingConstants.SERVER_SCAN, serverConfig);
987+
p.put(ProcessingConstants.CLUSTER_SCAN, ssi.clusterConfig);
988+
p.put(ProcessingConstants.ENVVARS, ssi.envVars);
989+
990+
DomainSpec spec = info.getDomain().getSpec();
991+
Integer nodePort = null;
992+
if (serverStartup == null) {
993+
List<ServerStartup> ssl = spec.getServerStartup();
994+
if (ssl != null) {
995+
for (ServerStartup ss : ssl) {
996+
if (serverName.equals(ss.getServerName())) {
997+
serverStartup = ss;
998+
break;
999+
}
1000+
}
1001+
}
1002+
}
1003+
1004+
if (serverStartup != null) {
1005+
nodePort = serverStartup.getNodePort();
1006+
}
1007+
if (nodePort == null && serverName.equals(spec.getAsName())) {
1008+
nodePort = spec.getAsNodePort();
1009+
}
1010+
1011+
p.put(ProcessingConstants.SERVER_NAME, serverName);
1012+
if (ssi.clusterConfig != null) {
1013+
p.put(ProcessingConstants.CLUSTER_NAME, ssi.clusterConfig.getClusterName());
1014+
}
1015+
p.put(ProcessingConstants.PORT, serverConfig.getListenPort());
1016+
p.put(ProcessingConstants.NODE_PORT, nodePort);
1017+
1018+
startDetails.add(new StepAndPacket(ServiceHelper.createForServerStep(null), p));
1019+
}
1020+
1021+
// Add cluster services
1022+
WlsDomainConfig scan = info.getScan();
1023+
if (scan != null) {
1024+
for (Map.Entry<String, WlsClusterConfig> entry : scan.getClusterConfigs().entrySet()) {
1025+
Packet p = packet.clone();
1026+
WlsClusterConfig clusterConfig = entry.getValue();
1027+
p.put(ProcessingConstants.CLUSTER_SCAN, clusterConfig);
1028+
p.put(ProcessingConstants.CLUSTER_NAME, clusterConfig.getClusterName());
1029+
for (WlsServerConfig serverConfig : clusterConfig.getServerConfigs()) {
1030+
p.put(ProcessingConstants.PORT, serverConfig.getListenPort());
1031+
break;
1032+
}
1033+
1034+
startDetails.add(new StepAndPacket(
1035+
ServiceHelper.createForClusterStep(
1036+
IngressHelper.createClusterStep(null)), p));
1037+
}
1038+
}
1039+
1040+
if (startDetails.isEmpty()) {
1041+
return doNext(packet);
1042+
}
1043+
return doForkJoin(next, packet, startDetails);
1044+
}
1045+
}
1046+
9361047
private static Step scaleDownIfNecessary(DomainPresenceInfo info, Collection<String> servers, Step next) {
9371048
Domain dom = info.getDomain();
9381049
DomainSpec spec = dom.getSpec();
@@ -1082,9 +1193,7 @@ public ServerDownStep(String serverName, ServerKubernetesObjects sko, Step next)
10821193

10831194
@Override
10841195
public NextAction apply(Packet packet) {
1085-
return doNext(IngressHelper.createRemoveServerStep(serverName,
1086-
ServiceHelper.deleteServicesStep(sko,
1087-
PodHelper.deletePodStep(sko, new ServerDownFinalizeStep(serverName, next)))), packet);
1196+
return doNext(PodHelper.deletePodStep(sko, new ServerDownFinalizeStep(serverName, next)), packet);
10881197
}
10891198
}
10901199

@@ -1110,40 +1219,9 @@ public NextAction apply(Packet packet) {
11101219
// "clusterScan"
11111220
// "envVars"
11121221
private static Step bringManagedServerUp(ServerStartupInfo ssi, Step next) {
1113-
return PodHelper.createManagedPodStep(
1114-
new BeforeManagedServerStep(
1115-
ssi,
1116-
ServiceHelper.createForServerStep(
1117-
IngressHelper.createAddServerStep(next))));
1222+
return PodHelper.createManagedPodStep(next);
11181223
}
11191224

1120-
private static class BeforeManagedServerStep extends Step {
1121-
private final ServerStartupInfo ssi;
1122-
1123-
public BeforeManagedServerStep(ServerStartupInfo ssi, Step next) {
1124-
super(next);
1125-
this.ssi = ssi;
1126-
}
1127-
1128-
@Override
1129-
public NextAction apply(Packet packet) {
1130-
WlsServerConfig scan = ssi.serverConfig;
1131-
ServerStartup ss = ssi.serverStartup;
1132-
Integer nodePort = null;
1133-
if (ss != null) {
1134-
nodePort = ss.getNodePort();
1135-
}
1136-
1137-
packet.put(ProcessingConstants.SERVER_NAME, scan.getName());
1138-
if (ssi.clusterConfig != null) {
1139-
packet.put(ProcessingConstants.CLUSTER_NAME, ssi.clusterConfig.getClusterName());
1140-
}
1141-
packet.put(ProcessingConstants.PORT, scan.getListenPort());
1142-
packet.put(ProcessingConstants.NODE_PORT, nodePort);
1143-
return doNext(packet);
1144-
}
1145-
}
1146-
11471225
private static void deleteDomainPresence(Domain dom) {
11481226
V1ObjectMeta meta = dom.getMetadata();
11491227
DomainSpec spec = dom.getSpec();
@@ -1464,22 +1542,30 @@ private static void dispatchServiceWatch(Watch.Response<V1Service> item) {
14641542
String domainUID = metadata.getLabels().get(LabelConstants.DOMAINUID_LABEL);
14651543
String serverName = metadata.getLabels().get(LabelConstants.SERVERNAME_LABEL);
14661544
String channelName = metadata.getLabels().get(LabelConstants.CHANNELNAME_LABEL);
1545+
String clusterName = metadata.getLabels().get(LabelConstants.CLUSTERNAME_LABEL);
14671546
if (domainUID != null) {
14681547
DomainPresenceInfo info = domains.get(domainUID);
1469-
if (info != null && serverName != null) {
1470-
ServerKubernetesObjects created = new ServerKubernetesObjects();
1471-
ServerKubernetesObjects current = info.getServers().putIfAbsent(serverName, created);
1472-
ServerKubernetesObjects sko = current != null ? current : created;
1473-
if (sko != null) {
1474-
switch (item.type) {
1475-
case "ADDED":
1548+
ServerKubernetesObjects sko = null;
1549+
if (info != null) {
1550+
if (serverName != null) {
1551+
ServerKubernetesObjects created = new ServerKubernetesObjects();
1552+
ServerKubernetesObjects current = info.getServers().putIfAbsent(serverName, created);
1553+
sko = current != null ? current : created;
1554+
}
1555+
switch (item.type) {
1556+
case "ADDED":
1557+
if (sko != null) {
14761558
if (channelName != null) {
14771559
sko.getChannels().put(channelName, s);
14781560
} else {
14791561
sko.getService().set(s);
14801562
}
1481-
break;
1482-
case "MODIFIED":
1563+
} else if (clusterName != null) {
1564+
info.getClusters().put(clusterName, s);
1565+
}
1566+
break;
1567+
case "MODIFIED":
1568+
if (sko != null) {
14831569
if (channelName != null) {
14841570
V1Service skoService = sko.getChannels().get(channelName);
14851571
if (skoService != null) {
@@ -1491,28 +1577,42 @@ private static void dispatchServiceWatch(Watch.Response<V1Service> item) {
14911577
sko.getService().compareAndSet(skoService, s);
14921578
}
14931579
}
1494-
break;
1495-
case "DELETED":
1580+
} else if (clusterName != null) {
1581+
V1Service clusterService = info.getClusters().get(clusterName);
1582+
if (clusterService != null) {
1583+
info.getClusters().replace(clusterName, clusterService, s);
1584+
}
1585+
}
1586+
break;
1587+
case "DELETED":
1588+
if (sko != null) {
14961589
if (channelName != null) {
14971590
V1Service oldService = sko.getChannels().put(channelName, null);
14981591
if (oldService != null) {
14991592
// Service was deleted, but sko still contained a non-null entry
1500-
LOGGER.info(MessageKeys.SERVICE_DELETED, domainUID, metadata.getNamespace(), serverName);
1593+
LOGGER.info(MessageKeys.SERVER_SERVICE_DELETED, domainUID, metadata.getNamespace(), serverName);
15011594
doCheckAndCreateDomainPresence(info.getDomain(), true);
15021595
}
15031596
} else {
15041597
V1Service oldService = sko.getService().getAndSet(null);
15051598
if (oldService != null) {
15061599
// Service was deleted, but sko still contained a non-null entry
1507-
LOGGER.info(MessageKeys.SERVICE_DELETED, domainUID, metadata.getNamespace(), serverName);
1600+
LOGGER.info(MessageKeys.SERVER_SERVICE_DELETED, domainUID, metadata.getNamespace(), serverName);
15081601
doCheckAndCreateDomainPresence(info.getDomain(), true);
15091602
}
15101603
}
1511-
break;
1604+
} else if (clusterName != null) {
1605+
V1Service oldService = info.getClusters().put(clusterName, null);
1606+
if (oldService != null) {
1607+
// Service was deleted, but clusters still contained a non-null entry
1608+
LOGGER.info(MessageKeys.CLUSTER_SERVICE_DELETED, domainUID, metadata.getNamespace(), clusterName);
1609+
doCheckAndCreateDomainPresence(info.getDomain(), true);
1610+
}
1611+
}
1612+
break;
15121613

1513-
case "ERROR":
1514-
default:
1515-
}
1614+
case "ERROR":
1615+
default:
15161616
}
15171617
}
15181618
}

‎src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
import io.kubernetes.client.models.V1EnvVar;
1515
import io.kubernetes.client.models.V1PersistentVolumeClaimList;
16+
import io.kubernetes.client.models.V1Service;
1617
import io.kubernetes.client.models.V1beta1Ingress;
1718
import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.Domain;
1819
import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.DomainSpec;
@@ -32,6 +33,7 @@ public class DomainPresenceInfo {
3233
private final AtomicReference<Collection<ServerStartupInfo>> serverStartupInfo;
3334

3435
private final ConcurrentMap<String, ServerKubernetesObjects> servers = new ConcurrentHashMap<>();
36+
private final ConcurrentMap<String, V1Service> clusters = new ConcurrentHashMap<>();
3537
private final ConcurrentMap<String, V1beta1Ingress> ingresses = new ConcurrentHashMap<>();
3638

3739
private V1PersistentVolumeClaimList claims = null;
@@ -139,6 +141,14 @@ public ConcurrentMap<String, ServerKubernetesObjects> getServers() {
139141
return servers;
140142
}
141143

144+
/**
145+
* Map from cluster name to Service objects
146+
* @return Cluster object map
147+
*/
148+
public ConcurrentMap<String, V1Service> getClusters() {
149+
return clusters;
150+
}
151+
142152
/**
143153
* Map from cluster name to Ingress
144154
* @return Cluster object map

0 commit comments

Comments
 (0)