Merge pull request #821 from oracle/owls-70947

OWLS-70974 - cyclic scaling test using REST API gets stuck after some time

Author: rjeberhard

operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java

@@ -1,9 +1,11 @@
-// Copyright 2017, 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator.helpers;
 
+import static java.net.HttpURLConnection.HTTP_CONFLICT;
+
 import com.squareup.okhttp.Call;
 import io.kubernetes.client.ApiCallback;
 import io.kubernetes.client.ApiClient;
@@ -46,6 +48,9 @@
 import oracle.kubernetes.operator.calls.RequestParams;
 import oracle.kubernetes.operator.calls.SynchronousCallDispatcher;
 import oracle.kubernetes.operator.calls.SynchronousCallFactory;
+import oracle.kubernetes.operator.logging.LoggingFacade;
+import oracle.kubernetes.operator.logging.LoggingFactory;
+import oracle.kubernetes.operator.logging.MessageKeys;
 import oracle.kubernetes.operator.work.Step;
 import oracle.kubernetes.weblogic.domain.v2.Domain;
 import oracle.kubernetes.weblogic.domain.v2.DomainList;
@@ -58,6 +63,8 @@ public class CallBuilder {
   /** HTTP status code for "Not Found" */
   public static final int NOT_FOUND = 404;
 
+  private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator");
+
   private static SynchronousCallDispatcher DISPATCHER =
       new SynchronousCallDispatcher() {
         @Override
@@ -159,6 +166,65 @@ public VersionInfo readVersionCode() throws ApiException {
         requestParams, ((client, params) -> new VersionApi(client).getCode()));
   }
 
+  /**
+   * Class extended by callers to {@link
+   * #executeSynchronousCallWithConflictRetry(RequestParamsBuilder, SynchronousCallFactory,
+   * ConflictRetry)} for building the RequestParams to be passed to {@link
+   * #executeSynchronousCall(RequestParams, SynchronousCallFactory)}.
+   *
+   * @param <T> Type of kubernetes object to be passed to the API
+   */
+  abstract static class RequestParamsBuilder<T> {
+    T body;
+
+    public RequestParamsBuilder(T body) {
+      this.body = body;
+    }
+
+    abstract RequestParams buildRequestParams();
+
+    void setBody(T body) {
+      this.body = body;
+    }
+  }
+
+  private <T> T executeSynchronousCallWithConflictRetry(
+      RequestParamsBuilder requestParamsBuilder,
+      SynchronousCallFactory<T> factory,
+      ConflictRetry<T> conflictRetry)
+      throws ApiException {
+    int retryCount = 0;
+    while (retryCount == 0 || retryCount < maxRetryCount) {
+      retryCount++;
+      RequestParams requestParams = requestParamsBuilder.buildRequestParams();
+      try {
+        return executeSynchronousCall(requestParams, factory);
+      } catch (ApiException apiException) {
+        boolean retry = false;
+        if (apiException.getCode() == HTTP_CONFLICT
+            && conflictRetry != null
+            && retryCount < maxRetryCount) {
+          T body = conflictRetry.getUpdatedObject();
+          if (body != null) {
+            requestParamsBuilder.setBody(body);
+            retry = true;
+            LOGGER.fine(
+                MessageKeys.SYNC_RETRY,
+                requestParams.call,
+                apiException.getCode(),
+                apiException.getMessage(),
+                retryCount,
+                maxRetryCount);
+          }
+        }
+        if (!retry) {
+          throw apiException;
+        }
+      }
+    }
+    return null;
+  }
+
   private <T> T executeSynchronousCall(
       RequestParams requestParams, SynchronousCallFactory<T> factory) throws ApiException {
     return DISPATCHER.execute(factory, requestParams, helper);
@@ -295,6 +361,33 @@ public Step readDomainAsync(String name, String namespace, ResponseStep<Domain>
               .replaceWebLogicOracleV2NamespacedDomain(
                   requestParams.name, requestParams.namespace, (Domain) requestParams.body, pretty);
 
+  /**
+   * Replace domain
+   *
+   * @param uid the domain uid (unique within the k8s cluster)
+   * @param namespace Namespace
+   * @param body Body
+   * @param conflictRetry ConflictRetry implementation to be called to obtain the latest version of
+   *     the Domain for retrying the replaceDomain synchronous call if previous call failed with
+   *     Conflict response code (409)
+   * @return Replaced domain
+   * @throws ApiException APIException
+   */
+  public Domain replaceDomainWithConflictRetry(
+      String uid, String namespace, Domain body, ConflictRetry<Domain> conflictRetry)
+      throws ApiException {
+    return executeSynchronousCallWithConflictRetry(
+        new RequestParamsBuilder<Domain>(body) {
+
+          @Override
+          RequestParams buildRequestParams() {
+            return new RequestParams("replaceDomain", namespace, uid, body);
+          }
+        },
+        REPLACE_DOMAIN_CALL,
+        conflictRetry);
+  }
+
   /**
    * Replace domain
    *

operator/src/main/java/oracle/kubernetes/operator/helpers/ConflictRetry.java

@@ -0,0 +1,22 @@
+// Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Licensed under the Universal Permissive License v 1.0 as shown at
+// http://oss.oracle.com/licenses/upl.
+
+package oracle.kubernetes.operator.helpers;
+
+/**
+ * Interface used by CallBuilder to obtain the latest version of the Kubernetes object for retrying
+ * synchronous API calls that previously failed with Conflict response code (409). This indicates an
+ * optimistic locking failure and the kubernetes object has since been modified. The synchoronus API
+ * can be retried with the latest version of the kubernetes object.
+ *
+ * @param <T> Type of kubernetes object to be passed to the API
+ */
+public interface ConflictRetry<T> {
+
+  /**
+   * @return The latest version of the kubernetes object for passing to the kubernetes API, or null
+   *     if the API should not be retried
+   */
+  T getUpdatedObject();
+}

operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java

@@ -1,4 +1,4 @@
-// Copyright 2017, 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 
@@ -147,4 +147,5 @@ private MessageKeys() {}
   public static final String CANNOT_START_DOMAIN_AFTER_MAX_RETRIES = "WLSKO-0144";
   public static final String CYCLING_POD = "WLSKO-0145";
   public static final String REPLICAS_EXCEEDS_TOTAL_CLUSTER_SERVER_COUNT = "WLSKO-0146";
+  public static final String SYNC_RETRY = "WLSKO-0147";
 }

operator/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java

@@ -1,4 +1,4 @@
-// Copyright 2017, 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 
@@ -27,6 +27,7 @@
 import oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource;
 import oracle.kubernetes.operator.helpers.AuthorizationProxy.Scope;
 import oracle.kubernetes.operator.helpers.CallBuilder;
+import oracle.kubernetes.operator.helpers.ConflictRetry;
 import oracle.kubernetes.operator.logging.LoggingFacade;
 import oracle.kubernetes.operator.logging.LoggingFactory;
 import oracle.kubernetes.operator.logging.MessageKeys;
@@ -237,23 +238,30 @@ public void scaleCluster(String domainUID, String cluster, int managedServerCoun
 
     verifyWLSConfiguredClusterCapacity(domain, cluster, managedServerCount);
 
-    updateReplicasForDomain(namespace, domain, cluster, managedServerCount);
+    if (updateReplicasForDomain(domain, cluster, managedServerCount)) {
+      overwriteDomain(
+          namespace,
+          domain,
+          () -> getDomainForConflictRetry(domainUID, cluster, managedServerCount));
+    }
     LOGGER.exiting();
   }
 
-  private void updateReplicasForDomain(
-      String namespace, Domain domain, String cluster, int newReplicaCount) {
+  private boolean updateReplicasForDomain(Domain domain, String cluster, int newReplicaCount) {
     if (newReplicaCount != domain.getReplicaCount(cluster)) {
       domain.setReplicaCount(cluster, newReplicaCount);
-      overwriteDomain(namespace, domain);
+      return true;
     }
+    return false;
   }
 
-  private void overwriteDomain(String namespace, Domain domain) {
+  private void overwriteDomain(
+      String namespace, final Domain domain, ConflictRetry<Domain> conflictRetry) {
     try {
       // Write out the Domain with updated replica values
       // TODO: Can we patch instead of replace?
-      new CallBuilder().replaceDomain(domain.getDomainUID(), namespace, domain);
+      new CallBuilder()
+          .replaceDomainWithConflictRetry(domain.getDomainUID(), namespace, domain, conflictRetry);
     } catch (ApiException e) {
       LOGGER.finer(
           String.format(
@@ -264,6 +272,14 @@ private void overwriteDomain(String namespace, Domain domain) {
     }
   }
 
+  Domain getDomainForConflictRetry(String domainUid, String cluster, int newReplicaCount) {
+    Domain domain = findDomain(domainUid, getDomainsList());
+    if (updateReplicasForDomain(domain, cluster, newReplicaCount)) {
+      return domain;
+    }
+    return null;
+  }
+
   private void verifyWLSConfiguredClusterCapacity(
       Domain domain, String cluster, int requestedSize) {
     // Query WebLogic Admin Server for current configured WebLogic Cluster size

operator/src/main/resources/Operator.properties

@@ -145,3 +145,4 @@ WLSKO-0143=Failed to parse file {0} from domain introspector for domain {1} due
 WLSKO-0144=Unable to start domain with domainUID {0} in namespace {1} after {2} attempts due to exception: {3}
 WLSKO-0145=Replacing pod {0} with {1}
 WLSKO-0146=Replica request of {0} exceeds the maximum dynamic server count + server count of {1} configured for cluster {2}
+WLSKO-0147=Call {0} has failed with code {1}: message {2}. It has failed {3} times and will be retried up to {4} times.