Merge pull request #479 from Countly/back_of_mech

Back of mechanism

Author: turtledreams

CHANGELOG.md

@@ -1,7 +1,9 @@
 ## XX.XX.XX
-* The feedback widgets now have transparent backgrounds for a cleaner look.
-* Extended the notification button URL handler to allow custom handling of URLs when notification buttons are clicked in the background.
+* Improved request queue handling, added a backoff mechanism to the SDK to better handle cases where the server responds slowly, enabled by default.
+* Added a config method to disable backoff mechanism "disableBackoffMechanism()"
 * Added a config method to disable server config updates in the initialization "disableSDKBehaviorSettingsUpdates()".
+* The feedback widgets now have transparent backgrounds and fullscreen for a cleaner look.
+* Extended the notification button URL handler to allow custom handling of URLs when notification buttons are clicked in the background.
 
 * Deprecated "presentFeedbackWidget(widgetInfo, context, closeButtonText, devCallback)", replaced with "presentFeedbackWidget(widgetInfo, context, devCallback)" in the feedbacks.
 

gradle.properties

@@ -22,7 +22,7 @@ org.gradle.configureondemand=true
 android.useAndroidX=true
 android.enableJetifier=true
 # RELEASE FIELD SECTION
-VERSION_NAME=25.4.1-RC2
+VERSION_NAME=25.4.1-RC3
 GROUP=ly.count.android
 POM_URL=https://github.com/Countly/countly-sdk-android
 POM_SCM_URL=https://github.com/Countly/countly-sdk-android

sdk/src/androidTest/java/ly/count/android/sdk/ConnectionProcessorTests.java

@@ -34,6 +34,7 @@ of this software and associated documentation files (the "Software"), to deal
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.Mockito;
 
 import static ly.count.android.sdk.UtilsNetworking.sha256Hash;
 import static org.junit.Assert.assertEquals;
@@ -105,6 +106,26 @@ public void setUp() {
             @Override public boolean getRefreshContentZoneEnabled() {
                 return true;
             }
+
+            @Override public boolean getBOMEnabled() {
+                return true;
+            }
+
+            @Override public int getBOMAcceptedTimeoutSeconds() {
+                return 10;
+            }
+
+            @Override public double getBOMRQPercentage() {
+                return 0.5;
+            }
+
+            @Override public int getBOMRequestAge() {
+                return 24;
+            }
+
+            @Override public int getBOMDuration() {
+                return 60;
+            }
         };
 
         Countly.sharedInstance().setLoggingEnabled(true);
@@ -136,7 +157,7 @@ public void setUp() {
             }
         };
 
-        connectionProcessor = new ConnectionProcessor("http://server", mockStore, mockDeviceId, configurationProviderFake, rip, null, null, moduleLog, healthTrackerMock);
+        connectionProcessor = new ConnectionProcessor("http://server", mockStore, mockDeviceId, configurationProviderFake, rip, null, null, moduleLog, healthTrackerMock, Mockito.mock(Runnable.class));
         testDeviceId = "123";
     }
 
@@ -145,7 +166,7 @@ public void testConstructorAndGetters() {
         final String serverURL = "https://secureserver";
         final CountlyStore mockStore = mock(CountlyStore.class);
         final DeviceIdProvider mockDeviceId = mock(DeviceIdProvider.class);
-        final ConnectionProcessor connectionProcessor1 = new ConnectionProcessor(serverURL, mockStore, mockDeviceId, configurationProviderFake, rip, null, null, moduleLog, healthTrackerMock);
+        final ConnectionProcessor connectionProcessor1 = new ConnectionProcessor(serverURL, mockStore, mockDeviceId, configurationProviderFake, rip, null, null, moduleLog, healthTrackerMock, Mockito.mock(Runnable.class));
         assertEquals(serverURL, connectionProcessor1.getServerURL());
         assertSame(mockStore, connectionProcessor1.getCountlyStore());
     }
@@ -212,7 +233,7 @@ public void urlConnectionCustomHeaderValues() throws IOException {
         customValues.put("5", "");
         customValues.put("6", null);
 
-        ConnectionProcessor connectionProcessor = new ConnectionProcessor("http://server", mockStore, mockDeviceId, configurationProviderFake, rip, null, customValues, moduleLog, healthTrackerMock);
+        ConnectionProcessor connectionProcessor = new ConnectionProcessor("http://server", mockStore, mockDeviceId, configurationProviderFake, rip, null, customValues, moduleLog, healthTrackerMock, Mockito.mock(Runnable.class));
         final URLConnection urlConnection = connectionProcessor.urlConnectionForServerRequest("eventData", null);
 
         assertEquals("bb", urlConnection.getRequestProperty("aa"));

sdk/src/androidTest/java/ly/count/android/sdk/TestUtils.java

@@ -44,7 +44,7 @@ public class TestUtils {
     public final static String commonAppKey = "appkey";
     public final static String commonDeviceId = "1234";
     public final static String SDK_NAME = "java-native-android";
-    public final static String SDK_VERSION = "25.4.1-RC2";
+    public final static String SDK_VERSION = "25.4.1-RC3";
     public static final int MAX_THREAD_COUNT_PER_STACK_TRACE = 50;
 
     public static class Activity2 extends Activity {

sdk/src/main/java/ly/count/android/sdk/ConfigurationProvider.java

@@ -18,4 +18,15 @@ interface ConfigurationProvider {
     boolean getLocationTrackingEnabled();
 
     boolean getRefreshContentZoneEnabled();
+
+    // BACKOFF MECHANISM
+    boolean getBOMEnabled();
+
+    int getBOMAcceptedTimeoutSeconds();
+
+    double getBOMRQPercentage();
+
+    int getBOMRequestAge();
+
+    int getBOMDuration();
 }

sdk/src/main/java/ly/count/android/sdk/ConnectionProcessor.java

@@ -34,6 +34,7 @@ of this software and associated documentation files (the "Software"), to deal
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLContext;
@@ -49,6 +50,7 @@ of this software and associated documentation files (the "Software"), to deal
  */
 public class ConnectionProcessor implements Runnable {
     private static final int CONNECT_TIMEOUT_IN_MILLISECONDS = 30_000;
+    // used in backoff mechanism to accept half of the CONNECT_TIMEOUT_IN_MILLISECONDS
     private static final int READ_TIMEOUT_IN_MILLISECONDS = 30_000;
 
     private static final String CRLF = "\r\n";
@@ -65,6 +67,7 @@ public class ConnectionProcessor implements Runnable {
     private final SSLContext sslContext_;
 
     private final Map<String, String> requestHeaderCustomValues_;
+    private final Runnable backoffCallback_;
 
     static String endPointOverrideTag = "&new_end_point=";
 
@@ -79,14 +82,15 @@ private enum RequestResult {
 
     ConnectionProcessor(final String serverURL, final StorageProvider storageProvider, final DeviceIdProvider deviceIdProvider, final ConfigurationProvider configProvider,
         final RequestInfoProvider requestInfoProvider, final SSLContext sslContext, final Map<String, String> requestHeaderCustomValues, ModuleLog logModule,
-        HealthTracker healthTracker) {
+        HealthTracker healthTracker, Runnable backoffCallback) {
         serverURL_ = serverURL;
         storageProvider_ = storageProvider;
         deviceIdProvider_ = deviceIdProvider;
         configProvider_ = configProvider;
         sslContext_ = sslContext;
         requestHeaderCustomValues_ = requestHeaderCustomValues;
         requestInfoProvider_ = requestInfoProvider;
+        backoffCallback_ = backoffCallback;
         L = logModule;
         this.healthTracker = healthTracker;
     }
@@ -96,7 +100,6 @@ private enum RequestResult {
         if (customEndpoint != null) {
             urlEndpoint = customEndpoint;
         }
-
         // determine whether or not request has a binary image file, if it has request will be sent as POST request
         boolean hasPicturePath = requestData.contains(ModuleUserProfile.PICTURE_PATH_KEY);
         boolean usingHttpPost = requestData.contains("&crash=") || requestData.length() >= 2048 || requestInfoProvider_.isHttpPostForced() || hasPicturePath;
@@ -229,7 +232,7 @@ private enum RequestResult {
                     break;
                 }
                 String value = conn.getHeaderField(headerIndex++);
-                approximateDateSize += key.getBytes("US-ASCII").length + value.getBytes("US-ASCII").length + 2L;
+                approximateDateSize += key.getBytes(StandardCharsets.US_ASCII).length + value.getBytes(StandardCharsets.US_ASCII).length + 2L;
             }
         } catch (Exception e) {
             L.e("[Connection Processor] urlConnectionForServerRequest, exception while calculating header field size: " + e);
@@ -434,6 +437,7 @@ public void run() {
                     conn = urlConnectionForServerRequest(requestData, customEndpoint);
                     long setupServerRequestTime = UtilsTime.getNanoTime() - pccTsStartGetURLConnection;
                     L.d("[ConnectionProcessor] run, TIMING Setup server request took:[" + setupServerRequestTime / 1000000.0d + "] ms");
+
                     if (pcc != null) {
                         pcc.TrackCounterTimeNs("ConnectionProcessorRun_07_SetupServerRequest", setupServerRequestTime);
                         pccTsStartOnlyInternet = UtilsTime.getNanoTime();
@@ -472,7 +476,7 @@ public void run() {
                     }
 
                     final RequestResult rRes;
-        
+
                     if (responseCode >= 200 && responseCode < 300) {
 
                         if (responseString.isEmpty()) {
@@ -525,6 +529,11 @@ public void run() {
                         // successfully submitted event data to Count.ly server, so remove
                         // this one from the stored events collection
                         storageProvider_.removeRequest(originalRequest);
+
+                        if (configProvider_.getBOMEnabled() && backoff(setupServerRequestTime, storedRequestCount, requestData)) {
+                            backoffCallback_.run();
+                            break;
+                        }
                     } else {
                         // will retry later
                         // warning was logged above, stop processing, let next tick take care of retrying
@@ -582,6 +591,41 @@ public void run() {
         L.v("[ConnectionProcessor] run, TIMING Whole queue took:[" + wholeQueueTime / 1000000.0d + "] ms");
     }
 
+    /**
+     * Backoff mechanism to prevent flooding the server with requests when server is not able to respond
+     * Needs 3 conditions to met:
+     * - Request has a timestamp younger than 12 hrs
+     * - The number of requests inside the queue is less than 10% of the max queue size
+     * - The response time from the server is greater than or equal to ACCEPTED_TIMEOUT_SECONDS
+     *
+     * @param responseTimeMillis response time  in milliseconds
+     * @param storedRequestCount number of requests in the queue
+     * @param requestData request data
+     * @return true if the backoff mechanism is triggered
+     */
+    private boolean backoff(long responseTimeMillis, int storedRequestCount, String requestData) {
+        long responseTimeSeconds = responseTimeMillis / 1_000_000_000L;
+        boolean result = false;
+
+        if (responseTimeSeconds >= configProvider_.getBOMAcceptedTimeoutSeconds()) {
+            // FLAG 1
+            if (storedRequestCount <= storageProvider_.getMaxRequestQueueSize() * configProvider_.getBOMRQPercentage()) {
+                // FLAG 2
+                if (!Utils.isRequestTooOld(requestData, configProvider_.getBOMRequestAge(), "[ConnectionProcessor] backoff", L)) {
+                    // FLAG 3
+                    result = true;
+                    healthTracker.logBackoffRequest();
+                }
+            }
+        }
+
+        if (!result) {
+            healthTracker.logConsecutiveBackoffRequest();
+        }
+
+        return result;
+    }
+
     String getServerURL() {
         return serverURL_;
     }

sdk/src/main/java/ly/count/android/sdk/ConnectionQueue.java

@@ -31,6 +31,7 @@ of this software and associated documentation files (the "Software"), to deal
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
 import org.json.JSONException;
@@ -52,6 +53,9 @@ class ConnectionQueue implements RequestQueueProvider {
     private Future<?> connectionProcessorFuture_;
     private DeviceIdProvider deviceIdProvider_;
     private SSLContext sslContext_;
+    private final ScheduledExecutorService backoffScheduler_ = Executors.newSingleThreadScheduledExecutor();
+    private final AtomicBoolean backoff_ = new AtomicBoolean(false);
+
     BaseInfoProvider baseInfoProvider;
     HealthTracker healthTracker;
     public PerformanceCounterCollector pcc;
@@ -874,6 +878,10 @@ void ensureExecutor() {
     public void tick() {
         //todo enable later
         //assert storageProvider != null;
+        if (backoff_.get()) {
+            L.i("[ConnectionQueue] tick, currently backed off, skipping tick");
+            return;
+        }
 
         boolean rqEmpty = isRequestQueueEmpty(); // this is a heavy operation, do it only once. Why heavy? reading storage
         boolean cpDoneIfOngoing = connectionProcessorFuture_ != null && connectionProcessorFuture_.isDone();
@@ -893,7 +901,21 @@ public void tick() {
     }
 
     public ConnectionProcessor createConnectionProcessor() {
-        ConnectionProcessor cp = new ConnectionProcessor(baseInfoProvider.getServerURL(), storageProvider, deviceIdProvider_, configProvider, requestInfoProvider, sslContext_, requestHeaderCustomValues, L, healthTracker);
+
+        ConnectionProcessor cp = new ConnectionProcessor(baseInfoProvider.getServerURL(), storageProvider, deviceIdProvider_, configProvider, requestInfoProvider, sslContext_, requestHeaderCustomValues, L, healthTracker, new Runnable() {
+            @Override
+            public void run() {
+                L.d("[ConnectionQueue] createConnectionProcessor:run, backed off, countdown started for " + configProvider.getBOMDuration() + " seconds");
+                backoff_.set(true);
+                backoffScheduler_.schedule(new Runnable() {
+                    @Override public void run() {
+                        L.d("[ConnectionQueue] createConnectionProcessor:run, countdown finished, running tick in background thread");
+                        backoff_.set(false);
+                        tick();
+                    }
+                }, configProvider.getBOMDuration(), TimeUnit.SECONDS);
+            }
+        });
         cp.pcc = pcc;
         return cp;
     }

sdk/src/main/java/ly/count/android/sdk/Countly.java

@@ -47,7 +47,7 @@ of this software and associated documentation files (the "Software"), to deal
  */
 public class Countly {
 
-    private final String DEFAULT_COUNTLY_SDK_VERSION_STRING = "25.4.1-RC2";
+    private final String DEFAULT_COUNTLY_SDK_VERSION_STRING = "25.4.1-RC3";
     /**
      * Used as request meta data on every request
      */

sdk/src/main/java/ly/count/android/sdk/CountlyConfig.java

@@ -202,6 +202,7 @@ public class CountlyConfig {
     // Requests older than this value in hours would be dropped (0 means this feature is disabled)
     int dropAgeHours = 0;
     String sdkBehaviorSettings;
+    boolean backOffMechanismEnabled = true;
     boolean sdkBehaviorSettingsRequestsDisabled = false;
 
     /**
@@ -1013,6 +1014,16 @@ public synchronized CountlyConfig setSDKBehaviorSettings(String sdkBehaviorSetti
         return this;
     }
 
+    /**
+     * Disable the back off mechanism
+     *
+     * @return Returns the same config object for convenient linking
+     */
+    public synchronized CountlyConfig disableBackoffMechanism() {
+        this.backOffMechanismEnabled = false;
+        return this;
+    }
+
     /**
      * Disable the SDK behavior settings update calls to the server
      *

sdk/src/main/java/ly/count/android/sdk/CountlyStore.java

@@ -992,4 +992,8 @@ public void setHealthCheckCounterState(@NonNull String counterState) {
             editor.apply();
         }
     }
+
+    public int getMaxRequestQueueSize() {
+        return maxRequestQueueSize;
+    }
 }