Add Telemetry for OOM Errors, Fixes AB#3124149

changelog.txt

@@ -19,6 +19,7 @@ Version 19.0.0
 - [MINOR] Managed profile Android util method (#2561)
 - [PATCH] Make userHandle response field optional (#2560)
 - [MINOR] Nonce redirect changes (#2552)
+- [PATCH] Fix OOM error handling (#2567)
 
 Version 18.2.2
 ----------

common/src/main/java/com/microsoft/identity/common/internal/platform/AndroidPlatformUtil.java

@@ -50,11 +50,14 @@
 import com.microsoft.identity.common.java.commands.ICommand;
 import com.microsoft.identity.common.java.commands.InteractiveTokenCommand;
 import com.microsoft.identity.common.java.commands.parameters.InteractiveTokenCommandParameters;
+import com.microsoft.identity.common.java.exception.BaseException;
 import com.microsoft.identity.common.java.exception.ClientException;
 import com.microsoft.identity.common.java.exception.ErrorStrings;
 import com.microsoft.identity.common.java.flighting.CommonFlight;
 import com.microsoft.identity.common.java.flighting.CommonFlightsManager;
 import com.microsoft.identity.common.java.logging.Logger;
+import com.microsoft.identity.common.java.opentelemetry.AttributeName;
+import com.microsoft.identity.common.java.opentelemetry.SpanExtension;
 import com.microsoft.identity.common.java.util.IPlatformUtil;
 import com.microsoft.identity.common.java.util.StringUtil;
 
@@ -67,6 +70,8 @@
 import javax.net.ssl.KeyManagerFactory;
 
 import edu.umd.cs.findbugs.annotations.Nullable;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.StatusCode;
 import lombok.AllArgsConstructor;
 import lombok.NonNull;
 
@@ -328,4 +333,34 @@ private boolean isValidHubRedirectURIForNAATests(String redirectUri) {
                 || redirectUri.equals("msauth://com.microsoft.teams/fcg80qvoM1YMKJZibjBwQcDfOno=")
                 || redirectUri.equals("https://login.microsoftonline.com/common/oauth2/nativeclient"));
     }
+
+    public void handleShutdownForOutOfMemoryError(@NonNull final BaseException exception, @NonNull final String tag) {
+        // When receiving an out of memory error, instead of gracefully returning a failure result, we should shut down
+        // current broker process, to allow a new broker process to be launched at the next request from client app.
+        // This will result in a new broker process with fresh memory allocation, rather than repeating out of memory
+        // errors. In the case that this is an msal-only scenario, and this code is running inside client app, then
+        // client app will be shut down and will need to be launched again.
+
+        // Log status code and record exception in span
+        final Span currentSpan = SpanExtension.current();
+        currentSpan.setStatus(StatusCode.ERROR);
+        currentSpan.recordException(exception);
+
+        // Attach the stack trace, to debug in telemetry later
+        currentSpan.setAttribute(AttributeName.out_of_memory_exception_stacktrace.name(),
+                StringUtil.getStacktraceAsStringFromElementArray(exception.getStackTrace()));
+
+        // End the span
+        currentSpan.end();
+
+        Logger.error(tag, "Received an out of memory error, shutting broker process so a new one can be launched.", exception);
+
+        // Shut down current process
+        // Calling client app will receive an MsalClientException with "Activity killed unexpectedly" from MSAL if broker is shut down with this statement
+        // Calling application should not crash, and should be able to make another call, which will result in a new broker process.
+
+        // killProcess preferable to system.exit, as it cleans up resources more thoroughly
+        android.os.Process.killProcess(android.os.Process.myPid());
+    }
+
 }

common4j/src/main/com/microsoft/identity/common/java/controllers/CommandDispatcher.java

@@ -550,6 +550,8 @@ private static CommandResult executeCommand(@SuppressWarnings(WarningType.rawtyp
             if (baseException instanceof UserCancelException) {
                 commandResult = CommandResult.ofNull(CommandResult.ResultStatus.CANCEL,
                         correlationId);
+            } else if (baseException.getErrorCode().equals(ClientException.OUT_OF_MEMORY)) { // If we receive an out of memory error
+                command.getParameters().getPlatformComponents().getPlatformUtil().handleShutdownForOutOfMemoryError(baseException, TAG);
             } else {
                 //Post On Error
                 commandResult = CommandResult.of(CommandResult.ResultStatus.ERROR, baseException,

common4j/src/main/com/microsoft/identity/common/java/opentelemetry/AttributeName.java

@@ -312,5 +312,10 @@ public enum AttributeName {
     /**
      * Indicates the new refresh token credential header attached in the eSTS request.
      */
-    is_new_refresh_token_cred_header_attached
+    is_new_refresh_token_cred_header_attached,
+
+    /**
+     * Records the stacktrace for an out-of-memory exception.
+     */
+    out_of_memory_exception_stacktrace
 }

common4j/src/main/com/microsoft/identity/common/java/util/IPlatformUtil.java

@@ -23,6 +23,7 @@
 package com.microsoft.identity.common.java.util;
 
 import com.microsoft.identity.common.java.commands.ICommand;
+import com.microsoft.identity.common.java.exception.BaseException;
 import com.microsoft.identity.common.java.exception.ClientException;
 import com.microsoft.identity.common.java.exception.ErrorStrings;
 
@@ -129,4 +130,12 @@ public interface IPlatformUtil {
      */
     @Nullable
     List<Map.Entry<String, String>> updateWithAndGetPlatformSpecificExtraQueryParameters(@Nullable List<Map.Entry<String, String>> originalList);
+
+    /**
+     * Handles the shutdown of the current process if we receive an out of memory error
+     * @param exception out of memory exception
+     * @param tag tag to be included in log statement
+     */
+    void handleShutdownForOutOfMemoryError(@NonNull final BaseException exception, @NonNull final String tag);
+
 }

common4j/src/main/com/microsoft/identity/common/java/util/StringUtil.java

@@ -463,4 +463,24 @@ public static void overwriteWithNull(final char[] chars) {
             chars[i] = '\0';
         }
     }
+
+    /**
+     * Converts an array of stack trace elements (one method invocation) into one concatenated string object.
+     *
+     * @param elements list of stacktrace elements
+     * @return concatenated string list
+     */
+    @NonNull
+    public static String getStacktraceAsStringFromElementArray(@NonNull final StackTraceElement[] elements) {
+        final StringBuilder builder = new StringBuilder();
+
+        builder.append(elements[0]);
+
+        for (int i = 1; i < elements.length; i++) {
+            builder.append("\n");
+            builder.append(elements[i]);
+        }
+
+        return builder.toString();
+    }
 }

common4j/src/testFixtures/java/com/microsoft/identity/common/components/MockPlatformComponentsFactory.java

@@ -27,6 +27,7 @@
 import com.microsoft.identity.common.java.browser.NoopBrowserSelector;
 import com.microsoft.identity.common.java.commands.ICommand;
 import com.microsoft.identity.common.java.crypto.IDevicePopManager;
+import com.microsoft.identity.common.java.exception.BaseException;
 import com.microsoft.identity.common.java.exception.ClientException;
 import com.microsoft.identity.common.java.interfaces.IPopManagerSupplier;
 import com.microsoft.identity.common.java.interfaces.PlatformComponents;
@@ -188,6 +189,11 @@ public String getPackageNameFromUid(int uid) {
         public List<Map.Entry<String, String>> updateWithAndGetPlatformSpecificExtraQueryParameters(@Nullable List<Map.Entry<String, String>> originalList) {
             return originalList;
         }
+
+        @Override
+        public void handleShutdownForOutOfMemoryError(@NonNull final BaseException exception, @NonNull final String tag) {
+            throw new UnsupportedOperationException();
+        }
     };
 
     public static final IBrowserSelector NON_FUNCTIONAL_BROWSER_SELECTOR = new NoopBrowserSelector();