Debug errors in Keystore layer , Fixes AB#3092835 (#2544)

Issue : We got a couple of IcMs
(https://portal.microsofticm.com/imp/v5/incidents/details/561690347/summary)
in which we found that the keystore operation "unwrap" is throwing
InvalidKey Exception.
This is only happening on Pixel 5 devices so far and we have not noticed
any recent OS update or security patch released. One of the IcMs says
that this has been happening for 6 months now. Although the impacted
devices are very low, Auth app team has been receiving at-least one
similar IcM every other week.

Fix : 
- Synchronizing the unwrap method. I am not sure if this would work as I
have not been able to reproduce the same error on the test devices i
have. Hence based on some online resources where other type of keystore
errors were observed due to multiple threads accessing the keystore at
the same time, I have added synchronized keyword on "unwrap" and 2 other
methods.
- Also added a telemetry metric to get the exceptions (if it is still
happening even after my fix)


Fixes
[AB#3092835](https://identitydivision.visualstudio.com/fac9d424-53d2-45c0-91b5-ef6ba7a6bf26/_workitems/edit/3092835)

Author: somalaya

changelog.txt

@@ -1,6 +1,11 @@
 vNext
 ----------
 
+Version 18.2.2
+----------
+(common4j 15.2.2)
+- [PATCH] Debug errors in Keystore layer (#2544)
+
 Version 18.2.0
 ----------
 (common4j 15.2.0)

common/src/main/java/com/microsoft/identity/common/crypto/AndroidWrappedKeyLoader.java

@@ -186,7 +186,7 @@ protected SecretKey generateRandomKey() throws ClientException {
      * @return SecretKey. Null if there isn't any.
      */
     @Nullable
-    /* package */ SecretKey readSecretKeyFromStorage() throws ClientException {
+    /* package */ synchronized SecretKey readSecretKeyFromStorage() throws ClientException {
         final String methodTag = TAG + ":readSecretKeyFromStorage";
         try {
             final KeyPair keyPair = AndroidKeyStoreUtil.readKey(mAlias);

common/src/main/java/com/microsoft/identity/common/internal/util/AndroidKeyStoreUtil.java

@@ -26,6 +26,9 @@
 
 import com.microsoft.identity.common.java.exception.ClientException;
 import com.microsoft.identity.common.java.logging.Logger;
+import com.microsoft.identity.common.java.opentelemetry.AttributeName;
+import com.microsoft.identity.common.java.opentelemetry.OTelUtility;
+import com.microsoft.identity.common.java.util.ThrowableUtil;
 import com.microsoft.identity.common.java.util.ported.DateUtilities;
 
 import java.io.IOException;
@@ -51,6 +54,8 @@
 import javax.crypto.SecretKey;
 
 import edu.umd.cs.findbugs.annotations.Nullable;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongCounter;
 import lombok.NonNull;
 
 import static com.microsoft.identity.common.java.exception.ClientException.UNKNOWN_CRYPTO_ERROR;
@@ -84,6 +89,11 @@ private AndroidKeyStoreUtil() {
 
     private static KeyStore mKeyStore;
 
+    private static final LongCounter sFailedAndroidKeyStoreUnwrapOperationCount = OTelUtility.createLongCounter(
+            "failed_keystore_key_unwrap_operation_count",
+            "Number of failed Android KeyStore unwrap operations"
+    );
+
     private static synchronized KeyStore getKeyStore()
             throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException {
         if (mKeyStore == null){
@@ -342,7 +352,7 @@ public static synchronized void deleteKey(
      * @param wrapAlgorithm the algorithm used to wrap the key.
      * @return the wrapped key data blob.
      */
-    public static byte[] wrap(@NonNull final SecretKey key,
+    public static synchronized byte[] wrap(@NonNull final SecretKey key,
                               @NonNull final KeyPair keyToWrap,
                               @NonNull final String wrapAlgorithm)
             throws ClientException {
@@ -397,7 +407,7 @@ public static byte[] wrap(@NonNull final SecretKey key,
      * @param wrapAlgorithm        the algorithm used to wrap the key.
      * @return the unwrapped key.
      */
-    public static SecretKey unwrap(@NonNull final byte[] wrappedKeyBlob,
+    public static synchronized SecretKey unwrap(@NonNull final byte[] wrappedKeyBlob,
                                    @NonNull final String wrappedKeyAlgorithm,
                                    @NonNull final KeyPair keyPairForUnwrapping,
                                    @NonNull final String wrapAlgorithm) throws ClientException {
@@ -440,7 +450,15 @@ public static SecretKey unwrap(@NonNull final byte[] wrappedKeyBlob,
                 exception.getMessage(),
                 exception
         );
-
+        if (exception instanceof InvalidKeyException) {
+            final Attributes attributes = Attributes.builder()
+                    .put(AttributeName.keystore_operation.name(), "unwrap")
+                    .put(AttributeName.error_code.name(), errCode)
+                    .put(AttributeName.error_type.name(), clientException.getClass().getSimpleName())
+                    .put(AttributeName.keystore_exception_stack_trace.name(), ThrowableUtil.getStackTraceAsString(clientException))
+                    .build();
+            sFailedAndroidKeyStoreUnwrapOperationCount.add(1, attributes);
+        }
         Logger.error(
                 methodTag,
                 errCode,

common4j/src/main/com/microsoft/identity/common/java/opentelemetry/AttributeName.java

@@ -292,5 +292,15 @@ public enum AttributeName {
     /**
      * Specify the result (or error stack trace) when determining if RT should be returned with AT response.
      */
-    stop_returning_rt_result
+    stop_returning_rt_result,
+
+    /**
+     * Indicates the operation name for Android KeyStore.
+     */
+    keystore_operation,
+
+    /**
+     * Indicates the stack trace from a Android KeyStore operation exception.
+     */
+    keystore_exception_stack_trace,
 }