Fixes for 5.8.2 FIPS ready, including updated private key lock/unlock handling and updates for RSA min keysize change. (#266)

Author: ColtonWilley

src/wp_ecc_kmgmt.c

@@ -733,7 +733,9 @@ static int wp_ecc_get_params_enc_pub_key(wp_Ecc* ecc, OSSL_PARAM params[],
                 outLen = 1 + 2 * ((ecc->bits + 7) / 8);
             }
             else {
+                PRIVATE_KEY_UNLOCK();
                 rc = wc_ecc_export_x963_ex(&ecc->key, p->data, &outLen, 0);
+                PRIVATE_KEY_LOCK();
                 if (rc != 0) {
                     ok = 0;
                 }
@@ -1433,7 +1435,9 @@ static int wp_ecc_export_keypair(wp_Ecc* ecc, OSSL_PARAM* params, int* pIdx,
     WOLFPROV_ENTER(WP_LOG_ECC, "wp_ecc_export_keypair");
 
     outLen = WP_ECC_PUBLIC_KEY_SIZE(ecc);
+    PRIVATE_KEY_UNLOCK();
     rc = wc_ecc_export_x963_ex(&ecc->key, data + *idx, &outLen, 0);
+    PRIVATE_KEY_LOCK();
     if (rc != 0) {
         ok = 0;
     }
@@ -2389,8 +2393,9 @@ static int wp_ecc_encode_pub_size(const wp_Ecc *ecc, size_t* keyLen)
     word32 len;
 
     WOLFPROV_ENTER(WP_LOG_ECC, "wp_ecc_encode_pub_size");
-
+    PRIVATE_KEY_UNLOCK();
     rc = wc_ecc_export_x963_ex((ecc_key*)&ecc->key, NULL, &len, 0);
+    PRIVATE_KEY_LOCK();
     if (rc != LENGTH_ONLY_E) {
         ok = 0;
     }
@@ -2421,7 +2426,9 @@ static int wp_ecc_encode_pub(const wp_Ecc *ecc, unsigned char* keyData,
 
     WOLFPROV_ENTER(WP_LOG_ECC, "wp_ecc_encode_pub");
 
+    PRIVATE_KEY_UNLOCK();
     rc = wc_ecc_export_x963_ex((ecc_key*)&ecc->key, keyData, &len, 0);
+    PRIVATE_KEY_LOCK();
     if (rc != 0) {
         ok = 0;
     }

src/wp_ecx_kmgmt.c

@@ -544,8 +544,10 @@ static int wp_ecx_get_params_priv_key(wp_Ecx* ecx, OSSL_PARAM params[])
             outLen = ecx->data->len;
         }
         else {
+            PRIVATE_KEY_UNLOCK();
             int rc = (*ecx->data->exportPriv)((void*)&ecx->key, p->data,
                 &outLen);
+            PRIVATE_KEY_LOCK();
             if (rc != 0) {
                 ok = 0;
             }
@@ -662,14 +664,18 @@ static int wp_ecx_match_priv_key(const wp_Ecx* ecx1, const wp_Ecx* ecx2)
     ok &= ecx1->hasPriv && ecx2->hasPriv;
     if (ok) {
         len1 = ecx1->data->len;
+        PRIVATE_KEY_UNLOCK();
         rc = (*ecx1->data->exportPriv)((void*)&ecx1->key, key1, &len1);
+        PRIVATE_KEY_LOCK();
         if (rc != 0) {
              ok = 0;
         }
     }
     if (ok) {
         len2 = ecx2->data->len;
+        PRIVATE_KEY_UNLOCK();
         rc = (*ecx2->data->exportPriv)((void*)&ecx2->key, key2, &len2);
+        PRIVATE_KEY_LOCK();
         if (rc != 0) {
             ok = 0;
         }
@@ -1066,7 +1072,12 @@ static int wp_ecx_export_keypair(wp_Ecx* ecx, OSSL_PARAM* params, int* pIdx,
     }
     if (ok && priv) {
         outLen = ecx->data->len;
+        PRIVATE_KEY_UNLOCK();
         rc = (*ecx->data->exportPriv)((void*)&ecx->key, data + *idx, &outLen);
+        PRIVATE_KEY_LOCK();
+        if (rc != 0) {
+            ok = 0;
+        }
         if (ok) {
             if (ecx->clamped) {
                 data[*idx + 0         ] = ecx->unclamped[0];

src/wp_rsa_kmgmt.c

@@ -36,6 +36,12 @@
 
 #ifdef WP_HAVE_RSA
 
+/* In 5.8.2 RSA_MIN_SIZE was changed from 1024 to 2048. We still need to
+ * allow 1024 in some cases, and have extended logic in place for it already.
+ * For FIPS 1024 bit keys, use existing checks and let wolfssl throw us back */
+#define WP_RSA_MIN_SIZE 1024
+#define WP_RSA_MAX_SIZE RSA_MAX_SIZE
+
 /** Supported selections (key parts) in this key manager for RSA. */
 #define WP_RSA_POSSIBLE_SELECTIONS                                             \
     (OSSL_KEYMGMT_SELECT_KEYPAIR | OSSL_KEYMGMT_SELECT_OTHER_PARAMETERS)
@@ -357,7 +363,7 @@ static int wp_rsa_check_key_size_int(int keySize, int allow1024)
 
     WOLFPROV_ENTER(WP_LOG_RSA, "wp_rsa_check_key_size_int");
 
-    if ((keySize < RSA_MIN_SIZE) || (keySize > RSA_MAX_SIZE)) {
+    if ((keySize < WP_RSA_MIN_SIZE) || (keySize > WP_RSA_MAX_SIZE)) {
         WOLFPROV_MSG(WP_LOG_RSA, "RSA key size invalid: %d\n", keySize);
         ok = 0;
     }