Fix to handle compatibility layer RSA PSS salt special cases like RSA_PSS_SALTLEN_DIGEST, RSA_PSS_SALTLEN_MAX. Fix for "error: ‘RSA_PSS_SALT_LEN_DEFAULT’ undeclared". For builds that don't support it just use 0 and mark not used.

Author: dgarske

src/pk.c

@@ -3526,9 +3526,39 @@ int wolfSSL_RSA_generate_key_ex(WOLFSSL_RSA* rsa, int bits, WOLFSSL_BIGNUM* e,
  * RSA padding APIs
  */
 
-#if defined(WC_RSA_PSS) && (defined(OPENSSL_ALL) || defined(WOLFSSL_ASIO) || \
-        defined(WOLFSSL_HAPROXY) || defined(WOLFSSL_NGINX))
+#ifdef WC_RSA_PSS
+
+static int rsa_pss_calc_salt(int saltLen, int hashLen, int emLen)
+{
+    /* Calculate the salt length to use for special cases. */
+    switch (saltLen) {
+        /* Negative saltLen values are treated differently. */
+        case RSA_PSS_SALTLEN_DIGEST:
+            saltLen = hashLen;
+            break;
+        case RSA_PSS_SALTLEN_MAX_SIGN:
+        case RSA_PSS_SALTLEN_MAX:
+        #ifdef WOLFSSL_PSS_LONG_SALT
+            saltLen = emLen - hashLen - 2;
+        #else
+            saltLen = hashLen;
+        #endif
+            break;
+        default:
+            if (saltLen < 0) {
+                /* log invalid salt, let wolfCrypt report error */
+                WOLFSSL_ERROR_MSG("invalid saltLen");
+                saltLen = -3; /* for wolfCrypt to produce error must be < -2 */
+            }
+            break;
+    }
+    return saltLen;
+}
+
+#if defined(OPENSSL_ALL) || defined(WOLFSSL_ASIO) || \
+    defined(WOLFSSL_HAPROXY) || defined(WOLFSSL_NGINX)
 #if !defined(HAVE_FIPS) || FIPS_VERSION_GT(2,0)
+
 /* Add PKCS#1 PSS padding to hash.
  *
  *
@@ -3646,28 +3676,7 @@ int wolfSSL_RSA_padding_add_PKCS1_PSS_mgf1(WOLFSSL_RSA *rsa, unsigned char *em,
     }
 
     if (ret == 1) {
-        /* Calculate the salt length to use for special cases. */
-        /* TODO: use special case wolfCrypt values? */
-        switch (saltLen) {
-        /* Negative saltLen values are treated differently. */
-        case RSA_PSS_SALTLEN_DIGEST:
-            saltLen = hashLen;
-            break;
-        case RSA_PSS_SALTLEN_MAX_SIGN:
-        case RSA_PSS_SALTLEN_MAX:
-        #ifdef WOLFSSL_PSS_LONG_SALT
-            saltLen = emLen - hashLen - 2;
-        #else
-            saltLen = hashLen;
-        #endif
-            break;
-        default:
-            if (saltLen < 0) {
-                /* No other negative values implemented. */
-                WOLFSSL_ERROR_MSG("invalid saltLen");
-                ret = 0;
-            }
-        }
+        saltLen = rsa_pss_calc_salt(saltLen, hashLen, emLen);
     }
 
     if (ret == 1) {
@@ -3759,31 +3768,7 @@ int wolfSSL_RSA_verify_PKCS1_PSS_mgf1(WOLFSSL_RSA *rsa,
     }
 
     if (ret == 1) {
-        /* Calculate the salt length to use for special cases. */
-        switch (saltLen) {
-        /* Negative saltLen values are treated differently */
-        case RSA_PSS_SALTLEN_DIGEST:
-            saltLen = hashLen;
-            break;
-        case RSA_PSS_SALTLEN_AUTO:
-        #ifdef WOLFSSL_PSS_SALT_LEN_DISCOVER
-            saltLen = RSA_PSS_SALT_LEN_DISCOVER;
-            break;
-        #endif
-        case RSA_PSS_SALTLEN_MAX:
-        #ifdef WOLFSSL_PSS_LONG_SALT
-            saltLen = emLen - hashLen - 2;
-        #else
-            saltLen = hashLen;
-        #endif
-            break;
-        default:
-            if (saltLen < 0) {
-                /* No other negative values implemented. */
-                WOLFSSL_ERROR_MSG("invalid saltLen");
-                ret = 0;
-            }
-        }
+        saltLen = rsa_pss_calc_salt(saltLen, hashLen, emLen);
     }
 
     if (ret == 1) {
@@ -3849,17 +3834,22 @@ int wolfSSL_RSA_verify_PKCS1_PSS(WOLFSSL_RSA *rsa, const unsigned char *mHash,
             saltLen);
 }
 #endif /* !HAVE_FIPS || FIPS_VERSION_GT(2,0) */
-#endif /* WC_RSA_PSS && (OPENSSL_ALL || WOLFSSL_ASIO || WOLFSSL_HAPROXY ||
-        *                WOLFSSL_NGINX) */
+#endif /* OPENSSL_ALL || WOLFSSL_ASIO || WOLFSSL_HAPROXY || WOLFSSL_NGINX) */
+#endif /* WC_RSA_PSS */
 
 /*
  * RSA sign/verify APIs
  */
 
-#ifndef WOLFSSL_PSS_SALT_LEN_DISCOVER
-    #define DEF_PSS_SALT_LEN    RSA_PSS_SALT_LEN_DEFAULT
+#if defined(WC_RSA_PSS) && !defined(HAVE_SELFTEST) && \
+    (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5,1))
+    #ifndef WOLFSSL_PSS_SALT_LEN_DISCOVER
+        #define DEF_PSS_SALT_LEN    RSA_PSS_SALT_LEN_DEFAULT
+    #else
+        #define DEF_PSS_SALT_LEN    RSA_PSS_SALT_LEN_DISCOVER
+    #endif
 #else
-    #define DEF_PSS_SALT_LEN    RSA_PSS_SALT_LEN_DISCOVER
+    #define DEF_PSS_SALT_LEN 0 /* not used */
 #endif
 
 #if defined(OPENSSL_EXTRA)
@@ -4098,16 +4088,19 @@ int wolfSSL_RSA_sign_mgf(int hashAlg, const unsigned char* hash,
         (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5,1))
         case WC_RSA_PKCS1_PSS_PADDING:
         {
+            RsaKey* key = (RsaKey*)rsa->internal;
             enum wc_HashType mgf1, hType;
             hType = wc_OidGetHash((int)nid2oid(hashAlg, oidHashType));
             if (mgf1Hash == WC_NID_undef)
                 mgf1Hash = hashAlg;
             mgf1 = wc_OidGetHash((int)nid2oid(mgf1Hash, oidHashType));
+            /* handle compat layer salt special cases */
+            saltLen = rsa_pss_calc_salt(saltLen, wc_HashGetDigestSize(hType),
+                wolfSSL_RSA_size(rsa));
 
             /* Create RSA PSS signature. */
             if ((signSz = wc_RsaPSS_Sign_ex(encodedSig, encSz, sigRet, outLen,
-                hType, wc_hash2mgf(mgf1), saltLen,
-                (RsaKey*)rsa->internal, rng)) <= 0) {
+                hType, wc_hash2mgf(mgf1), saltLen, key, rng)) <= 0) {
                 WOLFSSL_ERROR_MSG("Bad RSA Sign");
                 ret = 0;
             }
@@ -4244,19 +4237,23 @@ int wolfSSL_RSA_verify_mgf(int hashAlg, const unsigned char* hash,
     if (ret == 1 && padding == WC_RSA_PKCS1_PSS_PADDING) {
         #if defined(WC_RSA_PSS) && !defined(HAVE_SELFTEST) && \
         (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5,1))
+        RsaKey* key = (RsaKey*)rsa->internal;
         enum wc_HashType mgf1;
         hType = wc_OidGetHash((int)nid2oid(hashAlg, oidHashType));
         if (mgf1Hash == WC_NID_undef)
             mgf1Hash = hashAlg;
         mgf1 = wc_OidGetHash((int)nid2oid(mgf1Hash, oidHashType));
 
+        /* handle compat layer salt special cases */
+        saltLen = rsa_pss_calc_salt(saltLen, wc_HashGetDigestSize(hType),
+            wolfSSL_RSA_size(rsa));
+
         verLen = wc_RsaPSS_Verify_ex((byte*)sig, sigLen, sigDec, sigLen,
-            hType, wc_hash2mgf(mgf1), saltLen, (RsaKey*)rsa->internal);
+            hType, wc_hash2mgf(mgf1), saltLen, key);
         if (verLen > 0) {
             /* Check PSS padding is valid. */
             if (wc_RsaPSS_CheckPadding_ex(hash, hLen, sigDec, (word32)verLen,
-                hType, saltLen,
-                mp_count_bits(&((RsaKey*)rsa->internal)->n)) != 0) {
+                hType, saltLen, mp_count_bits(&key->n)) != 0) {
                 WOLFSSL_ERROR_MSG("wc_RsaPSS_CheckPadding_ex error");
                 ret = WOLFSSL_FAILURE;
             }