Fix coexist and improve rsa_pss_calc_salt invalid salt length checking.

Author: dgarske

src/pk.c

@@ -3534,25 +3534,25 @@ static int rsa_pss_calc_salt(int saltLen, int hashLen, int emLen)
     /* Calculate the salt length to use for special cases. */
     switch (saltLen) {
         /* Negative saltLen values are treated differently. */
-        case RSA_PSS_SALTLEN_DIGEST:
+        case WC_RSA_PSS_SALTLEN_DIGEST:
             saltLen = hashLen;
             break;
-        case RSA_PSS_SALTLEN_MAX_SIGN:
-        case RSA_PSS_SALTLEN_MAX:
+        case WC_RSA_PSS_SALTLEN_MAX_SIGN:
+        case WC_RSA_PSS_SALTLEN_MAX:
         #ifdef WOLFSSL_PSS_LONG_SALT
             saltLen = emLen - hashLen - 2;
         #else
             saltLen = hashLen;
         #endif
             break;
         default:
-            if (saltLen < 0) {
-                /* log invalid salt, let wolfCrypt report error */
-                WOLFSSL_ERROR_MSG("invalid saltLen");
-                saltLen = -3; /* for wolfCrypt to produce error must be < -2 */
-            }
             break;
     }
+    if (saltLen < 0) {
+        /* log invalid salt, let wolfCrypt handle error */
+        WOLFSSL_ERROR_MSG("invalid saltLen");
+        saltLen = -3; /* for wolfCrypt to produce error must be < -2 */
+    }
     return saltLen;
 }
 #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */

wolfssl/openssl/rsa.h

@@ -42,6 +42,17 @@
 #define WC_RSA_PKCS1_OAEP_PADDING 1
 #define WC_RSA_PKCS1_PSS_PADDING  2
 
+/* RSA PSS Salt special cases */
+/* Salt length same as digest length */
+#define WC_RSA_PSS_SALTLEN_DIGEST   (-1)
+/* Old max salt length */
+#define WC_RSA_PSS_SALTLEN_MAX_SIGN (-2)
+/* Verification only value to indicate to discover salt length. */
+#define WC_RSA_PSS_SALTLEN_AUTO     (-2)
+/* Max salt length */
+#define WC_RSA_PSS_SALTLEN_MAX      (-3)
+
+
 #ifndef OPENSSL_COEXIST
 
 /* Padding types */
@@ -60,14 +71,10 @@
 #define RSA_FLAG_NO_BLINDING            (1 << 7)
 #define RSA_FLAG_NO_CONSTTIME           (1 << 8)
 
-/* Salt length same as digest length */
-#define RSA_PSS_SALTLEN_DIGEST   (-1)
-/* Old max salt length */
-#define RSA_PSS_SALTLEN_MAX_SIGN (-2)
-/* Verification only value to indicate to discover salt length. */
-#define RSA_PSS_SALTLEN_AUTO     (-2)
-/* Max salt length */
-#define RSA_PSS_SALTLEN_MAX      (-3)
+#define RSA_PSS_SALTLEN_DIGEST   WC_RSA_PSS_SALTLEN_DIGEST
+#define RSA_PSS_SALTLEN_MAX_SIGN WC_RSA_PSS_SALTLEN_MAX_SIGN
+#define RSA_PSS_SALTLEN_AUTO     WC_RSA_PSS_SALTLEN_AUTO
+#define RSA_PSS_SALTLEN_MAX      WC_RSA_PSS_SALTLEN_MAX
 #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */
 
 #endif /* !OPENSSL_COEXIST */