fix(#114260): in rsa signatures, configure digest before padding mode (#115695)

Co-authored-by: Raphael Catolino <[email protected]>
Co-authored-by: Kevin Jones <[email protected]>
Co-authored-by: Jeremy Barton <[email protected]>

Author: 4 people

src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c

@@ -215,6 +215,15 @@ int32_t CryptoNative_RsaEncrypt(EVP_PKEY* pkey,
 
 static bool ConfigureSignature(EVP_PKEY_CTX* ctx, RsaPaddingMode padding, const EVP_MD* digest)
 {
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-qual"
+    if (EVP_PKEY_CTX_set_signature_md(ctx, digest) <= 0)
+#pragma clang diagnostic pop
+    {
+        return false;
+    }
+
     if (padding == RsaPaddingPkcs1)
     {
         if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0)
@@ -233,14 +242,6 @@ static bool ConfigureSignature(EVP_PKEY_CTX* ctx, RsaPaddingMode padding, const
         }
     }
 
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wcast-qual"
-    if (EVP_PKEY_CTX_set_signature_md(ctx, digest) <= 0)
-#pragma clang diagnostic pop
-    {
-        return false;
-    }
-
     return true;
 }