pkcs7: add RSA-PSS support for SignedData

Add full RSA-PSS (RSASSA-PSS) support to PKCS#7 SignedData
encoding and verification.

This change enables SignerInfo.signatureAlgorithm to use
id-RSASSA-PSS with explicit RSASSA-PSS-params (hash, MGF1,
salt length), as required by RFC 4055 and CMS profiles.

Key changes:
- Add RSA-PSS encode and verify paths for PKCS7 SignedData
- Encode full RSASSA-PSS AlgorithmIdentifier parameters
- Decode RSA-PSS parameters from SignerInfo for verification
- Treat RSA-PSS like ECDSA (sign raw digest, not DigestInfo)
- Fix certificate signatureAlgorithm parameter length handling
- Add API test coverage for RSA-PSS SignedData

This resolves failures when using RSA-PSS signer certificates
(e.g. -173 invalid signature algorithm) and maintains backward
compatibility with RSA PKCS#1 v1.5 and ECDSA.

Signed-off-by: Sameeh Jubran <sameeh@wolfssl.com>

Author: sameehj

.github/workflows/os-check.yml

@@ -53,6 +53,8 @@ jobs:
           '--enable-opensslall --enable-opensslextra CPPFLAGS=-DWC_RNG_SEED_CB',
           '--enable-opensslall --enable-opensslextra
             CPPFLAGS=''-DWC_RNG_SEED_CB -DWOLFSSL_NO_GETPID'' ',
+          # PKCS#7 with RSA-PSS (CMS RSASSA-PSS signers)
+          '--enable-pkcs7 CPPFLAGS=-DWC_RSA_PSS',
           '--enable-opensslextra CPPFLAGS=''-DWOLFSSL_NO_CA_NAMES'' ',
           '--enable-opensslextra=x509small',
           'CPPFLAGS=''-DWOLFSSL_EXTRA'' ',

doc/dox_comments/header_files/cryptocb.h

@@ -52,6 +52,17 @@
                 }
             }
         #endif
+        #if defined(WC_RSA_PSS) && !defined(NO_RSA)
+            if (info->pk.type == WC_PK_TYPE_RSA_PSS) {
+                // RSA-PSS sign/verify
+                ret = wc_RsaPSS_Sign_ex(
+                    info->pk.rsa.in, info->pk.rsa.inLen,
+                    info->pk.rsa.out, *info->pk.rsa.outLen,
+                    WC_HASH_TYPE_SHA256, WC_MGF1SHA256,
+                    RSA_PSS_SALT_LEN_DEFAULT,
+                    info->pk.rsa.key, info->pk.rsa.rng);
+            }
+        #endif
         #ifdef HAVE_ECC
             if (info->pk.type == WC_PK_TYPE_ECDSA_SIGN) {
                 // ECDSA

doc/dox_comments/header_files/doxygen_pages.h

@@ -51,6 +51,7 @@
         <li>\ref MD5</li>
         <li>\ref Password</li>
         <li>\ref PKCS7</li>
+        <li>\ref PKCS7_RSA_PSS</li>
         <li>\ref PKCS11</li>
         <li>\ref Poly1305</li>
         <li>\ref RIPEMD</li>
@@ -97,4 +98,13 @@
     \sa wc_CryptoCb_AesSetKey
     \sa \ref Crypto Callbacks
 */
+/*!
+    \page PKCS7_RSA_PSS PKCS#7 RSA-PSS (CMS)
+    PKCS#7 SignedData supports RSA-PSS signers (CMS RSASSA-PSS). When WC_RSA_PSS
+    is defined, use wc_PKCS7_InitWithCert with a signer certificate that has
+    RSA-PSS (id-RSASSA-PSS) and set hashOID and optional rng; encode produces
+    full RSASSA-PSS-params (hashAlgorithm, mgfAlgorithm, saltLength,
+    trailerField). Verify accepts NULL, empty, or absent parameters with
+    RFC defaults. See \ref PKCS7 for the main API.
+*/
 

doc/dox_comments/header_files/pkcs7.h

@@ -147,7 +147,7 @@ int  wc_PKCS7_EncodeData(wc_PKCS7* pkcs7, byte* output,
 
     \brief This function builds the PKCS7 signed data content type, encoding
     the PKCS7 structure into a buffer containing a parsable PKCS7
-    signed data packet.
+    signed data packet. For RSA-PSS signers (WC_RSA_PSS), see \ref PKCS7_RSA_PSS.
 
     \return Success On successfully encoding the PKCS7 data into the buffer,
     returns the index parsed up to in the PKCS7 structure. This index also

examples/configs/README.md

@@ -23,7 +23,7 @@ Example wolfSSL configuration file templates for use when autoconf is not availa
 * `user_settings_openssl_compat.h`: OpenSSL compatibility layer for drop-in replacement. Enables OPENSSL_ALL and related APIs.
 * `user_settings_baremetal.h`: Bare metal configuration. No filesystem, static memory only, minimal footprint.
 * `user_settings_rsa_only.h`: RSA-only configuration (no ECC). For legacy systems requiring RSA cipher suites.
-* `user_settings_pkcs7.h`: PKCS#7/CMS configuration for signing and encryption. S/MIME, firmware signing.
+* `user_settings_pkcs7.h`: PKCS#7/CMS configuration for signing and encryption. S/MIME, firmware signing. For RSA-PSS SignedData (CMS RSASSA-PSS), define `WC_RSA_PSS`; see doxygen \ref PKCS7_RSA_PSS.
 * `user_settings_ca.h`: Certificate Authority / PKI operations. Certificate generation, signing, CRL, OCSP.
 * `user_settings_wolfboot_keytools.h`: wolfBoot key generation and signing tool. Supports ECC, RSA, ED25519, ED448, and post-quantum (ML-DSA/Dilithium, LMS, XMSS).
 * `user_settings_wolfssh.h`: Minimum options for building wolfSSH. See comment at top for ./configure used to generate.

examples/configs/user_settings_pkcs7.h

@@ -115,6 +115,7 @@ extern "C" {
     #undef NO_RSA
     #define WOLFSSL_KEY_GEN
     #define WC_RSA_NO_PADDING
+    #define WC_RSA_PSS  /* RSA-PSS SignedData (id-RSASSA-PSS); see PKCS7_RSA_PSS */
 #else
     #define NO_RSA
 #endif

tests/api/test_asn.c

@@ -24,6 +24,7 @@
 #include <tests/api/test_asn.h>
 
 #include <wolfssl/wolfcrypt/asn.h>
+#include <wolfssl/wolfcrypt/rsa.h>
 
 #if defined(WC_ENABLE_ASYM_KEY_EXPORT) && defined(HAVE_ED25519)
 static int test_SetAsymKeyDer_once(byte* privKey, word32 privKeySz, byte* pubKey,
@@ -787,3 +788,135 @@ int test_wolfssl_local_MatchBaseName(void)
 
     return EXPECT_RESULT();
 }
+
+/*
+ * Testing wc_DecodeRsaPssParams with known DER byte arrays.
+ * Exercises both WOLFSSL_ASN_TEMPLATE and non-template paths.
+ */
+int test_wc_DecodeRsaPssParams(void)
+{
+    EXPECT_DECLS;
+#if defined(WC_RSA_PSS) && !defined(NO_RSA) && !defined(NO_ASN)
+    enum wc_HashType hash;
+    int mgf;
+    int saltLen;
+
+    /* SHA-256 / MGF1-SHA-256 / saltLen=32 */
+    static const byte pssParamsSha256[] = {
+        0x30, 0x34,
+          0xA0, 0x0F,
+            0x30, 0x0D,
+              0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03,
+                          0x04, 0x02, 0x01,
+              0x05, 0x00,
+          0xA1, 0x1C,
+            0x30, 0x1A,
+              0x06, 0x09, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D,
+                          0x01, 0x01, 0x08,
+              0x30, 0x0D,
+                0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03,
+                            0x04, 0x02, 0x01,
+                0x05, 0x00,
+          0xA2, 0x03,
+            0x02, 0x01, 0x20,
+    };
+
+    /* Hash-only: SHA-256 hash, defaults for MGF and salt */
+    static const byte pssParamsHashOnly[] = {
+        0x30, 0x11,
+          0xA0, 0x0F,
+            0x30, 0x0D,
+              0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03,
+                          0x04, 0x02, 0x01,
+              0x05, 0x00,
+    };
+
+    /* Salt-only: default hash/mgf, saltLen=48 */
+    static const byte pssParamsSaltOnly[] = {
+        0x30, 0x05,
+          0xA2, 0x03,
+            0x02, 0x01, 0x30,
+    };
+
+    /* NULL tag (05 00) means all defaults */
+    static const byte pssParamsNull[] = { 0x05, 0x00 };
+
+    /* Empty SEQUENCE means all non-default fields omitted => defaults */
+    static const byte pssParamsEmptySeq[] = { 0x30, 0x00 };
+
+    /* --- Test 1: sz=0 => all defaults --- */
+    hash = WC_HASH_TYPE_NONE;
+    mgf = 0;
+    saltLen = 0;
+    ExpectIntEQ(wc_DecodeRsaPssParams((const byte*)"", 0,
+        &hash, &mgf, &saltLen), 0);
+    ExpectIntEQ((int)hash, (int)WC_HASH_TYPE_SHA);
+    ExpectIntEQ(mgf, WC_MGF1SHA1);
+    ExpectIntEQ(saltLen, 20);
+
+    /* --- Test 2: NULL tag => all defaults --- */
+    hash = WC_HASH_TYPE_NONE;
+    mgf = 0;
+    saltLen = 0;
+    ExpectIntEQ(wc_DecodeRsaPssParams(pssParamsNull,
+        (word32)sizeof(pssParamsNull), &hash, &mgf, &saltLen), 0);
+    ExpectIntEQ((int)hash, (int)WC_HASH_TYPE_SHA);
+    ExpectIntEQ(mgf, WC_MGF1SHA1);
+    ExpectIntEQ(saltLen, 20);
+
+    /* --- Test 3: Empty SEQUENCE => all defaults --- */
+    hash = WC_HASH_TYPE_NONE;
+    mgf = 0;
+    saltLen = 0;
+    ExpectIntEQ(wc_DecodeRsaPssParams(pssParamsEmptySeq,
+        (word32)sizeof(pssParamsEmptySeq), &hash, &mgf, &saltLen), 0);
+    ExpectIntEQ((int)hash, (int)WC_HASH_TYPE_SHA);
+    ExpectIntEQ(mgf, WC_MGF1SHA1);
+    ExpectIntEQ(saltLen, 20);
+
+#ifndef NO_SHA256
+    /* --- Test 4: SHA-256 / MGF1-SHA-256 / salt=32 --- */
+    hash = WC_HASH_TYPE_NONE;
+    mgf = 0;
+    saltLen = 0;
+    ExpectIntEQ(wc_DecodeRsaPssParams(pssParamsSha256,
+        (word32)sizeof(pssParamsSha256), &hash, &mgf, &saltLen), 0);
+    ExpectIntEQ((int)hash, (int)WC_HASH_TYPE_SHA256);
+    ExpectIntEQ(mgf, WC_MGF1SHA256);
+    ExpectIntEQ(saltLen, 32);
+
+    /* --- Test 5: Hash only => SHA-256, default MGF/salt --- */
+    hash = WC_HASH_TYPE_NONE;
+    mgf = 0;
+    saltLen = 0;
+    ExpectIntEQ(wc_DecodeRsaPssParams(pssParamsHashOnly,
+        (word32)sizeof(pssParamsHashOnly), &hash, &mgf, &saltLen), 0);
+    ExpectIntEQ((int)hash, (int)WC_HASH_TYPE_SHA256);
+    ExpectIntEQ(mgf, WC_MGF1SHA1);
+    ExpectIntEQ(saltLen, 20);
+#endif
+
+    /* --- Test 6: Salt only => default hash/MGF, salt=48 --- */
+    hash = WC_HASH_TYPE_NONE;
+    mgf = 0;
+    saltLen = 0;
+    ExpectIntEQ(wc_DecodeRsaPssParams(pssParamsSaltOnly,
+        (word32)sizeof(pssParamsSaltOnly), &hash, &mgf, &saltLen), 0);
+    ExpectIntEQ((int)hash, (int)WC_HASH_TYPE_SHA);
+    ExpectIntEQ(mgf, WC_MGF1SHA1);
+    ExpectIntEQ(saltLen, 48);
+
+    /* --- Test 7: NULL pointer → BAD_FUNC_ARG --- */
+    ExpectIntEQ(wc_DecodeRsaPssParams(NULL, 10, &hash, &mgf, &saltLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+
+    /* --- Test 8: Bad leading tag => ASN_PARSE_E --- */
+    {
+        static const byte badTag[] = { 0x01, 0x00 };
+        ExpectIntEQ(wc_DecodeRsaPssParams(badTag, (word32)sizeof(badTag),
+            &hash, &mgf, &saltLen), WC_NO_ERR_TRACE(ASN_PARSE_E));
+    }
+
+#endif /* WC_RSA_PSS && !NO_RSA && !NO_ASN */
+    return EXPECT_RESULT();
+}

tests/api/test_asn.h

@@ -28,11 +28,13 @@ int test_SetAsymKeyDer(void);
 int test_GetSetShortInt(void);
 int test_wc_IndexSequenceOf(void);
 int test_wolfssl_local_MatchBaseName(void);
+int test_wc_DecodeRsaPssParams(void);
 
 #define TEST_ASN_DECLS                                              \
     TEST_DECL_GROUP("asn", test_SetAsymKeyDer),                     \
     TEST_DECL_GROUP("asn", test_GetSetShortInt),                    \
     TEST_DECL_GROUP("asn", test_wc_IndexSequenceOf),                \
-    TEST_DECL_GROUP("asn", test_wolfssl_local_MatchBaseName)
+    TEST_DECL_GROUP("asn", test_wolfssl_local_MatchBaseName),       \
+    TEST_DECL_GROUP("asn", test_wc_DecodeRsaPssParams)
 
 #endif /* WOLFCRYPT_TEST_ASN_H */

tests/api/test_pkcs7.c

@@ -947,6 +947,98 @@ int test_wc_PKCS7_EncodeSignedData(void)
 } /* END test_wc_PKCS7_EncodeSignedData */
 
 
+/*
+ * Testing wc_PKCS7_EncodeSignedData() with RSA-PSS signer certificate.
+ * Uses certs/rsapss/client-rsapss.der and client-rsapss-priv.der.
+ * Requires both encode and round-trip verify to succeed.
+ */
+#if defined(HAVE_PKCS7) && defined(WC_RSA_PSS) && !defined(NO_RSA) && \
+    !defined(NO_FILESYSTEM) && !defined(NO_SHA256)
+int test_wc_PKCS7_EncodeSignedData_RSA_PSS(void)
+{
+    EXPECT_DECLS;
+    PKCS7*    pkcs7 = NULL;
+    WC_RNG    rng;
+    byte      output[FOURK_BUF];
+    byte      cert[FOURK_BUF];
+    byte      key[FOURK_BUF];
+    word32    outputSz = (word32)sizeof(output);
+    word32    certSz = 0;
+    word32    keySz = 0;
+    XFILE     fp = XBADFILE;
+    byte      data[] = "Test data for RSA-PSS SignedData.";
+
+    XMEMSET(&rng, 0, sizeof(WC_RNG));
+    XMEMSET(output, 0, outputSz);
+    XMEMSET(cert, 0, sizeof(cert));
+    XMEMSET(key, 0, sizeof(key));
+
+    ExpectIntEQ(wc_InitRng(&rng), 0);
+    ExpectNotNull(pkcs7 = wc_PKCS7_New(HEAP_HINT, testDevId));
+
+    ExpectTrue((fp = XFOPEN("./certs/rsapss/client-rsapss.der", "rb")) != XBADFILE);
+    if (fp != XBADFILE) {
+        ExpectIntGT(certSz = (word32)XFREAD(cert, 1, sizeof(cert), fp), 0);
+        XFCLOSE(fp);
+        fp = XBADFILE;
+    }
+
+    ExpectTrue((fp = XFOPEN("./certs/rsapss/client-rsapss-priv.der", "rb")) != XBADFILE);
+    if (fp != XBADFILE) {
+        ExpectIntGT(keySz = (word32)XFREAD(key, 1, sizeof(key), fp), 0);
+        XFCLOSE(fp);
+        fp = XBADFILE;
+    }
+
+    ExpectIntEQ(wc_PKCS7_InitWithCert(pkcs7, cert, certSz), 0);
+
+    if (pkcs7 != NULL) {
+        /* Force RSA-PSS so SignerInfo uses id-RSASSA-PSS (cert may use RSA
+         * in subjectPublicKeyInfo).  WC_RSA_PSS is guaranteed by outer guard. */
+        pkcs7->publicKeyOID = RSAPSSk;
+
+        pkcs7->content       = data;
+        pkcs7->contentSz     = (word32)sizeof(data);
+        pkcs7->contentOID    = DATA;
+        pkcs7->hashOID       = SHA256h;
+        pkcs7->encryptOID    = RSAk;
+        pkcs7->privateKey    = key;
+        pkcs7->privateKeySz  = keySz;
+        pkcs7->rng           = &rng;
+        pkcs7->signedAttribs = NULL;
+        pkcs7->signedAttribsSz = 0;
+    }
+
+    /* EncodeSignedData with RSA-PSS cert: require encode and verify success */
+    {
+        int outLen = wc_PKCS7_EncodeSignedData(pkcs7, output, outputSz);
+        ExpectIntGT(outLen, 0);
+        if (outLen > 0) {
+            int verifyRet = wc_PKCS7_VerifySignedData(pkcs7, output,
+                                                       (word32)outLen);
+            ExpectIntEQ(verifyRet, 0);
+
+            if (pkcs7 != NULL) {
+                /* Verify decoded RSASSA-PSS parameters match what we
+                 * encoded:
+                 *   hashAlgorithm    = SHA-256
+                 *   maskGenAlgorithm = MGF1-SHA-256
+                 *   saltLength       = 32 (== SHA-256 digest length) */
+                ExpectIntEQ(pkcs7->pssHashType, (int)WC_HASH_TYPE_SHA256);
+                ExpectIntEQ(pkcs7->pssMgf, WC_MGF1SHA256);
+                ExpectIntEQ(pkcs7->pssSaltLen, 32);
+            }
+        }
+    }
+
+    wc_PKCS7_Free(pkcs7);
+    DoExpectIntEQ(wc_FreeRng(&rng), 0);
+
+    return EXPECT_RESULT();
+} /* END test_wc_PKCS7_EncodeSignedData_RSA_PSS */
+#endif
+
+
 /*
  * Testing wc_PKCS7_EncodeSignedData_ex() and wc_PKCS7_VerifySignedData_ex()
  */

tests/api/test_pkcs7.h

@@ -29,6 +29,10 @@ int test_wc_PKCS7_Init(void);
 int test_wc_PKCS7_InitWithCert(void);
 int test_wc_PKCS7_EncodeData(void);
 int test_wc_PKCS7_EncodeSignedData(void);
+#if defined(HAVE_PKCS7) && defined(WC_RSA_PSS) && !defined(NO_RSA) && \
+    !defined(NO_FILESYSTEM) && !defined(NO_SHA256)
+int test_wc_PKCS7_EncodeSignedData_RSA_PSS(void);
+#endif
 int test_wc_PKCS7_EncodeSignedData_ex(void);
 int test_wc_PKCS7_VerifySignedData_RSA(void);
 int test_wc_PKCS7_VerifySignedData_ECC(void);
@@ -55,10 +59,19 @@ int test_wc_PKCS7_VerifySignedData_PKCS7ContentSeq(void);
     TEST_DECL_GROUP("pkcs7", test_wc_PKCS7_New),                \
     TEST_DECL_GROUP("pkcs7", test_wc_PKCS7_Init)
 
+#if defined(HAVE_PKCS7) && defined(WC_RSA_PSS) && !defined(NO_RSA) && \
+    !defined(NO_FILESYSTEM) && !defined(NO_SHA256)
+#define TEST_PKCS7_RSA_PSS_SD_DECL \
+    TEST_DECL_GROUP("pkcs7_sd", test_wc_PKCS7_EncodeSignedData_RSA_PSS),
+#else
+#define TEST_PKCS7_RSA_PSS_SD_DECL
+#endif
+
 #define TEST_PKCS7_SIGNED_DATA_DECLS                                    \
     TEST_DECL_GROUP("pkcs7_sd", test_wc_PKCS7_InitWithCert),            \
     TEST_DECL_GROUP("pkcs7_sd", test_wc_PKCS7_EncodeData),              \
     TEST_DECL_GROUP("pkcs7_sd", test_wc_PKCS7_EncodeSignedData),        \
+    TEST_PKCS7_RSA_PSS_SD_DECL                                           \
     TEST_DECL_GROUP("pkcs7_sd", test_wc_PKCS7_EncodeSignedData_ex),     \
     TEST_DECL_GROUP("pkcs7_sd", test_wc_PKCS7_VerifySignedData_RSA),    \
     TEST_DECL_GROUP("pkcs7_sd", test_wc_PKCS7_VerifySignedData_ECC),    \