wc_CheckPrivateKey() returns NOT_COMPILED_IN for certain gating flags

Author: anhu

tests/api.c

@@ -20743,6 +20743,23 @@ static int test_EccSigFailure_cm(void)
 #endif /* !NO_RSA || HAVE_ECC */
 #endif /* NO_CERTS */
 
+static int test_wc_CheckPrivateKey_RSA_pub_only(void)
+{
+    EXPECT_DECLS;
+#if !defined(NO_RSA) && !defined(NO_ASN_CRYPT) && \
+    !defined(NO_CHECK_PRIVATE_KEY) && \
+    (defined(WOLFSSL_RSA_PUBLIC_ONLY) || defined(WOLFSSL_RSA_VERIFY_ONLY))
+    /* With RSA public-only or verify-only, wc_CheckPrivateKey should return
+     * NOT_COMPILED_IN for RSA key types since private key operations are not
+     * available. */
+    ExpectIntEQ(wc_CheckPrivateKey(server_key_der_2048,
+        sizeof_server_key_der_2048, server_cert_der_2048,
+        sizeof_server_cert_der_2048, RSAk, NULL),
+        WC_NO_ERR_TRACE(NOT_COMPILED_IN));
+#endif
+    return EXPECT_RESULT();
+}
+
 #if defined(HAVE_PK_CALLBACKS) && !defined(WOLFSSL_NO_TLS12)
 #if !defined(NO_FILESYSTEM) && !defined(NO_DH) && \
         !defined(NO_AES) && defined(HAVE_AES_CBC) && \
@@ -31782,6 +31799,7 @@ TEST_CASE testCases[] = {
     TEST_DECL(test_EccSigFailure_cm),
     TEST_DECL(test_RsaSigFailure_cm),
 #endif
+    TEST_DECL(test_wc_CheckPrivateKey_RSA_pub_only),
 
     /* PKCS8 testing */
     TEST_DECL(test_wolfSSL_no_password_cb),

wolfcrypt/src/asn.c

@@ -8565,6 +8565,9 @@ int wc_CheckPrivateKey(const byte* privKey, word32 privKeySz,
         || ks == RSAPSSk
     #endif
         ) {
+    #if defined(WOLFSSL_RSA_PUBLIC_ONLY) || defined(WOLFSSL_RSA_VERIFY_ONLY)
+        ret = NOT_COMPILED_IN;
+    #else
     #ifdef WOLFSSL_SMALL_STACK
         RsaKey* a;
         RsaKey* b = NULL;
@@ -8619,6 +8622,7 @@ int wc_CheckPrivateKey(const byte* privKey, word32 privKeySz,
         wc_FreeRsaKey(a);
         WC_FREE_VAR_EX(b, NULL, DYNAMIC_TYPE_RSA);
         WC_FREE_VAR_EX(a, NULL, DYNAMIC_TYPE_RSA);
+    #endif /* !WOLFSSL_RSA_PUBLIC_ONLY && !WOLFSSL_RSA_VERIFY_ONLY */
     }
     else
     #endif /* !NO_RSA && !NO_ASN_CRYPT */