test: crypto-rsa-dsa testing for dynamic openssl

Fixes: #52537

Signed-off-by: Michael Dawson <[email protected]>
PR-URL: #52781
Reviewed-By: Luigi Pinca <[email protected]>
Reviewed-By: James M Snell <[email protected]>

Author: mhdawson

test/parallel/test-crypto-rsa-dsa.js

@@ -223,8 +223,6 @@ function test_rsa(padding, encryptOaepHash, decryptOaepHash) {
 
 
   if (padding === constants.RSA_PKCS1_PADDING) {
-    // TODO(richardlau): see if it's possible to determine implicit rejection
-    // support when dynamically linked against OpenSSL.
     if (!process.config.variables.node_shared_openssl) {
       assert.throws(() => {
         crypto.privateDecrypt({
@@ -240,6 +238,55 @@ function test_rsa(padding, encryptOaepHash, decryptOaepHash) {
           oaepHash: decryptOaepHash
         }, encryptedBuffer);
       }, { code: 'ERR_INVALID_ARG_VALUE' });
+    } else {
+      // The version of a linked against OpenSSL. May
+      // or may not support implicit rejection. Figuring
+      // this out in the test is not feasible but we
+      // require that it pass based on one of the two
+      // cases of supporting it or not.
+      try {
+        // The expected exceptions should be thrown if implicit rejection
+        // is not supported
+        assert.throws(() => {
+          crypto.privateDecrypt({
+            key: rsaKeyPem,
+            padding: padding,
+            oaepHash: decryptOaepHash
+          }, encryptedBuffer);
+        }, { code: 'ERR_INVALID_ARG_VALUE' });
+        assert.throws(() => {
+          crypto.privateDecrypt({
+            key: rsaPkcs8KeyPem,
+            padding: padding,
+            oaepHash: decryptOaepHash
+          }, encryptedBuffer);
+        }, { code: 'ERR_INVALID_ARG_VALUE' });
+      } catch (e) {
+        if (e.toString() ===
+            'AssertionError [ERR_ASSERTION]: Missing expected exception.') {
+          // Implicit rejection must be supported since
+          // we did not get the exceptions that are thrown
+          // when it is not, we should be able to decrypt
+          let decryptedBuffer = crypto.privateDecrypt({
+            key: rsaKeyPem,
+            padding: padding,
+            oaepHash: decryptOaepHash
+          }, encryptedBuffer);
+          assert.deepStrictEqual(decryptedBuffer, input);
+
+          decryptedBuffer = crypto.privateDecrypt({
+            key: rsaPkcs8KeyPem,
+            padding: padding,
+            oaepHash: decryptOaepHash
+          }, encryptedBuffer);
+          assert.deepStrictEqual(decryptedBuffer, input);
+        } else {
+          // There was an exception but it is not the one we expect if implicit
+          // rejection is not supported so there was some other failure,
+          // re-throw it so the test fails
+          throw e;
+        }
+      }
     }
   } else {
     let decryptedBuffer = crypto.privateDecrypt({