Check for supported feature if decryption has failed

Also fix reference files in encryption tests, add certificate file in a format to be imported in Acrobat.

DEVSIX-7519

Author: vitali-pr

kernel/src/main/java/com/itextpdf/kernel/crypto/securityhandler/EncryptionUtils.java

@@ -126,6 +126,10 @@ static byte[] fetchEnvelopedData(Key certificateKey, Certificate certificate, St
                         }
                     }
                 } catch (Exception f) {
+                    // First check if the feature is supported, it will throw if not
+                    // Exact algorithm doesn't matter currently
+                    BouncyCastleFactoryCreator.getFactory().isEncryptionFeatureSupported(0, true);
+                    // Throw the original exception if the feature is supported
                     throw new PdfException(KernelExceptionMessageConstant.PDF_DECRYPTION, f);
                 }
             }

kernel/src/test/java/com/itextpdf/kernel/crypto/PdfEncryptionManuallyPortedTest.java

@@ -27,6 +27,8 @@ This file is part of the iText (R) project.
 import com.itextpdf.commons.bouncycastle.operator.AbstractOperatorCreationException;
 import com.itextpdf.commons.bouncycastle.pkcs.AbstractPKCSException;
 import com.itextpdf.io.font.constants.StandardFonts;
+import com.itextpdf.kernel.exceptions.KernelExceptionMessageConstant;
+import com.itextpdf.kernel.exceptions.PdfException;
 import com.itextpdf.kernel.font.PdfFontFactory;
 import com.itextpdf.kernel.logs.KernelLogMessageConstant;
 import com.itextpdf.kernel.pdf.CompressionConstants;
@@ -82,6 +84,8 @@ public class PdfEncryptionManuallyPortedTest extends ExtendedITextTest {
     public static final String sourceFolder = "./src/test/resources/com/itextpdf/kernel/crypto/PdfEncryptionManuallyPortedTest/";
 
     public static final char[] PRIVATE_KEY_PASS = "testpassphrase".toCharArray();
+    // There is also test.pfx to add to Acrobat to be able to open result pdf files. Password for it is also
+    // testpassphrase
     public static final String CERT = sourceFolder + "test.cer";
     public static final String PRIVATE_KEY = sourceFolder + "test.pem";
 
@@ -180,6 +184,27 @@ public void encryptWithCertificateAes256NoCompression() throws IOException, Inte
         encryptWithCertificate(filename, encryptionType, CompressionConstants.NO_COMPRESSION);
     }
 
+    @Test
+    @LogMessages(messages = @LogMessage(messageTemplate = KernelLogMessageConstant.MD5_IS_NOT_FIPS_COMPLIANT,
+            ignore = true))
+    public void openEncryptedDocWithWrongPrivateKey()
+            throws IOException, GeneralSecurityException, AbstractPKCSException, AbstractOperatorCreationException {
+        try (PdfReader reader = new PdfReader(sourceFolder + "encryptedWithCertificateAes128.pdf",
+                new ReaderProperties()
+                        .setPublicKeySecurityParams(
+                                getPublicCertificate(CERT),
+                                PemFileHelper.readPrivateKeyFromPemFile(
+                                        new FileInputStream(sourceFolder + "wrong.pem"), PRIVATE_KEY_PASS),
+                                FACTORY.getProviderName(),
+                                null))) {
+
+            Exception e = Assert.assertThrows(PdfException.class,
+                    () -> new PdfDocument(reader)
+            );
+            Assert.assertEquals(KernelExceptionMessageConstant.PDF_DECRYPTION, e.getMessage());
+        }
+    }
+
     public void encryptWithCertificate(String filename, int encryptionType, int compression) throws IOException,
             InterruptedException, GeneralSecurityException, AbstractPKCSException, AbstractOperatorCreationException {
         ITextTest.removeCryptographyRestrictions();

kernel/src/test/java/com/itextpdf/kernel/crypto/PdfEncryptionTest.java

@@ -269,27 +269,6 @@ public void openEncryptedDocWithWrongCertificate()
         }
     }
 
-    @Test
-    @LogMessages(messages = @LogMessage(messageTemplate = KernelLogMessageConstant.MD5_IS_NOT_FIPS_COMPLIANT,
-            ignore = true))
-    public void openEncryptedDocWithWrongPrivateKey()
-            throws IOException, GeneralSecurityException, AbstractPKCSException, AbstractOperatorCreationException {
-        try (PdfReader reader = new PdfReader(sourceFolder + "encryptedWithCertificateAes128.pdf",
-                new ReaderProperties()
-                        .setPublicKeySecurityParams(
-                                getPublicCertificate(CERT),
-                                PemFileHelper.readPrivateKeyFromPemFile(
-                                        new FileInputStream(sourceFolder + "wrong.pem"), PRIVATE_KEY_PASS),
-                                FACTORY.getProviderName(),
-                                null))) {
-
-            Exception e = Assert.assertThrows(PdfException.class,
-                    () -> new PdfDocument(reader)
-            );
-            Assert.assertEquals(KernelExceptionMessageConstant.PDF_DECRYPTION, e.getMessage());
-        }
-    }
-
     @Test
     @LogMessages(messages = @LogMessage(messageTemplate = KernelLogMessageConstant.MD5_IS_NOT_FIPS_COMPLIANT,
             ignore = true))
@@ -330,6 +309,9 @@ public void metadataReadingInEncryptedDoc() throws IOException, XMPException {
             ignore = true))
     public void copyEncryptedDocument() throws GeneralSecurityException, IOException, InterruptedException,
             AbstractPKCSException, AbstractOperatorCreationException {
+        // I don't know how this source doc was created. Currently it's not opening by Acrobat and Foxit.
+        // If I recreate it using iText, decrypting it in bc-fips on dotnet will start failing. But we probably still
+        // want this test.
         PdfDocument srcDoc = new PdfDocument(new PdfReader(sourceFolder + "encryptedWithCertificateAes128.pdf",
                 new ReaderProperties().
                         setPublicKeySecurityParams(getPublicCertificate(CERT), getPrivateKey(),