Tolerate RSA-PSS in OpenJCEPlus signature

The mask generation function (MGF) based on a hash algorithm
is recommended to use the same hash function as the hash function
fingerprinting the message for RSA-PSS. However, the structures
in [PKCS#1v2.1] allow for separate parameterization of the MGF
and the message digest.

OpenJCEPlus restricts to use the same hash algorithm for both MGF
and message digest. This will cause test failures since the tests
are expecting different MGF and message digest behavior.

Signed-off-by: Jinhang Zhang <[email protected]>

Author: JinhangZhang

test/jdk/javax/xml/crypto/dsig/PSSSpec.java

@@ -21,6 +21,12 @@
  * questions.
  */
 
+/*
+ * ===========================================================================
+ * (c) Copyright IBM Corp. 2025, 2025 All Rights Reserved
+ * ===========================================================================
+ */
+
 import jdk.test.lib.Asserts;
 import jdk.test.lib.Utils;
 import jdk.test.lib.security.XMLUtils;
@@ -34,6 +40,7 @@
 import javax.xml.crypto.dsig.dom.DOMValidateContext;
 import javax.xml.crypto.dsig.spec.RSAPSSParameterSpec;
 import java.security.KeyPairGenerator;
+import java.security.Signature;
 import java.security.spec.MGF1ParameterSpec;
 import java.security.spec.PSSParameterSpec;
 
@@ -54,7 +61,20 @@ public class PSSSpec {
 
     public static void main(String[] args) throws Exception {
         unmarshal();
-        marshal();
+        try {
+            marshal();
+        } catch (javax.xml.crypto.dsig.XMLSignatureException xmlse) {
+            Throwable cause = xmlse.getCause();
+            if (cause instanceof java.security.InvalidAlgorithmParameterException) {
+                if (Signature.getInstance("RSA-PSS").getProvider().getName().equals("OpenJCEPlus")
+                && cause.getMessage().equals("The message digest within the PSSParameterSpec does not match the MGF message digest.")
+                ) {
+                    System.out.println("Expected error message is caught for OpenJCEPlus provider.");
+                    return;
+                }
+            }
+            throw xmlse;
+        }
         spec();
     }