Tolerate RSASSA-PSS in OpenJCEPlus signature

The mask generation function (MGF) based on a hash algorithm
is recommended to use the same hash function as the hash function
fingerprinting the message for RSASSA-PSS. However, the structures
in [PKCS#1v2.1] allow for separate parameterization of the MGF
and the message digest.

OpenJCEPlus restricts to use the same hash algorithm for both MGF
and message digest. This will cause test failures since the tests
are expecting different MGF and message digest behavior.

Signed-off-by: Jinhang Zhang <[email protected]>

Author: JinhangZhang

test/jdk/javax/xml/crypto/dsig/SecureValidation.java

@@ -31,6 +31,13 @@
  *          java.base/sun.security.x509
  * @run main/othervm SecureValidation
  */
+
+/*
+ * ===========================================================================
+ * (c) Copyright IBM Corp. 2025, 2025 All Rights Reserved
+ * ===========================================================================
+ */
+
 import jdk.test.lib.Asserts;
 import jdk.test.lib.security.XMLUtils;
 import jdk.test.lib.Utils;
@@ -48,6 +55,7 @@
 import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathFactory;
 import java.security.PrivateKey;
+import java.security.Signature;
 import java.security.cert.X509Certificate;
 import java.security.spec.MGF1ParameterSpec;
 import java.security.spec.PSSParameterSpec;
@@ -70,10 +78,24 @@ public static void main(String[] args) throws Exception {
                 MGF1ParameterSpec.SHA512, 48, TRAILER_FIELD_BC);
 
         // Sign with PSS with SHA-384 and SHA-512
-        Document signed = XMLUtils.signer(privateKey, cert)
+        var signer = XMLUtils.signer(privateKey, cert)
                 .dm(DigestMethod.SHA384)
-                .sm(SignatureMethod.RSA_PSS, new RSAPSSParameterSpec(pspec))
-                .sign(doc);
+                .sm(SignatureMethod.RSA_PSS, new RSAPSSParameterSpec(pspec));
+        Document signed;
+        try {
+            signed = signer.sign(doc);
+        } catch (javax.xml.crypto.dsig.XMLSignatureException xmlse) {
+            Throwable cause = xmlse.getCause();
+            if (cause instanceof java.security.InvalidAlgorithmParameterException) {
+                if (Signature.getInstance("RSASSA-PSS").getProvider().getName().equals("OpenJCEPlus")
+                && cause.getMessage().equals("The message digest within the PSSParameterSpec does not match the MGF message digest.")
+                ) {
+                    System.out.println("Expected error message is caught for OpenJCEPlus provider.");
+                    return;
+                }
+            }
+            throw xmlse;
+        }
 
         XPath xp = XPathFactory.newInstance().newXPath();
         xp.setNamespaceContext(new NamespaceContext() {