8312428: PKCS11 tests fail with NSS 3.91

Valerie Peng · Valerie Peng · commit 1c598c2245c5 · 2023-08-29T22:08:34.000Z
Reviewed-by: ssahoo, rhalade
diff --git a/test/jdk/sun/security/pkcs11/MessageDigest/TestCloning.java b/test/jdk/sun/security/pkcs11/MessageDigest/TestCloning.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,8 @@
 import java.util.Random;
 import java.util.List;
 
+import jtreg.SkippedException;
+
 public class TestCloning extends PKCS11Test {
 
     public static void main(String[] args) throws Exception {
@@ -57,15 +59,31 @@ public void main(Provider p) throws Exception {
         r.nextBytes(data1);
         r.nextBytes(data2);
         System.out.println("Testing against provider " + p.getName());
+
+        boolean skipTest = true;
+
         for (String alg : ALGS) {
-            System.out.println("Testing " + alg);
+            System.out.println("Digest algo: " + alg);
             MessageDigest md = MessageDigest.getInstance(alg, p);
-            md = testCloning(md, p);
+            try {
+                md = testCloning(md, p);;
+            } catch (CloneNotSupportedException cnse) {
+                // skip test if clone isn't supported
+                System.out.println("=> Clone not supported; skip!");
+                continue;
+            }
+
+            // start testing below
+            skipTest = false;
+
             // repeat the test again after generating digest once
             for (int j = 0; j < 10; j++) {
                 md = testCloning(md, p);
             }
         }
+        if (skipTest) {
+            throw new SkippedException("Test Skipped!");
+        }
     }
 
     private static MessageDigest testCloning(MessageDigest mdObj, Provider p)
@@ -125,4 +143,3 @@ private static void check(byte[] d1, byte[] d2, String copyName)
         }
     }
 }
-
diff --git a/test/jdk/sun/security/pkcs11/PSSUtil.java b/test/jdk/sun/security/pkcs11/PSSUtil.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.Provider;
+import java.security.Signature;
+
+public class PSSUtil {
+
+    /**
+     * ALGORITHM name, fixed as RSA for PKCS11
+     */
+    private static final String KEYALG = "RSA";
+    private static final String SIGALG = "RSASSA-PSS";
+
+    public static enum AlgoSupport {
+        NO, MAYBE, YES
+    };
+
+    public static boolean isSignatureSupported(Provider p) {
+        try {
+            Signature.getInstance(SIGALG, p);
+            return true;
+        } catch (NoSuchAlgorithmException e) {
+            System.out.println("Skip testing " + SIGALG +
+                " due to no support");
+            return false;
+        }
+    }
+
+    public static AlgoSupport isHashSupported(Provider p, String... hashAlgs) {
+
+        AlgoSupport status = AlgoSupport.YES;
+        for (String h : hashAlgs) {
+            String sigAlg = (h.startsWith("SHA3-") ?
+                    h : h.replace("-", "")) + "with" + SIGALG;
+            try {
+                Signature.getInstance(sigAlg, p);
+                // Yes, proceed to check next hash algorithm
+                continue;
+            } catch (NoSuchAlgorithmException e) {
+                // continue trying other checks
+            }
+            try {
+                MessageDigest.getInstance(h, p);
+                status = AlgoSupport.MAYBE;
+            } catch (NoSuchAlgorithmException e) {
+                // if not supported as a standalone digest algo, chance of it
+                // being supported by PSS is very very low
+                return AlgoSupport.NO;
+            }
+        }
+        return status;
+    }
+
+    public static KeyPair generateKeys(Provider p, int size)
+            throws NoSuchAlgorithmException {
+        KeyPairGenerator kpg = KeyPairGenerator.getInstance(KEYALG, p);
+        kpg.initialize(size);
+        return kpg.generateKeyPair();
+    }
+}
diff --git a/test/jdk/sun/security/pkcs11/Signature/KeyAndParamCheckForPSS.java b/test/jdk/sun/security/pkcs11/Signature/KeyAndParamCheckForPSS.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,8 @@
 import java.security.interfaces.*;
 import java.security.spec.*;
 
+import jtreg.SkippedException;
+
 /**
  * @test
  * @bug 8080462 8226651 8242332
@@ -35,25 +37,19 @@
  */
 public class KeyAndParamCheckForPSS extends PKCS11Test {
 
-    /**
-     * ALGORITHM name, fixed as RSA for PKCS11
-     */
-    private static final String KEYALG = "RSA";
     private static final String SIGALG = "RSASSA-PSS";
 
     public static void main(String[] args) throws Exception {
         main(new KeyAndParamCheckForPSS(), args);
     }
 
+    private static boolean skipTest = true;
+
     @Override
     public void main(Provider p) throws Exception {
-        Signature sig;
-        try {
-            sig = Signature.getInstance(SIGALG, p);
-        } catch (NoSuchAlgorithmException e) {
-            System.out.println("Skip testing RSASSA-PSS" +
-                " due to no support");
-            return;
+        if (!PSSUtil.isSignatureSupported(p)) {
+            throw new SkippedException("Skip due to no support for " +
+                    SIGALG);
         }
 
         // NOTE: key length >= (digest length + 2) in bytes
@@ -76,27 +72,26 @@ public void main(Provider p) throws Exception {
         runTest(p, 1040, "SHA3-512", "SHA3-256");
         runTest(p, 1040, "SHA3-512", "SHA3-384");
         runTest(p, 1040, "SHA3-512", "SHA3-512");
+
+        if (skipTest) {
+            throw new SkippedException("Test Skipped");
+        }
     }
 
-    private void runTest(Provider p, int keySize, String hashAlg,
+    private static void runTest(Provider p, int keySize, String hashAlg,
             String mgfHashAlg) throws Exception {
 
-        // skip further test if this provider does not support hashAlg or
-        // mgfHashAlg
-        try {
-            MessageDigest.getInstance(hashAlg, p);
-            MessageDigest.getInstance(mgfHashAlg, p);
-        } catch (NoSuchAlgorithmException nsae) {
-            System.out.println("No support for " + hashAlg + ", skip");
+        System.out.println("Testing " + hashAlg + " and MGF1" + mgfHashAlg);
+        PSSUtil.AlgoSupport s = PSSUtil.isHashSupported(p, hashAlg, mgfHashAlg);
+        if (s == PSSUtil.AlgoSupport.NO) {
+            System.out.println("=> Skip; no support");
             return;
         }
 
-        System.out.println("Testing [" + keySize + " " + hashAlg + "]");
+        Signature sig = Signature.getInstance(SIGALG, p);
 
         // create a key pair with the supplied size
-        KeyPairGenerator kpg = KeyPairGenerator.getInstance(KEYALG, p);
-        kpg.initialize(keySize);
-        KeyPair kp = kpg.generateKeyPair();
+        KeyPair kp = PSSUtil.generateKeys(p, keySize);
 
         int bigSaltLen = keySize/8 - 14;
         AlgorithmParameterSpec paramsBad = new PSSParameterSpec(hashAlg,
@@ -108,58 +103,71 @@ private void runTest(Provider p, int keySize, String hashAlg,
         PublicKey pub = kp.getPublic();
 
         // test#1 - setParameter then initSign
-        Signature sig = Signature.getInstance("RSASSA-PSS", p);
-        sig.setParameter(paramsBad);
+        sig = Signature.getInstance(SIGALG, p);
+        try {
+            sig.setParameter(paramsGood);
+            sig.initSign(priv);
+            // algorithm support confirmed
+            skipTest = false;
+        } catch (Exception ex) {
+            if (s == PSSUtil.AlgoSupport.MAYBE) {
+                // confirmed to be unsupported; skip the rest of the test
+                System.out.println("=> Skip; no PSS support");
+                return;
+            } else {
+                throw new RuntimeException("Unexpected Exception", ex);
+            }
+        }
+
+        sig = Signature.getInstance(SIGALG, p);
         try {
+            sig.setParameter(paramsBad);
             sig.initSign(priv);
             throw new RuntimeException("Expected IKE not thrown");
         } catch (InvalidKeyException ike) {
-            System.out.println("test#1: got expected IKE");
+            // expected
         }
 
+        // test#2 - setParameter then initVerify
+        sig = Signature.getInstance(SIGALG, p);
         sig.setParameter(paramsGood);
-        sig.initSign(priv);
-        System.out.println("test#1: pass");
+        sig.initVerify(pub);
 
-        // test#2 - setParameter then initVerify
-        sig = Signature.getInstance("RSASSA-PSS", p);
-        sig.setParameter(paramsBad);
+        sig = Signature.getInstance(SIGALG, p);
         try {
+            sig.setParameter(paramsBad);
             sig.initVerify(pub);
             throw new RuntimeException("Expected IKE not thrown");
         } catch (InvalidKeyException ike) {
-            System.out.println("test#2: got expected IKE");
+            // expected
         }
 
-        sig.setParameter(paramsGood);
-        sig.initVerify(pub);
-
-        System.out.println("test#2: pass");
-
         // test#3 - initSign, then setParameter
-        sig = Signature.getInstance("RSASSA-PSS", p);
+        sig = Signature.getInstance(SIGALG, p);
         sig.initSign(priv);
+        sig.setParameter(paramsGood);
+
+        sig = Signature.getInstance(SIGALG, p);
         try {
+            sig.initSign(priv);
             sig.setParameter(paramsBad);
             throw new RuntimeException("Expected IAPE not thrown");
         } catch (InvalidAlgorithmParameterException iape) {
-            System.out.println("test#3: got expected IAPE");
+            // expected
         }
 
-        sig.setParameter(paramsGood);
-        System.out.println("test#3: pass");
-
         // test#4 - initVerify, then setParameter
-        sig = Signature.getInstance("RSASSA-PSS", p);
+        sig = Signature.getInstance(SIGALG, p);
+        sig.setParameter(paramsGood);
         sig.initVerify(pub);
+
+        sig = Signature.getInstance(SIGALG, p);
         try {
+            sig.initVerify(pub);
             sig.setParameter(paramsBad);
             throw new RuntimeException("Expected IAPE not thrown");
         } catch (InvalidAlgorithmParameterException iape) {
-            System.out.println("test#4: got expected IAPE");
+            // expected
         }
-
-        sig.setParameter(paramsGood);
-        System.out.println("test#4: pass");
     }
 }
diff --git a/test/jdk/sun/security/pkcs11/Signature/SignatureTestPSS.java b/test/jdk/sun/security/pkcs11/Signature/SignatureTestPSS.java
