8298310: Enhance TLS session negotiation

Reviewed-by: mbalao
Backport-of: 9a14b363feaaa1a1831fcc8620d41b4db2e0110a

Author: Alexey Bakhtin

src/java.base/share/classes/sun/security/provider/certpath/AdjacencyList.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -87,7 +87,7 @@ public class AdjacencyList {
     // the actual set of steps the AdjacencyList represents
     private ArrayList<BuildStep> mStepList;
 
-    // the original list, just for the toString method
+    // the original list
     private List<List<Vertex>> mOrigList;
 
     /**
@@ -114,6 +114,13 @@ public Iterator<BuildStep> iterator() {
         return Collections.unmodifiableList(mStepList).iterator();
     }
 
+    /**
+     * Returns the number of attempted paths (useful for debugging).
+     */
+    public int numAttemptedPaths() {
+        return mOrigList.size();
+    }
+
     /**
      * Recursive, private method which actually builds the step list from
      * the given adjacency list. <code>Follow</code> is the parent BuildStep

src/java.base/share/classes/sun/security/provider/certpath/Builder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -412,8 +412,7 @@ Set<String> getMatchingPolicies() {
 
     /**
      * Search the specified CertStores and add all certificates matching
-     * selector to resultCerts. Self-signed certs are not useful here
-     * and therefore ignored.
+     * selector to resultCerts.
      *
      * If the targetCert criterion of the selector is set, only that cert
      * is examined and the CertStores are not searched.
@@ -432,8 +431,7 @@ boolean addMatchingCerts(X509CertSelector selector,
         X509Certificate targetCert = selector.getCertificate();
         if (targetCert != null) {
             // no need to search CertStores
-            if (selector.match(targetCert) && !X509CertImpl.isSelfSigned
-                (targetCert, buildParams.sigProvider())) {
+            if (selector.match(targetCert)) {
                 if (debug != null) {
                     debug.println("Builder.addMatchingCerts: " +
                         "adding target cert" +
@@ -452,11 +450,8 @@ boolean addMatchingCerts(X509CertSelector selector,
                 Collection<? extends Certificate> certs =
                                         store.getCertificates(selector);
                 for (Certificate cert : certs) {
-                    if (!X509CertImpl.isSelfSigned
-                        ((X509Certificate)cert, buildParams.sigProvider())) {
-                        if (resultCerts.add((X509Certificate)cert)) {
-                            add = true;
-                        }
+                    if (resultCerts.add((X509Certificate)cert)) {
+                        add = true;
                     }
                 }
                 if (!checkAll && add) {

src/java.base/share/classes/sun/security/provider/certpath/ForwardBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -48,6 +48,7 @@
 import sun.security.x509.AuthorityInfoAccessExtension;
 import sun.security.x509.AuthorityKeyIdentifierExtension;
 import static sun.security.x509.PKIXExtensions.*;
+import sun.security.x509.SubjectAlternativeNameExtension;
 import sun.security.x509.X500Name;
 import sun.security.x509.X509CertImpl;
 
@@ -294,9 +295,7 @@ private void getMatchingCACerts(ForwardState currentState,
                         "\n  Issuer: " +
                             trustedCert.getIssuerX500Principal());
                 }
-                if (caCerts.add(trustedCert) && !searchAllCertStores) {
-                    return;
-                }
+                caCerts.add(trustedCert);
             }
         }
 
@@ -675,8 +674,7 @@ public int compare(X509Certificate oCert1, X509Certificate oCert2) {
      * only be executed in a reverse direction are deferred until the
      * complete path has been built.
      *
-     * Trust anchor certs are not validated, but are used to verify the
-     * signature and revocation status of the previous cert.
+     * Trust anchor certs are not validated.
      *
      * If the last certificate is being verified (the one whose subject
      * matches the target subject, then steps in 6.1.4 of the PKIX
@@ -707,17 +705,15 @@ void verifyCert(X509Certificate cert, State currentState,
         currState.untrustedChecker.check(cert, Collections.<String>emptySet());
 
         /*
-         * check for looping - abort a loop if we encounter the same
-         * certificate twice
+         * Abort if we encounter the same certificate or a certificate with
+         * the same public key, subject DN, and subjectAltNames as a cert
+         * that is already in path.
          */
-        if (certPathList != null) {
-            for (X509Certificate cpListCert : certPathList) {
-                if (cert.equals(cpListCert)) {
-                    if (debug != null) {
-                        debug.println("loop detected!!");
-                    }
-                    throw new CertPathValidatorException("loop detected");
-                }
+        for (X509Certificate cpListCert : certPathList) {
+            if (repeated(cpListCert, cert)) {
+                throw new CertPathValidatorException(
+                    "cert with repeated subject, public key, and " +
+                    "subjectAltNames detected");
             }
         }
 
@@ -796,21 +792,48 @@ void verifyCert(X509Certificate cert, State currentState,
              */
             KeyChecker.verifyCAKeyUsage(cert);
         }
+    }
 
-        /*
-         * the following checks are performed even when the cert
-         * is a trusted cert, since we are only extracting the
-         * subjectDN, and publicKey from the cert
-         * in order to verify a previous cert
-         */
+    /**
+     * Return true if two certificates are equal or have the same subject,
+     * public key, and subject alternative names.
+     */
+    private static boolean repeated(
+            X509Certificate currCert, X509Certificate nextCert) {
+        if (currCert.equals(nextCert)) {
+            return true;
+        }
+        return (currCert.getSubjectX500Principal().equals(
+            nextCert.getSubjectX500Principal()) &&
+            currCert.getPublicKey().equals(nextCert.getPublicKey()) &&
+            altNamesEqual(currCert, nextCert));
+    }
 
-        /*
-         * Check signature only if no key requiring key parameters has been
-         * encountered.
-         */
-        if (!currState.keyParamsNeeded()) {
-            (currState.cert).verify(cert.getPublicKey(),
-                                    buildParams.sigProvider());
+    /**
+     * Return true if two certificates have the same subject alternative names.
+     */
+    private static boolean altNamesEqual(
+            X509Certificate currCert, X509Certificate nextCert) {
+        X509CertImpl curr, next;
+        try {
+            curr = X509CertImpl.toImpl(currCert);
+            next = X509CertImpl.toImpl(nextCert);
+        } catch (CertificateException ce) {
+            return false;
+        }
+
+        SubjectAlternativeNameExtension currAltNameExt =
+            curr.getSubjectAlternativeNameExtension();
+        SubjectAlternativeNameExtension nextAltNameExt =
+            next.getSubjectAlternativeNameExtension();
+        if (currAltNameExt != null) {
+            if (nextAltNameExt == null) {
+                return false;
+            }
+            return Arrays.equals(currAltNameExt.getExtensionValue(),
+                nextAltNameExt.getExtensionValue());
+        } else {
+            return (nextAltNameExt == null);
         }
     }
 

src/java.base/share/classes/sun/security/provider/certpath/ForwardState.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -80,10 +80,8 @@ class ForwardState implements State {
     /* The list of user-defined checkers that support forward checking */
     ArrayList<PKIXCertPathChecker> forwardCheckers;
 
-    /* Flag indicating if key needing to inherit key parameters has been
-     * encountered.
-     */
-    boolean keyParamsNeededFlag = false;
+    /* Flag indicating if last cert in path is self-issued */
+    boolean selfIssued;
 
     /**
      * Returns a boolean flag indicating if the state is initial
@@ -96,18 +94,6 @@ public boolean isInitial() {
         return init;
     }
 
-    /**
-     * Return boolean flag indicating whether a public key that needs to inherit
-     * key parameters has been encountered.
-     *
-     * @return boolean true if key needing to inherit parameters has been
-     * encountered; false otherwise.
-     */
-    @Override
-    public boolean keyParamsNeeded() {
-        return keyParamsNeededFlag;
-    }
-
     /**
      * Display state for debugging purposes
      */
@@ -118,10 +104,10 @@ public String toString() {
         sb.append("\n  issuerDN of last cert: ").append(issuerDN);
         sb.append("\n  traversedCACerts: ").append(traversedCACerts);
         sb.append("\n  init: ").append(String.valueOf(init));
-        sb.append("\n  keyParamsNeeded: ").append
-                 (String.valueOf(keyParamsNeededFlag));
         sb.append("\n  subjectNamesTraversed: \n").append
                  (subjectNamesTraversed);
+        sb.append("\n  selfIssued: ").append
+                 (String.valueOf(selfIssued));
         sb.append("]\n");
         return sb.toString();
     }
@@ -166,18 +152,14 @@ public void updateState(X509Certificate cert)
 
         X509CertImpl icert = X509CertImpl.toImpl(cert);
 
-        /* see if certificate key has null parameters */
-        if (PKIX.isDSAPublicKeyWithoutParams(icert.getPublicKey())) {
-            keyParamsNeededFlag = true;
-        }
-
         /* update certificate */
         this.cert = icert;
 
         /* update issuer DN */
         issuerDN = cert.getIssuerX500Principal();
 
-        if (!X509CertImpl.isSelfIssued(cert)) {
+        selfIssued = X509CertImpl.isSelfIssued(cert);
+        if (!selfIssued) {
 
             /*
              * update traversedCACerts only if this is a non-self-issued
@@ -190,7 +172,7 @@ public void updateState(X509Certificate cert)
 
         /* update subjectNamesTraversed only if this is the EE cert or if
            this cert is not self-issued */
-        if (init || !X509CertImpl.isSelfIssued(cert)){
+        if (init || !selfIssued) {
             X500Principal subjName = cert.getSubjectX500Principal();
             subjectNamesTraversed.add(X500Name.asX500Name(subjName));
 

src/java.base/share/classes/sun/security/provider/certpath/State.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2001, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -62,12 +62,4 @@ public void updateState(X509Certificate cert)
      * @return boolean flag indicating if the state is initial (just starting)
      */
     public boolean isInitial();
-
-    /**
-     * Returns a boolean flag indicating if a key lacking necessary key
-     * algorithm parameters has been encountered.
-     *
-     * @return boolean flag indicating if key lacking parameters encountered.
-     */
-    public boolean keyParamsNeeded();
 }