8360937: Enhance certificate handling

Reviewed-by: fferrari, andrew
Backport-of: d3b1c2be9e87aad07cac29d94679130fe5807c17

Author: Alexey Bakhtin

jdk/src/share/classes/sun/security/util/DerValue.java

@@ -822,6 +822,22 @@ public boolean equals(Object o) {
                 doEquals(other, this);
     }
 
+    /**
+     * Checks that the BMPString does not contain any surrogate characters,
+     * which are outside the Basic Multilingual Plane.
+     *
+     * @throws IOException if illegal characters are detected
+     */
+    public void validateBMPString() throws IOException {
+        String bmpString = getBMPString();
+        for (int i = 0; i < bmpString.length(); i++) {
+            if (Character.isSurrogate(bmpString.charAt(i))) {
+                throw new IOException(
+                    "Illegal character in BMPString, index: " + i);
+            }
+         }
+     }
+
     /**
      * Helper for public method equals()
      */

jdk/src/share/classes/sun/security/x509/AVA.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,9 +30,14 @@
 import java.io.OutputStream;
 import java.io.Reader;
 import java.security.AccessController;
+import java.nio.charset.Charset;
 import java.text.Normalizer;
 import java.util.*;
 
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.charset.StandardCharsets.UTF_16BE;
+
 import sun.security.action.GetBooleanAction;
 import sun.security.util.*;
 import sun.security.pkcs.PKCS9Attribute;
@@ -606,6 +611,10 @@ private static boolean trailingSpace(Reader in) throws IOException {
             throw new IOException("AVA, extra bytes = "
                 + derval.data.available());
         }
+
+        if (value.tag == DerValue.tag_BMPString) {
+            value.validateBMPString();
+        }
     }
 
     AVA(DerInputStream in) throws IOException {
@@ -753,7 +762,7 @@ public String toRFC2253String(Map<String, String> oidMap) {
              */
             String valStr = null;
             try {
-                valStr = new String(value.getDataBytes(), "UTF8");
+                valStr = new String(value.getDataBytes(), getCharset(value, false));
             } catch (IOException ie) {
                 throw new IllegalArgumentException("DER Value conversion");
             }
@@ -906,7 +915,7 @@ public String toRFC2253CanonicalString() {
              */
             String valStr = null;
             try {
-                valStr = new String(value.getDataBytes(), "UTF8");
+                valStr = new String(value.getDataBytes(), getCharset(value, true));
             } catch (IOException ie) {
                 throw new IllegalArgumentException("DER Value conversion");
             }
@@ -1026,6 +1035,46 @@ private static boolean isDerString(DerValue value, boolean canonical) {
         }
     }
 
+    /*
+     * Returns the charset that should be used to decode each DN string type.
+     *
+     * This method ensures that multi-byte (UTF8String and BMPString) types
+     * are decoded using the correct charset and the String forms represent
+     * the correct characters. For 8-bit ASCII-based types (PrintableString
+     * and IA5String), we return ISO_8859_1 rather than ASCII, so that the
+     * complete range of characters can be represented, as many certificates
+     * do not comply with the Internationalized Domain Name ACE format.
+     *
+     * NOTE: this method only supports DirectoryStrings of the types returned
+     * by isDerString().
+     */
+    private static Charset getCharset(DerValue value, boolean canonical) {
+        if (canonical) {
+            switch (value.tag) {
+                case DerValue.tag_PrintableString:
+                    return ISO_8859_1;
+                case DerValue.tag_UTF8String:
+                    return UTF_8;
+                default:
+                    throw new Error("unexpected tag: " + value.tag);
+            }
+        }
+
+        switch (value.tag) {
+            case DerValue.tag_PrintableString:
+            case DerValue.tag_T61String:
+            case DerValue.tag_IA5String:
+            case DerValue.tag_GeneralString:
+                 return ISO_8859_1;
+            case DerValue.tag_BMPString:
+                 return UTF_16BE;
+            case DerValue.tag_UTF8String:
+                 return UTF_8;
+            default:
+                 throw new Error("unexpected tag: " + value.tag);
+        }
+    }
+
     boolean hasRFC2253Keyword() {
         return AVAKeyword.hasKeyword(oid, RFC2253);
     }

jdk/test/java/security/testlibrary/CertificateBuilder.java

@@ -53,6 +53,7 @@
 import sun.security.x509.SubjectAlternativeNameExtension;
 import sun.security.x509.URIName;
 import sun.security.x509.KeyIdentifier;
+import sun.security.x509.X500Name;
 
 /**
  * Helper class that builds and signs X.509 certificates.
@@ -89,7 +90,7 @@
 public class CertificateBuilder {
     private final CertificateFactory factory;
 
-    private X500Principal subjectName = null;
+    private X500Name subjectName = null;
     private BigInteger serialNumber = null;
     private PublicKey publicKey = null;
     private Date notBefore = null;
@@ -114,17 +115,35 @@ public CertificateBuilder() throws CertificateException {
      * @param name An {@link X500Principal} to be used as the subject name
      * on this certificate.
      */
-    public void setSubjectName(X500Principal name) {
-        subjectName = name;
+    public CertificateBuilder setSubjectName(X500Principal name) {
+        subjectName = X500Name.asX500Name(name);
+        return this;
     }
 
     /**
      * Set the subject name for the certificate.
      *
      * @param name The subject name in RFC 2253 format
      */
-    public void setSubjectName(String name) {
-        subjectName = new X500Principal(name);
+    public CertificateBuilder setSubjectName(String name) {
+        try {
+            subjectName = new X500Name(name);
+        } catch (IOException ioe) {
+            throw new IllegalArgumentException(ioe);
+        }
+        return this;
+    }
+
+    /**
+     * Set the subject name for the certificate. This method is useful when
+     * you need more control over the contents of the subject name.
+     *
+     * @param name an {@code X500Name} to be used as the subject name
+     * on this certificate
+     */
+    public CertificateBuilder setSubjectName(X500Name name) {
+        subjectName = name;
+        return this;
     }
 
     /**