8307383: Enhance DTLS connections

Alexey Bakhtin · RealCLanger · commit d719385bfd9d · 2024-10-03T21:52:40.000+02:00
Reviewed-by: mbaesken, andrew
Backport-of: 362dbbaa952b3d4a5270c6bfae879a12e9bdf4d1
diff --git a/src/java.base/share/classes/sun/security/ssl/ClientHello.java b/src/java.base/share/classes/sun/security/ssl/ClientHello.java
@@ -217,8 +217,6 @@ byte[] getHelloCookieBytes() {
                 // ignore cookie
                 hos.putBytes16(getEncodedCipherSuites());
                 hos.putBytes8(compressionMethod);
-                extensions.send(hos);       // In TLS 1.3, use of certain
-                                            // extensions is mandatory.
             } catch (IOException ioe) {
                 // unlikely
             }
@@ -1427,6 +1425,9 @@ public void consume(ConnectionContext context,
             shc.handshakeProducers.put(SSLHandshake.SERVER_HELLO.id,
                     SSLHandshake.SERVER_HELLO);
 
+            // Reset the ClientHello non-zero offset fragment allowance
+            shc.acceptCliHelloFragments = false;
+
             //
             // produce
             //
diff --git a/src/java.base/share/classes/sun/security/ssl/DTLSInputRecord.java b/src/java.base/share/classes/sun/security/ssl/DTLSInputRecord.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,13 +28,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.security.GeneralSecurityException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
+import java.util.*;
 import javax.crypto.BadPaddingException;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLProtocolException;
@@ -46,12 +40,23 @@
 final class DTLSInputRecord extends InputRecord implements DTLSRecord {
     private DTLSReassembler reassembler = null;
     private int             readEpoch;
+    private SSLContextImpl  sslContext;
 
     DTLSInputRecord(HandshakeHash handshakeHash) {
         super(handshakeHash, SSLReadCipher.nullDTlsReadCipher());
         this.readEpoch = 0;
     }
 
+    // Method to set TransportContext
+    public void setTransportContext(TransportContext tc) {
+        this.tc = tc;
+    }
+
+    // Method to set SSLContext
+    public void setSSLContext(SSLContextImpl sslContext) {
+        this.sslContext = sslContext;
+    }
+
     @Override
     void changeReadCiphers(SSLReadCipher readCipher) {
         this.readCipher = readCipher;
@@ -544,6 +549,27 @@ public int compareTo(RecordFragment o) {
         }
     }
 
+    /**
+     * Turn a sufficiently-large initial ClientHello fragment into one that
+     * stops immediately after the compression methods.  This is only used
+     * for the initial CH message fragment at offset 0.
+     *
+     * @param srcFrag the fragment actually received by the DTLSReassembler
+     * @param limit the size of the new, cloned/truncated handshake fragment
+     *
+     * @return a truncated handshake fragment that is sized to look like a
+     * complete message, but actually contains only up to the compression
+     * methods (no extensions)
+     */
+    private static HandshakeFragment truncateChFragment(HandshakeFragment srcFrag,
+            int limit) {
+        return new HandshakeFragment(Arrays.copyOf(srcFrag.fragment, limit),
+                srcFrag.contentType, srcFrag.majorVersion,
+                srcFrag.minorVersion, srcFrag.recordEnS, srcFrag.recordEpoch,
+                srcFrag.recordSeq, srcFrag.handshakeType, limit,
+                srcFrag.messageSeq, srcFrag.fragmentOffset, limit);
+    }
+
     private static final class HoleDescriptor {
         int offset;             // fragment_offset
         int limit;              // fragment_offset + fragment_length
@@ -647,10 +673,17 @@ void expectingFinishFlight() {
         // Queue up a handshake message.
         void queueUpHandshake(HandshakeFragment hsf) throws SSLProtocolException {
             if (!isDesirable(hsf)) {
-                // Not a dedired record, discard it.
+                // Not a desired record, discard it.
                 return;
             }
 
+            if (hsf.handshakeType == SSLHandshake.CLIENT_HELLO.id) {
+                // validate the first or subsequent ClientHello message
+                if ((hsf = valHello(hsf, hsf.messageSeq == 0)) == null) {
+                    return;
+                }
+            }
+
             // Clean up the retransmission messages if necessary.
             cleanUpRetransmit(hsf);
 
@@ -783,6 +816,100 @@ void queueUpHandshake(HandshakeFragment hsf) throws SSLProtocolException {
             }
         }
 
+        private HandshakeFragment valHello(HandshakeFragment hsf,
+                boolean firstHello) {
+            ServerHandshakeContext shc =
+                    (ServerHandshakeContext) tc.handshakeContext;
+            // Drop any fragment that is not a zero offset until we've received
+            // a second (or possibly later) CH message that passes the cookie
+            // check.
+            if (shc == null || !shc.acceptCliHelloFragments) {
+                if (hsf.fragmentOffset != 0) {
+                    return null;
+                }
+            } else {
+                // Let this fragment through to the DTLSReassembler as-is
+                return hsf;
+            }
+
+            try {
+                ByteBuffer fragmentData = ByteBuffer.wrap(hsf.fragment);
+
+                ProtocolVersion pv = ProtocolVersion.valueOf(
+                        Record.getInt16(fragmentData));
+                if (!pv.isDTLS) {
+                    return null;
+                }
+                // Read the random (32 bytes)
+                if (fragmentData.remaining() < 32) {
+                    if (SSLLogger.isOn && SSLLogger.isOn("verbose")) {
+                        SSLLogger.fine("Rejected client hello fragment (bad random len) " +
+                                "fo=" + hsf.fragmentOffset + " fl=" + hsf.fragmentLength);
+                    }
+                    return null;
+                }
+                fragmentData.position(fragmentData.position() + 32);
+
+                // SessionID
+                byte[] sessId = Record.getBytes8(fragmentData);
+                if (sessId.length > 0  &&
+                        !SSLConfiguration.enableDtlsResumeCookie) {
+                    // If we are in a resumption it is possible that the cookie
+                    // exchange will be skipped.  This is a server-side setting
+                    // and it is NOT the default.  If enableDtlsResumeCookie is
+                    // false though, then we will buffer fragments since there
+                    // is no cookie exchange to execute prior to performing
+                    // reassembly.
+                    return hsf;
+                }
+
+                // Cookie
+                byte[] cookie = Record.getBytes8(fragmentData);
+                if (firstHello && cookie.length != 0) {
+                    if (SSLLogger.isOn && SSLLogger.isOn("verbose")) {
+                        SSLLogger.fine("Rejected initial client hello fragment (bad cookie len) " +
+                                "fo=" + hsf.fragmentOffset + " fl=" + hsf.fragmentLength);
+                    }
+                    return null;
+                }
+                // CipherSuites
+                Record.getBytes16(fragmentData);
+                // Compression methods
+                Record.getBytes8(fragmentData);
+
+                // If it's the first fragment, we'll truncate it and push it
+                // through the reassembler.
+                if (firstHello) {
+                    return truncateChFragment(hsf, fragmentData.position());
+                } else {
+                    HelloCookieManager hcMgr = sslContext.
+                            getHelloCookieManager(ProtocolVersion.DTLS10);
+                    ByteBuffer msgFragBuf = ByteBuffer.wrap(hsf.fragment, 0,
+                            fragmentData.position());
+                    ClientHello.ClientHelloMessage chMsg =
+                            new ClientHello.ClientHelloMessage(shc, msgFragBuf, null);
+                    if (!hcMgr.isCookieValid(shc, chMsg, cookie)) {
+                        // Bad cookie check, truncate it and let the ClientHello
+                        // consumer recheck, fail and take the appropriate action.
+                        return truncateChFragment(hsf, fragmentData.position());
+                    } else {
+                        // It's a good cookie, return the original handshake
+                        // fragment and let it go into the DTLSReassembler like
+                        // any other fragment so we can wait for the rest of
+                        // the CH message.
+                        shc.acceptCliHelloFragments = true;
+                        return hsf;
+                    }
+                }
+            } catch (IOException ioe) {
+                if (SSLLogger.isOn && SSLLogger.isOn("verbose")) {
+                    SSLLogger.fine("Rejected client hello fragment " +
+                            "fo=" + hsf.fragmentOffset + " fl=" + hsf.fragmentLength);
+                }
+                return null;
+            }
+        }
+
         // Queue up a ChangeCipherSpec message
         void queueUpChangeCipherSpec(RecordFragment rf)
                 throws SSLProtocolException {
diff --git a/src/java.base/share/classes/sun/security/ssl/ServerHandshakeContext.java b/src/java.base/share/classes/sun/security/ssl/ServerHandshakeContext.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,6 +55,7 @@ class ServerHandshakeContext extends HandshakeContext {
     CertificateMessage.CertificateEntry currentCertEntry;
     private static final long DEFAULT_STATUS_RESP_DELAY = 5000L;
     final long statusRespTimeout;
+    boolean acceptCliHelloFragments = false;
 
 
     ServerHandshakeContext(SSLContextImpl sslContext,
diff --git a/src/java.base/share/classes/sun/security/ssl/TransportContext.java b/src/java.base/share/classes/sun/security/ssl/TransportContext.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -156,6 +156,11 @@ private TransportContext(SSLContextImpl sslContext, SSLTransport transport,
 
         this.acc = AccessController.getContext();
         this.consumers = new HashMap<>();
+
+        if (inputRecord instanceof DTLSInputRecord dtlsInputRecord) {
+            dtlsInputRecord.setTransportContext(this);
+            dtlsInputRecord.setSSLContext(this.sslContext);
+        }
     }
 
     // Dispatch plaintext to a specific consumer.
diff --git a/test/jdk/javax/net/ssl/DTLS/InvalidRecords.java b/test/jdk/javax/net/ssl/DTLS/InvalidRecords.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
 
 /*
  * @test
- * @bug 8043758
+ * @bug 8043758 8307383
  * @summary Datagram Transport Layer Security (DTLS)
  * @modules java.base/sun.security.util
  * @library /test/lib
@@ -36,6 +36,7 @@
 
 import java.net.DatagramPacket;
 import java.net.SocketAddress;
+import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -73,11 +74,34 @@ DatagramPacket createHandshakePacket(byte[] ba, SocketAddress socketAddr) {
             // ClientHello with cookie
             needInvalidRecords.set(false);
             System.out.println("invalidate ClientHello message");
-            if (ba[ba.length - 1] == (byte)0xFF) {
-                ba[ba.length - 1] = (byte)0xFE;
+            // We will alter the compression method field in order to make the cookie
+            // check fail.
+            ByteBuffer chRec = ByteBuffer.wrap(ba);
+            // Skip 59 bytes past the record header (13), the handshake header (12),
+            // the protocol version (2), and client random (32)
+            chRec.position(59);
+            // Jump past the session ID
+            int len = Byte.toUnsignedInt(chRec.get());
+            chRec.position(chRec.position() + len);
+            // Skip the cookie
+            len = Byte.toUnsignedInt(chRec.get());
+            chRec.position(chRec.position() + len);
+            // Skip past cipher suites
+            len = Short.toUnsignedInt(chRec.getShort());
+            chRec.position(chRec.position() + len);
+            // Read the data on the compression methods, should be at least 1
+            len = Byte.toUnsignedInt(chRec.get());
+            if (len >= 1) {
+                System.out.println("Detected compression methods (count = " + len + ")");
             } else {
                 ba[ba.length - 1] = (byte)0xFF;
+                throw new RuntimeException("Got zero length comp methods");
             }
+            // alter the first comp method.
+            int compMethodVal = Byte.toUnsignedInt(chRec.get(chRec.position()));
+            System.out.println("Changing value at position " + chRec.position() +
+                    " from " + compMethodVal + " to " + ++compMethodVal);
+            chRec.put(chRec.position(), (byte)compMethodVal);
         }
 
         return super.createHandshakePacket(ba, socketAddr);
diff --git a/test/jdk/javax/net/ssl/TLSCommon/MFLNTest.java b/test/jdk/javax/net/ssl/TLSCommon/MFLNTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,7 +34,15 @@ public class MFLNTest extends SSLEngineTestCase {
     public static void main(String[] args) {
         setUpAndStartKDCIfNeeded();
         System.setProperty("jsse.enableMFLNExtension", "true");
-        for (int mfl = 4096; mfl >= 256; mfl /= 2) {
+        String testMode = System.getProperty("test.mode", "norm");
+        int mflLen;
+        if (testMode.equals("norm_sni")) {
+            mflLen = 512;
+        } else {
+            mflLen = 256;
+        }
+
+        for (int mfl = 4096; mfl >= mflLen; mfl /= 2) {
             System.out.println("=============================================="
                     + "==============");
             System.out.printf("Testsing DTLS handshake with MFL = %d%n", mfl);
