DTLS: Keep record layer in handshake state

peterdettman · peterdettman · commit 12d7e771d1b3 · 2025-07-25T19:45:39.000+07:00
diff --git a/tls/src/main/java/org/bouncycastle/tls/DTLSClientProtocol.java b/tls/src/main/java/org/bouncycastle/tls/DTLSClientProtocol.java
@@ -34,10 +34,6 @@ public DTLSTransport connect(TlsClient client, DatagramTransport transport)
 
         TlsClientContextImpl clientContext = new TlsClientContextImpl(client.getCrypto());
 
-        ClientHandshakeState state = new ClientHandshakeState();
-        state.client = client;
-        state.clientContext = clientContext;
-
         client.init(clientContext);
         clientContext.handshakeBeginning(client);
 
@@ -47,9 +43,14 @@ public DTLSTransport connect(TlsClient client, DatagramTransport transport)
         DTLSRecordLayer recordLayer = new DTLSRecordLayer(clientContext, client, transport);
         client.notifyCloseHandle(recordLayer);
 
+        ClientHandshakeState state = new ClientHandshakeState();
+        state.client = client;
+        state.clientContext = clientContext;
+        state.recordLayer = recordLayer;
+
         try
         {
-            return clientHandshake(state, recordLayer);
+            return clientHandshake(state);
         }
         catch (TlsFatalAlertReceived fatalAlertReceived)
         {
@@ -59,17 +60,17 @@ public DTLSTransport connect(TlsClient client, DatagramTransport transport)
         }
         catch (TlsFatalAlert fatalAlert)
         {
-            abortClientHandshake(state, recordLayer, fatalAlert.getAlertDescription());
+            abortClientHandshake(state, fatalAlert.getAlertDescription());
             throw fatalAlert;
         }
         catch (IOException e)
         {
-            abortClientHandshake(state, recordLayer, AlertDescription.internal_error);
+            abortClientHandshake(state, AlertDescription.internal_error);
             throw e;
         }
         catch (RuntimeException e)
         {
-            abortClientHandshake(state, recordLayer, AlertDescription.internal_error);
+            abortClientHandshake(state, AlertDescription.internal_error);
             throw new TlsFatalAlert(AlertDescription.internal_error, e);
         }
         finally
@@ -78,17 +79,18 @@ public DTLSTransport connect(TlsClient client, DatagramTransport transport)
         }
     }
 
-    protected void abortClientHandshake(ClientHandshakeState state, DTLSRecordLayer recordLayer, short alertDescription)
+    protected void abortClientHandshake(ClientHandshakeState state, short alertDescription)
     {
-        recordLayer.fail(alertDescription);
+        state.recordLayer.fail(alertDescription);
         invalidateSession(state);
     }
 
-    protected DTLSTransport clientHandshake(ClientHandshakeState state, DTLSRecordLayer recordLayer)
+    protected DTLSTransport clientHandshake(ClientHandshakeState state)
         throws IOException
     {
         TlsClient client = state.client;
         TlsClientContextImpl clientContext = state.clientContext;
+        DTLSRecordLayer recordLayer = state.recordLayer;
         SecurityParameters securityParameters = clientContext.getSecurityParametersHandshake();
 
         DTLSReliableHandshake handshake = new DTLSReliableHandshake(clientContext, recordLayer,
@@ -1142,6 +1144,7 @@ protected static class ClientHandshakeState
     {
         TlsClient client = null;
         TlsClientContextImpl clientContext = null;
+        DTLSRecordLayer recordLayer = null;
         TlsSession tlsSession = null;
         SessionParameters sessionParameters = null;
         TlsSecret sessionMasterSecret = null;
diff --git a/tls/src/main/java/org/bouncycastle/tls/DTLSServerProtocol.java b/tls/src/main/java/org/bouncycastle/tls/DTLSServerProtocol.java
@@ -50,10 +50,6 @@ public DTLSTransport accept(TlsServer server, DatagramTransport transport, DTLSR
 
         TlsServerContextImpl serverContext = new TlsServerContextImpl(server.getCrypto());
 
-        ServerHandshakeState state = new ServerHandshakeState();
-        state.server = server;
-        state.serverContext = serverContext;
-
         server.init(serverContext);
         serverContext.handshakeBeginning(server);
 
@@ -63,9 +59,14 @@ public DTLSTransport accept(TlsServer server, DatagramTransport transport, DTLSR
         DTLSRecordLayer recordLayer = new DTLSRecordLayer(serverContext, server, transport);
         server.notifyCloseHandle(recordLayer);
 
+        ServerHandshakeState state = new ServerHandshakeState();
+        state.server = server;
+        state.serverContext = serverContext;
+        state.recordLayer = recordLayer;
+
         try
         {
-            return serverHandshake(state, recordLayer, request);
+            return serverHandshake(state, request);
         }
         catch (TlsFatalAlertReceived fatalAlertReceived)
         {
@@ -75,17 +76,17 @@ public DTLSTransport accept(TlsServer server, DatagramTransport transport, DTLSR
         }
         catch (TlsFatalAlert fatalAlert)
         {
-            abortServerHandshake(state, recordLayer, fatalAlert.getAlertDescription());
+            abortServerHandshake(state, fatalAlert.getAlertDescription());
             throw fatalAlert;
         }
         catch (IOException e)
         {
-            abortServerHandshake(state, recordLayer, AlertDescription.internal_error);
+            abortServerHandshake(state, AlertDescription.internal_error);
             throw e;
         }
         catch (RuntimeException e)
         {
-            abortServerHandshake(state, recordLayer, AlertDescription.internal_error);
+            abortServerHandshake(state, AlertDescription.internal_error);
             throw new TlsFatalAlert(AlertDescription.internal_error, e);
         }
         finally
@@ -94,17 +95,17 @@ public DTLSTransport accept(TlsServer server, DatagramTransport transport, DTLSR
         }
     }
 
-    protected void abortServerHandshake(ServerHandshakeState state, DTLSRecordLayer recordLayer, short alertDescription)
+    protected void abortServerHandshake(ServerHandshakeState state, short alertDescription)
     {
-        recordLayer.fail(alertDescription);
+        state.recordLayer.fail(alertDescription);
         invalidateSession(state);
     }
 
-    protected DTLSTransport serverHandshake(ServerHandshakeState state, DTLSRecordLayer recordLayer,
-        DTLSRequest request) throws IOException
+    protected DTLSTransport serverHandshake(ServerHandshakeState state, DTLSRequest request) throws IOException
     {
         TlsServer server = state.server;
         TlsServerContextImpl serverContext = state.serverContext;
+        DTLSRecordLayer recordLayer = state.recordLayer;
         SecurityParameters securityParameters = serverContext.getSecurityParametersHandshake();
 
         DTLSReliableHandshake handshake = new DTLSReliableHandshake(serverContext, recordLayer,
@@ -138,7 +139,7 @@ protected DTLSTransport serverHandshake(ServerHandshakeState state, DTLSRecordLa
         }
 
         {
-            byte[] serverHelloBody = generateServerHello(state, recordLayer);
+            byte[] serverHelloBody = generateServerHello(state);
 
             // TODO[dtls13] Ideally, move this into generateServerHello once legacy_record_version clarified
             {
@@ -452,7 +453,7 @@ protected byte[] generateNewSessionTicket(ServerHandshakeState state, NewSession
         return buf.toByteArray();
     }
 
-    protected byte[] generateServerHello(ServerHandshakeState state, DTLSRecordLayer recordLayer)
+    protected byte[] generateServerHello(ServerHandshakeState state)
         throws IOException
     {
         TlsServer server = state.server;
@@ -710,7 +711,7 @@ else if (TlsUtils.hasExpectedEmptyExtensionData(state.serverExtensions,
 
         state.clientHello = null;
 
-        applyMaxFragmentLengthExtension(recordLayer, securityParameters.getMaxFragmentLength());
+        applyMaxFragmentLengthExtension(state.recordLayer, securityParameters.getMaxFragmentLength());
 
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
         serverHello.encode(serverContext, buf);
@@ -1020,6 +1021,7 @@ protected static class ServerHandshakeState
     {
         TlsServer server = null;
         TlsServerContextImpl serverContext = null;
+        DTLSRecordLayer recordLayer = null;
         TlsSession tlsSession = null;
         SessionParameters sessionParameters = null;
         TlsSecret sessionMasterSecret = null;

Original file line number	Diff line number	Diff line change
`@@ -34,10 +34,6 @@ public DTLSTransport connect(TlsClient client, DatagramTransport transport)`
`34`	`34`
`35`	`35`	`TlsClientContextImpl clientContext = new TlsClientContextImpl(client.getCrypto());`
`36`	`36`
`37`		`- ClientHandshakeState state = new ClientHandshakeState();`
`38`		`- state.client = client;`
`39`		`- state.clientContext = clientContext;`
`40`		`-`
`41`	`37`	`client.init(clientContext);`
`42`	`38`	`clientContext.handshakeBeginning(client);`
`43`	`39`
`@@ -47,9 +43,14 @@ public DTLSTransport connect(TlsClient client, DatagramTransport transport)`
`47`	`43`	`DTLSRecordLayer recordLayer = new DTLSRecordLayer(clientContext, client, transport);`
`48`	`44`	`client.notifyCloseHandle(recordLayer);`
`49`	`45`
	`46`	`+ ClientHandshakeState state = new ClientHandshakeState();`
	`47`	`+ state.client = client;`
	`48`	`+ state.clientContext = clientContext;`
	`49`	`+ state.recordLayer = recordLayer;`
	`50`	`+`
`50`	`51`	`try`
`51`	`52`	`{`
`52`		`- return clientHandshake(state, recordLayer);`
	`53`	`+ return clientHandshake(state);`
`53`	`54`	`}`
`54`	`55`	`catch (TlsFatalAlertReceived fatalAlertReceived)`
`55`	`56`	`{`
`@@ -59,17 +60,17 @@ public DTLSTransport connect(TlsClient client, DatagramTransport transport)`
`59`	`60`	`}`
`60`	`61`	`catch (TlsFatalAlert fatalAlert)`
`61`	`62`	`{`
`62`		`- abortClientHandshake(state, recordLayer, fatalAlert.getAlertDescription());`
	`63`	`+ abortClientHandshake(state, fatalAlert.getAlertDescription());`
`63`	`64`	`throw fatalAlert;`
`64`	`65`	`}`
`65`	`66`	`catch (IOException e)`
`66`	`67`	`{`
`67`		`- abortClientHandshake(state, recordLayer, AlertDescription.internal_error);`
	`68`	`+ abortClientHandshake(state, AlertDescription.internal_error);`
`68`	`69`	`throw e;`
`69`	`70`	`}`
`70`	`71`	`catch (RuntimeException e)`
`71`	`72`	`{`
`72`		`- abortClientHandshake(state, recordLayer, AlertDescription.internal_error);`
	`73`	`+ abortClientHandshake(state, AlertDescription.internal_error);`
`73`	`74`	`throw new TlsFatalAlert(AlertDescription.internal_error, e);`
`74`	`75`	`}`
`75`	`76`	`finally`
`@@ -78,17 +79,18 @@ public DTLSTransport connect(TlsClient client, DatagramTransport transport)`
`78`	`79`	`}`
`79`	`80`	`}`
`80`	`81`
`81`		`- protected void abortClientHandshake(ClientHandshakeState state, DTLSRecordLayer recordLayer, short alertDescription)`
	`82`	`+ protected void abortClientHandshake(ClientHandshakeState state, short alertDescription)`
`82`	`83`	`{`
`83`		`- recordLayer.fail(alertDescription);`
	`84`	`+ state.recordLayer.fail(alertDescription);`
`84`	`85`	`invalidateSession(state);`
`85`	`86`	`}`
`86`	`87`
`87`		`- protected DTLSTransport clientHandshake(ClientHandshakeState state, DTLSRecordLayer recordLayer)`
	`88`	`+ protected DTLSTransport clientHandshake(ClientHandshakeState state)`
`88`	`89`	`throws IOException`
`89`	`90`	`{`
`90`	`91`	`TlsClient client = state.client;`
`91`	`92`	`TlsClientContextImpl clientContext = state.clientContext;`
	`93`	`+ DTLSRecordLayer recordLayer = state.recordLayer;`
`92`	`94`	`SecurityParameters securityParameters = clientContext.getSecurityParametersHandshake();`
`93`	`95`
`94`	`96`	`DTLSReliableHandshake handshake = new DTLSReliableHandshake(clientContext, recordLayer,`
`@@ -1142,6 +1144,7 @@ protected static class ClientHandshakeState`
`1142`	`1144`	`{`
`1143`	`1145`	`TlsClient client = null;`
`1144`	`1146`	`TlsClientContextImpl clientContext = null;`
	`1147`	`+ DTLSRecordLayer recordLayer = null;`
`1145`	`1148`	`TlsSession tlsSession = null;`
`1146`	`1149`	`SessionParameters sessionParameters = null;`
`1147`	`1150`	`TlsSecret sessionMasterSecret = null;`
Original file line number	Diff line number	Diff line change
`@@ -50,10 +50,6 @@ public DTLSTransport accept(TlsServer server, DatagramTransport transport, DTLSR`
`50`	`50`
`51`	`51`	`TlsServerContextImpl serverContext = new TlsServerContextImpl(server.getCrypto());`
`52`	`52`
`53`		`- ServerHandshakeState state = new ServerHandshakeState();`
`54`		`- state.server = server;`
`55`		`- state.serverContext = serverContext;`
`56`		`-`
`57`	`53`	`server.init(serverContext);`
`58`	`54`	`serverContext.handshakeBeginning(server);`
`59`	`55`
`@@ -63,9 +59,14 @@ public DTLSTransport accept(TlsServer server, DatagramTransport transport, DTLSR`
`63`	`59`	`DTLSRecordLayer recordLayer = new DTLSRecordLayer(serverContext, server, transport);`
`64`	`60`	`server.notifyCloseHandle(recordLayer);`
`65`	`61`
	`62`	`+ ServerHandshakeState state = new ServerHandshakeState();`
	`63`	`+ state.server = server;`
	`64`	`+ state.serverContext = serverContext;`
	`65`	`+ state.recordLayer = recordLayer;`
	`66`	`+`
`66`	`67`	`try`
`67`	`68`	`{`
`68`		`- return serverHandshake(state, recordLayer, request);`
	`69`	`+ return serverHandshake(state, request);`
`69`	`70`	`}`
`70`	`71`	`catch (TlsFatalAlertReceived fatalAlertReceived)`
`71`	`72`	`{`
`@@ -75,17 +76,17 @@ public DTLSTransport accept(TlsServer server, DatagramTransport transport, DTLSR`
`75`	`76`	`}`
`76`	`77`	`catch (TlsFatalAlert fatalAlert)`
`77`	`78`	`{`
`78`		`- abortServerHandshake(state, recordLayer, fatalAlert.getAlertDescription());`
	`79`	`+ abortServerHandshake(state, fatalAlert.getAlertDescription());`
`79`	`80`	`throw fatalAlert;`
`80`	`81`	`}`
`81`	`82`	`catch (IOException e)`
`82`	`83`	`{`
`83`		`- abortServerHandshake(state, recordLayer, AlertDescription.internal_error);`
	`84`	`+ abortServerHandshake(state, AlertDescription.internal_error);`
`84`	`85`	`throw e;`
`85`	`86`	`}`
`86`	`87`	`catch (RuntimeException e)`
`87`	`88`	`{`
`88`		`- abortServerHandshake(state, recordLayer, AlertDescription.internal_error);`
	`89`	`+ abortServerHandshake(state, AlertDescription.internal_error);`
`89`	`90`	`throw new TlsFatalAlert(AlertDescription.internal_error, e);`
`90`	`91`	`}`
`91`	`92`	`finally`
`@@ -94,17 +95,17 @@ public DTLSTransport accept(TlsServer server, DatagramTransport transport, DTLSR`
`94`	`95`	`}`
`95`	`96`	`}`
`96`	`97`
`97`		`- protected void abortServerHandshake(ServerHandshakeState state, DTLSRecordLayer recordLayer, short alertDescription)`
	`98`	`+ protected void abortServerHandshake(ServerHandshakeState state, short alertDescription)`
`98`	`99`	`{`
`99`		`- recordLayer.fail(alertDescription);`
	`100`	`+ state.recordLayer.fail(alertDescription);`
`100`	`101`	`invalidateSession(state);`
`101`	`102`	`}`
`102`	`103`
`103`		`- protected DTLSTransport serverHandshake(ServerHandshakeState state, DTLSRecordLayer recordLayer,`
`104`		`- DTLSRequest request) throws IOException`
	`104`	`+ protected DTLSTransport serverHandshake(ServerHandshakeState state, DTLSRequest request) throws IOException`
`105`	`105`	`{`
`106`	`106`	`TlsServer server = state.server;`
`107`	`107`	`TlsServerContextImpl serverContext = state.serverContext;`
	`108`	`+ DTLSRecordLayer recordLayer = state.recordLayer;`
`108`	`109`	`SecurityParameters securityParameters = serverContext.getSecurityParametersHandshake();`
`109`	`110`
`110`	`111`	`DTLSReliableHandshake handshake = new DTLSReliableHandshake(serverContext, recordLayer,`
`@@ -138,7 +139,7 @@ protected DTLSTransport serverHandshake(ServerHandshakeState state, DTLSRecordLa`
`138`	`139`	`}`
`139`	`140`
`140`	`141`	`{`
`141`		`- byte[] serverHelloBody = generateServerHello(state, recordLayer);`
	`142`	`+ byte[] serverHelloBody = generateServerHello(state);`
`142`	`143`
`143`	`144`	`// TODO[dtls13] Ideally, move this into generateServerHello once legacy_record_version clarified`
`144`	`145`	`{`
`@@ -452,7 +453,7 @@ protected byte[] generateNewSessionTicket(ServerHandshakeState state, NewSession`
`452`	`453`	`return buf.toByteArray();`
`453`	`454`	`}`
`454`	`455`
`455`		`- protected byte[] generateServerHello(ServerHandshakeState state, DTLSRecordLayer recordLayer)`
	`456`	`+ protected byte[] generateServerHello(ServerHandshakeState state)`
`456`	`457`	`throws IOException`
`457`	`458`	`{`
`458`	`459`	`TlsServer server = state.server;`
`@@ -710,7 +711,7 @@ else if (TlsUtils.hasExpectedEmptyExtensionData(state.serverExtensions,`
`710`	`711`
`711`	`712`	`state.clientHello = null;`
`712`	`713`
`713`		`- applyMaxFragmentLengthExtension(recordLayer, securityParameters.getMaxFragmentLength());`
	`714`	`+ applyMaxFragmentLengthExtension(state.recordLayer, securityParameters.getMaxFragmentLength());`
`714`	`715`
`715`	`716`	`ByteArrayOutputStream buf = new ByteArrayOutputStream();`
`716`	`717`	`serverHello.encode(serverContext, buf);`
`@@ -1020,6 +1021,7 @@ protected static class ServerHandshakeState`
`1020`	`1021`	`{`
`1021`	`1022`	`TlsServer server = null;`
`1022`	`1023`	`TlsServerContextImpl serverContext = null;`
	`1024`	`+ DTLSRecordLayer recordLayer = null;`
`1023`	`1025`	`TlsSession tlsSession = null;`
`1024`	`1026`	`SessionParameters sessionParameters = null;`
`1025`	`1027`	`TlsSecret sessionMasterSecret = null;`