Add key logging for TLS 1.3 and TLS 1.2 (#523)

SSLKEYLOGFILE implementation and tests

Author: nickrabbott

tests/tlstest.py

@@ -2853,12 +2853,17 @@ def connect():
         testConnServer(connection)
         connection.close()
     finally:
-        try:
-            os.remove(db_name)
-        except FileNotFoundError:
-            # dbm module may create files with different names depending on
-            # platform
-            os.remove(db_name + ".dat")
+        def quiet_remove(db_name):
+            try:
+                os.remove(db_name)
+            except OSError:
+                pass
+
+        # dbm module may create files with different names depending on
+        # platform
+        candidates = [db_name, db_name + ".dat", db_name + ".db"]
+        for candidate in candidates:
+            quiet_remove(candidate)
 
     test_no += 1
 

tlslite/session.py

@@ -112,7 +112,9 @@ def create(self, masterSecret, sessionID, cipherSuite,
                sr_app_secret=bytearray(0), exporterMasterSecret=bytearray(0),
                resumptionMasterSecret=bytearray(0), tickets=None,
                tls_1_0_tickets=None, ec_point_format=None,
-               delegated_credential=None):
+               delegated_credential=None,
+               cl_hs_traffic_secret=bytearray(0),
+               sr_hs_traffic_secret=bytearray(0)):
         self.masterSecret = masterSecret
         self.sessionID = sessionID
         self.cipherSuite = cipherSuite
@@ -130,6 +132,8 @@ def create(self, masterSecret, sessionID, cipherSuite,
         self.sr_app_secret = sr_app_secret
         self.exporterMasterSecret = exporterMasterSecret
         self.resumptionMasterSecret = resumptionMasterSecret
+        self.cl_handshake_traffic_secret = cl_hs_traffic_secret
+        self.sr_handshake_traffic_secret = sr_hs_traffic_secret
         # NOTE we need a reference copy not a copy of object here!
         self.tickets = tickets
         self.tls_1_0_tickets = tls_1_0_tickets

tlslite/sslkeylogging.py

@@ -0,0 +1,76 @@
+import os
+import sys
+import threading
+from .utils.compat import b2a_hex
+
+
+def posix_lock_write(file_path, lines):
+    import fcntl
+    with open(file_path, 'a') as f:
+        fcntl.flock(f, fcntl.LOCK_EX)
+        try:
+            f.writelines(lines)
+            f.flush()
+        finally:
+            fcntl.flock(f, fcntl.LOCK_UN)
+
+
+def unsafe_write(file_path, lines):
+    with open(file_path, 'a') as f:
+        f.writelines(lines)
+        f.flush()
+
+
+class SSLKeyLogger:
+    """
+    Write session secrets to the file pointed to by the SSLKEYLOGFILE
+    environment variable. Implemented to be thread-safe at the class level and
+    uses OS level file-locking. The file-locking implementation is a function
+    assigned to self.lock_write and determined by the value of sys.platform.
+
+    Currently, POSIX is supported for thread and process safety. If enabled for
+    other systems, safety is not guaranteed.
+
+    :param logfile_override: specify the filepath for logging
+    :type logfile_override: str
+    """
+    _lock = threading.Lock()
+
+    def __init__(self, logfile_override=None):
+        self.ssl_key_logfile = os.environ.get('SSLKEYLOGFILE')
+        if logfile_override:
+            self.ssl_key_logfile = logfile_override
+        self.platform = sys.platform
+        if self.platform in ['darwin', 'linux']:
+            self.lock_write = posix_lock_write
+        else:
+            self.lock_write = unsafe_write
+
+    def log_session_keys(self, keys):
+        """
+        Log session keys to the SSL key log file, if configured.
+
+        Write session keys in the `SSLKEYLOGFILE` format, allowing tools like
+        Wireshark to decrypt captured traffic. Each entry is written as a line
+        with the format: ``<LABEL> <CLIENT_RANDOM> <SECRET>``.
+
+        If neither the `SSLKEYLOGFILE` environment variable nor a
+        `logfile_override` is set, this method is a no-op.
+
+        :param keys: List of (label, client_random, secret) tuples.
+        :type keys: list[tuple[str, bytes, bytes]]
+        """
+        if not self.ssl_key_logfile:
+            return
+
+        lines = [
+            "{0} {1} {2}\n".format(
+                label,
+                b2a_hex(client_random).upper(),
+                b2a_hex(secret).upper()
+            )
+            for label, client_random, secret in keys
+        ]
+
+        with self._lock:
+            self.lock_write(self.ssl_key_logfile, lines)

tlslite/tlsconnection.py

@@ -43,6 +43,7 @@
 from .utils.cipherfactory import createAESCCM, createAESCCM_8, \
         createAESGCM, createCHACHA20
 from .utils.compression import choose_compression_send_algo
+from .sslkeylogging import SSLKeyLogger
 
 
 class TLSConnection(TLSRecordLayer):
@@ -80,14 +81,20 @@ class TLSConnection(TLSRecordLayer):
         compression wasn't used then it is set to None.
     """
 
-    def __init__(self, sock):
+    def __init__(self, sock, ssl_key_log_file=None):
         """Create a new TLSConnection instance.
 
         :param sock: The socket data will be transmitted on.  The
             socket should already be connected.  It may be in blocking or
             non-blocking mode.
 
         :type sock: socket.socket
+
+        :param ssl_key_log_file: override location for logging session secrets.
+            If not provided the filepath pointed to by the SSLKEYLOGFILE env
+            variable will be used.
+
+        :type ssl_key_log_file: str
         """
         TLSRecordLayer.__init__(self, sock)
         self.serverSigAlg = None
@@ -105,6 +112,7 @@ def __init__(self, sock):
         self._pha_supported = False
         self.client_cert_compression_algo = None
         self.server_cert_compression_algo = None
+        self.ssl_key_logger = SSLKeyLogger(ssl_key_log_file)
 
     def keyingMaterialExporter(self, label, length=20):
         """Return keying material as described in RFC 5705
@@ -418,7 +426,6 @@ def _handshakeClientAsync(self, srpParams=(), certParams=(), anonParams=(),
                               session=None, settings=None, checker=None,
                               nextProtos=None, serverName=None, reqTack=True,
                               alpn=None):
-
         handshaker = self._handshakeClientAsyncHelper(srpParams=srpParams,
                 certParams=certParams,
                 anonParams=anonParams,
@@ -431,6 +438,16 @@ def _handshakeClientAsync(self, srpParams=(), certParams=(), anonParams=(),
         for result in self._handshakeWrapperAsync(handshaker, checker):
             yield result
 
+        # Log client random and master secret for version < TLS1.3
+        # in the case of SRP fault, the session instance will be None
+        if self.session is not None:
+            if self.version < (3, 4):
+                self.ssl_key_logger.log_session_keys([(
+                    'CLIENT_RANDOM',
+                    self._clientRandom,
+                    self.session.masterSecret
+                )])
+
 
     def _handshakeClientAsyncHelper(self, srpParams, certParams, anonParams,
                                session, settings, serverName, nextProtos,
@@ -1332,6 +1349,14 @@ def _clientTLS13Handshake(self, settings, session, clientHello,
                                                     self._handshake_hash,
                                                     prfName)
 
+        # TLS1.3 log Client and Server traffic secrets for SSLKEYLOGFILE
+        self.ssl_key_logger.log_session_keys([
+            ('CLIENT_HANDSHAKE_TRAFFIC_SECRET',
+             clientHello.random, cl_handshake_traffic_secret),
+            ('SERVER_HANDSHAKE_TRAFFIC_SECRET',
+             clientHello.random, sr_handshake_traffic_secret)
+        ])
+
         # prepare for reading encrypted messages
         self._recordLayer.calcTLS1_3PendingState(
             serverHello.cipher_suite,
@@ -1656,6 +1681,15 @@ def _clientTLS13Handshake(self, settings, session, clientHello,
                                                bytearray(b'exp master'),
                                                self._handshake_hash, prfName)
 
+
+        # Now that we have all the TLS1.3 secrets during the handshake,
+        # log them if necessary
+        self.ssl_key_logger.log_session_keys([
+            ('EXPORTER_SECRET', clientHello.random, exporter_master_secret),
+            ('CLIENT_TRAFFIC_SECRET_0', clientHello.random, cl_app_traffic),
+            ('SERVER_TRAFFIC_SECRET_0', clientHello.random, sr_app_traffic)
+        ])
+
         self._recordLayer.calcTLS1_3PendingState(
             serverHello.cipher_suite,
             cl_app_traffic,
@@ -1752,7 +1786,9 @@ def _clientTLS13Handshake(self, settings, session, clientHello,
                             resumptionMasterSecret=resumption_master_secret,
                             # NOTE it must be a reference, not a copy!
                             tickets=self.tickets,
-                            delegated_credential=delegated_credential)
+                            delegated_credential=delegated_credential,
+                            cl_hs_traffic_secret=cl_handshake_traffic_secret,
+                            sr_hs_traffic_secret=sr_handshake_traffic_secret)
 
         yield "finished" if not resuming else "resumed_and_finished"
 
@@ -2305,6 +2341,16 @@ def handshakeServerAsync(self, verifierDB=None,
         for result in self._handshakeWrapperAsync(handshaker, checker):
             yield result
 
+        # Log client random and master secret for version < TLS1.3
+        # in the case of SRP fault, the session instance will be None
+        if self.session is not None:
+            if self.version < (3, 4):
+                self.ssl_key_logger.log_session_keys([(
+                    'CLIENT_RANDOM',
+                    self._clientRandom,
+                    self.session.masterSecret
+                )])
+
 
     def _handshakeServerAsyncHelper(self, verifierDB,
                                     cert_chain, privateKey, reqCert,
@@ -3032,6 +3078,15 @@ def _serverTLS13Handshake(self, settings, clientHello, cipherSuite,
                                                     bytearray(b'c hs traffic'),
                                                     self._handshake_hash,
                                                     prf_name)
+
+        # TLS1.3 log Client and Server traffic secrets for SSLKEYLOGFILE
+        self.ssl_key_logger.log_session_keys([
+            ('CLIENT_HANDSHAKE_TRAFFIC_SECRET',
+             clientHello.random, cl_handshake_traffic_secret),
+            ('SERVER_HANDSHAKE_TRAFFIC_SECRET',
+             clientHello.random, sr_handshake_traffic_secret)
+        ])
+
         self.version = version
         self._recordLayer.calcTLS1_3PendingState(
             cipherSuite,
@@ -3310,6 +3365,19 @@ def _serverTLS13Handshake(self, settings, clientHello, cipherSuite,
                                                self._handshake_hash,
                                                prf_name)
 
+
+        # Now that we have all the TLS1.3 secrets during the handshake,
+        # log them if necessary
+        self.ssl_key_logger.log_session_keys([
+            ('EXPORTER_SECRET',
+             clientHello.random, exporter_master_secret),
+            ('CLIENT_TRAFFIC_SECRET_0',
+             clientHello.random, cl_app_traffic),
+            ('SERVER_TRAFFIC_SECRET_0',
+             clientHello.random, sr_app_traffic)
+        ])
+
+
         # verify Finished of client
         cl_finished_key = HKDF_expand_label(cl_handshake_traffic_secret,
                                             b"finished", b'',