Merge pull request #8 from kjelle/master

Allow for out-of-order TLS Handshake processing and partial outputs

Author: gallypette

d4-tlsf.go

@@ -36,6 +36,7 @@ var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors")
 var verbose = flag.Bool("verbose", false, "Be verbose")
 var debug = flag.Bool("debug", false, "Display debug information")
 var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
+var partial = flag.Bool("partial", true, "Output partial TLS Sessions, e.g. where we don't see all three of client hello, server hello and certificate")
 
 // capture
 var iface = flag.String("i", "eth0", "Interface to read packets from")
@@ -101,6 +102,11 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 		optchecker: reassembly.NewTCPOptionCheck(),
 		tlsSession: d4tls.TLSSession{},
 	}
+
+	// Set network information in the TLSSession
+	cip, sip, cp, sp := getIPPorts(stream)
+	stream.tlsSession.SetNetwork(cip, sip, cp, sp)
+
 	return stream
 }
 
@@ -134,6 +140,7 @@ type tcpStream struct {
 	urls           []string
 	ident          string
 	tlsSession     d4tls.TLSSession
+	queued         bool
 	ignorefsmerr   bool
 	nooptcheck     bool
 	checksum       bool
@@ -181,6 +188,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
 	}
 	data := sg.Fetch(length)
 	if t.isTLS {
+
 		if length > 0 {
 			// We attempt to decode TLS
 			tls := &etls.ETLS{}
@@ -195,13 +203,18 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
 				//Debug("TLS: %s\n", gopacket.LayerDump(tls))
 				//		Debug("TLS: %s\n", gopacket.LayerGoString(tls))
 				if tls.Handshake != nil {
+
+					// If the timestamp has not been set, we set it for the first time.
+					if t.tlsSession.Record.Timestamp.IsZero() {
+						info := sg.CaptureInfo(0)
+						t.tlsSession.SetTimestamp(info.Timestamp)
+					}
+
 					for _, tlsrecord := range tls.Handshake {
 						switch tlsrecord.ETLSHandshakeMsgType {
 						// Client Hello
 						case 1:
-							info := sg.CaptureInfo(0)
-							cip, sip, cp, sp := getIPPorts(t)
-							t.tlsSession.PopulateClientHello(tlsrecord.ETLSHandshakeClientHello, cip, sip, cp, sp, info.Timestamp)
+							t.tlsSession.PopulateClientHello(tlsrecord.ETLSHandshakeClientHello)
 							t.tlsSession.D4Fingerprinting("ja3")
 						// Server Hello
 						case 2:
@@ -210,14 +223,15 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
 						// Server Certificate
 						case 11:
 							t.tlsSession.PopulateCertificate(tlsrecord.ETLSHandshakeCertificate)
-
 							t.tlsSession.D4Fingerprinting("tlsh")
-							// If we get a cert, we consider the handshake as finished and ready to ship to D4
-							queueSession(t.tlsSession)
-						default:
-							break
 						}
 					}
+
+					// If the handshake is considered finished and we have not yet outputted it we ship it to output.
+					if t.tlsSession.HandshakeComplete() && !t.queued {
+						t.queueSession()
+					}
+
 				}
 			}
 		}
@@ -235,6 +249,12 @@ func getIPPorts(t *tcpStream) (string, string, string, string) {
 }
 
 func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {
+	// If the handshakre has not yet been outputted, but there are some information such as
+	// either the client hello or the server hello, we also output a partial.
+	if *partial && !t.queued && t.tlsSession.HandshakeAny() {
+		t.queueSession()
+	}
+
 	// remove connection from the pool
 	return true
 }
@@ -288,7 +308,7 @@ func main() {
 	signal.Notify(signalChan, os.Interrupt)
 
 	// Job chan to hold Completed sessions to write
-	jobQ = make(chan d4tls.TLSSession, 100)
+	jobQ = make(chan d4tls.TLSSession, 4096)
 	cancelC := make(chan string)
 
 	// We start a worker to send the processed TLS connection the outside world
@@ -387,10 +407,13 @@ func main() {
 	w.Wait()
 }
 
-// Tries to enqueue or false
-func queueSession(t d4tls.TLSSession) bool {
+// queueSession tries to enqueue the tlsSession for output
+// returns true if it succeeded or false if it failed to publish the
+// tlsSession for output
+func (t *tcpStream) queueSession() bool {
+	t.queued = true
 	select {
-	case jobQ <- t:
+	case jobQ <- t.tlsSession:
 		return true
 	default:
 		return false
@@ -414,7 +437,6 @@ func processCompletedSession(cancelC <-chan string, jobQ <-chan d4tls.TLSSession
 }
 
 func output(t d4tls.TLSSession) {
-
 	jsonRecord, _ := json.MarshalIndent(t.Record, "", "    ")
 
 	// If an output folder was specified for certificates

d4tls/fingerprinter.go

@@ -49,6 +49,7 @@ func (t *TLSSession) d4fg() string {
 
 func (t *TLSSession) ja3s() bool {
 	var buf []byte
+
 	buf = strconv.AppendInt(buf, int64(t.handShakeRecord.ETLSHandshakeServerHello.Vers), 10)
 	// byte (44) is ","
 	buf = append(buf, byte(44))
@@ -81,6 +82,7 @@ func (t *TLSSession) ja3s() bool {
 
 func (t *TLSSession) ja3() bool {
 	var buf []byte
+
 	buf = strconv.AppendInt(buf, int64(t.handShakeRecord.ETLSHandshakeClientHello.Vers), 10)
 	// byte (44) is ","
 	buf = append(buf, byte(44))

d4tls/fingerprinter_test.go

@@ -203,7 +203,8 @@ func TestJA3(t *testing.T) {
 			for _, etlsrecord := range tls.Handshake {
 				if etlsrecord.ETLSHandshakeMsgType == 1 {
 					// Populate Client Hello related fields
-					tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now())
+					tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello)
+					tlss.SetTimestamp(time.Now())
 					tlss.D4Fingerprinting("ja3")
 					t.Logf("%v", tlss.Record.JA3)
 					t.Logf("%v", tlss.Record.JA3Digest)
@@ -279,7 +280,8 @@ func TestGreaseClientHelloExtensionExlusion(t *testing.T) {
 			for _, etlsrecord := range tls.Handshake {
 				if etlsrecord.ETLSHandshakeMsgType == 1 {
 					// Populate Client Hello related fields
-					tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now())
+					tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello)
+					tlss.SetTimestamp(time.Now())
 					tlss.D4Fingerprinting("ja3")
 					t.Logf("%v", tlss.Record.JA3Digest)
 					if strings.Index(tlss.Record.JA3, "2570") != -1 {
@@ -312,7 +314,8 @@ func TestGreaseClientHelloCipherExlusion(t *testing.T) {
 			for _, etlsrecord := range tls.Handshake {
 				if etlsrecord.ETLSHandshakeMsgType == 1 {
 					// Populate Client Hello related fields
-					tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now())
+					tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello)
+					tlss.SetTimestamp(time.Now())
 					tlss.D4Fingerprinting("ja3")
 					if strings.Index(tlss.Record.JA3, "2570") != -1 {
 						t.Logf("GREASE values should not end up in JA3\n")

d4tls/flags.go

@@ -0,0 +1,19 @@
+package d4tls
+
+// HandshakeState is a flag which keeps record of which handeshake message types
+// have been parsed.
+type HandshakeState uint8
+
+const (
+	StateClientHello = 1 << iota
+	StateServerHello
+	StateCertificate
+)
+
+func (s *HandshakeState) Set(flag HandshakeState) {
+	*s |= flag
+}
+
+func (s HandshakeState) Has(flag HandshakeState) bool {
+	return s&flag != 0
+}

d4tls/tlsdecoder.go

@@ -34,7 +34,35 @@ type sessionRecord struct {
 type TLSSession struct {
 	Record          sessionRecord
 	handShakeRecord etls.ETLSHandshakeRecord
-	stage           int
+	state           HandshakeState
+}
+
+// HandshakeComplete returns true if the TLS session has seen all three client helo, server helo and the certificate.
+func (t *TLSSession) HandshakeComplete() bool {
+	return t.state.Has(StateClientHello) &&
+		t.state.Has(StateServerHello) &&
+		t.state.Has(StateCertificate)
+}
+
+// HandshakePartially returns true if the client hello and server hello is set, but not the certificate.
+func (t *TLSSession) HandshakePartially() bool {
+	return t.state.Has(StateClientHello) &&
+		t.state.Has(StateServerHello) &&
+		!t.state.Has(StateCertificate)
+}
+
+// HandshakeAny returns true if any of the client or server has been seen
+func (t *TLSSession) HandshakeAny() bool {
+	return t.state.Has(StateClientHello) ||
+		t.state.Has(StateServerHello)
+}
+
+func (t *TLSSession) HandshakeState() string {
+	return fmt.Sprintf("ClientHello:%t ServerHello:%t Certificate:%t",
+		t.state.Has(StateClientHello),
+		t.state.Has(StateServerHello),
+		t.state.Has(StateCertificate),
+	)
 }
 
 // String returns a string that describes a TLSSession
@@ -59,40 +87,44 @@ func (t *TLSSession) String() string {
 	return buf.String()
 }
 
+// SetNetwork sets the network part in the TLSSession.Record struct.
+func (t *TLSSession) SetNetwork(cip string, sip string, cp string, sp string) {
+	t.Record.ClientIP = cip
+	t.Record.ServerIP = sip
+	t.Record.ClientPort = cp
+	t.Record.ServerPort = sp
+}
+
+// SetTimestamp sets the timestamp of this TLSSession in its TLSSession.Record struct
+func (t *TLSSession) SetTimestamp(ti time.Time) {
+	t.Record.Timestamp = ti
+}
+
 // PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct
-func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) {
-	if t.stage < 1 {
-		t.Record.ClientIP = cip
-		t.Record.ServerIP = sip
-		t.Record.ClientPort = cp
-		t.Record.ServerPort = sp
-		t.Record.Timestamp = ti
-		t.handShakeRecord.ETLSHandshakeClientHello = h
-		t.stage = 1
-	}
+func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg) {
+	t.state.Set(StateClientHello)
+	t.handShakeRecord.ETLSHandshakeClientHello = h
 }
 
 // PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
 func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) {
-	if t.stage < 2 {
-		t.handShakeRecord.ETLSHandshakeServerHello = h
-		t.stage = 2
-	}
+	t.state.Set(StateServerHello)
+	t.handShakeRecord.ETLSHandshakeServerHello = h
 }
 
 // PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
 func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) {
-	if t.stage < 3 {
-		t.handShakeRecord.ETLSHandshakeCertificate = c
-		for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates {
-			cert, err := x509.ParseCertificate(asn1Data)
-			if err != nil {
-				//return err
-			} else {
-				h := sha256.New()
-				h.Write(cert.Raw)
-				t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))})
-			}
+	t.state.Set(StateCertificate)
+	t.handShakeRecord.ETLSHandshakeCertificate = c
+
+	for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates {
+		cert, err := x509.ParseCertificate(asn1Data)
+		if err != nil {
+			//return err
+		} else {
+			h := sha256.New()
+			h.Write(cert.Raw)
+			t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))})
 		}
 	}
 }