Merge pull request #227 from Zadamsa/tls-refactor

TLS refactor

Author: SiskaPavel

Makefile.am

@@ -99,6 +99,7 @@ ipfixprobe_process_src=\
 		process/tls.hpp \
 		process/tls_parser.cpp \
 		process/tls_parser.hpp \
+		process/sha256.hpp \
 		process/smtp.cpp \
 		process/smtp.hpp \
 		process/dns-utils.hpp \

include/ipfixprobe/ipfix-elements.hpp

@@ -185,10 +185,10 @@ namespace ipxp {
 #define TLS_VERSION(F)                F(39499,  333,    2,   nullptr)
 #define TLS_ALPN(F)                   F(39499,  337,   -1,   nullptr)
 #define TLS_JA3(F)                    F(39499,  357,   -1,   nullptr)
+#define TLS_JA4(F)                    F(39499,  358,   -1,   nullptr)
 #define TLS_EXT_TYPE(F)               F(0,      291,   -1,   nullptr)
 #define TLS_EXT_LEN(F)                F(0,      291,   -1,   nullptr)
 
-
 #define SMTP_COMMANDS(F)              F(8057,    810,   4,   nullptr)
 #define SMTP_MAIL_COUNT(F)            F(8057,    811,   4,   nullptr)
 #define SMTP_RCPT_COUNT(F)            F(8057,    812,   4,   nullptr)
@@ -389,6 +389,7 @@ namespace ipxp {
    F(TLS_SNI) \
    F(TLS_ALPN) \
    F(TLS_JA3) \
+   F(TLS_JA4) \
    F(TLS_EXT_TYPE) \
    F(TLS_EXT_LEN)
 

process/quic_parser.cpp

@@ -248,95 +248,65 @@ uint64_t QUICParser::quic_get_variable_length(const uint8_t* start, uint64_t& of
     }
 } // QUICParser::quic_get_variable_length
 
-bool QUICParser::quic_obtain_tls_data(TLSData& payload)
+bool QUICParser::quic_parse_tls_extensions()
 {
-    quic_tls_extension_lengths_pos = 0;
-    quic_tls_ext_type_pos = 0;
-    quic_tls_ext_pos = 0;
-    while (payload.start + sizeof(tls_ext) <= payload.end) {
-        tls_ext* ext = (tls_ext*) payload.start;
-        uint16_t type = ntohs(ext->type);
-        uint16_t length = ntohs(ext->length);
-
-        // Store extension type
-        if (quic_tls_ext_type_pos < MAX_QUIC_TLS_EXT_LEN) {
-            quic_tls_ext_type[quic_tls_ext_type_pos] = type;
-            quic_tls_ext_type_pos += 1;
-        }
-
-        // Store extension type length
-        if (quic_tls_extension_lengths_pos < MAX_QUIC_TLS_EXT_LEN) {
-            quic_tls_extension_lengths[quic_tls_extension_lengths_pos] = length;
-            quic_tls_extension_lengths_pos += 1;
-        }
-
-        //
-        payload.start += sizeof(tls_ext);
-
-        if (payload.start + length > payload.end) {
-            break;
-        }
-
-        // Save value payload except for length
-        if (quic_tls_ext_pos + length < CURRENT_BUFFER_SIZE) {
+    const bool extensions_parsed = tls_parser.parse_extensions([this](
+        uint16_t extension_type,
+        const uint8_t* extension_payload,
+        uint16_t extension_length) {
+            if (extension_type == TLS_EXT_SERVER_NAME && extension_length != 0) {
+                tls_parser.parse_server_names(extension_payload, extension_length);
+            } else if (
+                (extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1
+                 || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS
+                 || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2)
+                && extension_length != 0) {
+                tls_parser.parse_quic_user_agent(extension_payload, extension_length);
+            }
+            if (quic_tls_ext_pos + extension_length < CURRENT_BUFFER_SIZE) {
 #ifndef QUIC_CH_FULL_TLS_EXT
-            if (type == TLS_EXT_ALPN || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1
-                || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS
-                || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2) {
+                if (extension_type == TLS_EXT_ALPN || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1
+                    || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS
+                    || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2) {
 #endif
-                memcpy(quic_tls_ext + quic_tls_ext_pos, payload.start, length);
-                quic_tls_ext_pos += length;
+                    memcpy(quic_tls_ext + quic_tls_ext_pos, extension_payload, extension_length);
+                    quic_tls_ext_pos += extension_length;
 #ifndef QUIC_CH_FULL_TLS_EXT
-            }
+                }
 #endif
-        }
-
-        // Legacy extract specific fields
-        if (type == TLS_EXT_SERVER_NAME && length != 0) {
-            tls_parser.tls_get_server_name(payload, sni, BUFF_SIZE);
-        } else if (
-            (type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1
-             || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS
-             || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2)
-            && length != 0) {
-            tls_parser.tls_get_quic_user_agent(payload, user_agent, BUFF_SIZE);
-        }
-        payload.start += length;
-    }
-    return payload.obejcts_parsed != 0;
+            }
+            tls_parser.add_extension(extension_type, extension_length);
+    });
+    if (!extensions_parsed) {
+        return false;
+    }
+    tls_parser.save_server_names(sni, BUFF_SIZE);
+    tls_parser.save_quic_user_agent(user_agent, BUFF_SIZE);
+
+    const size_t copy_count = std::min<size_t>(tls_parser.get_extensions().size(), MAX_QUIC_TLS_EXT_LEN);
+    std::transform(tls_parser.get_extensions().begin(),
+                   tls_parser.get_extensions().begin() + static_cast<ssize_t>(copy_count),
+                   std::begin(quic_tls_ext_type),
+                   [](const TLSExtension& typeLength) {
+                       return typeLength.type;
+                   });
+    std::transform(tls_parser.get_extensions().begin(),
+                   tls_parser.get_extensions().begin() + static_cast<ssize_t>(copy_count),
+                   std::begin(quic_tls_extension_lengths),
+                   [](const TLSExtension& typeLength) {
+                       return typeLength.length;
+                   });
+    quic_tls_ext_type_pos = quic_tls_extension_lengths_pos = copy_count;
+    return true;
 }
 
 bool QUICParser::quic_parse_tls()
 {
-    TLSData payload = {
-        payload.start = final_payload + quic_crypto_start,
-        payload.end = final_payload + quic_crypto_start + quic_crypto_len,
-        payload.obejcts_parsed = 0,
-    };
-
-    if (!tls_parser.tls_check_handshake(payload)) {
-        return false;
-    }
-    if (!tls_parser.tls_skip_random(payload)) {
-        return false;
-    }
-    if (!tls_parser.tls_skip_sessid(payload)) {
-        return false;
-    }
-    if (!tls_parser.tls_skip_cipher_suites(payload)) {
+    if (!tls_parser.parse_quic_tls(final_payload + quic_crypto_start, quic_crypto_len)) {
         return false;
     }
-    if (!tls_parser.tls_skip_compression_met(payload)) {
-        return false;
-    }
-    if (!tls_parser.tls_check_ext_len(payload)) {
-        return false;
-    }
-    // If no parameters were extracted. We also accept the QUIC connection. (no error check here)
-    quic_obtain_tls_data(payload);
-
-    return true;
-} // QUICPlugin::quic_parse_tls
+    return quic_parse_tls_extensions();
+}
 
 uint8_t QUICParser::quic_draft_version(uint32_t version)
 {
@@ -1394,14 +1364,16 @@ bool QUICParser::quic_parse_headers(const Packet& pkt, bool forceInitialParsing)
 
 bool QUICParser::quic_set_server_port(const Packet& pkt)
 {
-    tls_handshake hs = tls_parser.tls_get_handshake();
+    if (!tls_parser.get_handshake().has_value()) {
+        return false;
+    }
 
     switch (packet_type) {
     case INITIAL:
-        tls_hs_type = hs.type;
-        if (hs.type == 1) {
+        tls_hs_type = tls_parser.get_handshake()->type;
+        if (tls_hs_type == 1) {
             server_port = pkt.dst_port;
-        } else if (hs.type == 2) {
+        } else if (tls_hs_type == 2) {
             // Won't be reached, since we don't supply the OCCID to quic_parser
             server_port = pkt.src_port;
         }

process/quic_parser.hpp

@@ -119,7 +119,7 @@ class QUICParser {
     uint64_t quic_get_variable_length(const uint8_t*, uint64_t&);
     bool quic_check_version(uint32_t, uint8_t);
     bool quic_check_pointer_pos(const uint8_t*, const uint8_t*);
-    bool quic_obtain_tls_data(TLSData&);
+    bool quic_parse_tls_extensions();
     bool quic_set_server_port(const Packet& pkt);
     bool quic_check_min_initial_size(const Packet& pkt);
     bool quic_check_supported_version(const uint32_t version);

process/sha256.hpp

@@ -0,0 +1,148 @@
+// Single file sha256
+// https://github.com/LekKit/sha256
+
+#include <stdio.h>
+#include <string.h>
+
+namespace sha256 {
+
+struct sha256_buff {
+    unsigned long data_size;
+    unsigned int h[8];
+    unsigned char last_chunk[64];
+    unsigned char chunk_size;
+};
+
+void sha256_init(struct sha256_buff* buff)
+{
+    buff->h[0] = 0x6a09e667;
+    buff->h[1] = 0xbb67ae85;
+    buff->h[2] = 0x3c6ef372;
+    buff->h[3] = 0xa54ff53a;
+    buff->h[4] = 0x510e527f;
+    buff->h[5] = 0x9b05688c;
+    buff->h[6] = 0x1f83d9ab;
+    buff->h[7] = 0x5be0cd19;
+    buff->data_size = 0;
+    buff->chunk_size = 0;
+}
+
+static const unsigned int k[64] = {
+    0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+    0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+    0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+    0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+    0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+    0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+    0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+    0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
+
+#define rotate_r(val, bits) (val >> bits | val << (32 - bits))
+
+static void sha256_calc_chunk(struct sha256_buff* buff, const unsigned char* chunk)
+{
+    unsigned int w[64];
+    unsigned int tv[8];
+    unsigned int i;
+
+    for (i = 0; i < 16; ++i) {
+        w[i] = (unsigned int) chunk[0] << 24 | (unsigned int) chunk[1] << 16
+            | (unsigned int) chunk[2] << 8 | (unsigned int) chunk[3];
+        chunk += 4;
+    }
+
+    for (i = 16; i < 64; ++i) {
+        unsigned int s0 = rotate_r(w[i - 15], 7) ^ rotate_r(w[i - 15], 18) ^ (w[i - 15] >> 3);
+        unsigned int s1 = rotate_r(w[i - 2], 17) ^ rotate_r(w[i - 2], 19) ^ (w[i - 2] >> 10);
+        w[i] = w[i - 16] + s0 + w[i - 7] + s1;
+    }
+
+    for (i = 0; i < 8; ++i)
+        tv[i] = buff->h[i];
+
+    for (i = 0; i < 64; ++i) {
+        unsigned int S1 = rotate_r(tv[4], 6) ^ rotate_r(tv[4], 11) ^ rotate_r(tv[4], 25);
+        unsigned int ch = (tv[4] & tv[5]) ^ (~tv[4] & tv[6]);
+        unsigned int temp1 = tv[7] + S1 + ch + k[i] + w[i];
+        unsigned int S0 = rotate_r(tv[0], 2) ^ rotate_r(tv[0], 13) ^ rotate_r(tv[0], 22);
+        unsigned int maj = (tv[0] & tv[1]) ^ (tv[0] & tv[2]) ^ (tv[1] & tv[2]);
+        unsigned int temp2 = S0 + maj;
+
+        tv[7] = tv[6];
+        tv[6] = tv[5];
+        tv[5] = tv[4];
+        tv[4] = tv[3] + temp1;
+        tv[3] = tv[2];
+        tv[2] = tv[1];
+        tv[1] = tv[0];
+        tv[0] = temp1 + temp2;
+    }
+
+    for (i = 0; i < 8; ++i)
+        buff->h[i] += tv[i];
+}
+
+void sha256_update(struct sha256_buff* buff, const void* data, unsigned long size)
+{
+    const unsigned char* ptr = (const unsigned char*) data;
+    buff->data_size += size;
+    /* If there is data left in buff, concatenate it to process as new chunk */
+    if (size + buff->chunk_size >= 64) {
+        unsigned char tmp_chunk[64];
+        memcpy(tmp_chunk, buff->last_chunk, buff->chunk_size);
+        memcpy(tmp_chunk + buff->chunk_size, ptr, 64 - buff->chunk_size);
+        ptr += (64 - buff->chunk_size);
+        size -= (64 - buff->chunk_size);
+        buff->chunk_size = 0;
+        sha256_calc_chunk(buff, tmp_chunk);
+    }
+    /* Run over data chunks */
+    while (size >= 64) {
+        sha256_calc_chunk(buff, ptr);
+        ptr += 64;
+        size -= 64;
+    }
+
+    /* Save remaining data in buff, will be reused on next call or finalize */
+    memcpy(buff->last_chunk + buff->chunk_size, ptr, size);
+    buff->chunk_size += size;
+}
+
+void sha256_finalize(struct sha256_buff* buff)
+{
+    buff->last_chunk[buff->chunk_size] = 0x80;
+    buff->chunk_size++;
+    memset(buff->last_chunk + buff->chunk_size, 0, 64 - buff->chunk_size);
+
+    /* If there isn't enough space to fit int64, pad chunk with zeroes and prepare next chunk */
+    if (buff->chunk_size > 56) {
+        sha256_calc_chunk(buff, buff->last_chunk);
+        memset(buff->last_chunk, 0, 64);
+    }
+
+    /* Add total size as big-endian int64 x8 */
+    unsigned long size = buff->data_size * 8;
+    int i;
+    for (i = 8; i > 0; --i) {
+        buff->last_chunk[55 + i] = size & 255;
+        size >>= 8;
+    }
+
+    sha256_calc_chunk(buff, buff->last_chunk);
+}
+
+void hash_it(const unsigned char* data, unsigned long data_size, unsigned char* hash)
+{
+    struct sha256_buff buff;
+    sha256_init(&buff);
+    sha256_update(&buff, data, data_size);
+    sha256_finalize(&buff);
+    for (int i = 0; i < 8; i++) {
+        hash[i * 4] = (buff.h[i] >> 24) & 255;
+        hash[i * 4 + 1] = (buff.h[i] >> 16) & 255;
+        hash[i * 4 + 2] = (buff.h[i] >> 8) & 255;
+        hash[i * 4 + 3] = buff.h[i] & 255;
+    }
+}
+
+}