Merge pull request #54 from CESNET/devel

Bugfixes and improvements

Author: cejkato2

Makefile.am

@@ -5,11 +5,11 @@ SUBDIRS+=input/nfbCInterface
 endif
 
 SUBDIRS+=. tests init
-bin_PROGRAMS=ipfixprobe
+bin_PROGRAMS=ipfixprobe ipfixprobe_stats
 
 DISTCHECK_CONFIGURE_FLAGS="--with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)"
 
-ipfixprobe_LDFLAGS=-lpthread -ldl
+ipfixprobe_LDFLAGS=-lpthread -ldl -latomic
 ipfixprobe_CFLAGS=-I$(srcdir)/include/
 ipfixprobe_CXXFLAGS=-std=gnu++11 -Wno-write-strings -I$(srcdir)/include/
 
@@ -181,6 +181,8 @@ ipfixprobe_src=\
 		ring.c \
 		workers.cpp \
 		workers.hpp \
+		stats.cpp \
+		stats.hpp \
 		ipfixprobe.hpp \
 		ipfixprobe.cpp
 
@@ -192,6 +194,15 @@ endif
 
 ipfixprobe_SOURCES=$(ipfixprobe_src) main.cpp
 
+ipfixprobe_stats_CXXFLAGS=-std=gnu++11 -Wno-write-strings -I$(srcdir)/include/
+ipfixprobe_stats_SOURCES=ipfixprobe_stats.cpp \
+		include/ipfixprobe/options.hpp \
+		include/ipfixprobe/utils.hpp \
+		stats.cpp \
+		stats.hpp \
+		options.cpp \
+		utils.cpp
+
 pkgdocdir=${docdir}/ipfixprobe
 pkgdoc_DATA=README.md
 EXTRA_DIST=README.md \

README.md

@@ -4,6 +4,7 @@
 This application creates biflows from packet input and exports them to output interface.
 
 ## Requirements
+- libatomic
 - kernel version at least 3.19 when using raw sockets input plugin enabled by default (disable with `--without-raw` parameter for `./configure`)
 - [libpcap](http://www.tcpdump.org/) when compiling with pcap plugin (`--with-pcap` parameter)
 - netcope-common [COMBO cards](https://www.liberouter.org/technologies/cards/) when compiling with ndp plugin (`--with-ndp` parameter)
@@ -231,10 +232,12 @@ List of unirec fields exported together with basic flow fields on interface by R
 ### TLS
 List of unirec fields exported together with basic flow fields on interface by TLS plugin.
 
-| UniRec field        | Type   | Description                  |
-|:-------------------:|:------:|:----------------------------:|
-| TLS_SNI             | string | TLS server name indication   |
-| TLS_JA3             | string | TLS client JA3 fingerprint   |
+| UniRec field        | Type   | Description                                                   |
+|:-------------------:|:------:|:-------------------------------------------------------------:|
+| TLS_SNI             | string | TLS server name indication field from client                  |
+| TLS_ALPN            | string | TLS application protocol layer negotiation field from server  |
+| TLS_VERSION         | uint16 | TLS client protocol version                                   |
+| TLS_JA3             | string | TLS client JA3 fingerprint                                    |
 
 ### DNS
 List of unirec fields exported together with basic flow fields on interface by DNS plugin.
@@ -411,7 +414,7 @@ Note: the following fields are UniRec arrays.
 
 ##### Example:
 ```
-ipfixprobe -p pstats:includezeros -r sample.pcap -i "f:output.trapcap"
+ipfixprobe 'pcap;file=pcaps/http.pcap' -p pstats:includezeros -o 'unirec;i=u:stats:timeout=WAIT;p=stats'"
 ```
 
 ### OSQUERY
@@ -522,7 +525,7 @@ The exported unirec fields and IPFIX basiclists is shown in following table:
 
 ##### Example:
 ```
-ipfixprobe -p phists:includezeros -r sample.pcap -i "f:output.trapcap"
+ipfixprobe 'pcap;file=pcaps/http.pcap' -p phists:includezeros -o 'unirec;i=u:hists:timeout=WAIT;p=phists'"
 ```
 ### BSTATS
 

configure.ac

@@ -73,6 +73,16 @@ AC_TYPE_UINT8_T
 AX_C_BIGENDIAN_CROSS
 
 
+AC_ARG_WITH([defaultsocketdir],
+   [AS_HELP_STRING([--with-defaultsocketdir=DIR], [Directory for UNIX&service IFCs [/tmp], for production set it to e.g. /var/run/ipfixprobe.])],
+   [],
+   [with_defaultsocketdir=/tmp])
+
+AC_SUBST([defaultsocketdir], [$with_defaultsocketdir])
+AC_DEFINE_DIR([DEFAULTSOCKETDIR], [defaultsocketdir], [Default path to socket directory])
+
+AC_CHECK_LIB(atomic, __atomic_store, [libatomic=yes], AC_MSG_ERROR([libatomic not found]))
+
 ### gtest
 AC_ARG_WITH([gtest],
         AC_HELP_STRING([--with-gtest],[Compile ipfixprobe with gtest framework]),

include/ipfixprobe/input.hpp

@@ -66,10 +66,11 @@ class InputPlugin : public Plugin
       ERROR
    };
 
-   uint64_t m_processed;
+   uint64_t m_seen;
    uint64_t m_parsed;
+   uint64_t m_dropped;
 
-   InputPlugin() : m_processed(0), m_parsed(0) {}
+   InputPlugin() : m_seen(0), m_parsed(0), m_dropped(0) {}
    virtual ~InputPlugin() {}
 
    virtual Result get(PacketBlock &packets) = 0;

include/ipfixprobe/ipfix-elements.hpp

@@ -194,6 +194,7 @@ namespace ipxp {
 
 #define TLS_SNI(F)                    F(8057,   808,   -1,   nullptr)
 #define TLS_ALPN(F)                   F(8057,   809,   -1,   nullptr)
+#define TLS_VERSION(F)                F(8057,   810,    2,   nullptr)
 #define TLS_JA3(F)                    F(8057,   830,   -1,   nullptr)
 
 #define SMTP_COMMANDS(F)              F(8057,    810,   4,   nullptr)
@@ -348,6 +349,7 @@ namespace ipxp {
 #define IPFIX_TLS_TEMPLATE(F) \
    F(TLS_SNI) \
    F(TLS_ALPN) \
+   F(TLS_VERSION) \
    F(TLS_JA3)
 
 #define IPFIX_NTP_TEMPLATE(F) \

input/benchmark.cpp

@@ -131,7 +131,7 @@ InputPlugin::Result Benchmark::get(PacketBlock &packets)
          break;
       }
    }
-   m_processed += packets.cnt;
+   m_seen += packets.cnt;
    m_parsed += packets.cnt;
    return res;
 }

input/parser.cpp

@@ -47,9 +47,9 @@
 #include <iostream>
 #include <sys/types.h>
 
-#ifndef WITH_NDP
+#ifdef WITH_PCAP
 #include <pcap/sll.h>
-#endif /* WITH_NDP */
+#endif /* WITH_PCAP */
 
 #include "parser.hpp"
 #include "headers.hpp"
@@ -141,7 +141,7 @@ inline uint16_t parse_eth_hdr(const u_char *data_ptr, uint16_t data_len, Packet
    return hdr_len;
 }
 
-#ifndef WITH_NDP
+#ifdef WITH_PCAP
 /**
  * \brief Parse specific fields from SLL frame header.
  * \param [in] data_ptr Pointer to begin of header.
@@ -178,7 +178,7 @@ inline uint16_t parse_sll(const u_char *data_ptr, uint16_t data_len, Packet *pkt
    pkt->ethertype = ntohs(sll->sll_protocol);
    return sizeof(struct sll_header);
 }
-#endif /* WITH_NDP */
+#endif /* WITH_PCAP */
 
 
 /**
@@ -614,15 +614,15 @@ void parse_packet(parser_opt_t *opt, struct timeval ts, const uint8_t *data, uin
    uint32_t l3_hdr_offset = 0;
    uint32_t l4_hdr_offset = 0;
    try {
-   #ifndef WITH_NDP
+   #ifdef WITH_PCAP
       if (opt->datalink == DLT_EN10MB) {
          data_offset = parse_eth_hdr(data, caplen, pkt);
       } else {
          data_offset = parse_sll(data, caplen, pkt);
       }
    #else
       data_offset = parse_eth_hdr(data, caplen, pkt);
-   #endif /* WITH_NDP */
+   #endif /* WITH_PCAP */
 
       if (pkt->ethertype == ETH_P_TRILL) {
          data_offset += parse_trill(data + data_offset, caplen - data_offset, pkt);

input/pcap.cpp

@@ -263,13 +263,13 @@ InputPlugin::Result PcapReader::get(PacketBlock &packets)
          return Result::TIMEOUT;
       }
       if (ret > 0) {
-         m_processed += ret;
+         m_seen += ret;
          m_parsed += opt.pblock->cnt;
          return opt.packet_valid ? Result::PARSED : Result::NOT_PARSED;
       }
    } else {
       if (opt.pblock->cnt) {
-         m_processed += ret ? ret : opt.pblock->cnt;
+         m_seen += ret ? ret : opt.pblock->cnt;
          m_parsed += opt.pblock->cnt;
          return Result::PARSED;
       } else if (ret == 0) {

input/pcap.hpp

@@ -82,7 +82,7 @@ class PcapOptParser : public OptionsParser
       register_option("f", "file", "PATH", "Path to a pcap file", [this](const char *arg){m_file = arg; return true;}, OptionFlags::RequiredArgument);
       register_option("i", "ifc", "IFC", "Network interface name", [this](const char *arg){m_ifc = arg; return true;}, OptionFlags::RequiredArgument);
       register_option("F", "filter", "STR", "Filter string", [this](const char *arg){m_filter = arg; return true;}, OptionFlags::RequiredArgument);
-      register_option("s", "snaplen", "SIZE", "Snapshot length in bytes",
+      register_option("s", "snaplen", "SIZE", "Snapshot length in bytes (live capture only)",
          [this](const char *arg){try {m_snaplen = str2num<decltype(m_snaplen)>(arg);} catch(std::invalid_argument &e) {return false;} return true;},
          OptionFlags::RequiredArgument);
       register_option("l", "list", "", "Print list of available interfaces", [this](const char *arg){m_list = true; return true;}, OptionFlags::NoArgument);

input/raw.cpp

@@ -369,7 +369,7 @@ InputPlugin::Result RawReader::get(PacketBlock &packets)
       throw PluginError("error during reading from socket");
    }
 
-   m_processed += ret;
+   m_seen += ret;
    m_parsed += packets.cnt;
    return packets.cnt ? Result::PARSED : Result::NOT_PARSED;
  }