Merge pull request #4 from bennySim/devel

JSON output: added option to print detailed info about ipfix packet header

Author: Lukas955

doc/data/configs/tcp2anon2json.xml

@@ -40,6 +40,7 @@
         <ignoreUnknown>true</ignoreUnknown>
         <ignoreOptions>false</ignoreOptions>
         <nonPrintableChar>true</nonPrintableChar>
+	<detailedInfo>true</detailedInfo>
 
         <!-- Output methods -->
         <outputs>

src/plugins/output/json/README.rst

@@ -70,6 +70,7 @@ Don't forget to remove (or comment) outputs that you don't want to use!
             <nonPrintableChar>true</nonPrintableChar>
             <numericNames>false</numericNames>
             <splitBiflow>false</splitBiflow>
+	    <detailedInfo>true</detailedInfo>
 
             <outputs>
                 <server>
@@ -142,6 +143,10 @@ Formatting parameters:
     In case of Biflow records, split the record to two unidirectional flow records. Non-biflow
     records are unaffected. [values: true/false, default: false]
 
+:``detailedInfo``:
+    Add detailed info about the IPFIX message (export time, sequence number, ...) to each record
+    under "ipfix:" prefix. [values: true/false, default: false]
+
 ----
 
 Output types: At least one of the following output must be configured. Multiple server/send/file

src/plugins/output/json/src/Config.cpp

@@ -59,6 +59,7 @@ enum params_xml_nodes {
     FMT_NONPRINT,      /**< Non-printable chars             */
     FMT_NUMERIC,       /**< Use numeric names               */
     FMT_BFSPLIT,       /**< Split biflow                    */
+    FMT_DETAILEDINFO,  /**< Detailed information            */
     // Common output
     OUTPUT_LIST,       /**< List of output types            */
     OUTPUT_PRINT,      /**< Print to standard output        */
@@ -139,6 +140,7 @@ static const struct fds_xml_args args_params[] = {
     FDS_OPTS_ELEM(FMT_NONPRINT,  "nonPrintableChar", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_ELEM(FMT_NUMERIC,   "numericNames",     FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_ELEM(FMT_BFSPLIT,   "splitBiflow",      FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
+    FDS_OPTS_ELEM(FMT_DETAILEDINFO,  "detailedInfo", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_NESTED(OUTPUT_LIST, "outputs",   args_outputs, 0),
     FDS_OPTS_END
 };
@@ -457,6 +459,10 @@ Config::parse_params(fds_xml_ctx_t *params)
             assert(content->type == FDS_OPTS_T_BOOL);
             format.split_biflow = content->val_bool;
             break;
+            case FMT_DETAILEDINFO: //Add detailed information about each record
+            assert(content->type == FDS_OPTS_T_BOOL);
+            format.detailed_info = content->val_bool;
+            break;
         case OUTPUT_LIST: // List of output plugin
             assert(content->type == FDS_OPTS_T_CONTEXT);
             parse_outputs(content->ptr_ctx);
@@ -481,6 +487,7 @@ Config::default_set()
     format.ignore_options = true;
     format.numeric_names = false;
     format.split_biflow = false;
+    format.detailed_info = false;
 
     outputs.prints.clear();
     outputs.files.clear();

src/plugins/output/json/src/Config.hpp

@@ -58,6 +58,8 @@ struct cfg_format {
     bool ignore_unknown;
     /** Convert white spaces in string (do not skip)                                             */
     bool white_spaces;
+    /** Add detailed information about each record                                            */
+    bool detailed_info;
     /** Ignore Options Template records                                                          */
     bool ignore_options;
     /** Use only numeric identifiers of Information Elements                                     */

src/plugins/output/json/src/Storage.cpp

@@ -81,6 +81,7 @@ Storage::Storage(const ipx_ctx_t *ctx, const struct cfg_format &fmt)
     if (m_format.split_biflow) {
         m_flags |= FDS_CD2J_REVERSE_SKIP;
     }
+
 }
 
 Storage::~Storage()
@@ -149,6 +150,9 @@ Storage::records_store(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr)
     // Process all data records
     const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(msg);
 
+    // Message header
+    auto hdr = (fds_ipfix_msg_hdr*) ipx_msg_ipfix_get_packet(msg);
+
     for (uint32_t i = 0; i < rec_cnt; ++i) {
         ipx_ipfix_record *ipfix_rec = ipx_msg_ipfix_get_drec(msg, i);
 
@@ -159,7 +163,7 @@ Storage::records_store(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr)
         }
 
         // Convert the record
-        convert(ipfix_rec->rec, iemgr, false);
+        convert(ipfix_rec->rec, iemgr, hdr, false);
 
         // Store it
         for (Output *output : m_outputs) {
@@ -174,7 +178,7 @@ Storage::records_store(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr)
         }
 
         // Convert the record from reverse point of view
-        convert(ipfix_rec->rec, iemgr, true);
+        convert(ipfix_rec->rec, iemgr, hdr, true);
 
         // Store it
         for (Output *output : m_outputs) {
@@ -198,20 +202,47 @@ Storage::records_store(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr)
  * \throw runtime_error if the JSON converter fails
  */
 void
-Storage::convert(struct fds_drec &rec, const fds_iemgr_t *iemgr, bool reverse)
+Storage::convert(struct fds_drec &rec, const fds_iemgr_t *iemgr, fds_ipfix_msg_hdr *hdr, bool reverse)
 {
     // Convert the record
     uint32_t flags = m_flags;
     flags |= reverse ? FDS_CD2J_BIFLOW_REVERSE : 0;
 
     int rc = fds_drec2json(&rec, flags, iemgr, &m_record.buffer, &m_record.size_alloc);
     if (rc < 0) {
-        throw std::runtime_error("Conversion to JSON failed (probably a memory allocation error!");
+        throw std::runtime_error("Conversion to JSON failed (probably a memory allocation error)!");
     }
 
     m_record.size_used = size_t(rc);
-    // Append the record with end of line character
-    buffer_append("\n");
+
+    if (m_format.detailed_info) {
+
+        // Remove '}' parenthesis at the end of the record
+        m_record.size_used--;
+
+        // Array for formatting detailed info fields
+        char field[64];
+        snprintf(field, 64, ",\"ipfix:exportTime\":\"%" PRIu32 "\"", ntohl(hdr->export_time));
+        buffer_append(field);
+
+        snprintf(field, 64, ",\"ipfix:seqNumber\":\"%" PRIu32 "\"", ntohl(hdr->seq_num));
+        buffer_append(field);
+
+        snprintf(field, 64, ",\"ipfix:odid\":\"%" PRIu32 "\"", ntohl(hdr->odid));
+        buffer_append(field);
+
+        snprintf(field, 32, ",\"ipfix:msgLength\":\"%" PRIu16 "\"", ntohs(hdr->length));
+        buffer_append(field);
+
+        snprintf(field, 32, ",\"ipfix:templateId\":\"%" PRIu16 "\"", rec.tmplt->id);
+        buffer_append(field);
+
+        // Append the record with '}' parenthesis removed before
+        buffer_append("}");
+    }
+
+         // Append the record with end of line character
+         buffer_append("\n");
 
     /* Note: additional information (e.g. ODID, Export Time, etc.) can be added here,
      * just use buffer_append() and buffer_reserve() to append and extend buffer, respectively.

src/plugins/output/json/src/Storage.hpp

@@ -95,7 +95,7 @@ class Storage {
     } m_record; /**< Converted JSON record                                                       */
 
     // Convert an IPFIX record to a JSON string
-    void convert(struct fds_drec &rec, const fds_iemgr_t *iemgr, bool reverse = false);
+    void convert(struct fds_drec &rec, const fds_iemgr_t *iemgr, struct fds_ipfix_msg_hdr *hdr, bool reverse = false);
 
     // Remaining buffer size
     size_t buffer_remain() const {return m_record.size_alloc - m_record.size_used;};