JSON converter: add (Options) templates records

Author: Simona Bennarova

doc/data/configs/tcp2anon2json.xml

@@ -40,7 +40,8 @@
         <ignoreUnknown>true</ignoreUnknown>
         <ignoreOptions>false</ignoreOptions>
         <nonPrintableChar>true</nonPrintableChar>
-	<detailedInfo>true</detailedInfo>
+	<detailedInfo>false</detailedInfo>
+	<templateInfo>false</templateInfo>
 
         <!-- Output methods -->
         <outputs>

src/plugins/output/json/README.rst

@@ -70,7 +70,8 @@ Don't forget to remove (or comment) outputs that you don't want to use!
             <nonPrintableChar>true</nonPrintableChar>
             <numericNames>false</numericNames>
             <splitBiflow>false</splitBiflow>
-	    <detailedInfo>true</detailedInfo>
+	    <detailedInfo>false</detailedInfo>
+	    <templateInfo>false</templateInfo>
 
             <outputs>
                 <server>
@@ -147,6 +148,10 @@ Formatting parameters:
     Add detailed info about the IPFIX message (export time, sequence number, ...) to each record
     under "ipfix:" prefix. [values: true/false, default: false]
 
+:``templateInfo``:
+    Add Template and Options Template records to the top of the exported JSON file with prefix
+    "ipfix.template" and "ipfix.optionsTemplate". [values: true/false, default: false]
+
 ----
 
 Output types: At least one of the following output must be configured. Multiple server/send/file
@@ -280,4 +285,49 @@ among the nested arrays.
         }
     }
 
-Keep on mind that all structures can be nested in each other.
+Keep on mind that all structures can be nested in each other.
+
+Template and Options Template records
+-------------------------------------
+
+Template and Options Template records are special records.
+
+*Template record* describes structure of flow records, and its type is "ipfix.template".
+Converted *Template record* contains "ipfix:templateId" field, which is unique to the Transport Session 
+and Observation Domain and "ipfix:fields" which is an array of JSON objects specifing fields of flow records. 
+
+.. code-block:: json
+{
+	"@type": "ipfix.template",
+	"ipfix:templateId": "49171",
+	"ipfix:fields": [{
+		"ipfix:elementId": "16399",
+		"ipfix:enterpriseId": "6871",
+		"ipfix:fieldLength": "1"
+	}, {
+		"ipfix:elementId": "184",
+		"ipfix:enterpriseId": "29305",
+		"ipfix:fieldLength": "4"
+	}]
+}
+
+*Options Template record* describes structure of additional information for the collector.
+These additional information are converted to record with type "ipfix.optionsEntry".
+Converted *Options Template record* is similar to previous type, however it contains also 
+"ipfix:scopeCount" field, which specifies number of scope fields in the record.
+  
+.. code-block:: json
+{
+	"@type": "ipfix.optionsTemplate",
+	"ipfix:templateId": "53252",
+	"ipfix:scopeCount": "3",
+	"ipfix:fields": [{
+		"ipfix:elementId": "322",
+		"ipfix:enterpriseId": "0",
+		"ipfix:fieldLength": "4"
+	}, {
+		"ipfix:elementId": "554",
+		"ipfix:enterpriseId": "6871",
+		"ipfix:fieldLength": "65535"
+	}]
+}

src/plugins/output/json/src/Config.cpp

@@ -60,6 +60,7 @@ enum params_xml_nodes {
     FMT_NUMERIC,       /**< Use numeric names               */
     FMT_BFSPLIT,       /**< Split biflow                    */
     FMT_DETAILEDINFO,  /**< Detailed information            */
+    FMT_TMPLTINFO,      /**< Template records                */
     // Common output
     OUTPUT_LIST,       /**< List of output types            */
     OUTPUT_PRINT,      /**< Print to standard output        */
@@ -141,6 +142,7 @@ static const struct fds_xml_args args_params[] = {
     FDS_OPTS_ELEM(FMT_NUMERIC,   "numericNames",     FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_ELEM(FMT_BFSPLIT,   "splitBiflow",      FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_ELEM(FMT_DETAILEDINFO,  "detailedInfo", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
+    FDS_OPTS_ELEM(FMT_TMPLTINFO, "templateInfo", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_NESTED(OUTPUT_LIST, "outputs",   args_outputs, 0),
     FDS_OPTS_END
 };
@@ -459,14 +461,18 @@ Config::parse_params(fds_xml_ctx_t *params)
             assert(content->type == FDS_OPTS_T_BOOL);
             format.split_biflow = content->val_bool;
             break;
-            case FMT_DETAILEDINFO: //Add detailed information about each record
+            case FMT_DETAILEDINFO: // Add detailed information about each record
             assert(content->type == FDS_OPTS_T_BOOL);
             format.detailed_info = content->val_bool;
             break;
         case OUTPUT_LIST: // List of output plugin
             assert(content->type == FDS_OPTS_T_CONTEXT);
             parse_outputs(content->ptr_ctx);
             break;
+            case FMT_TMPLTINFO: // Add template records
+            assert(content->type == FDS_OPTS_T_BOOL);
+            format.template_info = content->val_bool;
+            break;
         default:
             throw std::invalid_argument("Unexpected element within <params>!");
         }
@@ -488,6 +494,7 @@ Config::default_set()
     format.numeric_names = false;
     format.split_biflow = false;
     format.detailed_info = false;
+    format.template_info = false;
 
     outputs.prints.clear();
     outputs.files.clear();

src/plugins/output/json/src/Config.hpp

@@ -66,6 +66,8 @@ struct cfg_format {
     bool numeric_names;
     /** Split biflow records                                                                     */
     bool split_biflow;
+    /** Add template records                                                                     */
+    bool template_info;
 };
 
 /** Output configuration base structure                                                          */

src/plugins/output/json/src/Storage.cpp

@@ -144,17 +144,147 @@ Storage::output_add(Output *output)
     m_outputs.push_back(output);
 }
 
+/**
+ * \brief Convert template record to JSON string
+ *
+ * \param[in] tset_iter     (Options) Template set structure to convert
+ * \param[in] set_id        Id of the set
+ * \param[in] hdr           Message header of IPFIX record
+ * \throw runtime_error    if template parser failed
+ */
+
+void
+Storage::convert_tmplt_rec(struct fds_tset_iter *tset_iter, uint16_t set_id, fds_ipfix_msg_hdr* hdr)
+{
+    enum fds_template_type type;
+    void *ptr;
+    if (set_id == FDS_IPFIX_SET_TMPLT) {
+        buffer_append("{\"@type\":\"ipfix.template\",");
+        type = FDS_TYPE_TEMPLATE;
+        ptr = tset_iter->ptr.trec;
+    } else {
+        assert(set_id == FDS_IPFIX_SET_OPTS_TMPLT);
+        buffer_append("{\"@type\":\"ipfix.optionsTemplate\",");
+        type = FDS_TYPE_TEMPLATE_OPTS;
+        ptr = tset_iter->ptr.opts_trec;
+    }
+
+    // Filling the template structure with data from raw packet
+    uint16_t tmplt_size = tset_iter->size;
+    struct fds_template *tmplt;
+    int rc;
+    rc = fds_template_parse(type, ptr, &tmplt_size, &tmplt);
+    if (rc != FDS_OK) {
+        throw std::runtime_error("Parsing failed due to memory allocation error or the format of template is invalid!");
+    }
+
+    // Printing out the header
+    char field[64];
+    snprintf(field, 64, "\"ipfix:templateId\":\"%" PRIu16 "\"", tmplt->id);
+    buffer_append(field);
+    if (set_id == FDS_IPFIX_SET_OPTS_TMPLT) {
+        snprintf(field, 64, ",\"ipfix:scopeCount\":\"%" PRIu16 "\"", tmplt->fields_cnt_scope);
+        buffer_append(field);
+    }
+
+    // Add detailed info to record
+    if (m_format.detailed_info) {
+        addDetailedInfo(hdr);
+    }
+
+    buffer_append(",\"ipfix:fields\":[");
+
+    // Iteration through the fields and converting them to JSON string
+    for (uint16_t i = 0; i < tmplt->fields_cnt_total; i++) {
+        struct fds_tfield current = tmplt->fields[i];
+        if (i != 0) { // Not first field
+            buffer_append(",");
+        }
+        buffer_append("{");
+        snprintf(field, 64, "\"ipfix:elementId\":\"%" PRIu16 "\"", current.id);
+        buffer_append(field);
+        snprintf(field, 64, ",\"ipfix:enterpriseId\":\"%" PRIu32 "\"", current.en);
+        buffer_append(field);
+        snprintf(field, 64, ",\"ipfix:fieldLength\":\"%" PRIu16 "\"", current.length);
+        buffer_append(field);
+        buffer_append("}");
+    }
+    buffer_append("]}\n");
+
+    // Free allocated memory
+    fds_template_destroy(tmplt);
+}
+
+/**
+ * \brief Convert Template sets and Options Template sets
+ *
+ * From all sets in the Message, try to convert just Template and Options template sets.
+ * \param[in] set   All sets in the Message
+ * \param[in] hdr   Message header of IPFIX record
+ */
+void
+Storage::convert_set(struct ipx_ipfix_set *set, fds_ipfix_msg_hdr* hdr)
+{
+
+    bool flush = false;
+    int ret = IPX_OK;
+    uint16_t set_id = ntohs(set->ptr->flowset_id);
+    if (set_id == FDS_IPFIX_SET_TMPLT || set_id == FDS_IPFIX_SET_OPTS_TMPLT) {
+
+        // Template set
+        struct fds_tset_iter tset_iter;
+        fds_tset_iter_init(&tset_iter, set->ptr);
+
+        // Iteration through all templates in the set
+        while (fds_tset_iter_next(&tset_iter) == FDS_OK) {
+            flush = true;
+
+            // Read and print single template
+            convert_tmplt_rec(&tset_iter, set_id, hdr);
+
+            // Store it
+            for (Output *output : m_outputs) {
+                if (output->process(m_record.buffer, m_record.size_used) != IPX_OK) {
+                    ret = IPX_ERR_DENIED;
+                    goto endloop;
+                }
+            }
+
+            // Buffer is empty
+            m_record.size_used = 0;
+        }
+    }
+endloop:
+    if (flush) {
+        for (Output *output : m_outputs) {
+            output->flush();
+        }
+    }
+}
+
 int
 Storage::records_store(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr)
 {
+    // Message header
+    auto hdr = (fds_ipfix_msg_hdr*) ipx_msg_ipfix_get_packet(msg);
+
+    // Process template records if enabled
+    if (m_format.template_info) {
+        struct ipx_ipfix_set *sets;
+        size_t set_cnt;
+        ipx_msg_ipfix_get_sets(msg, &sets, &set_cnt);
+
+        // Iteration through all sets
+        for (uint32_t i = 0; i < set_cnt; i++) {
+            convert_set(&sets[i], hdr);
+        }
+    }
+
     // Process all data records
     const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(msg);
     bool flush = false;
     int ret = IPX_OK;
 
-    // Message header
-    auto hdr = (fds_ipfix_msg_hdr*) ipx_msg_ipfix_get_packet(msg);
-
     for (uint32_t i = 0; i < rec_cnt; ++i) {
         ipx_ipfix_record *ipfix_rec = ipx_msg_ipfix_get_drec(msg, i);
 
@@ -204,6 +334,30 @@ Storage::records_store(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr)
     return ret;
 }
 
+/**
+ * \brief Add fields with detailed info (export time, sequence number, ODID, message length) to record
+ *
+ * For each record, add detailed information if detailedInfo is enabled.
+ * @param[in] hdr   Message header of IPFIX record
+ */
+void
+Storage::addDetailedInfo(fds_ipfix_msg_hdr *hdr)
+{
+    // Array for formatting detailed info fields
+    char field[64];
+    snprintf(field, 64, ",\"ipfix:exportTime\":\"%" PRIu32 "\"", ntohl(hdr->export_time));
+    buffer_append(field);
+
+    snprintf(field, 64, ",\"ipfix:seqNumber\":\"%" PRIu32 "\"", ntohl(hdr->seq_num));
+    buffer_append(field);
+
+    snprintf(field, 64, ",\"ipfix:odid\":\"%" PRIu32 "\"", ntohl(hdr->odid));
+    buffer_append(field);
+
+    snprintf(field, 32, ",\"ipfix:msgLength\":\"%" PRIu16 "\"", ntohs(hdr->length));
+    buffer_append(field);
+}
+
 /**
  * \brief Convert an IPFIX record to JSON string
  *
@@ -228,25 +382,15 @@ Storage::convert(struct fds_drec &rec, const fds_iemgr_t *iemgr, fds_ipfix_msg_h
 
     m_record.size_used = size_t(rc);
 
-    if (m_format.detailed_info) {
-
+    if (m_format.detailed_info && !m_format.template_info) {
         // Remove '}' parenthesis at the end of the record
         m_record.size_used--;
 
-        // Array for formatting detailed info fields
-        char field[64];
-        snprintf(field, 64, ",\"ipfix:exportTime\":\"%" PRIu32 "\"", ntohl(hdr->export_time));
-        buffer_append(field);
-
-        snprintf(field, 64, ",\"ipfix:seqNumber\":\"%" PRIu32 "\"", ntohl(hdr->seq_num));
-        buffer_append(field);
-
-        snprintf(field, 64, ",\"ipfix:odid\":\"%" PRIu32 "\"", ntohl(hdr->odid));
-        buffer_append(field);
-
-        snprintf(field, 32, ",\"ipfix:msgLength\":\"%" PRIu16 "\"", ntohs(hdr->length));
-        buffer_append(field);
+        // Add detailed info to JSON string
+        addDetailedInfo(hdr);
 
+        // Add template ID to JSON string
+        char field[64];
         snprintf(field, 32, ",\"ipfix:templateId\":\"%" PRIu16 "\"", rec.tmplt->id);
         buffer_append(field);
 

src/plugins/output/json/src/Storage.hpp

@@ -110,6 +110,12 @@ class Storage {
     void buffer_append(const char *str);
     // Reserve memory for a JSON string
     void buffer_reserve(size_t n);
+    // Convert set to JSON string
+    void convert_set(struct ipx_ipfix_set *set, fds_ipfix_msg_hdr* hdr);
+    // Convert template record to a JSON string
+    void convert_tmplt_rec(struct fds_tset_iter *tset_iter, uint16_t set_id, fds_ipfix_msg_hdr* hdr);
+    // Add detailed info (templateId, ODID, seqNum, exportTime) to JSON string
+    void addDetailedInfo(fds_ipfix_msg_hdr *hdr);
 
 public:
     /**