Merge pull request #1 from CESNET/devel

JSON output: replace the internal JSON converter with libfds converter

Author: bennySim

src/plugins/output/json/CMakeLists.txt

@@ -5,8 +5,6 @@ add_library(json-output MODULE
     src/Config.hpp
     src/Storage.cpp
     src/Storage.hpp
-    src/protocols.hpp
-    src/protocols.cpp
     src/Printer.cpp
     src/Printer.hpp
     src/File.cpp

src/plugins/output/json/README.rst

@@ -10,13 +10,16 @@ Each flow record is converted individually and send to outputs where at least on
 method must be configured. All fields in an IPFIX record are formatted as: <field name>:<value>,
 where the *field name* represents a name of a corresponding Information Element as
 "<Enterprise Name>:<Information Element Name>" and *value* is a number, string,
-boolean or null (if conversion from IPFIX to JSON fails). The enterprise name is a name of an
+boolean or null (if conversion from IPFIX to JSON fails). The Enterprise Name is a name of an
 organization that manages the field, such as IANA, CISCO, etc.
 
 If the field name and type is unknown and "ignoreUnknown" option is disabled, the field name
 use format 'enXX:idYY' where XX is an Enterprise Number and YY is an Information Element ID.
 See notes for instructions on how to add missing definitions of Information Elements.
 
+Structured data types (basicList, subTemplateList and subTemplateMultiList) are supported. See
+the particular section below for information about their formatting.
+
 Example output
 --------------
 
@@ -65,6 +68,8 @@ Don't forget to remove (or comment) outputs that you don't want to use!
             <ignoreUnknown>true</ignoreUnknown>
             <ignoreOptions>true</ignoreOptions>
             <nonPrintableChar>true</nonPrintableChar>
+            <numericNames>false</numericNames>
+            <splitBiflow>false</splitBiflow>
 
             <outputs>
                 <server>
@@ -127,6 +132,16 @@ Formatting parameters:
     Ignore non-printable characters (newline, tab, control characters, etc.) in IPFIX strings.
     If disabled, these characters are escaped on output. [values: true/false, default: true]
 
+:``numericNames``:
+    Use only short identification of Information Elements (i.e. "enXX:idYY"). If enabled, the
+    short version is used even if the definition of the field is known. This option can help to
+    create a shorter JSON records with key identifiers which are independent on the internal
+    configuration. [values: true/false, default: false]
+
+:``splitBiflow``:
+    In case of Biflow records, split the record to two unidirectional flow records. Non-biflow
+    records are unaffected. [values: true/false, default: false]
+
 ----
 
 Output types: At least one of the following output must be configured. Multiple server/send/file
@@ -189,7 +204,75 @@ If one or more Information Element definitions are missing, you can easily add t
 All definitions are provided by `libfds <https://github.com/CESNET/libfds/>`_ library.
 See the project website for help.
 
+If a flow record contains multiple occurrences of the same Information Element,
+their values are stored into a single name/value pair as JSON array. Order of the values
+in the array corresponds to their order in the flow record.
+
 For higher performance, it is advisable to use non-formatted conversion of IPFIX data types.
-If performance matters, you should prefer, for example, timestamps as numbers over ISO 8601 strings.
+In that case, you should prefer, for example, timestamps as numbers over ISO 8601 strings
+and numeric identifiers of fields as they are usually shorted.
+
+Structured data types
+---------------------
+
+Flow records can be extended with structured data types (as described in RFC6313).
+Each of these types are formatted as JSON objects with "@type" field which helps to distinguish
+its formatting. Moreover, as the standard describes, the semantic of the list is also included.
+
+Converted *basicList* contains "fieldID" with the Information Element identifier of zero or more
+Information Element(s) contained in the list. All values are stored as a JSON array in "data" field.
+
+.. code-block:: json
+
+    {
+        "example:blField": {
+            "@type": "basicList",
+            "semantic": "anyOf",
+            "fieldID": "iana:octetDeltaCount",
+            "data": [23, 34, 23]
+        }
+    }
+
+Converted *subTemplateList* contains only additional "data" field with array of zero or more
+JSON objects. As all nested JSON object are described by the same IPFIX Template, it's guaranteed
+the their structure is also the same. The "semantic" field indicates the relationship among the
+different JSON objects.
+
+.. code-block:: json
+
+    {
+        "example:stlField": {
+            "@type": "subTemplateList",
+            "semantic": "allOf",
+            "data": [
+                {"keyA.1": "value1", "keyA.2": "value2"},
+                {"keyB.1": "value1", "keyB.2": "value2"},
+            ]
+        }
+    }
+
+Converted *subTemplateMultiList* is similar to the previous type, however, sub-records can be
+even more nested. The "data" field contains a JSON array with zero or more nested JSON arrays.
+Each nested array contains zero or more JSON objects and it's guaranteed that JSON objects in
+the same array have the same structure. The "semantic" field indicates top-level relationship
+among the nested arrays.
+
+.. code-block:: json
+
+    {
+        "example:stmlField": {
+            "@type": "subTemplateMultiList",
+            "semantic": "allOf",
+            "data" : [
+                [
+                    {"keyA.1": "value1", "keyA.2": "value2"},
+                    {"keyB.1": "value1", "keyB.2": "value2"},
+                ],
+                [
+                    {"idA.1": "something", "idB.1": 123}
+                ]
+            ]
+        }
+    }
 
-Structured data types (basicList, subTemplateList, etc.) are not supported yet.
+Keep on mind that all structures can be nested in each other.

src/plugins/output/json/doc/ipfixcol2-json-output.7.rst

@@ -9,8 +9,8 @@ JSON (output plugin)
 :Author: Lukáš Huták ([email protected])
 :Author: Petr Velan ([email protected])
 :Date:   2018-09-20
-:Copyright: Copyright © 2018 CESNET, z.s.p.o.
-:Version: 2.0
+:Copyright: Copyright © 2019 CESNET, z.s.p.o.
+:Version: 2.1
 :Manual section: 7
 :Manual group: IPFIXcol collector
 

src/plugins/output/json/src/Config.cpp

@@ -57,6 +57,8 @@ enum params_xml_nodes {
     FMT_UNKNOWN,       /**< Unknown definitions             */
     FMT_OPTIONS,       /**< Ignore Options Template Records */
     FMT_NONPRINT,      /**< Non-printable chars             */
+    FMT_NUMERIC,       /**< Use numeric names               */
+    FMT_BFSPLIT,       /**< Split biflow                    */
     // Common output
     OUTPUT_LIST,       /**< List of output types            */
     OUTPUT_PRINT,      /**< Print to standard output        */
@@ -135,6 +137,8 @@ static const struct fds_xml_args args_params[] = {
     FDS_OPTS_ELEM(FMT_UNKNOWN,   "ignoreUnknown",    FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_ELEM(FMT_OPTIONS,   "ignoreOptions",    FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_ELEM(FMT_NONPRINT,  "nonPrintableChar", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
+    FDS_OPTS_ELEM(FMT_NUMERIC,   "numericNames",     FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
+    FDS_OPTS_ELEM(FMT_BFSPLIT,   "splitBiflow",      FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT),
     FDS_OPTS_NESTED(OUTPUT_LIST, "outputs",   args_outputs, 0),
     FDS_OPTS_END
 };
@@ -445,6 +449,14 @@ Config::parse_params(fds_xml_ctx_t *params)
             assert(content->type == FDS_OPTS_T_BOOL);
             format.white_spaces = content->val_bool;
             break;
+        case FMT_NUMERIC:  // Use only numeric identifiers
+            assert(content->type == FDS_OPTS_T_BOOL);
+            format.numeric_names = content->val_bool;
+            break;
+        case FMT_BFSPLIT:  // Split biflow records
+            assert(content->type == FDS_OPTS_T_BOOL);
+            format.split_biflow = content->val_bool;
+            break;
         case OUTPUT_LIST: // List of output plugin
             assert(content->type == FDS_OPTS_T_CONTEXT);
             parse_outputs(content->ptr_ctx);
@@ -467,6 +479,8 @@ Config::default_set()
     format.white_spaces = true;
     format.ignore_unknown = true;
     format.ignore_options = true;
+    format.numeric_names = false;
+    format.split_biflow = false;
 
     outputs.prints.clear();
     outputs.files.clear();

src/plugins/output/json/src/Config.hpp

@@ -60,6 +60,10 @@ struct cfg_format {
     bool white_spaces;
     /** Ignore Options Template records                                                          */
     bool ignore_options;
+    /** Use only numeric identifiers of Information Elements                                     */
+    bool numeric_names;
+    /** Split biflow records                                                                     */
+    bool split_biflow;
 };
 
 /** Output configuration base structure                                                          */