Merge branch 'fds-output-selection' into 'devel'

FDS Output: Add support for selecting only a subset of fields to be saved

See merge request monitoring/ipfixcol2!17

Author: sedmicha

src/plugins/output/fds/README.rst

@@ -28,6 +28,21 @@ Example configuration
                 <timeWindow>300</timeWindow>
                 <align>yes</align>
             </dumpInterval>
+            <outputSelection>
+                <element>iana:octetDeltaCount</element>
+                <element>iana@reverse:octetDeltaCount@reverse</element>
+                <element>iana:packetDeltaCount</element>
+                <element>iana@reverse:packetDeltaCount@reverse</element>
+                <element>iana:destinationIPv4Address</element>
+                <element>iana:destinationIPv6Address</element>
+                <element>iana:sourceIPv4Address</element>
+                <element>iana:sourceIPv6Address</element>
+                <element>iana:destinationTransportPort</element>
+                <element>iana:sourceTransportPort</element>
+                <element>iana:protocolIdentifier</element>
+                <element>cesnet:quicSNI</element>
+                <element>cesnet:TLSSNI</element>
+            </outputSelection>
         </params>
     </output>
 
@@ -62,6 +77,20 @@ Parameters
         and window size is 5 minutes long, files will be created at 0, 5, 10, etc.
         [values: yes/no, default: yes]
 
+:``outputSelection``:
+    Select only a subset of fields that will be saved in output. A list of IPFIX
+    elements is provided. Any fields of an element that are not a part of this list
+    will be skipped and not included in the resulting FDS output file.
+    Note that when
+    [default: all fields]
+
+    :``element``:
+        The identifier of the IPFIX element in the form of ``SCOPE_NAME:ELEMENT_NAME``,
+        for example ``iana:sourceIPv4Address``. ``SCOPE_NAME:`` prefix can be omitted in
+        case of "iana:".
+        Note: Reverse elements use the following naming pattern
+        ``iana@reverse:sourceIPv4Address@reverse``
+
 :``asyncIO``:
     Allows to use asynchronous I/O for writing to the file. Usually when parts
     of the file are being written, the process is blocked on synchronous I/O

src/plugins/output/fds/src/Config.cpp

@@ -1,6 +1,7 @@
 /**
  * \file src/plugins/output/fds/src/Config.cpp
  * \author Lukas Hutak <[email protected]>
+ * \author Michal Sedlak <[email protected]>
  * \brief Parser of XML configuration (source file)
  * \date 2019
  *
@@ -21,6 +22,11 @@
  *     <align>...</align>                 <!-- optional -->
  *   </dumpInterval>
  *   <asyncIO>...</asyncIO>               <!-- optional -->
+ *   <outputSelection>                    <!-- optional -->
+ *     <element>...</element>
+ *     <element>...</element>
+ *     ...
+ *   </outputSelection>
  * </params>
  */
 
@@ -30,9 +36,12 @@ enum params_xml_nodes {
     NODE_COMPRESS,
     NODE_DUMP,
     NODE_ASYNCIO,
+    NODE_SELECTION,
 
     DUMP_WINDOW,
-    DUMP_ALIGN
+    DUMP_ALIGN,
+
+    SELECTION_ELEMENT,
 };
 
 /// Definition of the \<dumpInterval\> node
@@ -42,17 +51,24 @@ static const struct fds_xml_args args_dump[] = {
     FDS_OPTS_END
 };
 
+/// Definition of the \<outputSelection\> node
+static const struct fds_xml_args args_selection[] = {
+    FDS_OPTS_ELEM(SELECTION_ELEMENT,   "element",      FDS_OPTS_T_STRING, FDS_OPTS_P_MULTI),
+    FDS_OPTS_END
+};
+
 /// Definition of the \<params\> node
 static const struct fds_xml_args args_params[] = {
     FDS_OPTS_ROOT("params"),
-    FDS_OPTS_ELEM(NODE_STORAGE,  "storagePath",        FDS_OPTS_T_STRING, 0),
-    FDS_OPTS_ELEM(NODE_COMPRESS, "compression",        FDS_OPTS_T_STRING, FDS_OPTS_P_OPT),
-    FDS_OPTS_NESTED(NODE_DUMP,   "dumpInterval",       args_dump,         FDS_OPTS_P_OPT),
-    FDS_OPTS_ELEM(NODE_ASYNCIO,  "asyncIO",            FDS_OPTS_T_BOOL,   FDS_OPTS_P_OPT),
+    FDS_OPTS_ELEM(NODE_STORAGE,     "storagePath",        FDS_OPTS_T_STRING, 0),
+    FDS_OPTS_ELEM(NODE_COMPRESS,    "compression",        FDS_OPTS_T_STRING, FDS_OPTS_P_OPT),
+    FDS_OPTS_NESTED(NODE_DUMP,      "dumpInterval",       args_dump,         FDS_OPTS_P_OPT),
+    FDS_OPTS_ELEM(NODE_ASYNCIO,     "asyncIO",            FDS_OPTS_T_BOOL,   FDS_OPTS_P_OPT),
+    FDS_OPTS_NESTED(NODE_SELECTION, "outputSelection",    args_selection,    FDS_OPTS_P_OPT),
     FDS_OPTS_END
 };
 
-Config::Config(const char *params)
+Config::Config(const char *params, const fds_iemgr_t *iemgr)
 {
     set_default();
 
@@ -74,7 +90,7 @@ Config::Config(const char *params)
 
     // Parse parameters and check configuration
     try {
-        parse_root(params_ctx);
+        parse_root(params_ctx, iemgr);
         validate();
     } catch (std::exception &ex) {
         throw std::runtime_error("Failed to parse the configuration: " + std::string(ex.what()));
@@ -93,6 +109,9 @@ Config::set_default()
 
     m_window.align = true;
     m_window.size = WINDOW_SIZE;
+
+    m_selection_used = false;
+    m_selection.clear();
 }
 
 /**
@@ -114,10 +133,11 @@ Config::validate()
 /**
  * @brief Process \<params\> node
  * @param[in] ctx XML context to process
+ * @param[in] iemgr Information elements manager
  * @throw runtime_error if the parser fails
  */
 void
-Config::parse_root(fds_xml_ctx_t *ctx)
+Config::parse_root(fds_xml_ctx_t *ctx, const fds_iemgr_t *iemgr)
 {
     const struct fds_xml_cont *content;
     while (fds_xml_next(ctx, &content) != FDS_EOC) {
@@ -151,6 +171,11 @@ Config::parse_root(fds_xml_ctx_t *ctx)
             assert(content->type == FDS_OPTS_T_CONTEXT);
             parse_dump(content->ptr_ctx);
             break;
+        case NODE_SELECTION:
+            // Output selection
+            assert(content->type == FDS_OPTS_T_CONTEXT);
+            parse_selection(content->ptr_ctx, iemgr);
+            break;
         default:
             // Internal error
             throw std::runtime_error("Unknown XML node");
@@ -187,4 +212,38 @@ Config::parse_dump(fds_xml_ctx_t *ctx)
             throw std::runtime_error("Unknown XML node");
         }
     }
-}
+}
+
+/**
+ * @brief Auxiliary function for parsing \<outputSelection\> options
+ * @param[in] ctx XML context to process
+ * @param[in] iemgr Information elements manager
+ * @throw runtime_error if the parser fails
+ */
+void
+Config::parse_selection(fds_xml_ctx_t *ctx, const fds_iemgr_t* iemgr)
+{
+    m_selection_used = true;
+
+    const struct fds_xml_cont *content;
+    while(fds_xml_next(ctx, &content) != FDS_EOC) {
+        switch (content->id) {
+        case SELECTION_ELEMENT: {
+            // IPFIX element to select
+            assert(content->type == FDS_OPTS_T_STRING);
+
+            const fds_iemgr_elem* ie = fds_iemgr_elem_find_name(iemgr, content->ptr_string);
+            if (!ie) {
+                throw std::runtime_error("Element \"" + std::string(content->ptr_string) + "\" not found!");
+            }
+            element elem;
+            elem.pen = ie->scope->pen;
+            elem.id = ie->id;
+            m_selection.push_back(elem);
+        } break;
+        default:
+            // Internal error
+            throw std::runtime_error("Unknown XML node");
+        }
+    }
+}

src/plugins/output/fds/src/Config.hpp

@@ -1,6 +1,7 @@
 /**
  * \file src/plugins/output/fds/src/Config.hpp
  * \author Lukas Hutak <[email protected]>
+ * \author Michal Sedlak <[email protected]>
  * \brief Parser of XML configuration (header file)
  * \date 2019
  *
@@ -12,6 +13,7 @@
 #define IPFIXCOL2_FDS_CONFIG_HPP
 
 #include <string>
+#include <vector>
 #include <libfds.h>
 
 /**
@@ -22,9 +24,10 @@ class Config {
     /**
      * @brief Parse configuration of the plugin
      * @param[in] params XML parameters to parse
+     * @param[in] iemgr Information elements manager
      * @throw runtime_exception on error
      */
-    Config(const char *params);
+    Config(const char *params, const fds_iemgr_t *iemgr);
     ~Config() = default;
 
     enum class calg {
@@ -45,6 +48,13 @@ class Config {
         uint32_t size;    ///< Time window size
     } m_window;   ///< Window alignment
 
+    struct element {
+        uint32_t pen;
+        uint16_t id;
+    };
+    bool m_selection_used;
+    std::vector<element> m_selection;
+
 private:
     /// Default window size
     static const uint32_t WINDOW_SIZE = 300U;
@@ -55,9 +65,11 @@ class Config {
     validate();
 
     void
-    parse_root(fds_xml_ctx_t *ctx);
+    parse_root(fds_xml_ctx_t *ctx, const fds_iemgr_t *iemgr);
     void
     parse_dump(fds_xml_ctx_t *ctx);
+    void
+    parse_selection(fds_xml_ctx_t *ctx, const fds_iemgr_t *iemgr);
 };