UniRec output plugin: added internal implementation of link/dir_bit_field

Author: Lukas955

extra_plugins/output/unirec/README.rst

@@ -242,12 +242,15 @@ discarded. Otherwise, the value is saturated.
 To see conversion warnings, add the UniRec field to ``<uniRecFormat>`` and run the collector
 with increased verbosity level i.e. ``ipfixcol2 -v``.
 
-Note
-----
+Notes
+-----
 
 Bidirectional flows are not currently supported by UniRec, therefore, biflow records are
 automatically split into two unidirectional flow records during conversion.
 
 When multiple IPFIX Information Elements are mapped to the same UniRec field and those IPFIX fields
 are present in an IPFIX record, the last field occurrence (in the appropriate IPFIX Template)
 is converted to the UniRec field.
+
+TODO: describe "link_bit_field" + "dir_bit_field"
+

extra_plugins/output/unirec/config/unirec-elements.txt

@@ -10,6 +10,10 @@
 #      ("eXXidYY" where XX is Private Enterprise Number and YY is field ID)
 #
 # See ipfixcol2-unirec-output(1) for details
+#
+# Note:
+#   IPFIX IEs "_internal_dbf_" and "_internal_lbf_" represents internal
+#   conversion functions of UniRec plugin.
 
 #UNIREC NAME                   UNIREC TYPE   IPFIX IEs        DESCRIPTION
 # --- Basic fields ---
@@ -21,12 +25,12 @@ PROTOCOL                       uint8         e0id4            # Transport protoc
 TCP_FLAGS                      uint8         e0id6            # TCP flags
 BYTES                          uint64        e0id1            # Number of bytes in flow
 PACKETS                        uint32        e0id2            # Number of packets in flow
-TIME_FIRST                     time          e0id152          # Time of the first packet of a flow
-TIME_LAST                      time          e0id153          # Time of the last packet of a flow
-DIR_BIT_FIELD                  uint8         e0id10           # Bit field used for determining incoming/outgoing flow (1 => Incoming, 0 => Outgoing)
-LINK_BIT_FIELD                 uint64        e0id405          # Bit field of links on which was flow seen
 TTL                            uint8         e0id192          # IP time to live
 TOS                            uint8         e0id5            # IP type of service
+TIME_FIRST                     time          e0id150,e0id152,e0id154,e0id156 # Time of the first packet of a flow
+TIME_LAST                      time          e0id151,e0id153,e0id155,e0id157 # Time of the last packet of a flow
+DIR_BIT_FIELD                  uint8         _internal_dbf_   # Bit field used for determining incoming/outgoing flow (1 => Incoming, 0 => Outgoing)
+LINK_BIT_FIELD                 uint64        _internal_lbf_   # Bit field of links on which was flow seen
 
 # --- DNS specific fields ---
 DNS_ANSWERS                    uint16        e8057id0         # DNS answers

extra_plugins/output/unirec/src/map.c

@@ -189,7 +189,7 @@ map_rec_add(map_t *map, const struct map_rec *rec)
  * \return Pointer to the definition or NULL (unknown or invalid IE specifier)
  */
 static const struct fds_iemgr_elem *
-map_elem_get(const fds_iemgr_t *mgr, const char *elem)
+map_elem_get_ipfix(const fds_iemgr_t *mgr, const char *elem)
 {
     const struct fds_iemgr_elem *res;
 
@@ -211,6 +211,23 @@ map_elem_get(const fds_iemgr_t *mgr, const char *elem)
     return fds_iemgr_elem_find_id(mgr, ipfix_en, ipfix_id);
 }
 
+/**
+ * \brief Get the type of internal function
+ * \param[in] elem Identification of an internal function to find
+ * \return Function identifier or ::MAP_SRC_INVALID in case or failure
+ */
+static enum MAP_SRC
+map_elem_get_internal(const char *elem)
+{
+    if (strcmp(elem, "_internal_lbf_") == 0) {
+        return MAP_SRC_INTERNAL_LBF;
+    } else if (strcmp(elem, "_internal_dbf_") == 0) {
+        return MAP_SRC_INTERNAL_DBF;
+    } else {
+        return MAP_SRC_INVALID;
+    }
+}
+
 /**
  * \brief Parse list of IPFIX IE and add mapping records to the database
  * \param[in] map         Mapping database
@@ -259,20 +276,33 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str,
         }
 
         // Parse IPFIX specifier
-        const struct fds_iemgr_elem *elem_def = map_elem_get(map->iemgr, subtoken);
-        if (!elem_def) {
-            snprintf(map->err_buffer, ERR_SIZE, "Line %zu: IPFIX specifier '%s' is invalid or "
-                "a definition of the Information Element is missing! For more information, see "
-                "the plugin documentation.", line_id, subtoken);
-            rc = IPX_ERR_FORMAT;
-            break;
+        const struct fds_iemgr_elem *elem_def = map_elem_get_ipfix(map->iemgr, subtoken);
+        if (elem_def != NULL) {
+            // Store the "IPFIX element" record
+            rec.ipfix.source = MAP_SRC_IPFIX;
+            rec.ipfix.def = elem_def;
+            rec.ipfix.id = elem_def->id;
+            rec.ipfix.en = elem_def->scope->pen;
+            rc = map_rec_add(map, &rec);
+            continue;
+        }
+
+        enum MAP_SRC fn_id = map_elem_get_internal(subtoken);
+        if (fn_id != MAP_SRC_INVALID) {
+            // Store the "Internal function" record
+            rec.ipfix.source = fn_id;
+            rec.ipfix.def = NULL;
+            rec.ipfix.id = 0;
+            rec.ipfix.en = 0;
+            rc = map_rec_add(map, &rec);
+            continue;
         }
 
-        // Store the record
-        rec.ipfix.def = elem_def;
-        rec.ipfix.id = elem_def->id;
-        rec.ipfix.en = elem_def->scope->pen;
-        rc = map_rec_add(map, &rec);
+        snprintf(map->err_buffer, ERR_SIZE, "Line %zu: IPFIX specifier '%s' is invalid or "
+            "a definition of the Information Element is missing! For more information, see "
+            "the plugin documentation.", line_id, subtoken);
+        rc = IPX_ERR_FORMAT;
+        break;
     }
 
     free(defs_cpy);
@@ -374,6 +404,10 @@ map_sort_fn(const void *p1, const void *p2)
     struct map_rec *rec1 = *(struct map_rec **) p1;
     struct map_rec *rec2 = *(struct map_rec **) p2;
 
+    if (rec1->ipfix.source != rec2->ipfix.source) {
+        return (rec1->ipfix.source < rec2->ipfix.source) ? (-1) : 1;
+    }
+
     // Primary sort by PEN
     if (rec1->ipfix.en != rec2->ipfix.en) {
         return (rec1->ipfix.en < rec2->ipfix.en) ? (-1) : 1;
@@ -455,6 +489,11 @@ map_load(map_t *map, const char *file)
     for (size_t i = 1; i < map->rec_size; i++) {
         // Records are sorted, therefore, the same IPFIX IEs should be next to each other
         const struct map_rec *rec_now = map->rec_array[i];
+        if (rec_prev->ipfix.source != MAP_SRC_IPFIX || rec_now->ipfix.source != MAP_SRC_IPFIX) {
+            rec_prev = rec_now;
+            continue;
+        }
+
         if (rec_prev->ipfix.en != rec_now->ipfix.en || rec_prev->ipfix.id != rec_now->ipfix.id) {
             rec_prev = rec_now;
             continue;

extra_plugins/output/unirec/src/map.h

@@ -49,9 +49,27 @@
 /** Internal map type */
 typedef struct map_s map_t;
 
+/** Source field                                      */
+enum MAP_SRC {
+    /** Invalid (internal value)                      */
+    MAP_SRC_INVALID,
+    /** IPFIX field                                   */
+    MAP_SRC_IPFIX,
+    /** Internal "link bit field" converter           */
+    MAP_SRC_INTERNAL_LBF,
+    /** Internal "dir bit field" converter            */
+    MAP_SRC_INTERNAL_DBF
+};
+
 /** IPFIX-to-UniRec mapping record                    */
 struct map_rec {
     struct {
+        /**
+         * \brief Data source
+         * \note If the field is not ::MAP_SRC_IPFIX, parameters en, id and def are NOT defined!
+         */
+        enum MAP_SRC source;
+
         /** Private Enterprise Number                 */
         uint32_t en;
         /** Information Element ID                    */