Merge branch 'unirec-exporter-odid-ip' into 'master'

Extend UniRec output with ODID and EXPORTER_IP

See merge request monitoring/ipfixcol2!10

Author: sedmicha

extra_plugins/output/unirec/README.rst

@@ -270,3 +270,7 @@ and egress flows. For this reason, the lowest bit of IPFIX field ``iana:ingressI
 (PEN: 0, ID: 10) is copied to the particular UniRec field (Nemea modules expects that
 1 = ingress flow and 0 = egress flow).
 
+Other UniRec specific fields available are ``ODID`` and ``EXPORTER_IP``. The information for these
+fields is extracted from the IPFIX message session. The ``ODID`` contains raw Observation Domain ID
+of the exporter and ``EXPORTER_IP`` carries exporter IP address to identify which probe the flow
+was observed on.

extra_plugins/output/unirec/config/unirec-elements.txt

@@ -36,6 +36,8 @@ TIME_FIRST                     time          e0id150,e0id152,e0id154,e0id156
 TIME_LAST                      time          e0id151,e0id153,e0id155,e0id157    # Time of the last packet of a flow
 DIR_BIT_FIELD                  uint8         _internal_dbf_                     # Bit field used for determining incoming/outgoing flow (1 => Incoming, 0 => Outgoing)
 LINK_BIT_FIELD                 uint64        _internal_lbf_                     # Bit field of links on which was flow seen
+ODID                           uint32        _internal_odid_                    # Observation Domain ID (ODID) of exporter
+EXPORTER_IP                    ipaddr        _internal_exporter_ip_             # IP address of exporter from exporting session
 
 SRC_MAC                        macaddr       e0id56
 DST_MAC                        macaddr       e0id80

extra_plugins/output/unirec/src/map.c

@@ -232,6 +232,10 @@ map_elem_get_internal(const char *elem)
         return MAP_SRC_INTERNAL_LBF;
     } else if (strcmp(elem, "_internal_dbf_") == 0) {
         return MAP_SRC_INTERNAL_DBF;
+    } else if (strcmp(elem, "_internal_odid_") == 0) {
+        return MAP_SRC_INTERNAL_ODID;
+    } else if (strcmp(elem, "_internal_exporter_ip_") == 0) {
+        return MAP_SRC_INTERNAL_EXPORTER_IP;
     } else {
         return MAP_SRC_INVALID;
     }

extra_plugins/output/unirec/src/map.h

@@ -58,7 +58,11 @@ enum MAP_SRC {
     /** Internal "link bit field" converter           */
     MAP_SRC_INTERNAL_LBF,
     /** Internal "dir bit field" converter            */
-    MAP_SRC_INTERNAL_DBF
+    MAP_SRC_INTERNAL_DBF,
+    /** Internal "odid field" converter               */
+    MAP_SRC_INTERNAL_ODID,
+    /** Internal "exporter ip field" converter        */
+    MAP_SRC_INTERNAL_EXPORTER_IP
 };
 
 enum MAP_FLAGS {

extra_plugins/output/unirec/src/translator.c

@@ -137,8 +137,8 @@ struct translator_s {
     } progress; /**< Auxiliary conversion variables                     */
 
     struct {
-        /** IPFIX Message header                                        */
-        const struct fds_ipfix_msg_hdr *hdr;
+        /** IPFIX Message context                                        */
+        const struct ipx_msg_ctx *ctx;
     } msg_context; /**< IPFIX context of the record to translate        */
 
     struct {
@@ -151,6 +151,18 @@ struct translator_s {
             int  field_size;            /**< UniRec field size          */
             ur_field_id_t field_id;     /**< UniRec field ID            */
         } lbf; /**< ODID to 'link bit field' converter                  */
+        struct
+        {
+            bool en;                    /**< Enabled/disabled           */
+            int  req_idx;               /**< Index in the template      */
+            ur_field_id_t field_id;     /**< UniRec field ID            */
+        } odid; /**< ODID from IPFIX Message header                     */
+        struct
+        {
+            bool en;                    /**< Enabled/disabled           */
+            int  req_idx;               /**< Index in the template      */
+            ur_field_id_t field_id;     /**< UniRec field ID            */
+        } exporter_ip; /**< Exporter IP address from message session    */
     } extra_conv; /**< Special internal conversion functions            */
 };
 
@@ -802,12 +814,12 @@ translate_internal_dbf(translator_t *trans, const struct translator_rec *rec,
 static int
 translate_internal_lbf(translator_t *trans)
 {
-    if (!trans->msg_context.hdr) {
-        return 1; // Message header is not available!
+    if (!trans->msg_context.ctx) {
+        return 1; // Message context is not available!
     }
 
     // Get the ODID value
-    uint32_t odid = ntohl(trans->msg_context.hdr->odid);
+    uint32_t odid = trans->msg_context.ctx->odid;
 
     // Store the value is 'link bit field'
     ur_field_type_t ur_id = trans->extra_conv.lbf.field_id;
@@ -833,6 +845,83 @@ translate_internal_lbf(translator_t *trans)
     return 0;
 }
 
+/**
+ * \brief Convert (fill) ODID field
+ *
+ * \note This function uses a message context configured by the user. It's independent on the
+ *   content of an IPFIX record.
+ * \param[in] trans Internal translator structure
+ * \return On success returns 0. Otherwise returns non-zero value!
+ */
+static int
+translate_internal_odid(translator_t *trans)
+{
+    if (!trans->msg_context.ctx) {
+        return 1; // Message context is not available!
+    }
+
+    // Get the ODID value
+    uint32_t odid = trans->msg_context.ctx->odid;
+
+    ur_field_type_t ur_id = trans->extra_conv.odid.field_id;
+    void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id);
+
+    *((uint32_t *) field_ptr) = odid;
+
+    return 0;
+}
+
+/**
+ * \brief Convert (fill) EXPORTER_IP field with IPv4 or IPv6 address from IPFIX
+ * message session.
+ *
+ * \note This function uses a message context configured by the user. It's independent on the
+ *   content of an IPFIX record.
+ * \param[in] trans Internal translator structure
+ * \return On success returns 0. Otherwise returns non-zero value!
+ */
+static int
+translate_internal_exporter_ip(translator_t *trans)
+{
+    if (!trans->msg_context.ctx) {
+        return 1; // Message context is not available!
+    }
+
+    const struct ipx_session *session = trans->msg_context.ctx->session;
+
+    const struct ipx_session_net *session_net;
+    switch (session->type)
+    {
+    case FDS_SESSION_UDP:
+        session_net = &session->udp.net;
+        break;
+    case FDS_SESSION_TCP:
+        session_net = &session->tcp.net;
+        break;
+    case FDS_SESSION_SCTP:
+        session_net = &session->sctp.net;
+        break;
+    default:
+        // Uncompatible type of message session. E.g. messages from file.
+        return 1;
+    }
+
+    // Get pointer to save result IP
+    ur_field_type_t ur_id = trans->extra_conv.exporter_ip.field_id;
+    void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id);
+
+    if (session->udp.net.l3_proto == AF_INET) {
+        *((ip_addr_t *) field_ptr) = ip_from_4_bytes_be((char *) &session_net->addr_src.ipv4.s_addr);
+    } else if (session->udp.net.l3_proto == AF_INET6) {
+        *((ip_addr_t *) field_ptr) = ip_from_16_bytes_be((char *) session_net->addr_src.ipv6.s6_addr);
+    } else {
+        // Unknown IP protocol
+        return 1;
+    }
+
+    return 0;
+}
+
 /**
  * \brief Get size of UniRec numeric data types
  * \param type UniRec data types
@@ -1451,6 +1540,62 @@ translator_table_fill_internal(translator_t *trans, const struct map_rec *map_re
         return IPX_OK;
     }
 
+    /* Internal "ODID field" function
+     * Implemented as a special converter that must be enabled in the translator because
+     * ODID is not defined in the IPFIX record but in the IPFIX Message header
+     */
+    if (src == MAP_SRC_INTERNAL_ODID) {
+        if (trans->extra_conv.odid.en) {
+            // The function is already enabled!
+            IPX_CTX_ERROR(trans->ctx, "Internal 'odid field' function can be mapped only "
+                "to one UniRec field!", '\0');
+            return IPX_ERR_DENIED;
+        }
+
+        if (ur_type != UR_TYPE_UINT32) {
+            IPX_CTX_ERROR(trans->ctx, "Internal 'odid field' function supports only UniRec "
+                "uint32 type but UniRec field '%s' is '%s'!", map_rec->unirec.name,
+                map_rec->unirec.type_str);
+            return IPX_ERR_DENIED;
+        }
+
+        // Enable the internal converter
+        trans->extra_conv.odid.en = true;
+        trans->extra_conv.odid.req_idx = field_idx;
+        trans->extra_conv.odid.field_id = ur_id;
+        IPX_CTX_DEBUG(trans->ctx, "Added conversion from internal 'odid field' to UniRec '%s'",
+            map_rec->unirec.name);
+        return IPX_ERR_NOTFOUND; // Do NOT add the record!
+    }
+
+    /* Internal "exporter_ip field" function
+     * Implemented as a special converter that must be enabled in the translator because
+     * IPFIX Message session is not defined in IPFIX record
+     */
+    if (src == MAP_SRC_INTERNAL_EXPORTER_IP) {
+        if (trans->extra_conv.exporter_ip.en) {
+            // The function is already enabled!
+            IPX_CTX_ERROR(trans->ctx, "Internal 'exporter_ip field' function can be mapped only "
+                "to one UniRec field!", '\0');
+            return IPX_ERR_DENIED;
+        }
+
+        if (ur_type != UR_TYPE_IP) {
+            IPX_CTX_ERROR(trans->ctx, "Internal 'exporter_ip field' function supports only UniRec "
+                "ipaddress type but UniRec field '%s' is '%s'!", map_rec->unirec.name,
+                map_rec->unirec.type_str);
+            return IPX_ERR_DENIED;
+        }
+
+        // Enable the internal converter
+        trans->extra_conv.exporter_ip.en = true;
+        trans->extra_conv.exporter_ip.req_idx = field_idx;
+        trans->extra_conv.exporter_ip.field_id = ur_id;
+        IPX_CTX_DEBUG(trans->ctx, "Added conversion from internal 'exporter_ip field' to UniRec '%s'",
+            map_rec->unirec.name);
+        return IPX_ERR_NOTFOUND; // Do NOT add the record!
+    }
+
     IPX_CTX_ERROR(trans->ctx, "Unimplemented internal mapping function!", '\0');
     return IPX_ERR_DENIED;
 }
@@ -1598,6 +1743,32 @@ translator_call_internals(translator_t *trans)
         }
     }
 
+    if (trans->extra_conv.odid.en) {
+        // Call internal 'odid field' converter
+        int field_idx = trans->extra_conv.odid.req_idx;
+        if (translate_internal_odid(trans) == 0) {
+            // Success
+            trans->progress.req_fields[field_idx] = 0; // Filled!
+            converted_fields++;
+        } else {
+            IPX_CTX_WARNING(trans->ctx, "Internal function 'odid field' failed to fill "
+                "UniRec field '%s'", trans->progress.req_names[field_idx]);
+        }
+    }
+
+    if (trans->extra_conv.exporter_ip.en) {
+        // Call internal 'exporter_ip field' converter
+        int field_idx = trans->extra_conv.exporter_ip.req_idx;
+        if (translate_internal_exporter_ip(trans) == 0) {
+            // Success
+            trans->progress.req_fields[field_idx] = 0; // Filled!
+            converted_fields++;
+        } else {
+            IPX_CTX_WARNING(trans->ctx, "Internal function 'exporter_ip field' failed to fill "
+                "UniRec field '%s'", trans->progress.req_names[field_idx]);
+        }
+    }
+
     return converted_fields;
 }
 
@@ -1634,9 +1805,9 @@ translator_destroy(translator_t *trans)
 }
 
 void
-translator_set_context(translator_t *trans, const struct fds_ipfix_msg_hdr *hdr)
+translator_set_context(translator_t *trans, const struct ipx_msg_ctx *ctx)
 {
-    trans->msg_context.hdr = hdr;
+    trans->msg_context.ctx = ctx;
 }
 
 const void *

extra_plugins/output/unirec/src/translator.h

@@ -78,10 +78,10 @@ translator_destroy(translator_t *trans);
  * This function MUST be called before processing each IPFIX Message to set proper record
  * parameters. The message header is required for determine ODID or other parameters.
  * \param[in] trans Translator instance
- * \param[in] hdr   IPFIX message header
+ * \param[in] ctx   IPFIX message context
  */
 void
-translator_set_context(translator_t *trans, const struct fds_ipfix_msg_hdr *hdr);
+translator_set_context(translator_t *trans, const struct ipx_msg_ctx *ctx);
 
 /**
  * \brief Convert a IPFIX record to an UniRec message

extra_plugins/output/unirec/src/unirecplugin.c

@@ -344,8 +344,8 @@ ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg)
     const void *msg_data = NULL;
 
     ipx_msg_ipfix_t *ipfix = ipx_msg_base2ipfix(msg);
-    const uint8_t *ipfix_raw_msg = ipx_msg_ipfix_get_packet(ipfix);
-    translator_set_context(conf->trans, (const struct fds_ipfix_msg_hdr *) ipfix_raw_msg);
+    const struct ipx_msg_ctx *ipfix_msg_ctx = ipx_msg_ipfix_get_ctx(ipfix);
+    translator_set_context(conf->trans, ipfix_msg_ctx);
 
     const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(ipfix);
     for (uint32_t i = 0; i < rec_cnt; i++) {