fdsdump: flowProvider: define component

Author: Lukas Hutak

src/tools/fdsdump/src/flowProvider.cpp

@@ -0,0 +1,227 @@
+
+#include <new>
+#include <stdexcept>
+#include <iostream>
+
+#include "fieldView.hpp"
+#include "flowProvider.hpp"
+
+FlowProvider::FlowProvider(const shared_iemgr &iemgr)
+    : m_iemgr(iemgr)
+{
+    int ret;
+
+    m_file.reset(fds_file_init());
+    if (!m_file) {
+        throw std::runtime_error("fds_file_init() has failed");
+    }
+
+    ret = fds_file_set_iemgr(m_file.get(), m_iemgr.get());
+    if (ret != FDS_OK) {
+        throw std::runtime_error("fds_file_set_iemgr() has failed: " + std::to_string(ret));
+    }
+}
+
+void
+FlowProvider::add_file(const std::string &file)
+{
+    m_remains.push_back(file);
+}
+
+void
+FlowProvider::set_filter(const std::string &expr)
+{
+    fds_ipfix_filter_t *filter;
+    int ret;
+
+    ret = fds_ipfix_filter_create(&filter, m_iemgr.get(), expr.c_str());
+    m_filter.reset(filter);
+
+    if (ret != FDS_OK) {
+        const std::string err_msg = fds_ipfix_filter_get_error(filter);
+        throw std::runtime_error("fds_ipfix_filter_create() has failed: " + err_msg);
+    }
+}
+
+void
+FlowProvider::set_biflow_autoignore(bool enable)
+{
+    m_biflow_autoignore = enable;
+}
+
+bool
+FlowProvider::prepare_next_file()
+{
+    while (true) {
+        const std::string *next_file;
+        int ret;
+
+        // Check if there are more files to read...
+        if (m_remains.empty()) {
+            return false;
+        }
+
+        next_file = &m_remains.front();
+
+        // Try to open it
+        ret = fds_file_open(m_file.get(), next_file->c_str(), FDS_FILE_READ);
+        if (ret != FDS_OK) {
+            const std::string err_msg = fds_file_error(m_file.get());
+
+            std::cerr << "fds_file_open('" << *next_file << "') failed: " << err_msg << std::endl;
+            m_remains.pop_front();
+            continue;
+        }
+
+        m_remains.pop_front();
+        return true;
+    }
+}
+
+bool
+FlowProvider::prepare_next_record()
+{
+    int ret;
+
+    ret = fds_file_read_rec(m_file.get(), &m_flow.rec, NULL);
+    switch (ret) {
+    case FDS_OK:
+        return true;
+    case FDS_EOC:
+        return false;
+    default:
+        throw std::runtime_error("fds_file_read_rec() has failed: " + std::to_string(ret));
+    }
+}
+
+enum Direction
+FlowProvider::filter_record(struct fds_drec *rec)
+{
+    const fds_template_flag_t tflags = rec->tmplt->flags;
+    const bool is_biflow = (tflags & FDS_TEMPLATE_BIFLOW) != 0;
+
+    if (!m_filter) {
+        // No filter defined, match it
+        return is_biflow ? DIRECTION_BOTH : DIRECTION_FWD;
+    }
+
+    if (is_biflow) {
+        switch (fds_ipfix_filter_eval_biflow(m_filter.get(), rec)) {
+        case FDS_IPFIX_FILTER_NO_MATCH:
+            return DIRECTION_NONE;
+        case FDS_IPFIX_FILTER_MATCH_FWD:
+            return DIRECTION_FWD;
+        case FDS_IPFIX_FILTER_MATCH_REV:
+            return DIRECTION_REV;
+        case FDS_IPFIX_FILTER_MATCH_BOTH:
+            return DIRECTION_BOTH;
+        }
+
+        return DIRECTION_NONE;
+    } else {
+        return (fds_ipfix_filter_eval(m_filter.get(), rec))
+            ? DIRECTION_FWD
+            : DIRECTION_NONE;
+    }
+}
+
+bool
+FlowProvider::field_has_only_zero_value(struct fds_drec *rec, uint16_t id, bool reverse)
+{
+    const uint16_t flags = reverse ? FDS_DREC_BIFLOW_REV : FDS_DREC_BIFLOW_FWD;
+    struct fds_drec_iter iter;
+    int found = false;
+
+    fds_drec_iter_init(&iter, rec, flags);
+
+    while (fds_drec_iter_find(&iter, 0, id) != FDS_EOC) {
+        FieldView field(iter.field);
+
+        found = true;
+
+        if (field.as_uint() != 0) {
+            return false;
+        }
+    }
+
+    return found;
+}
+
+bool
+FlowProvider::direction_is_empty(struct fds_drec *rec, bool reverse)
+{
+    const uint16_t IPFIX_OCTET_DELTA = 1;
+    const uint16_t IPFIX_PACKET_DELTA = 2;
+
+    if (!field_has_only_zero_value(rec, IPFIX_OCTET_DELTA, reverse)) {
+        return false;
+    }
+
+    if (!field_has_only_zero_value(rec, IPFIX_PACKET_DELTA, reverse)) {
+        return false;
+    }
+
+    return true;
+}
+
+enum Direction
+FlowProvider::biflow_autoignore(struct fds_drec *rec)
+{
+    const fds_template_flag_t tflags = rec->tmplt->flags;
+    const bool is_biflow = (tflags & FDS_TEMPLATE_BIFLOW) != 0;
+    int result = 0;
+
+    if (!m_biflow_autoignore) {
+        // Disabled
+        return is_biflow ? DIRECTION_BOTH : DIRECTION_FWD;
+    }
+
+    if (!is_biflow) {
+        return DIRECTION_FWD;
+    }
+
+    if (!direction_is_empty(rec, false)) {
+        result |= DIRECTION_FWD;
+    }
+
+    if (!direction_is_empty(rec, true)) {
+        result |= DIRECTION_REV;
+    }
+
+    return static_cast<enum Direction>(result);
+}
+
+Flow *
+FlowProvider::next_record()
+{
+    int dir;
+
+    while (true) {
+        if (!m_file_ready) {
+            if (!prepare_next_file()) {
+                // No more file, no more records
+                return nullptr;
+            }
+
+            m_file_ready = true;
+        }
+
+        if (!prepare_next_record()) {
+            m_file_ready = false;
+            continue;
+        }
+
+        dir = filter_record(&m_flow.rec);
+        if (dir == DIRECTION_NONE) {
+            continue;
+        }
+
+        dir &= biflow_autoignore(&m_flow.rec);
+        if (dir == DIRECTION_NONE) {
+            continue;
+        }
+
+        m_flow.dir = static_cast<enum Direction>(dir);
+        return &m_flow;
+    }
+}

src/tools/fdsdump/src/flowProvider.hpp

@@ -0,0 +1,73 @@
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <list>
+
+#include <libfds.h>
+
+#include "common.hpp" // unique_file, unique_iemgr
+#include "flow.hpp"
+
+class FlowProvider {
+public:
+    FlowProvider(const shared_iemgr &iemgr);
+    ~FlowProvider() = default;
+
+    FlowProvider(const FlowProvider &) = delete;
+    FlowProvider &operator=(const FlowProvider &) = delete;
+
+    /**
+     * @brief Add a new file to process
+     * @param[in] file Path to the file
+     */
+    void
+    add_file(const std::string &file);
+
+    /**
+     * @brief Set a flow filter.
+     * @param[in] expr Filtration expression.
+     */
+    void
+    set_filter(const std::string &expr);
+
+    /**
+     * @brief Enable heuristic that automatically hides specific direction of
+     * biflow record that seems to be empty.
+     *
+     * Some expoters migth send uniflow records as biflow records but we usually
+     * don't want to show these values to a user. If enabled, the <em>match</em>
+     * variable of Record is modified to hide direction that seems to be empty
+     * even if it matches the filter.
+     * @param[in] enable True/false
+     */
+    void
+    set_biflow_autoignore(bool enable);
+
+    /**
+     * @brief Get the next flow record
+     * @return Pointer or NULL (no more records to process)
+     */
+    Flow *
+    next_record();
+
+private:
+    bool prepare_next_file();
+    bool prepare_next_record();
+    enum Direction filter_record(struct fds_drec *rec);
+    enum Direction biflow_autoignore(struct fds_drec *rec);
+
+    bool field_has_only_zero_value(struct fds_drec *rec, uint16_t id, bool reverse);
+    bool direction_is_empty(struct fds_drec *rec, bool reverse);
+
+    std::list<std::string> m_remains;
+
+    shared_iemgr m_iemgr;
+    unique_filter m_filter {nullptr, &fds_ipfix_filter_destroy};
+    unique_file m_file {nullptr, &fds_file_close};
+    bool m_file_ready = false;
+    bool m_biflow_autoignore = false;
+
+    Flow m_flow;
+};