fdsdump: introduce file statistics

Author: Lukas955

src/tools/fdsdump/src/CMakeLists.txt

@@ -4,13 +4,15 @@ include_directories("${CMAKE_CURRENT_SOURCE_DIR}")
 add_subdirectory(common)
 add_subdirectory(lister)
 add_subdirectory(aggregator)
+add_subdirectory(statistics)
 
 add_executable(fdsdump
     main.cpp
     options.cpp
     $<TARGET_OBJECTS:common_obj>
 	$<TARGET_OBJECTS:lister_obj>
     $<TARGET_OBJECTS:aggregator_obj>
+    $<TARGET_OBJECTS:statistics_obj>
 )
 
 target_include_directories(fdsdump

src/tools/fdsdump/src/main.cpp

@@ -10,6 +10,7 @@
 #include <common/filelist.hpp>
 #include <lister/lister.hpp>
 #include <aggregator/mode.hpp>
+#include <statistics/mode.hpp>
 
 #include "options.hpp"
 
@@ -55,6 +56,9 @@ main(int argc, char *argv[])
         case Options::Mode::aggregate:
             aggregator::mode_aggregate(iemgr, options);
             break;
+        case Options::Mode::stats:
+            statistics::mode_statistics(iemgr, options);
+            break;
         default:
             throw std::runtime_error("Invalid mode");
         }

src/tools/fdsdump/src/options.cpp

@@ -57,7 +57,7 @@ void Options::parse(int argc, char *argv[])
         {"no-biflow-autoignore", no_argument,       NULL, OPT_BIFLOW_AUTOIGNORE_OFF},
         {0, 0, 0, 0},
     };
-    const char *short_opts = "r:c:o:O:F:A:S:";
+    const char *short_opts = "r:c:o:O:F:A:S:I";
     int opt;
 
     while ((opt = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) {
@@ -80,6 +80,9 @@ void Options::parse(int argc, char *argv[])
         case 'A':
             m_aggregation_keys = optarg;
             break;
+        case 'I':
+            m_mode = Mode::stats;
+            break;
         case 'S':
             m_aggregation_fields = optarg;
             break;
@@ -101,7 +104,13 @@ void Options::parse(int argc, char *argv[])
 
 void Options::validate()
 {
-    if (!m_aggregation_keys.empty()) {
+    if (m_mode == Mode::stats) {
+        // File statistics
+        if (m_output_specifier.empty()) {
+            m_output_specifier = "table";
+        }
+
+    } else if (!m_aggregation_keys.empty()) {
         // Record aggregation
         m_mode = Mode::aggregate;
 

src/tools/fdsdump/src/options.hpp

@@ -31,6 +31,7 @@ class Options {
         undefined,
         list,
         aggregate,
+        stats,
     };
 
     /**

src/tools/fdsdump/src/statistics/CMakeLists.txt

@@ -0,0 +1,9 @@
+# Create a common "statistics" library
+set(STATISTICS_SRC
+    mode.cpp
+    printer.cpp
+    jsonPrinter.cpp
+    tablePrinter.cpp
+)
+
+add_library(statistics_obj OBJECT ${STATISTICS_SRC})

src/tools/fdsdump/src/statistics/jsonPrinter.cpp

@@ -0,0 +1,66 @@
+
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+#include "jsonPrinter.hpp"
+
+namespace fdsdump {
+namespace statistics {
+
+JsonPrinter::JsonPrinter(const std::string &args)
+{
+    if (!args.empty()) {
+        throw std::invalid_argument("JSON output: options are not supported");
+    }
+}
+
+JsonPrinter::~JsonPrinter()
+{
+}
+
+void
+JsonPrinter::print_prologue()
+{
+}
+
+void
+JsonPrinter::print_stats(const fds_file_stats &stats)
+{
+    const uint64_t flows_total = stats.recs_total - stats.recs_opts_total;
+
+    std::cout << "{";
+
+    std::cout << "\"recs_total\":" << stats.recs_total << ",";
+    std::cout << "\"recs_flow_total\":" << flows_total << ",";
+    std::cout << "\"recs_bf_total\":" << stats.recs_bf_total << ",";
+    std::cout << "\"recs_opts_total\":" << stats.recs_opts_total << ",";
+    std::cout << "\"recs_tcp\":" << stats.recs_tcp << ",";
+    std::cout << "\"recs_udp\":" << stats.recs_udp << ",";
+    std::cout << "\"recs_icmp\":" << stats.recs_icmp << ",";
+    std::cout << "\"recs_other\":" << stats.recs_other << ",";
+    std::cout << "\"recs_bf_tcp\":" << stats.recs_bf_tcp << ",";
+    std::cout << "\"recs_bf_udp\":" << stats.recs_bf_udp << ",";
+    std::cout << "\"recs_bf_icmp\":" << stats.recs_bf_icmp << ",";
+    std::cout << "\"recs_bf_other\":" << stats.recs_bf_other << ",";
+    std::cout << "\"pkts_total\":" << stats.pkts_total << ",";
+    std::cout << "\"pkts_tcp\":" << stats.pkts_tcp << ",";
+    std::cout << "\"pkts_udp\":" << stats.pkts_udp << ",";
+    std::cout << "\"pkts_icmp\":" << stats.pkts_icmp << ",";
+    std::cout << "\"pkts_other\":" << stats.pkts_other << ",";
+    std::cout << "\"bytes_total\":" << stats.bytes_total << ",";
+    std::cout << "\"bytes_tcp\":" << stats.bytes_tcp << ",";
+    std::cout << "\"bytes_udp\":" << stats.bytes_udp << ",";
+    std::cout << "\"bytes_icmp\":" << stats.bytes_icmp << ",";
+    std::cout << "\"bytes_other\":" << stats.bytes_other;
+
+    std::cout << "}\n";
+}
+
+void
+JsonPrinter::print_epilogue()
+{
+}
+
+} // statistics
+} // fdsdump

src/tools/fdsdump/src/statistics/jsonPrinter.hpp

@@ -0,0 +1,28 @@
+
+#pragma once
+
+#include <string>
+#include "printer.hpp"
+
+namespace fdsdump {
+namespace statistics {
+
+class JsonPrinter : public Printer {
+public:
+    JsonPrinter(const std::string &args);
+
+    virtual
+    ~JsonPrinter();
+
+    virtual void
+    print_prologue() override;
+
+    virtual void
+    print_stats(const fds_file_stats &stats) override;
+
+    virtual void
+    print_epilogue() override;
+};
+
+} // statistics
+} // fdsdump

src/tools/fdsdump/src/statistics/mode.cpp

@@ -0,0 +1,82 @@
+
+#include <iostream>
+
+#include "common/flowProvider.hpp"
+
+#include "mode.hpp"
+#include "printer.hpp"
+
+namespace fdsdump {
+namespace statistics {
+
+static void stats_merge(
+    fds_file_stats &dst,
+    const fds_file_stats &src)
+{
+    dst.recs_total += src.recs_total;
+    dst.recs_bf_total += src.recs_bf_total;
+    dst.recs_opts_total += src.recs_opts_total;
+    dst.bytes_total += src.bytes_total;
+    dst.pkts_total += src.pkts_total;
+    dst.recs_tcp += src.recs_tcp;
+    dst.recs_udp += src.recs_udp;
+    dst.recs_icmp += src.recs_icmp;
+    dst.recs_other += src.recs_other;
+    dst.recs_bf_tcp += src.recs_bf_tcp;
+    dst.recs_bf_udp += src.recs_bf_udp;
+    dst.recs_bf_icmp += src.recs_bf_icmp;
+    dst.recs_bf_other += src.recs_bf_other;
+    dst.bytes_tcp += src.bytes_tcp;
+    dst.bytes_udp += src.bytes_udp;
+    dst.bytes_icmp += src.bytes_icmp;
+    dst.bytes_other += src.bytes_other;
+    dst.pkts_tcp += src.pkts_tcp;
+    dst.pkts_udp += src.pkts_udp;
+    dst.pkts_icmp += src.pkts_icmp;
+    dst.pkts_other += src.pkts_other;
+}
+
+void
+mode_statistics(const shared_iemgr &iemgr, const Options &opts)
+{
+    auto printer = printer_factory(opts.get_output_specifier());
+    const FileList &file_names = opts.get_input_files();
+    unique_file file {nullptr, &fds_file_close};
+    fds_file_stats stats {};
+
+    file.reset(fds_file_init());
+    if (!file) {
+        throw std::runtime_error("fds_file_init() has failed");
+    }
+
+    for (const auto &file_name : file_names) {
+        const uint32_t flags = FDS_FILE_READ | FDS_FILE_NOASYNC;
+        const fds_file_stats *file_stats;
+        int ret;
+
+        ret = fds_file_open(file.get(), file_name.c_str(), flags);
+        if (ret != FDS_OK) {
+            const std::string err_msg = fds_file_error(file.get());
+            std::cerr << "fds_file_open('" << file_name << "') failed: " << err_msg << std::endl;
+            continue;
+        }
+
+        file_stats = fds_file_stats_get(file.get());
+        if (!file_stats) {
+            std::cerr << "fds_file_stats_get('" << file_name << "') failed" << std::endl;
+            continue;
+        }
+
+        stats_merge(stats, *file_stats);
+    }
+
+    printer->print_prologue();
+    printer->print_stats(stats);
+    printer->print_epilogue();
+
+    (void) iemgr;
+    return;
+}
+
+} // statistics
+} // fdsdump

src/tools/fdsdump/src/statistics/mode.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "common/common.hpp"
+#include "options.hpp"
+
+namespace fdsdump {
+namespace statistics {
+
+void
+mode_statistics(const shared_iemgr &iemgr, const Options &opts);
+
+} // statistics
+} // fdsdump

src/tools/fdsdump/src/statistics/printer.cpp

@@ -0,0 +1,56 @@
+#include <algorithm>
+#include <cassert>
+#include <functional>
+#include <stdexcept>
+#include <vector>
+
+#include "printer.hpp"
+#include "jsonPrinter.hpp"
+#include "tablePrinter.hpp"
+
+namespace fdsdump {
+namespace statistics {
+
+struct PrinterFactory {
+    const char *name;
+    std::function<Printer *(const std::string &)> create_fn;
+};
+
+static const std::vector<struct PrinterFactory> g_printers {
+    {"json", [](const std::string &args) {
+        return new JsonPrinter(args); }
+    },
+    {"table", [](const std::string &args){
+        return new TablePrinter(args); }
+    },
+};
+
+std::unique_ptr<Printer>
+printer_factory(const std::string &manual)
+{
+    std::string type;
+    std::string args;
+    size_t delim_pos;
+
+    delim_pos = manual.find(':');
+    type = manual.substr(0, delim_pos);
+    args = (delim_pos != std::string::npos)
+        ? manual.substr(delim_pos + 1, std::string::npos)
+        : "";
+
+    // Convert type to lower case
+    std::for_each(type.begin(), type.end(), [](char &c) { c = std::tolower(c); });
+
+    for (const auto &it : g_printers) {
+        if (type != it.name) {
+            continue;
+        }
+
+        return std::unique_ptr<Printer>(it.create_fn(args));
+    }
+
+    throw std::invalid_argument("Unsupported output type '" + type + "'");
+}
+
+} // statistics
+} // fdsdump