...

Author: sedmicha

extra_plugins/output/clickhouse/src/datatype.cpp

@@ -18,9 +18,17 @@ template <unsigned Precision> class ColumnDateTime64 : public clickhouse::Column
     ColumnDateTime64() : clickhouse::ColumnDateTime64(Precision) {}
 };
 
-DataType type_from_ipfix(fds_iemgr_element_type type)
+DataType type_from_ipfix(const fds_iemgr_elem *elem)
 {
-    switch (type) {
+    // Special cases
+    static constexpr uint32_t PEN_CESNET = 8057;
+    static constexpr uint16_t ID_FLOWUUID = 1300;
+    if (elem->scope->pen == PEN_CESNET && elem->id == ID_FLOWUUID) {
+        return DataType::Uuid;
+    }
+
+    // Otherwise
+    switch (elem->data_type) {
     case FDS_ET_STRING:                 return DataType::String;
     case FDS_ET_SIGNED_8:               return DataType::Int8;
     case FDS_ET_SIGNED_16:              return DataType::Int16;
@@ -36,7 +44,7 @@ DataType type_from_ipfix(fds_iemgr_element_type type)
     case FDS_ET_DATE_TIME_MILLISECONDS: return DataType::DatetimeMillisecs;
     case FDS_ET_DATE_TIME_MICROSECONDS: return DataType::DatetimeMicrosecs;
     case FDS_ET_DATE_TIME_NANOSECONDS:  return DataType::DatetimeNanosecs;
-    default:                            throw Error("unsupported IPFIX data type {}", type);
+    default:                            throw Error("unsupported IPFIX data type {}", elem->data_type);
     }
 }
 
@@ -91,9 +99,9 @@ DataType find_common_type(const fds_iemgr_alias &alias)
         throw Error("alias \"{}\" has no sources", alias.name);
     }
 
-    DataType common_type = type_from_ipfix(alias.sources[0]->data_type);
+    DataType common_type = type_from_ipfix(alias.sources[0]);
     for (size_t i = 1; i < alias.sources_cnt; i++) {
-        common_type = unify_type(common_type, type_from_ipfix(alias.sources[i]->data_type));
+        common_type = unify_type(common_type, type_from_ipfix(alias.sources[i]));
     }
     return common_type;
 }
@@ -199,6 +207,19 @@ static int64_t get_datetime64(fds_drec_field field)
     value = (static_cast<int64_t>(ts.tv_sec) * 1'000'000'000 + static_cast<int64_t>(ts.tv_nsec)) / Divisor;
     return value;
 }
+
+static std::pair<uint64_t, uint64_t> get_uuid(fds_drec_field field)
+{
+    if (field.size != 16) {
+        throw Error("invalid uuid field size. expected 16, got {}", field.size);
+    }
+
+    return {
+        *reinterpret_cast<uint64_t *>(&field.data[0]),
+        *reinterpret_cast<uint64_t *>(&field.data[8])
+    };
+}
+
 }
 
 template<DataType> struct DataTypeTraits {};
@@ -299,6 +320,12 @@ template<> struct DataTypeTraits<DataType::DatetimeNanosecs> {
     static constexpr auto Getter = &getters::get_datetime64<1>;
 };
 
+template<> struct DataTypeTraits<DataType::Uuid> {
+    using ColumnType = clickhouse::ColumnUUID;
+    static constexpr std::string_view ClickhouseTypeName = "UUID";
+    static constexpr auto Getter = &getters::get_uuid;
+};
+
 template <typename Func>
 static void visit(DataType type, Func func)
 {
@@ -319,6 +346,7 @@ static void visit(DataType type, Func func)
     case DataType::IPv4:              func(DataTypeTraits<DataType::IPv4>{});              break;
     case DataType::IPv6:              func(DataTypeTraits<DataType::IPv6>{});              break;
     case DataType::IP:                func(DataTypeTraits<DataType::IP>{});                break;
+    case DataType::Uuid:              func(DataTypeTraits<DataType::Uuid>{});              break;
     case DataType::Invalid:           throw std::runtime_error("invalid data type");
     }
 }

extra_plugins/output/clickhouse/src/datatype.h

@@ -47,6 +47,8 @@ enum class DataType {
     DatetimeMillisecs,
     DatetimeMicrosecs,
     DatetimeNanosecs,
+
+    Uuid,
 };
 
 enum class DataTypeNullable {
@@ -115,6 +117,7 @@ template <> struct fmt::formatter<DataType> : formatter<string_view> {
             case DataType::DatetimeMicrosecs: return "DatetimeMicrosecs";
             case DataType::DatetimeMillisecs: return "DatetimeMillisecs";
             case DataType::DatetimeSecs: return "DatetimeSecs";
+            case DataType::Uuid: return "Uuid";
             }
             return "???";
         }();
@@ -159,12 +162,12 @@ template <> struct fmt::formatter<fds_iemgr_element_type> : formatter<string_vie
 };
 
 /**
- * @brief Get intermediary data type for a corresponding IPFIX element type
+ * @brief Get intermediary data type for a corresponding IPFIX element
  *
- * @param type The IPFIX type
+ * @param elem The IPFIX element definition
  * @return The intermediary data type
  */
-DataType type_from_ipfix(fds_iemgr_element_type type);
+DataType type_from_ipfix(const fds_iemgr_elem *elem);
 
 /**
  * @brief Get Clickhouse data type for the intermediary data type
@@ -196,7 +199,8 @@ using ValueVariant = std::variant<
     int64_t,
     IP4Addr,
     IP6Addr,
-    std::string
+    std::string,
+    std::pair<uint64_t, uint64_t>
 >;
 
 using GetterFn = std::function<void (fds_drec_field field, ValueVariant &value)>;

extra_plugins/output/clickhouse/src/plugin.cpp

@@ -59,7 +59,7 @@ static std::vector<ColumnCtx> prepare_columns(std::vector<Config::Column> &colum
 
         if (std::holds_alternative<const fds_iemgr_elem *>(column_cfg.source)) {
             const fds_iemgr_elem *elem = std::get<const fds_iemgr_elem *>(column_cfg.source);
-            type = type_from_ipfix(elem->data_type);
+            type = type_from_ipfix(elem);
             column.extractor = make_extractor(*elem);
 
         } else if (std::holds_alternative<const fds_iemgr_alias *>(column_cfg.source)) {

src/plugins/intermediate/CMakeLists.txt

@@ -1,6 +1,6 @@
 # List of output plugin to build and install
 add_subdirectory(anonymization)
-add_subdirectory(dummy_enricher)
-add_subdirectory(asn)
-add_subdirectory(geoip)
+# add_subdirectory(dummy_enricher)
+# add_subdirectory(asn)
+# add_subdirectory(geoip)
 add_subdirectory(uuid)

src/plugins/intermediate/uuid/uuid.c

@@ -65,6 +65,7 @@ modifier_callback(const struct fds_drec *rec, struct ipx_modifier_output output[
 
     uint32_t r = (uint32_t)rand();
     fill_uuid(instance->unix_time_ms, instance->counter, r, output->raw);
+    output->length = 16;
     instance->counter++;
 
     return IPX_OK;
@@ -357,7 +358,7 @@ IPX_API struct ipx_plugin_info ipx_plugin_info = {
     // Plugin type
     .type = IPX_PT_INTERMEDIATE,
     // Plugin identification name
-    .name = "asn",
+    .name = "uuid",
     // Brief description of plugin
     .dsc = "IPv4/IPv6 autonomous system number (ASN) module",
     // Configuration flags (reserved for future use)