Skip to content

Commit aabf4c0

Browse files
sedmichasedmicha
committed
introduce clickhouse plugin
1 parent 6380631 commit aabf4c0

File tree

16 files changed

+1428
-0
lines changed

16 files changed

+1428
-0
lines changed

‎extra_plugins/output/clickhouse/CMakeLists.txt‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
cmake_minimum_required(VERSION 3.12)
2+
project(ipfixcol2-clichouse-output)
3+
4+
# Description of the project
5+
set(CH_DESCRIPTION
6+
"Output plugin for IPFIXcol2 that store flow records to ClickHouse database."
7+
)
8+
9+
set(CH_VERSION_MAJOR 1)
10+
set(CH_VERSION_MINOR 0)
11+
set(CH_VERSION_PATCH 0)
12+
set(CH_VERSION
13+
${CH_VERSION_MAJOR}.${CH_VERSION_MINOR}.${CH_VERSION_PATCH})
14+
15+
include(CMakeModules/install_dirs.cmake)
16+
include(CMakeModules/clickhouse-cpp.cmake)
17+
include(CMakeModules/fmt.cmake)
18+
include(CheckCCompilerFlag)
19+
include(CheckCXXCompilerFlag)
20+
# Include custom FindXXX modules
21+
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")
22+
23+
# Find IPFIXcol
24+
find_package(IPFIXcol2 2.3.0 REQUIRED)
25+
26+
# Set default build type if not specified by user
27+
if (NOT CMAKE_BUILD_TYPE)
28+
set (CMAKE_BUILD_TYPE Debug
29+
CACHE STRING "Choose type of build (Release/Debug)." FORCE)
30+
endif()
31+
32+
option(ENABLE_DOC_MANPAGE "Enable manual page building" ON)
33+
34+
set(CMAKE_CXX_STANDARD 17)
35+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
36+
set(CMAKE_CXX_EXTENSIONS ON)
37+
38+
# Hard coded definitions
39+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
40+
set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG")
41+
set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic")
42+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
43+
set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
44+
set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic")
45+
46+
# Header files for source code building
47+
include_directories(
48+
"${IPFIXCOL2_INCLUDE_DIRS}" # IPFIXcol2 header files
49+
)
50+
51+
# Create a linkable module
52+
add_library(clickhouse-output MODULE
53+
src/common.cpp
54+
src/config.cpp
55+
src/datatype.cpp
56+
src/main.cpp
57+
src/plugin.cpp
58+
)
59+
60+
target_link_libraries(clickhouse-output PRIVATE clickhouse::client fmt::fmt)
61+
62+
install(
63+
TARGETS clickhouse-output
64+
LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/"
65+
)
66+
67+
if (ENABLE_DOC_MANPAGE)
68+
find_package(Rst2Man)
69+
if (NOT RST2MAN_FOUND)
70+
message(FATAL_ERROR "rst2man is not available. Install python-docutils or disable manual page generation (-DENABLE_DOC_MANPAGE=False)")
71+
endif()
72+
73+
# Build a manual page
74+
set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-clickhouse-output.7.rst")
75+
set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-clickhouse-output.7")
76+
77+
add_custom_command(TARGET clickhouse-output PRE_BUILD
78+
COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE}
79+
DEPENDS ${SRC_FILE}
80+
VERBATIM
81+
)
82+
83+
install(
84+
FILES "${DST_FILE}"
85+
DESTINATION "${INSTALL_DIR_MAN}/man7"
86+
)
87+
endif()

‎extra_plugins/output/clickhouse/CMakeModules/FindIPFIXcol2.cmake‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# IPFIXCOL2_FOUND - System has IPFIXcol
2+
# IPFIXCOL2_INCLUDE_DIRS - The IPFIXcol include directories
3+
# IPFIXCOL2_DEFINITIONS - Compiler switches required for using IPFIXcol
4+
5+
# use pkg-config to get the directories and then use these values
6+
# in the find_path() and find_library() calls
7+
find_package(PkgConfig)
8+
pkg_check_modules(PC_IPFIXCOL QUIET ipfixcol2)
9+
set(IPFIXCOL2_DEFINITIONS ${PC_IPFIXCOL_CFLAGS_OTHER})
10+
11+
find_path(
12+
IPFIXCOL2_INCLUDE_DIR ipfixcol2.h
13+
HINTS ${PC_IPFIXCOL_INCLUDEDIR} ${PC_IPFIXCOL_INCLUDE_DIRS}
14+
PATH_SUFFIXES include
15+
)
16+
17+
if (PC_IPFIXCOL_VERSION)
18+
# Version extracted from pkg-config
19+
set(IPFIXCOL_VERSION_STRING ${PC_IPFIXCOL_VERSION})
20+
elseif(IPFIXCOL2_INCLUDE_DIR AND EXISTS "${IPFIXCOL2_INCLUDE_DIR}/ipfixcol2/api.h")
21+
# Try to extract library version from a header file
22+
file(STRINGS "${IPFIXCOL2_INCLUDE_DIR}/ipfixcol2/api.h" ipfixcol_version_str
23+
REGEX "^#define[\t ]+IPX_API_VERSION_STR[\t ]+\".*\"")
24+
25+
string(REGEX REPLACE "^#define[\t ]+IPX_API_VERSION_STR[\t ]+\"([^\"]*)\".*" "\\1"
26+
IPFIXCOL_VERSION_STRING "${ipfixcol_version_str}")
27+
unset(ipfixcol_version_str)
28+
endif()
29+
30+
# handle the QUIETLY and REQUIRED arguments and set IPFIXCOL2_FOUND to TRUE
31+
# if all listed variables are TRUE
32+
include(FindPackageHandleStandardArgs)
33+
find_package_handle_standard_args(IPFIXcol2
34+
REQUIRED_VARS IPFIXCOL2_INCLUDE_DIR
35+
VERSION_VAR IPFIXCOL_VERSION_STRING
36+
)
37+
38+
set(IPFIXCOL2_INCLUDE_DIRS ${IPFIXCOL2_INCLUDE_DIR})
39+
mark_as_advanced(IPFIXCOL2_INCLUDE_DIR)

‎extra_plugins/output/clickhouse/CMakeModules/FindRst2Man.cmake‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# RST2MAN_FOUND - true if the program was found
2+
# RST2MAN_VERSION - version of rst2man
3+
# RST2MAN_EXECUTABLE - path to the rst2man program
4+
5+
find_program(RST2MAN_EXECUTABLE
6+
NAMES rst2man rst2man.py rst2man-3 rst2man-3.py
7+
DOC "The Python Docutils generator of Unix Manpages from reStructuredText"
8+
)
9+
10+
if (RST2MAN_EXECUTABLE)
11+
# Get the version string
12+
execute_process(
13+
COMMAND ${RST2MAN_EXECUTABLE} --version
14+
OUTPUT_VARIABLE rst2man_version_str
15+
)
16+
# Expected format: rst2man (Docutils 0.13.1 [release], Python 2.7.15, on linux2)
17+
string(REGEX REPLACE "^rst2man[\t ]+\\(Docutils[\t ]+([^\t ]*).*" "\\1"
18+
RST2MAN_VERSION "${rst2man_version_str}")
19+
unset(rst2man_version_str)
20+
endif()
21+
22+
# handle the QUIETLY and REQUIRED arguments and set RST2MAN_FOUND to TRUE
23+
# if all listed variables are set
24+
include(FindPackageHandleStandardArgs)
25+
find_package_handle_standard_args(Rst2Man
26+
REQUIRED_VARS RST2MAN_EXECUTABLE
27+
VERSION_VAR RST2MAN_VERSION
28+
)
29+
30+
mark_as_advanced(RST2MAN_EXECUTABLE RST2MAN_VERSION)

‎extra_plugins/output/clickhouse/CMakeModules/clickhouse-cpp.cmake‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# clickhouse-cpp library (C++ client for ClickHouse)
2+
#
3+
# The project consists of a library that can be independently
4+
# added as dependency:
5+
# - clickhouse::client
6+
7+
include(FetchContent)
8+
set(FETCHCONTENT_QUIET OFF)
9+
10+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
11+
FetchContent_Declare(
12+
clickhouse
13+
GIT_REPOSITORY "https://github.com/ClickHouse/clickhouse-cpp.git"
14+
GIT_TAG "v2.5.1"
15+
GIT_SHALLOW ON
16+
)
17+
18+
FetchContent_MakeAvailable(clickhouse)
19+
add_library(clickhouse::client ALIAS clickhouse-cpp-lib)

‎extra_plugins/output/clickhouse/CMakeModules/fmt.cmake‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# fmt library
2+
#
3+
# The project consists of a library that can be independently
4+
# added as dependency:
5+
# - fmt::fmt
6+
7+
include(FetchContent)
8+
set(FETCHCONTENT_QUIET OFF)
9+
10+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
11+
FetchContent_Declare(
12+
fmt
13+
GIT_REPOSITORY "https://github.com/fmtlib/fmt"
14+
GIT_TAG "11.0.2"
15+
GIT_SHALLOW ON
16+
)
17+
18+
FetchContent_MakeAvailable(fmt)

‎extra_plugins/output/clickhouse/CMakeModules/install_dirs.cmake‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# The purpose of this file is to automatically determine install directories
2+
#
3+
# If no directories are defined, use GNU install directories by default.
4+
# However, in case of RPM build, install directories are typically passed
5+
# to CMake as definitions that overwrites the default paths.
6+
#
7+
8+
include(GNUInstallDirs)
9+
10+
# Binary directories
11+
set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR})
12+
13+
# Library directories
14+
if (DEFINED LIB_INSTALL_DIR)
15+
set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR})
16+
else()
17+
set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR})
18+
endif()
19+
20+
# Include directories
21+
if (DEFINED INCLUDE_INSTALL_DIR)
22+
set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR})
23+
else()
24+
set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR})
25+
endif()
26+
27+
# System configuration
28+
if (DEFINED SYSCONF_INSTALL_DIR)
29+
set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR})
30+
else()
31+
set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR})
32+
endif()
33+
34+
# Share files (architecture independend data)
35+
if (DEFINED SHARE_INSTALL_PREFIX)
36+
set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX})
37+
else()
38+
set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR})
39+
endif()
40+
41+
set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/")
42+
set(INSTALL_DIR_MAN "${INSTALL_DIR_SHARE}/man/")
43+
set(INSTALL_DIR_DOC "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/")

‎extra_plugins/output/clickhouse/doc/ipfixcol2-clickhouse-output.7.rst‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
=============================
2+
ipfixcol2-clickhouse-output
3+
=============================
4+
5+
--------------------------
6+
ClickHouse (output plugin)
7+
--------------------------
8+
9+
:Author: Michal Sedlak ([email protected])
10+
:Date: 2024-11-04
11+
:Copyright: Copyright © 2024 CESNET, z.s.p.o.
12+
:Version: 1.0
13+
:Manual section: 7
14+
:Manual group: IPFIXcol collector
15+
16+
Description
17+
-----------
18+
19+
.. .. include:: ../README.rst
20+
.. :start-line: 3
21+
.. :end-before: How to build
22+
..
23+
.. .. include:: ../README.rst
24+
.. :start-after: make install

‎extra_plugins/output/clickhouse/src/common.cpp‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/**
2+
* @file
3+
* @author Michal Sedlak <[email protected]>
4+
* @brief
5+
* @date 2024
6+
*
7+
* Copyright(c) 2024 CESNET z.s.p.o.
8+
* SPDX-License-Identifier: BSD-3-Clause
9+
*/
10+
11+
#include "common.h"
12+
13+
#include <iostream>
14+
15+
void print_block(const clickhouse::Block& block) {
16+
std::cout << "================================================================================\n";
17+
for (size_t i = 0; i < block.GetColumnCount(); i++) {
18+
std::cout << block.GetColumnName(i) << ":" << block[i]->GetType().GetName() << (i < block.GetColumnCount() - 1 ? '\t' : '\n');
19+
}
20+
std::cout << "--------------------------------------------------------------------------------\n";
21+
22+
for (size_t i = 0; i < block.GetRowCount(); i++) {
23+
for (size_t j = 0; j < block.GetColumnCount(); j++) {
24+
switch (block[j]->GetType().GetCode()) {
25+
case clickhouse::Type::Int8:
26+
std::cout << +block[j]->As<clickhouse::ColumnInt8>()->At(i);
27+
break;
28+
case clickhouse::Type::Int16:
29+
std::cout << block[j]->As<clickhouse::ColumnInt16>()->At(i);
30+
break;
31+
case clickhouse::Type::Int32:
32+
std::cout << block[j]->As<clickhouse::ColumnInt32>()->At(i);
33+
break;
34+
case clickhouse::Type::Int64:
35+
std::cout << block[j]->As<clickhouse::ColumnInt64>()->At(i);
36+
break;
37+
case clickhouse::Type::Int128:
38+
std::cout << block[j]->As<clickhouse::ColumnInt128>()->At(i);
39+
break;
40+
case clickhouse::Type::UInt8:
41+
std::cout << +block[j]->As<clickhouse::ColumnUInt8>()->At(i);
42+
break;
43+
case clickhouse::Type::UInt16:
44+
std::cout << block[j]->As<clickhouse::ColumnUInt16>()->At(i);
45+
break;
46+
case clickhouse::Type::UInt32:
47+
std::cout << block[j]->As<clickhouse::ColumnUInt32>()->At(i);
48+
break;
49+
case clickhouse::Type::UInt64:
50+
std::cout << block[j]->As<clickhouse::ColumnUInt64>()->At(i);
51+
break;
52+
case clickhouse::Type::String:
53+
std::cout << block[j]->As<clickhouse::ColumnString>()->At(i);
54+
break;
55+
default:
56+
std::cout << "-";
57+
break;
58+
}
59+
std::cout << (j < block.GetColumnCount() - 1 ? '\t' : '\n');
60+
}
61+
}
62+
std::cout << "================================================================================\n\n";
63+
};

0 commit comments

Comments
 (0)