Skip to content

Commit 4fcaf73

Browse files
sedmichasedmicha
committed
Merge branch 'sedlak-clickhouse' into 'master'
Clickhouse output plugin See merge request monitoring/ipfixcol2!20
2 parents 6f7f8df + fcc1cc8 commit 4fcaf73

30 files changed

+3114
-0
lines changed

‎README.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ network interface and a port. Multiple instances of these plugins can run concur
6464
format for long-term preservation
6565
- `UniRec <extra_plugins/output/unirec>`_ (*) - send flow records in UniRec format
6666
via TRAP communication interface (into Nemea modules)
67+
- `ClickHouse <extra_plugins/output/clickhouse>`_ (*) - insert flow records
68+
into a ClickHouse database
6769

6870
\* Must be installed individually due to extra dependencies
6971

‎extra_plugins/output/clickhouse/CMakeLists.txt‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
cmake_minimum_required(VERSION 3.12)
2+
project(ipfixcol2-clichouse-output)
3+
4+
# Description of the project
5+
set(CH_DESCRIPTION
6+
"Output plugin for IPFIXcol2 that store flow records to ClickHouse database."
7+
)
8+
9+
set(CH_VERSION_MAJOR 1)
10+
set(CH_VERSION_MINOR 0)
11+
set(CH_VERSION_PATCH 0)
12+
set(CH_VERSION
13+
${CH_VERSION_MAJOR}.${CH_VERSION_MINOR}.${CH_VERSION_PATCH})
14+
15+
include(CMakeModules/install_dirs.cmake)
16+
include(CMakeModules/clickhouse-cpp.cmake)
17+
include(CMakeModules/fmt.cmake)
18+
include(CheckCCompilerFlag)
19+
include(CheckCXXCompilerFlag)
20+
# Include custom FindXXX modules
21+
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")
22+
23+
# Find IPFIXcol
24+
find_package(IPFIXcol2 2.3.0 REQUIRED)
25+
26+
# Set default build type if not specified by user
27+
if (NOT CMAKE_BUILD_TYPE)
28+
set (CMAKE_BUILD_TYPE Debug
29+
CACHE STRING "Choose type of build (Release/Debug)." FORCE)
30+
endif()
31+
32+
option(ENABLE_DOC_MANPAGE "Enable manual page building" ON)
33+
34+
set(CMAKE_CXX_STANDARD 17)
35+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
36+
set(CMAKE_CXX_EXTENSIONS ON)
37+
38+
# Hard coded definitions
39+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
40+
set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG")
41+
set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic")
42+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
43+
set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
44+
set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic")
45+
46+
# Header files for source code building
47+
include_directories(
48+
"${IPFIXCOL2_INCLUDE_DIRS}" # IPFIXcol2 header files
49+
)
50+
51+
# Create a linkable module
52+
add_library(clickhouse-output MODULE
53+
src/common.cpp
54+
src/config.cpp
55+
src/datatype.cpp
56+
src/main.cpp
57+
src/plugin.cpp
58+
src/inserter.cpp
59+
src/recparser.cpp
60+
src/stats.cpp
61+
)
62+
63+
target_link_libraries(clickhouse-output PRIVATE clickhouse::client fmt::fmt)
64+
65+
install(
66+
TARGETS clickhouse-output
67+
LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/"
68+
)
69+
70+
if (ENABLE_DOC_MANPAGE)
71+
find_package(Rst2Man)
72+
if (NOT RST2MAN_FOUND)
73+
message(FATAL_ERROR "rst2man is not available. Install python-docutils or disable manual page generation (-DENABLE_DOC_MANPAGE=False)")
74+
endif()
75+
76+
# Build a manual page
77+
set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-clickhouse-output.7.rst")
78+
set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-clickhouse-output.7")
79+
80+
add_custom_command(TARGET clickhouse-output PRE_BUILD
81+
COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE}
82+
DEPENDS ${SRC_FILE}
83+
VERBATIM
84+
)
85+
86+
install(
87+
FILES "${DST_FILE}"
88+
DESTINATION "${INSTALL_DIR_MAN}/man7"
89+
)
90+
endif()

‎extra_plugins/output/clickhouse/CMakeModules/FindIPFIXcol2.cmake‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# IPFIXCOL2_FOUND - System has IPFIXcol
2+
# IPFIXCOL2_INCLUDE_DIRS - The IPFIXcol include directories
3+
# IPFIXCOL2_DEFINITIONS - Compiler switches required for using IPFIXcol
4+
5+
# use pkg-config to get the directories and then use these values
6+
# in the find_path() and find_library() calls
7+
find_package(PkgConfig)
8+
pkg_check_modules(PC_IPFIXCOL QUIET ipfixcol2)
9+
set(IPFIXCOL2_DEFINITIONS ${PC_IPFIXCOL_CFLAGS_OTHER})
10+
11+
find_path(
12+
IPFIXCOL2_INCLUDE_DIR ipfixcol2.h
13+
HINTS ${PC_IPFIXCOL_INCLUDEDIR} ${PC_IPFIXCOL_INCLUDE_DIRS}
14+
PATH_SUFFIXES include
15+
)
16+
17+
if (PC_IPFIXCOL_VERSION)
18+
# Version extracted from pkg-config
19+
set(IPFIXCOL_VERSION_STRING ${PC_IPFIXCOL_VERSION})
20+
elseif(IPFIXCOL2_INCLUDE_DIR AND EXISTS "${IPFIXCOL2_INCLUDE_DIR}/ipfixcol2/api.h")
21+
# Try to extract library version from a header file
22+
file(STRINGS "${IPFIXCOL2_INCLUDE_DIR}/ipfixcol2/api.h" ipfixcol_version_str
23+
REGEX "^#define[\t ]+IPX_API_VERSION_STR[\t ]+\".*\"")
24+
25+
string(REGEX REPLACE "^#define[\t ]+IPX_API_VERSION_STR[\t ]+\"([^\"]*)\".*" "\\1"
26+
IPFIXCOL_VERSION_STRING "${ipfixcol_version_str}")
27+
unset(ipfixcol_version_str)
28+
endif()
29+
30+
# handle the QUIETLY and REQUIRED arguments and set IPFIXCOL2_FOUND to TRUE
31+
# if all listed variables are TRUE
32+
include(FindPackageHandleStandardArgs)
33+
find_package_handle_standard_args(IPFIXcol2
34+
REQUIRED_VARS IPFIXCOL2_INCLUDE_DIR
35+
VERSION_VAR IPFIXCOL_VERSION_STRING
36+
)
37+
38+
set(IPFIXCOL2_INCLUDE_DIRS ${IPFIXCOL2_INCLUDE_DIR})
39+
mark_as_advanced(IPFIXCOL2_INCLUDE_DIR)

‎extra_plugins/output/clickhouse/CMakeModules/FindRst2Man.cmake‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# RST2MAN_FOUND - true if the program was found
2+
# RST2MAN_VERSION - version of rst2man
3+
# RST2MAN_EXECUTABLE - path to the rst2man program
4+
5+
find_program(RST2MAN_EXECUTABLE
6+
NAMES rst2man rst2man.py rst2man-3 rst2man-3.py
7+
DOC "The Python Docutils generator of Unix Manpages from reStructuredText"
8+
)
9+
10+
if (RST2MAN_EXECUTABLE)
11+
# Get the version string
12+
execute_process(
13+
COMMAND ${RST2MAN_EXECUTABLE} --version
14+
OUTPUT_VARIABLE rst2man_version_str
15+
)
16+
# Expected format: rst2man (Docutils 0.13.1 [release], Python 2.7.15, on linux2)
17+
string(REGEX REPLACE "^rst2man[\t ]+\\(Docutils[\t ]+([^\t ]*).*" "\\1"
18+
RST2MAN_VERSION "${rst2man_version_str}")
19+
unset(rst2man_version_str)
20+
endif()
21+
22+
# handle the QUIETLY and REQUIRED arguments and set RST2MAN_FOUND to TRUE
23+
# if all listed variables are set
24+
include(FindPackageHandleStandardArgs)
25+
find_package_handle_standard_args(Rst2Man
26+
REQUIRED_VARS RST2MAN_EXECUTABLE
27+
VERSION_VAR RST2MAN_VERSION
28+
)
29+
30+
mark_as_advanced(RST2MAN_EXECUTABLE RST2MAN_VERSION)

‎extra_plugins/output/clickhouse/CMakeModules/clickhouse-cpp.cmake‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# clickhouse-cpp library (C++ client for ClickHouse)
2+
#
3+
# The project consists of a library that can be independently
4+
# added as dependency:
5+
# - clickhouse::client
6+
7+
include(FetchContent)
8+
set(FETCHCONTENT_QUIET OFF)
9+
10+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
11+
set(BUILD_SHARED_LIBS OFF)
12+
FetchContent_Declare(
13+
clickhouse
14+
GIT_REPOSITORY "https://github.com/ClickHouse/clickhouse-cpp.git"
15+
GIT_TAG "v2.5.1"
16+
GIT_SHALLOW ON
17+
)
18+
19+
FetchContent_MakeAvailable(clickhouse)
20+
add_library(clickhouse::client ALIAS clickhouse-cpp-lib)
21+
set(BUILD_SHARED_LIBS ON)

‎extra_plugins/output/clickhouse/CMakeModules/fmt.cmake‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# fmt library
2+
#
3+
# The project consists of a library that can be independently
4+
# added as dependency:
5+
# - fmt::fmt
6+
7+
include(FetchContent)
8+
set(FETCHCONTENT_QUIET OFF)
9+
10+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
11+
set(BUILD_SHARED_LIBS OFF)
12+
FetchContent_Declare(
13+
fmt
14+
GIT_REPOSITORY "https://github.com/fmtlib/fmt"
15+
GIT_TAG "11.0.2"
16+
GIT_SHALLOW ON
17+
)
18+
19+
FetchContent_MakeAvailable(fmt)
20+
set(BUILD_SHARED_LIBS ON)

‎extra_plugins/output/clickhouse/CMakeModules/install_dirs.cmake‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# The purpose of this file is to automatically determine install directories
2+
#
3+
# If no directories are defined, use GNU install directories by default.
4+
# However, in case of RPM build, install directories are typically passed
5+
# to CMake as definitions that overwrites the default paths.
6+
#
7+
8+
include(GNUInstallDirs)
9+
10+
# Binary directories
11+
set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR})
12+
13+
# Library directories
14+
if (DEFINED LIB_INSTALL_DIR)
15+
set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR})
16+
else()
17+
set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR})
18+
endif()
19+
20+
# Include directories
21+
if (DEFINED INCLUDE_INSTALL_DIR)
22+
set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR})
23+
else()
24+
set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR})
25+
endif()
26+
27+
# System configuration
28+
if (DEFINED SYSCONF_INSTALL_DIR)
29+
set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR})
30+
else()
31+
set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR})
32+
endif()
33+
34+
# Share files (architecture independend data)
35+
if (DEFINED SHARE_INSTALL_PREFIX)
36+
set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX})
37+
else()
38+
set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR})
39+
endif()
40+
41+
set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/")
42+
set(INSTALL_DIR_MAN "${INSTALL_DIR_SHARE}/man/")
43+
set(INSTALL_DIR_DOC "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/")

0 commit comments

Comments
 (0)