Skip to content

Commit 5f8f72e

Browse files
Lukas955Lukas955
committed
Merge branch 'hutak-kafka' into devel
2 parents 9b382a5 + d580544 commit 5f8f72e

File tree

15 files changed

+835
-24
lines changed

15 files changed

+835
-24
lines changed

‎.github/workflows/main.yml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,18 +29,24 @@ jobs:
2929
apt-get update
3030
apt-get -y install git gcc g++ cmake make libxml2-dev liblz4-dev libzstd-dev
3131
apt-get -y install python3-docutils zlib1g-dev pkg-config
32+
apt-get -y install librdkafka-dev
33+
- name: Enable additional repositories (CentOS 8)
34+
if: startsWith(matrix.image, 'centos:8')
35+
run: |
36+
dnf -y install 'dnf-command(config-manager)'
37+
dnf config-manager --set-enabled AppStream PowerTools
3238
- name: Install dependencies for libfds and IPFIXcol2 (CentOS)
3339
if: startsWith(matrix.image, 'centos')
3440
run: |
3541
yum -y install epel-release
3642
yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel
37-
yum -y install zlib-devel pkgconfig
43+
yum -y install zlib-devel pkgconfig librdkafka-devel
3844
yum -y install python3-docutils || yum -y install python-docutils
39-
- name: Install depedencies for libfds and IPFIXcol2 (Fedora)
45+
- name: Install dependencies for libfds and IPFIXcol2 (Fedora)
4046
if: startsWith(matrix.image, 'fedora')
4147
run: |
4248
dnf -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel
43-
dnf -y install python3-docutils zlib-devel pkgconfig
49+
dnf -y install python3-docutils zlib-devel pkgconfig librdkafka-devel
4450
4551
# Build libfds library ------------------------------------------------------------------
4652
# Note: Master against master branch. Otherwise against debug branch.

‎.github/workflows/packages.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
run: |
3636
apt-get update
3737
apt-get -y install git gcc g++ cmake make libxml2-dev liblz4-dev libzstd-dev
38-
apt-get -y install python3-docutils zlib1g-dev pkg-config
38+
apt-get -y install python3-docutils zlib1g-dev pkg-config librdkafka-dev
3939
apt-get -y install debhelper devscripts build-essential fakeroot zip
4040
4141
# Build LIBFDS DEB package ---------------------------------------------------------------
@@ -99,18 +99,23 @@ jobs:
9999
id: vars
100100

101101
# Dependencies ---------------------------------------------------------------------------
102+
- name: Enable additional repositories (CentOS 8)
103+
if: startsWith(matrix.image, 'centos:8')
104+
run: |
105+
dnf -y install 'dnf-command(config-manager)'
106+
dnf config-manager --set-enabled AppStream PowerTools
102107
- name: Install dependencies for libfds and IPFIXcol2 (CentOS)
103108
if: startsWith(matrix.image, 'centos')
104109
run: |
105110
yum -y install epel-release
106111
yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel
107-
yum -y install zlib-devel pkgconfig rpm-build
112+
yum -y install zlib-devel pkgconfig rpm-build librdkafka-devel
108113
yum -y install python3-docutils || yum -y install python-docutils
109114
- name: Install depedencies for libfds and IPFIXcol2 (Fedora)
110115
if: startsWith(matrix.image, 'fedora')
111116
run: |
112117
dnf -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel
113-
dnf -y install python3-docutils zlib-devel pkgconfig rpm-build
118+
dnf -y install python3-docutils zlib-devel pkgconfig rpm-build librdkafka-devel
114119
115120
# Build LIBFDS RPM package ---------------------------------------------------------------
116121
- name: Checkout libfds library - master branch

‎CMakeModules/FindLibRDKafka.cmake‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# LIBRDKAFKA_FOUND - System has librdkafka
2+
# LIBRDKAFKA_INCLUDE_DIRS - The librdkafka include directories
3+
# LIBRDKAFKA_LIBRARIES - The libraries needed to use librdkafka
4+
# LIBRDKAFKA_DEFINITIONS - Compiler switches required for using librdkafka
5+
6+
# use pkg-config to get the directories and then use these values
7+
# in the find_path() and find_library() calls
8+
find_package(PkgConfig)
9+
if (PKG_CONFIG_FOUND)
10+
pkg_check_modules(PC_RDKAFKA QUIET rdkafka)
11+
set(LIBRDKAFKA_DEFINITIONS ${PC_RDKAFKA_CFLAGS_OTHER})
12+
endif()
13+
14+
find_path(
15+
KAFKA_INCLUDE_DIR librdkafka/rdkafka.h
16+
HINTS ${PC_RDKAFKA_INCLUDEDIR} ${PC_RDKAFKA_INCLUDE_DIRS}
17+
PATH_SUFFIXES include
18+
)
19+
20+
find_library(
21+
KAFKA_LIBRARY NAMES rdkafka librdkafka
22+
HINTS ${PC_RDKAFKA_LIBDIR} ${PC_RDKAFKA_LIBRARY_DIRS}
23+
PATH_SUFFIXES lib lib64
24+
)
25+
26+
if (PC_RDKAFKA_VERSION)
27+
# Version extracted from pkg-config
28+
set(KAFKA_VERSION_STRING ${PC_RDKAFKA_VERSION})
29+
elseif(KAFKA_INCLUDE_DIR AND KAFKA_LIBRARY)
30+
# Try to get the version of the installed library
31+
try_run(
32+
KAFKA_RES_RUN KAFKA_RES_COMP
33+
${CMAKE_CURRENT_BINARY_DIR}/try_run/kafka_version_test/
34+
${PROJECT_SOURCE_DIR}/CMakeModules/try_run/kafka_version.c
35+
CMAKE_FLAGS
36+
-DLINK_LIBRARIES=${KAFKA_LIBRARY}
37+
-DINCLUDE_DIRECTORIES=${KAFKA_INCLUDE_DIR}
38+
RUN_OUTPUT_VARIABLE KAFKA_VERSION_VAR
39+
)
40+
41+
if (KAFKA_RES_COMP AND KAFKA_RES_RUN EQUAL 0)
42+
# Successfully compiled and executed with return code 0
43+
set(KAFKA_VERSION_STRING ${KAFKA_VERSION_VAR})
44+
endif()
45+
endif()
46+
47+
# handle the QUIETLY and REQUIRED arguments and set LIBRDKAFKA_FOUND to TRUE
48+
# if all listed variables are TRUE
49+
include(FindPackageHandleStandardArgs)
50+
find_package_handle_standard_args(LibRDKafka
51+
REQUIRED_VARS KAFKA_LIBRARY KAFKA_INCLUDE_DIR
52+
VERSION_VAR KAFKA_VERSION_STRING
53+
)
54+
55+
set(LIBRDKAFKA_LIBRARIES ${KAFKA_LIBRARY})
56+
set(LIBRDKAFKA_INCLUDE_DIRS ${KAFKA_INCLUDE_DIR})
57+
mark_as_advanced(KAFKA_INCLUDE_DIR KAFKA_LIBRARY)

‎CMakeModules/try_run/kafka_version.c‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#include <librdkafka/rdkafka.h>
2+
#include <stdio.h>
3+
4+
int
5+
main(int argc, char *argv[])
6+
{
7+
const char *ver_str = rd_kafka_version_str();
8+
if (!ver_str) {
9+
return 1;
10+
}
11+
12+
printf("%s", ver_str);
13+
return 0;
14+
}

‎README.rst‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,18 +79,23 @@ Second, install build dependencies of the collector
7979

8080
.. code-block::
8181
82-
yum install gcc gcc-c++ cmake make python3-docutils zlib-devel
82+
yum install gcc gcc-c++ cmake make python3-docutils zlib-devel librdkafka-devel
8383
# Optionally: doxygen pkgconfig
8484
85-
* Note: latest systems (e.g. Fedora) use ``dnf`` instead of ``yum``.
85+
* Note: latest systems (e.g. Fedora/CentOS 8) use ``dnf`` instead of ``yum``.
8686
* Note: package ``python3-docutils`` may by also named as ``python-docutils`` or ``python2-docutils``
8787
* Note: package ``pkgconfig`` may by also named as ``pkg-config``
88+
* Note: CentOS 8 requires additional system repositories (``AppStream`` and ``PowerTools``) to be enabled:
89+
90+
.. code-block::
91+
92+
dnf config-manager --set-enabled AppStream PowerTools
8893
8994
**Debian/Ubuntu:**
9095

9196
.. code-block::
9297
93-
apt-get install gcc g++ cmake make python3-docutils zlib1g-dev
98+
apt-get install gcc g++ cmake make python3-docutils zlib1g-dev librdkafka-dev
9499
# Optionally: doxygen pkg-config
95100
96101
Finally, build and install the collector:

‎pkg/deb/templates/control.in‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@ Priority: optional
66
Standards-Version: 3.9.8
77
Build-Depends: debhelper (>= 9), cmake (>= 2.8.8), make (>= 4.0),
88
libfds-dev, gcc (>= 4.8), g++ (>= 4.8), pkg-config,
9-
zlib1g-dev, python3-docutils | python-docutils
9+
zlib1g-dev, python3-docutils | python-docutils,
10+
librdkafka-dev
1011

1112
Package: @CPACK_PACKAGE_NAME@
1213
Architecture: any
13-
Depends: ${shlibs:Depends}, ${misc:Depends}, libfds (>= 0.2.0), zlib1g
14+
Depends: ${shlibs:Depends}, ${misc:Depends}, libfds (>= 0.2.0), zlib1g,
15+
librdkafka1 (>= 0.9.3)
1416
Description: @CPACK_PACKAGE_DESCRIPTION_SUMMARY@
1517
IPFIXcol is a flexible IPFIX (RFC 7011) flow data collector designed to
1618
be extensible by plugins.

‎pkg/rpm/ipfixcol2.spec.in‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ Packager: @CPACK_PACKAGE_CONTACT@
1313
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}
1414
BuildRequires: gcc >= 4.8, gcc-c++ >= 4.8, cmake >= 2.8.8, make
1515
BuildRequires: libfds-devel, /usr/bin/rst2man, zlib-devel
16-
Requires: libfds >= 0.2.0, zlib
16+
BuildRequires: librdkafka-devel
17+
Requires: libfds >= 0.2.0, zlib, librdkafka >= 0.9.3
1718

1819
%description
1920
IPFIXcol is a flexible IPFIX (RFC 7011) flow data collector designed to

‎src/core/configurator/plugin_mgr.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ ipx_plugin_mgr::cache_add_file(const char *path)
207207
dlerror();
208208

209209
// Try to load the plugin (and unload it automatically)
210-
const int flags = RTLD_LAZY | RTLD_LOCAL;
210+
const int flags = RTLD_LAZY | RTLD_LOCAL | RTLD_DEEPBIND;
211211
auto delete_fn = [](void *handle) {dlclose(handle);};
212212
std::unique_ptr<void, std::function<void(void*)>> handle(dlopen(path, flags), delete_fn);
213213
if (!handle) {
@@ -308,7 +308,7 @@ ipx_plugin_mgr::plugin_list()
308308
dlerror();
309309

310310
const char *path = cache_entry.path.c_str();
311-
const int flags = RTLD_LAZY | RTLD_LOCAL;
311+
const int flags = RTLD_LAZY | RTLD_LOCAL | RTLD_DEEPBIND;
312312
auto delete_fn = [](void *handle) {dlclose(handle);};
313313

314314
std::unique_ptr<void, std::function<void(void*)>> handle(dlopen(path, flags), delete_fn);
@@ -512,7 +512,7 @@ ipx_plugin_mgr::plugin::plugin(const std::string &path, bool auto_unload)
512512

513513
// Load the plugin
514514
auto delete_fn = [](void *handle) {dlclose(handle);};
515-
int flags = RTLD_NOW | RTLD_LOCAL;
515+
int flags = RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND;
516516
dlerror(); // Clear all errors first
517517
std::unique_ptr<void, std::function<void(void*)>> handle_wrap(dlopen(path.c_str(), flags), delete_fn);
518518
if (!handle_wrap) {

‎src/plugins/output/json/CMakeLists.txt‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,25 @@ add_library(json-output MODULE
99
src/Printer.hpp
1010
src/File.cpp
1111
src/File.hpp
12+
src/Kafka.cpp
13+
src/Kafka.hpp
1214
src/Server.cpp
1315
src/Server.hpp
1416
src/Sender.cpp
1517
src/Server.hpp
1618
)
1719

20+
find_package(LibRDKafka 0.9.3 REQUIRED)
1821
find_package(ZLIB REQUIRED)
19-
include_directories(${ZLIB_INCLUDE_DIRS})
20-
target_link_libraries(json-output ${ZLIB_LIBRARIES})
22+
23+
include_directories(
24+
${ZLIB_INCLUDE_DIRS} # zlib
25+
${LIBRDKAFKA_INCLUDE_DIRS} # librdkafka
26+
)
27+
target_link_libraries(json-output
28+
${ZLIB_LIBRARIES}
29+
${LIBRDKAFKA_LIBRARIES}
30+
)
2131

2232
install(
2333
TARGETS json-output

‎src/plugins/output/json/README.rst‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ Don't forget to remove (or comment) outputs that you don't want to use!
9696
<templateInfo>false</templateInfo>
9797
9898
<outputs>
99+
<!-- Choose one or more of the following outputs -->
99100
<server>
100101
<name>Local server</name>
101102
<port>8000</port>
@@ -122,6 +123,20 @@ Don't forget to remove (or comment) outputs that you don't want to use!
122123
<timeAlignment>yes</timeAlignment>
123124
<compression>none</compression>
124125
</file>
126+
127+
<kafka>
128+
<name>Send to Kafka</name>
129+
<brokers>127.0.0.1</brokers>
130+
<topic>ipfix</topic>
131+
<blocking>false</blocking>
132+
<partition>unassigned</partition>
133+
134+
<!-- Zero or more additional properties -->
135+
<property>
136+
<key>compression.codec</key>
137+
<value>lz4</value>
138+
</property>
139+
</kafka>
125140
</outputs>
126141
</params>
127142
</output>
@@ -187,8 +202,9 @@ Formatting parameters:
187202

188203
----
189204

190-
Output types: At least one of the following output must be configured. Multiple server/send/file
191-
outputs can be used at the same time if the outputs are not in collision with each other.
205+
Output types: At least one of the following output must be configured. Multiple
206+
server/send/file/kafka outputs can be used at the same time if the outputs are not in collision
207+
with each other.
192208

193209
:``server``:
194210
TCP (push) server provides data on a local port. Converted records are automatically send to
@@ -241,6 +257,35 @@ outputs can be used at the same time if the outputs are not in collision with ea
241257
:``none``: Compression disabled [default]
242258
:``gzip``: GZIP compression
243259

260+
:``kafka``:
261+
Send data to Kafka i.e. Kafka producer.
262+
263+
:``name``: Identification name of the output. Used only for readability.
264+
:``brokers``:
265+
Initial list of brokers as a CSV list of broker "host" or "host:port".
266+
:``topic``:
267+
Kafka topic to produce to.
268+
:``partition``:
269+
Partition number to produce to. If the value is "unassigned", then the default random
270+
distribution is used. [default: "unassigned"]
271+
:``brokerVersion``:
272+
Older broker versions (before 0.10.0) provide no way for a client to query for
273+
supported protocol features making it impossible for the client to know what features
274+
it may use. As a workaround a user may set this property to the expected broker
275+
version and the client will automatically adjust its feature set.
276+
[default: <empty>]
277+
:``blocking``:
278+
Enable blocking on produce. If disabled and a cluster is down or not able
279+
to retrieve records fast enough, some flow records may be dropped. On the other hand,
280+
if enabled, no records are dropped. However, if the cluster is slow or not accessible
281+
at all, the plugin waits (i.e. blocks) until data are send. This can significantly slow
282+
down or block(!) the whole collector and other output plugins [true/false, default: false]
283+
:``property``:
284+
Additional configuration properties of librdkafka library as key/value pairs.
285+
Multiple <property> parameters, which can improve performance, can be defined.
286+
See the project website for the full list of supported options. Keep on mind that
287+
some options might not be available in all versions of the library.
288+
244289
:``print``:
245290
Write data on standard output.
246291

0 commit comments

Comments
 (0)