Merge pull request #179 from CESNET/lz4-compression

Ipfix plugin: Add Lz4 compression

Author: SiskaPavel

.github/workflows/c-cpp.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get -y install git build-essential autoconf libtool libpcap-dev pkg-config libxml2-dev libunwind-dev libfuse3-dev fuse3 cmake
+        sudo apt-get -y install git build-essential autoconf libtool libpcap-dev pkg-config libxml2-dev libunwind-dev libfuse3-dev fuse3 cmake liblz4-dev
         ( git clone --depth 1 https://github.com/CESNET/nemea-framework /tmp/nemea-framework; cd /tmp/nemea-framework; ./bootstrap.sh &&./configure --bindir=/usr/bin/nemea/ -q &&make -j10 && sudo make install; sudo ldconfig)
         ( git clone --depth 1 https://github.com/CESNET/nemea-modules /tmp/nemea-modules; cd /tmp/nemea-modules; ./bootstrap.sh &&./configure --bindir=/usr/bin/nemea/ -q &&make -j10 && sudo make install; )
         ( git clone -b release --depth 1 https://github.com/CESNET/telemetry /tmp/telemetry; cd /tmp/telemetry; mkdir build && cd build; cmake -DCMAKE_INSTALL_PREFIX=/usr .. &&make -j10 && sudo make install; )

.github/workflows/codeql-analysis.yml

@@ -30,7 +30,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get -y install git build-essential autoconf libtool libpcap-dev pkg-config libxml2-dev libfuse3-dev fuse3 cmake
+        sudo apt-get -y install git build-essential autoconf libtool libpcap-dev pkg-config libxml2-dev libfuse3-dev fuse3 cmake liblz4-dev
         ( git clone --depth 1 https://github.com/CESNET/nemea-framework /tmp/nemea-framework; cd /tmp/nemea-framework; ./bootstrap.sh &&./configure --bindir=/usr/bin/nemea/ -q &&make -j10 && sudo make install; sudo ldconfig)
         ( git clone -b release --depth 1 https://github.com/CESNET/telemetry /tmp/telemetry; cd /tmp/telemetry; mkdir build && cd build; cmake -DCMAKE_INSTALL_PREFIX=/usr .. &&make -j10 && sudo make install; )
     # Initializes the CodeQL tools for scanning.

.github/workflows/coverity.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get -y install git build-essential autoconf libtool libpcap-dev pkg-config libxml2-dev libfuse3-dev fuse3 cmake
+        sudo apt-get -y install git build-essential autoconf libtool libpcap-dev pkg-config libxml2-dev libfuse3-dev fuse3 cmake liblz4-dev
         ( git clone --depth 1 https://github.com/CESNET/nemea-framework /tmp/nemea-framework; cd /tmp/nemea-framework; ./bootstrap.sh &&./configure --bindir=/usr/bin/nemea/ -q &&make -j10 && sudo make install; sudo ldconfig)
         ( git clone --depth 1 https://github.com/CESNET/nemea-modules /tmp/nemea-modules; cd /tmp/nemea-modules; ./bootstrap.sh &&./configure --bindir=/usr/bin/nemea/ -q &&make -j10 && sudo make install; )
         ( git clone -b release --depth 1 https://github.com/CESNET/telemetry /tmp/telemetry; cd /tmp/telemetry; mkdir build && cd build; cmake -DCMAKE_INSTALL_PREFIX=/usr .. &&make -j10 && sudo make install; )

README.md

@@ -114,6 +114,9 @@ The output flow records are composed of information provided by the enabled plug
 
 See `ipfixprobe -h output` for more information and complete list of output plugins and their parameters.
 
+LZ4 compression:
+ipfix plugin supports LZ4 compression algorithm over tcp. See plugin's help for more information.
+
 ## Parameters
 ### Module specific parameters
 - `-i ARGS`       Activate input plugin  (-h input for help)

configure.ac

@@ -83,6 +83,11 @@ AC_DEFINE_DIR([DEFAULTSOCKETDIR], [defaultsocketdir], [Default path to socket di
 
 AC_CHECK_LIB(atomic, __atomic_store, [libatomic=yes], AC_MSG_ERROR([libatomic not found]))
 
+PKG_CHECK_MODULES([LIBLZ4], [liblz4])
+CFLAGS="$LIBLZ4_CFLAGS $CFLAGS"
+CXXFLAGS="$LIBLZ4_CFLAGS $CXXFLAGS"
+LIBS="$LIBLZ4_LIBS $LIBS"
+
 ### gtest
 AC_ARG_WITH([gtest],
         AC_HELP_STRING([--with-gtest],[Compile ipfixprobe with gtest framework]),

debian/control

@@ -3,7 +3,7 @@ Section: net
 Priority: standard
 Maintainer: Tomas Cejka <[email protected]> 
 Build-Depends: autoconf (>=2.69), pkg-config, libtool, make (>=4.2.1), debhelper (>=9),
-	openssl, libpcap-dev, libpcap0.8, libssl-dev, libatomic1
+	openssl, libpcap-dev, libpcap0.8, libssl-dev, libatomic1, liblz4-dev
 Standards-Version: 4.5.0
 Homepage: https://github.com/CESNET/ipfixprobe
 Vcs-Git: https://github.com/CESNET/ipfixprobe

init/ipfixprobed

@@ -118,7 +118,12 @@ if [ -e "$CONFFILE" ]; then
       output_affinity="@$OUTPUT_WORKER_CPU"
    fi
 
-   output="-o ipfix$output_affinity;host=${HOST:-127.0.0.1};port=${PORT:-4739};id=${LINK:-0};dir=${DIR:-0};${UDP_PARAM};${NON_BLOCKING_TCP_PARAM};template=${TEMPLATE_REFRESH_RATE:-300}"
+   LZ4_COMPRESSION_PARAM=""
+   if [[ $LZ4_COMPRESSION == "yes" ]]; then
+        LZ4_COMPRESSION_PARAM="lz4-compression";
+   fi
+
+   output="-o ipfix$output_affinity;host=${HOST:-127.0.0.1};port=${PORT:-4739};id=${LINK:-0};dir=${DIR:-0};${UDP_PARAM};${NON_BLOCKING_TCP_PARAM};${LZ4_COMPRESSION_PARAM};template=${TEMPLATE_REFRESH_RATE:-300}"
 
    telemetry=""
    if [ "$USE_FUSE" = "1" ]; then

init/link0.conf.example

@@ -197,6 +197,9 @@ TEMPLATE_REFRESH_RATE=300
 # Define output worker (thread) affinity, e.g. CPU core isolated from the scheduler
 #OUTPUT_WORKER_CPU=12
 
+# Enable LZ4 compression (only with TCP)
+LZ4_COMPRESSION=no
+
 ####### Fuse telemetry
 
 USE_FUSE=0

ipfixprobe.spec.in

@@ -57,8 +57,8 @@ Vendor: CESNET, z.s.p.o.
 Packager: @USERNAME@ <@USERMAIL@>
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}
 Summary: IPFIX flow exporter with various extending IPFIX elements exported by plugins.
-Requires: libatomic fuse3 telemetry
-BuildRequires: gcc gcc-c++ make doxygen pkgconfig libatomic telemetry
+Requires: libatomic fuse3 telemetry lz4
+BuildRequires: gcc gcc-c++ make doxygen pkgconfig libatomic telemetry lz4-devel
 Provides: ipfixprobe
 
 %if %{with ndp}