Merge branch 'rpm-dpdk' of gitlab.liberouter.org:monitoring/ipfixprobe

Author: Pavel Siska

.gitlab-ci.yml

@@ -0,0 +1,139 @@
+default:
+    image: $CI_REGISTRY/monitoring/ipfixprobe/rpmbuilder:latest
+
+stages:
+   - prepare
+   - build
+   - test
+   - packages
+   - deploy
+
+ci_rpmbuilder:
+  stage: prepare
+  image: docker:latest
+  services:
+    - docker:dind
+  variables:
+    REPOSITORY: $CI_REGISTRY/monitoring/ipfixprobe/rpmbuilder
+
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+    - cd docker/rpmbuilder; docker build -t $REPOSITORY:latest .
+    - docker push $REPOSITORY:latest
+  only:
+    changes:
+      - docker/rpmbuilder/Dockerfile
+
+ci_rpmbuilder_dpdk:
+  stage: prepare
+  image: docker:latest
+  services:
+    - docker:dind
+  variables:
+    REPOSITORY: $CI_REGISTRY/monitoring/ipfixprobe/rpmbuilder-dpdk
+
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+    - cd docker/rpmbuilder-dpdk; docker build -t $REPOSITORY:latest .
+    - docker push $REPOSITORY:latest
+  only:
+    changes:
+      - docker/rpmbuilder-dpdk/Dockerfile
+
+ci_rpmbuilder_el7:
+  stage: prepare
+  image: docker:latest
+  services:
+    - docker:dind
+  variables:
+    REPOSITORY: $CI_REGISTRY/monitoring/ipfixprobe/rpmbuilder-epel7
+
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+    - cd docker/rpmbuilder-epel7/; docker build -t $REPOSITORY:latest .
+    - docker push $REPOSITORY:latest
+  only:
+    changes:
+      - docker/rpmbuilder-epel7/Dockerfile
+
+basic-build:
+  stage: build
+  script:
+    - autoreconf -i
+    - ./configure -q
+    - make
+  artifacts:
+    paths:
+      - ipfixprobe
+  cache:
+    paths:
+      - "*.o"
+
+# run tests using the binary built before
+test:
+  stage: test
+  before_script:
+    - git submodule init
+    - git submodule update
+  script:
+    - autoreconf -i
+    - ./configure -q --with-pcap --with-nemea --with-gtest
+    - make check
+
+buildrpm:
+  stage: packages
+  script:
+    - autoreconf -i
+    - ./configure -q
+    - make rpm-nemea
+    - make rpm-ndp
+  artifacts:
+    paths:
+      - RPMBUILD/RPMS/
+      - RPMBUILD/SRPMS/
+
+buildrpmdpdk:
+  stage: packages
+  image: $CI_REGISTRY/monitoring/ipfixprobe/rpmbuilder-dpdk
+  script:
+    - autoreconf -i
+    - ./configure -q
+    - make rpm-nemea
+    - make rpm-dpdk
+  artifacts:
+    paths:
+      - RPMBUILD/RPMS/
+      - RPMBUILD/SRPMS/
+
+deploy:
+  stage: deploy
+  only:
+    - rpm-dpdk
+  before_script:
+    - microdnf -y install copr-cli git
+  script:
+    - echo "Prepare COPR config..."
+    - mkdir ~/.config
+    - echo -e "[copr-cli]\nlogin = LOGIN\nusername = cejkat\ncopr_url = https://copr.fedorainfracloud.org\ntoken = TOKEN" | sed "s/TOKEN/${COPR_TOKEN}/; s/LOGIN/${COPR_LOGIN}/" > ~/.config/copr
+    - echo "Run RPM build using COPR..."
+    - copr build --nowait @CESNET/NEMEA RPMBUILD/SRPMS/ipfixprobe-dpdk-*.src.rpm
+    - echo "Application successfully deployed."
+  dependencies:
+    - buildrpmdpdk
+
+buildrpm_epel7:
+  stage: packages
+  image: $CI_REGISTRY/monitoring/ipfixprobe/rpmbuilder-epel7
+  script:
+    - autoreconf -i
+    - ./configure -q
+    - make rpm-ndp
+    - make rpm-nemea
+  artifacts:
+    paths:
+      - RPMBUILD/RPMS/
+      - RPMBUILD/SRPMS/
+

Makefile.am

@@ -9,6 +9,8 @@ bin_PROGRAMS=ipfixprobe ipfixprobe_stats
 
 DISTCHECK_CONFIGURE_FLAGS="--with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)"
 
+AM_CPPFLAGS=-I$(srcdir)/include/
+
 ipfixprobe_LDFLAGS=-lpthread -ldl -latomic -ltelemetry -lappFs
 ipfixprobe_CFLAGS=-I$(srcdir)/include/ -fPIC -DFUSE_USE_VERSION=30
 ipfixprobe_CXXFLAGS=-std=gnu++17 -Wno-write-strings -I$(srcdir)/include/ -fPIC -DFUSE_USE_VERSION=30
@@ -33,13 +35,24 @@ ipfixprobe_input_src=\
 #example_la_SOURCES=example.cpp example.hpp
 #example_la_LDFLAGS=-module -shared -avoid-version
 
+if ENABLE_PCAP
+pkglib_LTLIBRARIES=input-pcap.la
+input_pcap_la_SOURCES= input/pcap.cpp input/pcap.hpp
+input_pcap_la_LDFLAGS=-module -shared -avoid-version -lpcap
+endif
 
 if WITH_NDP
 ipfixprobe_LDFLAGS+=-lnuma -lndpRI -L$(srcdir)/input/nfbCInterface/
 ipfixprobe_CXXFLAGS+=-I$(srcdir)/input/nfbCInterface/include/
 ipfixprobe_input_src+=\
 		input/ndp.cpp \
 		input/ndp.hpp
+
+pkglib_LTLIBRARIES=input-ndp.la
+input_ndp_la_CXXFLAGS=-I$(srcdir)/input/nfbCInterface/include/
+input_ndp_la_SOURCES= input/ndp.cpp input/ndp.hpp
+input_ndp_la_LDFLAGS=-module -shared -avoid-version -lnuma -lndpRI -L$(srcdir)/input/nfbCInterface/
+
 endif
 
 if WITH_RAW
@@ -175,8 +188,7 @@ ipfixprobe_input_src+=\
 endif
 
 if WITH_DPDK
-ipfixprobe_input_src+=\
-		input/dpdk/dpdkMbuf.hpp \
+dpdk_src_files=input/dpdk/dpdkMbuf.hpp \
 		input/dpdk/dpdkMbuf.cpp \
 		input/dpdk/dpdkDevice.hpp \
 		input/dpdk/dpdkDevice.cpp \
@@ -185,10 +197,19 @@ ipfixprobe_input_src+=\
 		input/dpdk/dpdkPortTelemetry.cpp \
 		input/dpdk/dpdkTelemetry.hpp \
 		input/dpdk/dpdkTelemetry.cpp \
-        input/dpdk.cpp \
-        input/dpdk.h \
-	input/dpdk-ring.cpp \
-	input/dpdk-ring.h
+		input/dpdk.cpp \
+		input/dpdk.h \
+		input/dpdk-ring.cpp \
+		input/dpdk-ring.h
+
+ipfixprobe_input_src+=$(dpdk_src_files)
+
+pkglib_LTLIBRARIES=input-dpdk.la
+input_dpdk_la_CFLAGS=-I$(srcdir)/include/ $(DPDK_CFLAGS)
+input_dpdk_la_CXXFLAGS=$(DPDK_CFLAGS)
+input_dpdk_la_SOURCES=$(dpdk_src_files)
+input_dpdk_la_LDFLAGS=-module -shared -avoid-version $(DPDK_LIBS)
+
 endif
 
 ipfixprobe_headers_src=\
@@ -305,6 +326,8 @@ srpm:
 	make ${AM_MAKEFLAGS} distdir='$(RPMDIR)/SOURCES/$(RPMFILENAME)' distdir
 	( cd "$(RPMDIR)/SOURCES/"; tar -z -c -f $(RPMFILENAME)-$(RELEASE).tar.gz $(RPMFILENAME); rm -rf $(RPMFILENAME); )
 	$(RPMBUILD) -bs $(PACKAGE_NAME).spec --define "_topdir `pwd`/$(RPMDIR)";
+	$(RPMBUILD) -bs $(PACKAGE_NAME)-dpdk.spec --define "_topdir `pwd`/$(RPMDIR)";
+	$(RPMBUILD) -bs $(PACKAGE_NAME)-ndp.spec --define "_topdir `pwd`/$(RPMDIR)";
 
 .PHONY: rpm
 rpm: srpm
@@ -316,7 +339,11 @@ rpm-nemea: srpm
 
 .PHONY: rpm-ndp
 rpm-ndp: srpm
-	$(RPMBUILD) --define "_topdir `pwd`/$(RPMDIR)" --rebuild $(RPMDIR)/SRPMS/$(RPMFILENAME)-$(RELEASE).src.rpm --with ndp --with unwind;
+	$(RPMBUILD) --define "_topdir `pwd`/$(RPMDIR)" --rebuild $(RPMDIR)/SRPMS/$(PACKAGE_NAME)-ndp-$(VERSION)-$(RELEASE).src.rpm --with unwind;
+
+.PHONY: rpm-dpdk
+rpm-dpdk: srpm
+	$(RPMBUILD) --define "_topdir `pwd`/$(RPMDIR)" --rebuild $(RPMDIR)/SRPMS/$(PACKAGE_NAME)-dpdk-$(VERSION)-$(RELEASE).src.rpm --with unwind;
 else
 endif
 

configure.ac

@@ -20,16 +20,17 @@ RPM_BUILDREQ=
 
 AC_CONFIG_MACRO_DIR([m4])
 # Must be checked before default -g -O2 is set:
+CXXFLAGS=" -std=gnu++17 "
 AC_ARG_ENABLE([debug],
         AC_HELP_STRING([--enable-debug],
         [Enable build with debug symbols and without optimizations.]),
         [if test "$enableval" = "yes"; then
-              CXXFLAGS="-Wall -g -O0 $CXXFLAGS"
+              CXXFLAGS+="-Wall -g -O0 $CXXFLAGS"
               CFLAGS="-Wall -g -O0 $CFLAGS"
          else
-              CXXFLAGS="-Wall -g -O3 $CXXFLAGS"
+              CXXFLAGS+="-Wall -g -O3 $CXXFLAGS"
               CFLAGS="-Wall -g -O3 $CFLAGS"
-        fi], [CXXFLAGS="-Wall -g -O3 $CXXFLAGS"
+        fi], [CXXFLAGS+="-Wall -g -O3 $CXXFLAGS"
               CPPFLAGS="-DNDEBUG=1 $CPPFLAGS"
               CFLAGS="-Wall -g -O3"])
 AM_CONDITIONAL(DEBUG, test x"$debug" = x"true")
@@ -81,6 +82,9 @@ AC_ARG_WITH([defaultsocketdir],
 AC_SUBST([defaultsocketdir], [$with_defaultsocketdir])
 AC_DEFINE_DIR([DEFAULTSOCKETDIR], [defaultsocketdir], [Default path to socket directory])
 
+pkglibdir=$libdir/ipfixprobe
+AC_DEFINE_DIR([PLUGINDIR], [pkglibdir], [Path to loadable plugins])
+
 AC_CHECK_LIB(atomic, __atomic_store, [libatomic=yes], AC_MSG_ERROR([libatomic not found]))
 
 PKG_CHECK_MODULES([LIBLZ4], [liblz4])
@@ -217,7 +221,7 @@ if test x${withndp} = xyes; then
         ), AC_MSG_ERROR([nfb/nfb.h not found. Try installing netcope-common-devel]))
 fi
 
-AM_CONDITIONAL(WITH_NDP,  test x${libnfb} = xyes && test x${withndp} = xyes)
+AM_CONDITIONAL(WITH_NDP,  test x${libnfb} = xyes -a x${withndp} = xyes)
 if [[ -z "$WITH_NDP_TRUE" ]]; then
    AC_DEFINE([WITH_NDP], [1], [Define to 1 if the ndp is available])
    CPPFLAGS="$CPPFLAGS -DIPXP_FLOW_CACHE_SIZE=19 -DIPXP_FLOW_LINE_SIZE=2" # 524288 cache records, 4 record per line
@@ -236,31 +240,47 @@ AC_ARG_WITH([pcap],
       fi
         ], [withpcap="no"]
 )
-
-if test x${withpcap} = xyes; then
-   AC_CHECK_HEADER(pcap.h,
-              AC_CHECK_LIB(pcap, pcap_open_live, [libpcap=yes],
-                           AC_CHECK_LIB(wpcap, pcap_open_live, [libwpcap=yes], AC_MSG_ERROR([libpcap not found. Try installing libpcap]))),
-                           AC_MSG_ERROR([pcap.h not found. Try installing libpcap-devel or libwpcap-devel]))
+AC_ARG_ENABLE([pcap-plugin],
+        AC_HELP_STRING([--disable-pcap-plugin],
+        [Disable build of input-pcap plugin.]),
+        [if test "$enableval" = "yes"; then
+              disablepcapplugin=no
+         else
+              disablepcapplugin=yes
+         fi], [disablepcapplugin=no])
+AM_CONDITIONAL(DISABLE_PCAP_MODULE, test x"$disablepcapplugin" = x"yes")
+
+BACKUPLIBS=$LIBS
+AC_CHECK_HEADER(pcap.h,
+           AC_CHECK_LIB(pcap, pcap_open_live, [libpcap=yes
+           have_pcap=yes],
+                        AC_CHECK_LIB(wpcap, pcap_open_live, [libwpcap=yes
+                        have_pcap=yes], [AC_MSG_WARN([libpcap not found. Try installing libpcap])
+                        have_pcap=no])),
+                        [AC_MSG_WARN([pcap.h not found. Try installing libpcap-devel or libwpcap-devel])
+                        have_pcap=no])
+
+if test x${withpcap} = xyes -a x${have_pcap} = xno; then
+   AC_MSG_ERROR([Cannot compile pcap plugin that was enabled by --with-pcap.])
 fi
 
-AM_CONDITIONAL(WITH_PCAP, test x${withpcap} && (test x${libpcap} = xyes || test x${libwpcap} = xyes))
-if [[ -z "$WITH_PCAP_TRUE" ]]; then
+AM_CONDITIONAL(ENABLE_PCAP, test x${have_pcap} = xyes -a x${disablepcapplugin} = xno)
+AM_CONDITIONAL(WITH_PCAP, test x${withpcap} = xyes -a x${have_pcap} = xyes)
+if test x${withpcap} = xyes -a x${have_pcap} = xyes; then
    AC_DEFINE([WITH_PCAP], [1], [Define to 1 if the libpcap is available])
-   if [[ -z "$WITH_PCAP_TRUE" ]]; then
-      if test x${libpcap} = xyes; then
-         LIBS="-lpcap $LIBS"
-         RPM_REQUIRES+=" libpcap"
-         RPM_BUILDREQ+=" libpcap-devel"
-      else
-         LIBS="-lwpcap $LIBS"
-         RPM_REQUIRES+=" libwpcap"
-         RPM_BUILDREQ+=" libwpcap-devel"
-      fi
+   if test x${libpcap} = xyes; then
+      LIBS="-lpcap $LIBS"
+      RPM_REQUIRES+=" libpcap"
+      RPM_BUILDREQ+=" libpcap-devel"
+   else
+      LIBS="-lwpcap $LIBS"
+      RPM_REQUIRES+=" libwpcap"
+      RPM_BUILDREQ+=" libwpcap-devel"
    fi
+else
+   LIBS=$BACKUPLIBS
 fi
 
-
 AC_ARG_WITH([unwind],
         AC_HELP_STRING([--with-unwind],[Compile ipfixprobe with libunwind to print stack on crash]),
         [
@@ -400,7 +420,7 @@ AM_CONDITIONAL(WITH_STEM, test x${withstem} = xyes)
 if [[ -z "$WITH_STEM_TRUE" ]]; then
    AC_DEFINE([WITH_STEM], [1], [Define to 1 to use flexprobe testing interface])
    CFLAGS="-I/usr/local/include/Stem $CFLAGS"
-   CXXFLAGS="-I/usr/local/include/Stem -std=gnu++17 -g -Wno-write-strings $CXXFLAGS"
+   CXXFLAGS="-I/usr/local/include/Stem -g -Wno-write-strings $CXXFLAGS"
    LIBS="-lstem $LIBS"
 fi
 
@@ -463,6 +483,8 @@ fi
 
 AC_CONFIG_FILES([Makefile
                  ipfixprobe.spec
+                 ipfixprobe-dpdk.spec
+                 ipfixprobe-ndp.spec
                  ipfixprobe.bash
                  input/nfbCInterface/Makefile
                  init/Makefile
@@ -499,6 +521,12 @@ echo "  LIBS...................: $LIBS"
 echo "Enforced NEMEA (for copr): $COPRRPM"
 echo "FlexProbe Data Interface.: $withflexprobe"
 echo "DPDK Interface...........: $withdpdk"
+echo "pcap Interface...........: $withpcap"
+if test -n "$ENABLE_PCAP_TRUE"; then
+echo "   pcap module...........: no"
+else
+echo "   pcap module...........: yes"
+fi
 echo
 echo "Installation.............: make install (as root if needed, with 'su' or 'sudo')"
 echo "  prefix.................: $prefix"

docker/README.md

@@ -1,3 +1,24 @@
+# How to build and push docker images
+
+To login:
+
+```
+podman login gitlab.liberouter.org:5050
+```
+
+To build new image with tag rpmbuilder:
+```
+cd rpmbuilder
+podman build -t rpmbuilder .
+```
+
+To push rpmbuilder image to repository:
+
+```
+podman push rpmbuilder:latest gitlab.liberouter.org:5050/monitoring/ipfixprobe
+```
+
+=======
 # ipfixprobe Docker wrapper 
 
 This repository contains a Docker container that processes network traffic from a pcap file using `ipfixprobe`. It accepts a pcap file and a processing script, runs it inside the container, and outputs the results in CSV format.