Merge branch 'develop'

Author: salcock

.github/workflows/deb-build.yaml

@@ -21,7 +21,6 @@ jobs:
            - "ubuntu:xenial"
            - "ubuntu:bionic"
            - "ubuntu:focal"
-           - "ubuntu:groovy"
 
      steps:
       - name: Checkout repo

README.md

@@ -1,6 +1,6 @@
 OpenLI -- open source ETSI-compliant Lawful Intercept software
 
-Version: 1.0.10
+Version: 1.0.11
 
 ---------------------------------------------------------------------------
 

configure.ac

@@ -1,6 +1,6 @@
 # Super primitive configure script
 
-AC_INIT(openli, 1.0.10, salcock@waikato.ac.nz)
+AC_INIT(openli, 1.0.11, salcock@waikato.ac.nz)
 
 AM_INIT_AUTOMAKE([subdir-objects])
 AC_CONFIG_SRCDIR(src/collector/collector.c)

debian/changelog

@@ -1,3 +1,26 @@
+openli (1.0.11-1) unstable; urgency=medium
+
+  * Improved collector encoding performance by saving and reusing
+    previously-encoded records that have the exact same layout.
+  * Improved encoding performance at high packet rates by sending encoded
+    records to the forwarding thread in batches.
+  * Mediators will now actively avoid splitting a record across multiple send
+    calls wherever possible.
+  * Fixed bug where a collector would simply stop forwarding records for an
+    LIID on to the mediator for no apparent reason, especially when more
+    encoder threads were being used.
+  * Fixed bug where encoding jobs would be lost without being seen by the
+    encoder thread.
+  * Fixed performance-related issue where an overwhelmed mediator would never
+    send data to its handovers.
+  * Fixed crash in mediator after a handover is disconnected for failing to
+    send a keep alive response.
+  * BER encoding optimization has been removed.
+  * Fix issue where collector memory usage would be extremely high when under
+    load.
+
+ -- Shane Alcock <shane.alcock@waikato.ac.nz>  Wed, 21 Jul 2021 10:56:13 +1200
+
 openli (1.0.10-1) unstable; urgency=medium
 
   * Intercepts can now be configured with a start and/or end time (unix

debian/compat

@@ -2,8 +2,8 @@ Source: openli
 Section: net
 Priority: optional
 Maintainer: Shane Alcock <shane.alcock@waikato.ac.nz>
-Build-Depends: debhelper (>= 9), dh-autoreconf, dh-systemd (>=1.5),
- libtrace4-dev (>= 4.0.14), libyaml-dev, uthash-dev, libwandder2-dev,
+Build-Depends: debhelper-compat (= 12), dh-autoreconf, dh-systemd (>=1.5),
+ libtrace4-dev (>= 4.0.16), libyaml-dev, uthash-dev, libwandder2-dev,
  libjudy-dev, libzmq3-dev, libgoogle-perftools-dev, libosip2-dev,
  libssl1.0-dev (>=1.0.2r) | libssl-dev, librabbitmq-dev,
  libmicrohttpd-dev, libjson-c-dev, libsqlcipher-dev

debian/control

@@ -1,7 +1,7 @@
 #!/usr/bin/make -f
 %:
 
-	dh $@ --with=systemd --with autoreconf
+	dh $@ --with autoreconf
 
 
 override_dh_auto_configure:

debian/rules

@@ -11,7 +11,7 @@ export SOURCENAME=`echo ${GITHUB_REF##*/} | cut -d '-' -f 1`
 apt-get update
 apt-get install -y equivs devscripts dpkg-dev quilt curl apt-transport-https \
     apt-utils ssl-cert ca-certificates gnupg lsb-release debhelper git \
-    pkg-config
+    pkg-config sed
 
 DISTRO=$(lsb_release -sc)
 
@@ -21,8 +21,23 @@ curl -1sLf 'https://dl.cloudsmith.io/public/wand/libtrace/cfg/setup/bash.deb.sh'
 curl -1sLf 'https://dl.cloudsmith.io/public/wand/openli/cfg/setup/bash.deb.sh' | bash
 
 case ${DISTRO} in
-        jessie | xenial | stretch )
+        xenial )
                 curl -1sLf 'https://dl.cloudsmith.io/public/wand/dpdk-wand/cfg/setup/bash.deb.sh' | bash
+                apt-get install -y debhelper dh-systemd -t xenial-backports
+                sed -i 's/debhelper-compat (= 12)/debhelper (>= 10)/' debian/control
+                sed -i 's/--with auto/--with=systemd --with auto/' debian/rules
+                echo "10" > debian/compat
+        ;;
+
+        stretch )
+                curl -1sLf 'https://dl.cloudsmith.io/public/wand/dpdk-wand/cfg/setup/bash.deb.sh' | bash
+                sed -i 's/debhelper-compat (= 12)/debhelper (>= 10)/' debian/control
+                sed -i 's/--with auto/--with=systemd --with auto/' debian/rules
+                echo "10" > debian/compat
+        ;;
+
+        bionic )
+                apt-get install -y debhelper -t bionic-backports
         ;;
 esac
 

debpkg-setup.sh

@@ -155,28 +155,6 @@ A collector only requires a small amount of configuration: a username and
 password that can be used to authenticate against a local RabbitMQ instance,
 and a flag to inform the collector that RabbitMQ output is enabled.
 
-### Encoding Methods
-OpenLI supports two methods for encoding the intercepted records and meta-data
-so that they conform to the ETSI standard. The first method is DER encoding,
-whereby all fields are encoded in the most space-efficient way. The advantage
-of DER is that there is only one "correct" result for encoding a given input,
-so it is less likely to be incompatible with what the receiver is expecting.
-However, this encoding method is slower and therefore may limit your
-maximum interception capability.
-
-The second method is BER encoding, which allows us to encode numeric values
-using a fixed-width field, e.g. every integer consumes eight bytes, regardless
-of whether it is a small number or a large one. This is less space-efficient
-and relies on the receiver being prepared for fields that consume more space
-than they might need, but is much faster for OpenLI to encode because each
-record will have a relatively static structure.
-
-OpenLI versions prior to 1.0.9 supported DER encoding only, and this is the
-default encoding method as it has been well tested in production deployments.
-BER has the potential to be much more efficient, but we recommend doing some
-specific testing with your local LEAs before switching over to BER as not all
-LEA equipment may be able to decode it.
-
 ### Target Identification for VOIP Intercepts
 By default, OpenLI does NOT trust the "From:" field in SIP packets when it is
 determining whether a SIP packet has been sent by an intercept target. This
@@ -209,8 +187,6 @@ The basic option keys are:
 * interceptpointid  -- set the interception point ID
 * seqtrackerthreads -- set the number of threads to use for sequence number
                        tracking (defaults to 1).
-* encoding          -- choose the encoding method to use, either `ber` or
-                       `der` (defaults to `der`).
 * encoderthreads    -- set the number of threads to use for encoding ETSI
                        records (defaults to 2).
 * forwardingthreads -- set the number of threads to use for forwarding

doc/CollectorDoc.md

@@ -28,11 +28,6 @@ encoderthreads: 2
 # mediators. You probably don't need to change this.
 forwardingthreads: 1
 
-# The encoding method to use for generating ETSI records. DER is the default
-# method. BER is an alternative that should be more efficient, but not all
-# LEAs may be able to decode it.
-encoding: der
-
 # Set this to yes if you want to override the policy of not trusting the
 # contents of the "From:" field in SIP packets (as this field is not
 # validated and can be easily spoofed).