Biflow aggregator - Add tests

Damir Zainullin · Damir Zainullin · commit d9fde3d6755f · 2025-07-12T15:40:42.000+02:00
diff --git a/biflow_aggregator/Makefile.am b/biflow_aggregator/Makefile.am
@@ -5,3 +5,5 @@ biflow_aggregator_SOURCES=main.cpp fields.c fields.h configuration.cpp configura
                           rapidxml.hpp
 biflow_aggregator_LDADD=-lunirec -ltrap
 include ../aminclude.am
+
+TESTS = tests/test.sh
diff --git a/biflow_aggregator/tests/config.xml b/biflow_aggregator/tests/config.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0"?>
+<aggregator>
+    <id name="packet_aggregation">
+        <field>
+            <type>SUM</type>
+            <name>PACKETS</name>
+        </field>
+    </id>
+    <id name="generic_flow_key_min_ports">
+		<field>
+			<name>FLOW_ID</name>
+			<type>KEY</type>
+		</field>
+
+		<field>
+			<name>SRC_PORT</name>
+			<type>MIN</type>
+		</field>
+		<field>
+			<name>DST_PORT</name>
+			<type>MIN</type>
+		</field>
+    </id>
+    <id name="src_dst_ip_all_aggregations">
+		<field>
+			<name>SRC_IP</name>
+			<type>KEY</type>
+		</field>
+		<field>
+			<name>DST_IP</name>
+			<type>KEY</type>
+		</field>
+
+		<field>
+			<name>SUM</name>
+			<type>SUM</type>
+		</field>
+		<field>
+			<name>MIN</name>
+			<type>MIN</type>
+		</field>
+        <field>
+			<name>MAX</name>
+			<type>MAX</type>
+		</field>
+        <field>
+			<name>FIRST_NON_EMPTY</name>
+			<type>FIRST_NON_EMPTY</type>
+		</field>
+        <field>
+			<name>LAST_NON_EMPTY</name>
+			<type>LAST_NON_EMPTY</type>
+		</field>
+        <field>
+			<name>FIRST</name>
+			<type>FIRST</type>
+		</field>
+        <field>
+			<name>LAST</name>
+			<type>LAST</type>
+		</field>
+        <field>
+			<name>AVG</name>
+			<type>AVG</type>
+		</field>
+        <field>
+			<name>BITOR</name>
+			<type>BITOR</type>
+		</field>
+        <field>
+			<name>STR_APPEND</name>
+			<type>APPEND</type>
+			<delimiter>:</delimiter>
+			<size>10</size>
+		</field>
+        <field>
+			<name>SORTED_MERGE_VALUE</name>
+			<type>SORTED_MERGE</type>
+			<delimiter>:</delimiter>
+			<sort_key>SORTED_MERGE_KEY</sort_key>
+			<sort_type>ASCENDING</sort_type>
+			<size>10</size>
+		</field>
+    </id>
+</aggregator>
diff --git a/biflow_aggregator/tests/inputs/input1_packet_aggregation b/biflow_aggregator/tests/inputs/input1_packet_aggregation
@@ -0,0 +1,3 @@
+ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST
+192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0
+192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0
diff --git a/biflow_aggregator/tests/inputs/input2_packet_aggregation b/biflow_aggregator/tests/inputs/input2_packet_aggregation
@@ -0,0 +1,6 @@
+ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST
+192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0
+192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0
+192.168.1.8,192.168.1.9,10,2016-10-28T17:00:13.0,2016-10-28T17:00:17.0
+192.168.1.8,192.168.1.9,17,2016-10-28T17:00:21.0,2016-10-28T17:00:23.0
+192.168.1.8,192.168.1.9,53,2016-10-28T17:00:26.0,2016-10-28T17:00:41.0
diff --git a/biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports b/biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports
@@ -0,0 +1,6 @@
+ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST,uint32 FLOW_ID,uint16 SRC_PORT,uint16 DST_PORT
+192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0,1,0,6666
+192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0,1,6666,0
+192.168.1.8,192.168.1.9,10,2016-10-28T17:00:13.0,2016-10-28T17:00:17.0,2,6666,0
+192.168.1.8,192.168.1.9,17,2016-10-28T17:00:21.0,2016-10-28T17:00:23.0,2,3333,3333
+192.168.1.8,192.168.1.9,53,2016-10-28T17:00:26.0,2016-10-28T17:00:41.0,2,0,6666
diff --git a/biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations b/biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations
@@ -0,0 +1,3 @@
+ipaddr DST_IP,ipaddr SRC_IP,time TIME_FIRST,time TIME_LAST,uint32 SUM,uint32 MIN,uint32 MAX,string FIRST_NON_EMPTY,string LAST_NON_EMPTY,uint32 FIRST,uint32 LAST,double AVG,string STR_APPEND,uint32 BITOR,uint32* SORTED_MERGE_KEY,uint32* SORTED_MERGE_VALUE
+192.168.1.1,192.168.1.2,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0,333,5,5,,222,16,32,7,test1,1,[3|2|1],[1|2|3]
+192.168.1.1,192.168.1.2,2016-10-28T17:00:1.0,2016-10-28T17:00:11.0,333,1,4,test,,555,33,9,test2,3,[8|7|6],[6|5|4]
diff --git a/biflow_aggregator/tests/references/reference1_gt0 b/biflow_aggregator/tests/references/reference1_gt0
@@ -0,0 +1 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667
diff --git a/biflow_aggregator/tests/references/reference1_gt5a b/biflow_aggregator/tests/references/reference1_gt5a
@@ -0,0 +1,2 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1
+2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,666
diff --git a/biflow_aggregator/tests/references/reference1_gt5r b/biflow_aggregator/tests/references/reference1_gt5r
@@ -0,0 +1 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667
diff --git a/biflow_aggregator/tests/references/reference2_gt0 b/biflow_aggregator/tests/references/reference2_gt0
@@ -0,0 +1,2 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:23.000000,4,694
+2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,53
diff --git a/biflow_aggregator/tests/references/reference2_gt5a b/biflow_aggregator/tests/references/reference2_gt5a
@@ -0,0 +1,5 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1
+2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,666
+2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,10
+2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,17
+2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,53
diff --git a/biflow_aggregator/tests/references/reference2_gt5r b/biflow_aggregator/tests/references/reference2_gt5r
@@ -0,0 +1,4 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667
+2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,10
+2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,17
+2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,53
diff --git a/biflow_aggregator/tests/references/reference3_gt0 b/biflow_aggregator/tests/references/reference3_gt0
@@ -0,0 +1,2 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,1,0,0
+2016-10-28T17:00:13.000000,2016-10-28T17:00:41.000000,3,2,0,0
diff --git a/biflow_aggregator/tests/references/reference3_gt5a b/biflow_aggregator/tests/references/reference3_gt5a
@@ -0,0 +1,5 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1,6666,0
+2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,1,0,6666
+2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,2,0,6666
+2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,2,3333,3333
+2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,2,6666,0
diff --git a/biflow_aggregator/tests/references/reference3_gt5r b/biflow_aggregator/tests/references/reference3_gt5r
@@ -0,0 +1,4 @@
+2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,1,0,0
+2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,2,0,6666
+2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,2,3333,3333
+2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,2,6666,0
diff --git a/biflow_aggregator/tests/references/reference4_gt0 b/biflow_aggregator/tests/references/reference4_gt0
@@ -0,0 +1 @@
+192.168.1.1,192.168.1.2,8.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,2,16,33,5,1,666,"test","222","test1:",[3|2|1|4|5|6]
diff --git a/biflow_aggregator/tests/references/reference4_gt5a b/biflow_aggregator/tests/references/reference4_gt5a
@@ -0,0 +1,2 @@
+192.168.1.1,192.168.1.2,7.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1,16,32,5,5,333,"","222","test1:",[3|2|1]
+192.168.1.1,192.168.1.2,9.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,1,555,33,4,1,333,"test","","test2:",[4|5|6]
diff --git a/biflow_aggregator/tests/references/reference4_gt5r b/biflow_aggregator/tests/references/reference4_gt5r
@@ -0,0 +1 @@
+192.168.1.1,192.168.1.2,8.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,2,16,33,5,1,666,"test","222","test1:",[3|2|1|4|5|6]
diff --git a/biflow_aggregator/tests/test.sh b/biflow_aggregator/tests/test.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+#set -e
+
+die_if_not_running() {
+    local pid=$1
+    local error_message=$2
+    if not kill -0 "$pid" 2>/dev/null; then
+        echo $error_message
+        exit 1
+    fi
+}
+
+run_test_with_global_timeout() {
+    local input=$1
+    local output=$2
+    local reference=$3
+    local config=$4
+    local global_timeout=$5
+
+    (../biflow_aggregator -i "u:lr,u:ba" -e -c config.xml -n $config -g $global_timeout & ) || true
+    local AGGREGATOR_PID=$!
+    sleep 0.5
+    die_if_not_running $AGGREGATOR_PID "Failed to start biflow aggregator"
+
+    ../../logger/logger -w $output -i "u:ba" &  
+    local LOGGER_PID=$!
+    sleep 0.5
+    die_if_not_running $LOGGER_PID "Failed to start logger"
+
+    ../../logreplay/logreplay -f $input -i "u:lr" &  
+    local LOGREPLAY_PID=$!
+    sleep 0.5
+    die_if_not_running $LOGREPLAY_PID "Failed to start logreplay"
+
+    sleep 2
+    wait $LOGREPLAY_PID
+    kill $AGGREGATOR_PID 2>/dev/null
+    sleep 0.2
+    kill $LOGGER_PID 2>/dev/null
+    wait $LOGGER_PID
+    
+    if ! colordiff $output $reference; then
+        echo $output doesnt match $reference
+        success="false"
+    fi
+}
+
+script_path="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+cd $script_path
+success="true"
+mkdir -p outputs 
+for input in inputs/input*; do
+    basename=$(basename "$input")
+    echo Found $basename
+    if [[ $basename =~ input([0-9]+)_(.+) ]]; then
+        index="${BASH_REMATCH[1]}"
+        config="${BASH_REMATCH[2]}"
+        output=outputs/output$index
+        reference=references/reference$index
+    else    
+        echo $input
+        echo "Incorrect input name. Must be input<index>_<config_name>"
+        exit 1
+    fi
+
+    echo "Running test without global timeout..."
+    run_test_with_global_timeout $input ${output}_gt0 ${reference}_gt0 $config "0"
+
+    echo "Running test with relative global timeout..."
+    run_test_with_global_timeout $input ${output}_gt5r ${reference}_gt5r $config "5r"
+
+    echo "Running test with absolute global timeout..."
+    run_test_with_global_timeout $input ${output}_gt5a ${reference}_gt5a $config "5a"
+
+done
+
+if [ "$success" = "true" ]; then
+    echo "All tests passed successfully."
+else
+    echo "Some tests failed."
+    exit 1
+fi

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST`
	`2`	`+192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0`
	`3`	`+192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+ipaddr DST_IP,ipaddr SRC_IP,time TIME_FIRST,time TIME_LAST,uint32 SUM,uint32 MIN,uint32 MAX,string FIRST_NON_EMPTY,string LAST_NON_EMPTY,uint32 FIRST,uint32 LAST,double AVG,string STR_APPEND,uint32 BITOR,uint32* SORTED_MERGE_KEY,uint32* SORTED_MERGE_VALUE`
	`2`	`+192.168.1.1,192.168.1.2,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0,333,5,5,,222,16,32,7,test1,1,[3\|2\|1],[1\|2\|3]`
	`3`	`+192.168.1.1,192.168.1.2,2016-10-28T17:00:1.0,2016-10-28T17:00:11.0,333,1,4,test,,555,33,9,test2,3,[8\|7\|6],[6\|5\|4]`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1`
	`2`	`+2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,666`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+2016-10-28T17:00:01.000000,2016-10-28T17:00:23.000000,4,694`
	`2`	`+2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,53`