sp: add tests for Stream Processor hopping window

Signed-off-by: Masoud Koleini <[email protected]>

Author: koleini

tests/internal/data/stream_processor/gen_msgpack.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-echo "=== Just hit ctrl-c after 2 seconds... ==="
+echo "=== Hit ctrl-c if it runs forever!... ==="
 echo
 
 # Delete old samples file
@@ -11,4 +11,30 @@ rm -rf samples.mp samples-subkeys.mp
  -i tail -t samples         -p path=samples.json          -p parser=json       \
  -i tail -t samples-subkeys -p path=samples-subkeys.json  -p parser=json       \
  -o file -m samples         -p format=msgpack  -p path=samples.mp              \
- -o file -m samples-subkeys -p format=msgpack  -p path=samples-subkeys.mp -f 1
+ -o file -m samples-subkeys -p format=msgpack  -p path=samples-subkeys.mp -f 1 &
+ pid=$!
+ sleep 2
+ kill -9 $pid
+
+ # Generate new msgpack files for hopping window
+ files=$(find "samples-hw/" -type f -name "*.mp")
+
+ for file in $files
+ do
+     rm -f $file
+ done
+
+ files=$(find "samples-hw/" -type f -name "*.json")
+
+ for file in $files
+ do
+     # Generate new msgpack files for hopping window
+     echo ""
+     echo "=== Generating MessagePack file for $file ... ==="
+     ../../../../build/bin/fluent-bit -R ../../../../conf/parsers.conf   \
+      -i tail -p path=$file  -p parser=json                              \
+      -o file -p format=msgpack  -p path=$(echo $file | sed s/json/mp/) -f 1 &
+      pid=$!
+      sleep 2
+      kill -9 $pid
+ done

tests/internal/data/stream_processor/samples-hw/1.json

@@ -0,0 +1,4 @@
+{"id": 0, "word1": "fluent"          , "word2": "logging"         , "bytes": 10     , "bool": true}
+{"id": 1, "word1": "fluentd"         , "word2": "rlz"             , "bytes": 10.0   , "bool": true}
+{"id": 2, "word1": "fluent-bit"      , "word3": "rlz"             , "bytes": 10     , "bool": true}
+{"id": 3, "word1": "fluent-logger"   , "word3": ""                , "bytes": 10     , "bool": true}

tests/internal/data/stream_processor/samples-hw/1.mp

@@ -0,0 +1,4 @@
+{"id": 4, "word1": "forward"         , "word3": "plain"           , "bytes": 10     , "bool": true}
+{"id": 5, "word5": "forward-protocol", "word6": "secure"          , "bytes": 10     , "bool": true}
+{"id": 6, "word1": "stream"          , "word3": "processing"      , "bytes": 10.20  , "bool": false}
+{"id": 7, "word1": "edge-rocks"      , "word6": ""                , "bytes": 10     , "bool": true}

tests/internal/data/stream_processor/samples-hw/2.json

@@ -0,0 +1,3 @@
+{"id": 8, "word1": "treasure-data"   , "word3": "cncf"            , "bytes": 10     , "bool": true}
+{"id": 9, "word1": "arm"             , "word3": "linux foundation", "bytes": "10.30", "bool": false}
+{"id": 10, "word1": "fluent-bit"     , "word3": null              , "bytes": 10     , "bool": true}

tests/internal/data/stream_processor/samples-hw/2.mp

@@ -0,0 +1,4 @@
+{"id": 0, "word1": "fluent"          , "word2": "logging"         , "bytes": 10     , "bool": true}
+{"id": 1, "word1": "fluentd"         , "word2": "rlz"             , "bytes": 10.0   , "bool": true}
+{"id": 2, "word1": "fluent-bit"      , "word3": "rlz"             , "bytes": 10     , "bool": true}
+{"id": 3, "word1": "fluent-logger"   , "word3": ""                , "bytes": 10     , "bool": true}

tests/internal/data/stream_processor/samples-hw/3.json

@@ -0,0 +1,4 @@
+{"id": 4, "word1": "forward"         , "word3": "plain"           , "bytes": 10     , "bool": true}
+{"id": 5, "word5": "forward-protocol", "word6": "secure"          , "bytes": 10     , "bool": true}
+{"id": 6, "word1": "stream"          , "word3": "processing"      , "bytes": 10.20  , "bool": false}
+{"id": 7, "word1": "edge-rocks"      , "word6": ""                , "bytes": 10     , "bool": true}