Skip to content

Commit 923f014

Browse files
PavelShilin89PavelShilin89
authored
Test/add logstash 9.2 version testing (#3888)
* Split the logstash test into two separate tests, create personal folders for each integration * Update nightly_integration.yml
1 parent 75f2b34 commit 923f014

11 files changed

+106
-100
lines changed

‎.github/workflows/nightly_integration.yml‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
uses: actions/checkout@v3
3535
- uses: manticoresoftware/[email protected]
3636
with:
37-
test_prefix: test/clt-tests/integrations/test-integrations-dbeaver
37+
test_prefix: test/clt-tests/integrations/dbeaver/test-integrations-dbeaver
3838
image: ghcr.io/manticoresoftware/manticoresearch:test-kit-latest
3939
comment_mode: failures
4040
run_args: --privileged
@@ -49,7 +49,7 @@ jobs:
4949
uses: actions/checkout@v3
5050
- uses: manticoresoftware/[email protected]
5151
with:
52-
test_prefix: test/clt-tests/integrations/test-integrations-filebeat
52+
test_prefix: test/clt-tests/integrations/filebeat/test-integrations-filebeat
5353
image: ghcr.io/manticoresoftware/manticoresearch:test-kit-latest
5454
comment_mode: failures
5555
run_args: --privileged
@@ -64,7 +64,7 @@ jobs:
6464
uses: actions/checkout@v3
6565
- uses: manticoresoftware/[email protected]
6666
with:
67-
test_prefix: test/clt-tests/integrations/test-integrations-fluentbit
67+
test_prefix: test/clt-tests/integrations/fluentbit/test-integrations-fluentbit
6868
image: ghcr.io/manticoresoftware/manticoresearch:test-kit-latest
6969
comment_mode: failures
7070
run_args: --privileged
@@ -78,7 +78,7 @@ jobs:
7878
uses: actions/checkout@v3
7979
- uses: manticoresoftware/[email protected]
8080
with:
81-
test_prefix: test/clt-tests/integrations/test-integrations-logstash
81+
test_prefix: test/clt-tests/integrations/logstash/test-integrations-logstash
8282
image: ghcr.io/manticoresoftware/manticoresearch:test-kit-latest
8383
comment_mode: failures
8484
run_args: --privileged
@@ -93,22 +93,37 @@ jobs:
9393
uses: actions/checkout@v3
9494
- uses: manticoresoftware/[email protected]
9595
with:
96-
test_prefix: test/clt-tests/integrations/test-integrations-support-filebeat-versions
96+
test_prefix: test/clt-tests/integrations/filebeat/test-integrations-support-filebeat-versions
9797
image: ghcr.io/manticoresoftware/manticoresearch:test-kit-latest
9898
comment_mode: failures
9999
run_args: --privileged
100100
ui_host: "https://clt.manticoresearch.com"
101101

102-
test_support_logstash_versions:
102+
test_check_logstash_versions:
103+
name: Check for new Logstash versions
104+
runs-on: ubuntu-22.04
105+
timeout-minutes: 10
106+
steps:
107+
- name: Checkout repository
108+
uses: actions/checkout@v3
109+
- uses: manticoresoftware/[email protected]
110+
with:
111+
test_prefix: test/clt-tests/integrations/logstash/test-integrations-check-logstash-versions
112+
image: ghcr.io/manticoresoftware/manticoresearch:test-kit-latest
113+
comment_mode: failures
114+
run_args: --privileged
115+
ui_host: "https://clt.manticoresearch.com"
116+
117+
test_logstash_versions:
103118
name: Test Logstash versions support
104119
runs-on: ubuntu-22.04
105-
timeout-minutes: 60
120+
timeout-minutes: 180
106121
steps:
107122
- name: Checkout repository
108123
uses: actions/checkout@v3
109124
- uses: manticoresoftware/[email protected]
110125
with:
111-
test_prefix: test/clt-tests/integrations/test-integrations-support-logstash-versions
126+
test_prefix: test/clt-tests/integrations/logstash/test-integrations-test-logstash-versions
112127
image: ghcr.io/manticoresoftware/manticoresearch:test-kit-latest
113128
comment_mode: failures
114129
run_args: --privileged
@@ -123,7 +138,7 @@ jobs:
123138
uses: actions/checkout@v3
124139
- uses: manticoresoftware/[email protected]
125140
with:
126-
test_prefix: test/clt-tests/integrations/test-integrations-vector
141+
test_prefix: test/clt-tests/integrations/vector/test-integrations-vector
127142
image: ghcr.io/manticoresoftware/manticoresearch:test-kit-latest
128143
comment_mode: failures
129144
run_args: --privileged
@@ -153,7 +168,7 @@ jobs:
153168
uses: actions/checkout@v3
154169
- uses: manticoresoftware/[email protected]
155170
with:
156-
test_prefix: test/clt-tests/integrations/test-integrations-grafana
171+
test_prefix: test/clt-tests/integrations/grafana/test-integrations-grafana
157172
image: manticoresearch/dind:v1
158173
comment_mode: failures
159174
run_args: --privileged

‎test/clt-tests/integrations/test-integrations-dbeaver.rec‎ renamed to ‎test/clt-tests/integrations/dbeaver/test-integrations-dbeaver.rec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
––– block: ../base/start-searchd –––
1+
––– block: ../../base/start-searchd –––
22
––– input –––
33
mysql -h0 -P9306 -e "CREATE TABLE products (id INT, name TEXT, price FLOAT, category_id INT)"
44
––– output –––

‎test/clt-tests/integrations/test-integrations-filebeat.rec‎ renamed to ‎test/clt-tests/integrations/filebeat/test-integrations-filebeat.rec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ apt-get -y install gnupg2 openjdk-8-jre-headless curl jq filebeat > /dev/null 2>
2222
––– input –––
2323
echo -e "filebeat.inputs:\n- type: filestream\n id: example\n paths:\n - /var/log/dpkg.log\n\noutput.elasticsearch:\n hosts: [\"http://localhost:9308\"]\n index: \"dpkg_log\"\n allow_older_versions: true\n compression_level: 0\n\nsetup.ilm:\n enabled: false\n\nsetup.template:\n name: \"dpkg_log\"\n pattern: \"dpkg_log\"" > /filebeat.yml
2424
––– output –––
25-
––– block: ../base/start-searchd –––
25+
––– block: ../../base/start-searchd –––
2626
––– input –––
2727
echo -e "2023-05-31 10:42:55 status triggers-awaited ca-certificates-java:all 20190405ubuntu1.1\n2023-05-31 10:42:55 trigproc libc-bin:amd64 2.31-0ubuntu9.9 <none>\n2023-05-31 10:42:55 status half-configured libc-bin:amd64 2.31-0ubuntu9.9\n2023-05-31 10:42:55 status installed libc-bin:amd64 2.31-0ubuntu9.9\n2023-05-31 10:42:55 trigproc systemd:amd64 245.4-4ubuntu3.21 <none>" > /var/log/dpkg.log; echo $?
2828
––– output –––

‎test/clt-tests/integrations/test-integrations-support-filebeat-versions.rec‎ renamed to ‎test/clt-tests/integrations/filebeat/test-integrations-support-filebeat-versions.rec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
––– block: ../base/start-searchd-with-buddy –––
1+
––– block: ../../base/start-searchd-with-buddy –––
22
––– input –––
33
set -b
44
––– output –––

‎test/clt-tests/integrations/test-integrations-fluentbit.rec‎ renamed to ‎test/clt-tests/integrations/fluentbit/test-integrations-fluentbit.rec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ echo -e "2023-05-31 10:42:55 status triggers-awaited ca-certificates-java:all 20
3737
cat /var/log/dpkg.log | wc -l
3838
––– output –––
3939
5
40-
––– block: ../base/start-searchd –––
40+
––– block: ../../base/start-searchd –––
4141
––– input –––
4242
fluent-bit -c /usr/share/fluentbit/fluentbit/fluentbit.conf > /dev/null 2>&1; echo $?
4343
––– output –––

‎test/clt-tests/integrations/test-integrations-grafana.rec‎ renamed to ‎test/clt-tests/integrations/grafana/test-integrations-grafana.rec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
––– block: ../base/dind/init –––
1+
––– block: ../../base/dind/init –––
22
––– comment –––
33
Create Docker network
44
––– input –––

‎test/clt-tests/integrations/logstash/test-integrations-check-logstash-versions.rec‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
––– block: ../../base/start-searchd-with-buddy –––
2+
––– input –––
3+
set -b
4+
––– output –––
5+
––– input –––
6+
export PATH=/usr/bin:/usr/local/bin:/usr/sbin:/sbin:/bin
7+
––– output –––
8+
––– input –––
9+
apt-get update > /dev/null 2>&1 && apt-get install -y curl jq > /dev/null 2>&1; echo $?
10+
––– output –––
11+
0
12+
––– input –––
13+
bash << 'SCRIPT'
14+
# Static list of TESTED versions
15+
TESTED_VERSIONS="7.17
16+
8.0
17+
8.1
18+
8.2
19+
8.3
20+
8.4
21+
8.5
22+
8.6
23+
8.7
24+
8.8
25+
8.9
26+
8.10
27+
8.11
28+
8.12
29+
8.13
30+
8.14
31+
8.15
32+
8.16
33+
8.17
34+
8.18
35+
8.19
36+
9.0
37+
9.1"
38+
39+
# Check for NEW versions (after latest tested)
40+
LATEST_TESTED_VERSION="9.1"
41+
NEW_VERSIONS=$(curl -s "https://hub.docker.com/v2/repositories/library/logstash/tags/?page_size=100" \
42+
| jq -r ".results[].name" \
43+
| grep "^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$" \
44+
| sed "s/\.[0-9]*$//" \
45+
| awk '!/rc|beta|alpha/' \
46+
| sort -t. -k1,1n -k2,2n | uniq \
47+
| awk -v latest="$LATEST_TESTED_VERSION" '
48+
function version_compare(v1, v2) {
49+
split(v1, a, ".")
50+
split(v2, b, ".")
51+
if (a[1] != b[1]) return a[1] - b[1]
52+
return a[2] - b[2]
53+
}
54+
version_compare($0, latest) > 0')
55+
56+
if [ -n "$NEW_VERSIONS" ]; then
57+
echo "🆕 NEW Logstash versions detected:"
58+
echo "$NEW_VERSIONS"
59+
echo "❌ Need to test new versions and update the list!"
60+
exit 1
61+
else
62+
echo "✅ No new versions found after $LATEST_TESTED_VERSION"
63+
fi
64+
65+
# Use static list for testing
66+
echo "Using tested versions:"
67+
echo "$TESTED_VERSIONS"
68+
SCRIPT
69+
––– output –––
70+
🆕 NEW Logstash versions detected:
71+
9.2
72+
❌ Need to test new versions and update the list!

‎test/clt-tests/integrations/test-integrations-logstash.rec‎ renamed to ‎test/clt-tests/integrations/logstash/test-integrations-logstash.rec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ cd /usr/local/src
2828
––– input –––
2929
ln -s /usr/share/logstash/logstash-7.10.0/bin/logstash /usr/bin/logstash
3030
––– output –––
31-
––– block: ../base/start-searchd –––
31+
––– block: ../../base/start-searchd –––
3232
––– input –––
3333
echo 'input { file { path => ["/var/log/dpkg.log"] start_position => "beginning" sincedb_path => "/dev/null" mode => "read" exit_after_read => true file_completed_action => "log" file_completed_log_path => "/dev/null" } } output { elasticsearch { index => "dpkg_log" hosts => ["http://localhost:9308"] ilm_enabled => false manage_template => false } }' > logstash.conf
3434
––– output –––

‎test/clt-tests/integrations/test-integrations-support-logstash-versions.rec‎ renamed to ‎test/clt-tests/integrations/logstash/test-integrations-test-logstash-versions.rec‎

Lines changed: 1 addition & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
––– block: ../base/start-searchd-with-buddy –––
1+
––– block: ../../base/start-searchd-with-buddy –––
22
––– input –––
33
set -b
44
––– output –––
@@ -10,87 +10,6 @@ apt-get update > /dev/null 2>&1 && apt-get install -y gnupg openjdk-11-jre-headl
1010
––– output –––
1111
0
1212
––– input –––
13-
# Static list of TESTED versions (never changes)
14-
TESTED_VERSIONS="7.17
15-
8.0
16-
8.1
17-
8.2
18-
8.3
19-
8.4
20-
8.5
21-
8.6
22-
8.7
23-
8.8
24-
8.9
25-
8.10
26-
8.11
27-
8.12
28-
8.13
29-
8.14
30-
8.15
31-
8.16
32-
8.17
33-
8.18
34-
8.19
35-
9.0
36-
9.1"
37-
38-
# Check for NEW versions (after latest tested)
39-
LATEST_TESTED_VERSION="9.1"
40-
NEW_VERSIONS=$(curl -s "https://hub.docker.com/v2/repositories/library/logstash/tags/?page_size=100" \
41-
| jq -r ".results[].name" \
42-
| grep "^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$" \
43-
| sed "s/\.[0-9]*$//" \
44-
| awk '!/rc|beta|alpha/' \
45-
| sort -t. -k1,1n -k2,2n | uniq \
46-
| awk -v latest="$LATEST_TESTED_VERSION" '
47-
function version_compare(v1, v2) {
48-
split(v1, a, ".")
49-
split(v2, b, ".")
50-
if (a[1] != b[1]) return a[1] - b[1]
51-
return a[2] - b[2]
52-
}
53-
version_compare($0, latest) > 0')
54-
55-
if [ -n "$NEW_VERSIONS" ]; then
56-
echo "🆕 NEW Logstash versions detected:"
57-
echo "$NEW_VERSIONS"
58-
echo "❌ Need to test new versions and update the list!"
59-
exit 1
60-
else
61-
echo "✅ No new versions found after $LATEST_TESTED_VERSION"
62-
fi
63-
64-
# Use static list for testing
65-
echo "Using tested versions:"
66-
echo "$TESTED_VERSIONS"
67-
––– output –––
68-
✅ No new versions found after 9.1
69-
Using tested versions:
70-
7.17
71-
8.0
72-
8.1
73-
8.2
74-
8.3
75-
8.4
76-
8.5
77-
8.6
78-
8.7
79-
8.8
80-
8.9
81-
8.10
82-
8.11
83-
8.12
84-
8.13
85-
8.14
86-
8.15
87-
8.16
88-
8.17
89-
8.18
90-
8.19
91-
9.0
92-
9.1
93-
––– input –––
9413
cat << 'EOF' > /tmp/logstash-single-test.sh
9514
#!/bin/bash
9615
set -euo pipefail

‎test/clt-tests/integrations/test-integrations-vector.rec‎ renamed to ‎test/clt-tests/integrations/vector/test-integrations-vector.rec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ cat /var/log/dpkg.log | wc -l
3838
echo -e "[sources.test_file]\ntype = \"file\"\ninclude = [ \"/var/log/dpkg.log\" ]\n\n[transforms.modify_test_file]\ntype = \"remap\"\ninputs = [ \"test_file\" ]\nsource = \"\"\".vec_timestamp = del(.timestamp)\"\"\"\n\n[sinks.manticore]\ntype = \"elasticsearch\"\ninputs = [ \"modify_test_file\" ]\nendpoints = [\"http://127.0.0.1:9308\"]\nbulk.index = \"dpkg_log\"" | tee /usr/share/vector/vector.toml > /dev/null 2>&1; echo $?
3939
––– output –––
4040
0
41-
––– block: ../base/start-searchd –––
41+
––– block: ../../base/start-searchd –––
4242
––– input –––
4343
vector -c vector.toml > /var/log/vector.log 2>&1 &
4444
––– output –––

0 commit comments

Comments
 (0)