Skip to content

Commit b6a92e1

Browse files
jotakjotak
committed
Fix e2e tests with Kafka
- Update strimzi to 0.43 - Refactor tests: avoid using external address+node port+kafka client to inject and read data. Instead, leverage FLP entirely with 2 FLPs: - FLP1: ingest from static file, stamp, output to kafka - FLP2: ingest from flp1/kafka, stamp, output to logs
1 parent 8ded3b0 commit b6a92e1

File tree

16 files changed

+16175
-11408
lines changed

16 files changed

+16175
-11408
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -955,6 +955,7 @@ Images
955955
image-push Push MULTIARCH_TARGETS images
956956
manifest-build Build MULTIARCH_TARGETS manifest
957957
manifest-push Push MULTIARCH_TARGETS manifest
958+
extract-binaries Extract all MULTIARCH_TARGETS binaries
958959
goyacc Regenerate filters query langage
959960
960961
kubernetes

‎docs/api.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,7 @@ Following is the supported API format for filter transformations:
229229
input: entry input field
230230
value: specified value of input field:
231231
castInt: set true to cast the value field as an int (numeric values are float64 otherwise)
232+
samplingField: sampling field name to be set when sampling is used; if the field already exists in flows, its value is multiplied with the new sampling
232233
</pre>
233234
## Transform Network API
234235
Following is the supported API format for network transformations:

‎hack/update-docs.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ update-animated-gif() {
2828

2929
main() {
3030
update-readme
31-
update-animated-gif
31+
# update-animated-gif
3232
}
3333

3434
main

‎pkg/test/e2e/kafka/flp-1.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
---
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
name: flp-1
6+
labels:
7+
app: flp-1
8+
spec:
9+
template:
10+
spec:
11+
containers:
12+
- name: flowlogs-pipeline
13+
image: quay.io/netobserv/flowlogs-pipeline:e2e-amd64
14+
args:
15+
- "--config=/etc/flowlogs-pipeline/flowlogs-pipeline.conf.yaml"
16+
imagePullPolicy: Never
17+
volumeMounts:
18+
- name: configuration
19+
mountPath: "/etc/flowlogs-pipeline/"
20+
volumes:
21+
- name: configuration
22+
configMap:
23+
name: flp-config-1
24+
restartPolicy: Never
25+
---

‎pkg/test/e2e/kafka/flp-2.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
apiVersion: v1
3+
kind: Pod
4+
metadata:
5+
name: flp-2
6+
labels:
7+
app: flp-2
8+
spec:
9+
containers:
10+
- name: flowlogs-pipeline
11+
image: quay.io/netobserv/flowlogs-pipeline:e2e-amd64
12+
args:
13+
- "--config=/etc/flowlogs-pipeline/flowlogs-pipeline.conf.yaml"
14+
imagePullPolicy: Never
15+
volumeMounts:
16+
- name: configuration
17+
mountPath: "/etc/flowlogs-pipeline/"
18+
volumes:
19+
- name: configuration
20+
configMap:
21+
name: flp-config-2
22+
---

‎pkg/test/e2e/kafka/flp-config.yaml‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎pkg/test/e2e/kafka/flp-configs.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: flp-config-1
5+
data:
6+
flowlogs-pipeline.conf.yaml: |
7+
pipeline:
8+
- name: file_ingest
9+
- name: transform
10+
follows: file_ingest
11+
- name: kafka_encode
12+
follows: transform
13+
- name: logs
14+
follows: transform
15+
parameters:
16+
- name: file_ingest
17+
ingest:
18+
type: file
19+
file:
20+
filename: /etc/flowlogs-pipeline/ingest.jsonlogs
21+
decoder:
22+
type: json
23+
- name: transform
24+
transform:
25+
type: filter
26+
filter:
27+
rules:
28+
- type: add_field
29+
addField:
30+
input: ProcessedByFLP1
31+
value: true
32+
- name: kafka_encode
33+
encode:
34+
type: kafka
35+
kafka:
36+
address: my-cluster-kafka-bootstrap.default.svc:9092
37+
topic: test
38+
- name: logs
39+
write:
40+
type: stdout
41+
ingest.jsonlogs: |
42+
{"Bytes":20800,"DstAddr":"10.130.2.2","DstPort":36936}
43+
{"Bytes":116000,"DstAddr":"10.0.156.12","DstPort":9537}
44+
{"Bytes":20800,"DstAddr":"10.130.2.13","DstPort":3100}
45+
{"Bytes":20800,"DstAddr":"10.128.4.12","DstPort":3100}
46+
---
47+
apiVersion: v1
48+
kind: ConfigMap
49+
metadata:
50+
name: flp-config-2
51+
data:
52+
flowlogs-pipeline.conf.yaml: |
53+
pipeline:
54+
- name: kafka_ingest
55+
- name: transform
56+
follows: kafka_ingest
57+
- name: logs
58+
follows: transform
59+
parameters:
60+
- name: kafka_ingest
61+
ingest:
62+
type: kafka
63+
kafka:
64+
brokers: [my-cluster-kafka-bootstrap.default.svc:9092]
65+
topic: test
66+
groupid: group_test_in
67+
decoder:
68+
type: json
69+
- name: transform
70+
transform:
71+
type: filter
72+
filter:
73+
rules:
74+
- type: add_field
75+
addField:
76+
input: ProcessedByFLP2
77+
value: true
78+
- name: logs
79+
write:
80+
type: stdout
81+
---

‎pkg/test/e2e/kafka/flp.yaml‎

Lines changed: 0 additions & 99 deletions
This file was deleted.

‎pkg/test/e2e/kafka/kafka.strimzi.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
# Based on
2-
# https://raw.githubusercontent.com/strimzi/strimzi-kafka-operator/0.31.0/examples/kafka/kafka-ephemeral.yaml
2+
# https://raw.githubusercontent.com/strimzi/strimzi-kafka-operator/0.43.0/examples/kafka/kafka-ephemeral.yaml
33
apiVersion: kafka.strimzi.io/v1beta2
44
kind: Kafka
55
metadata:
66
name: my-cluster
77
spec:
88
kafka:
9-
version: 3.2.1
9+
version: 3.8.0
1010
replicas: 1
1111
listeners:
1212
- name: plain
@@ -27,7 +27,7 @@ spec:
2727
transaction.state.log.min.isr: 1
2828
default.replication.factor: 1
2929
min.insync.replicas: 1
30-
inter.broker.protocol.version: "3.2"
30+
inter.broker.protocol.version: "3.8"
3131
storage:
3232
type: ephemeral
3333
zookeeper:

‎pkg/test/e2e/kafka/kafka_suite_test.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ var TestEnv env.Environment
3131

3232
var manifestDeployDefinitions = e2e.ManifestDeployDefinitions{
3333
e2e.ManifestDeployDefinition{
34-
YamlFile: "strimzi-cluster-operator-0.31.0.yaml",
34+
YamlFile: "strimzi-cluster-operator-0.43.0.yaml",
3535
PostFunction: func(_ context.Context, _ *envconf.Config, _ string) error {
3636
// Wait few seconds to allow the CRD to be registered so the next step does not fail
3737
time.Sleep(5 * time.Second)
@@ -42,12 +42,16 @@ var manifestDeployDefinitions = e2e.ManifestDeployDefinitions{
4242
YamlFile: "kafka.strimzi.yaml",
4343
PostFunction: postStrimziDeploy,
4444
},
45+
e2e.ManifestDeployDefinition{YamlFile: "topic.yaml"},
46+
e2e.ManifestDeployDefinition{YamlFile: "flp-configs.yaml"},
47+
// flp-2 receives from flp-1 (which starts producing immediately), so start the receiver first
4548
e2e.ManifestDeployDefinition{
46-
YamlFile: "flp-config.yaml",
49+
YamlFile: "flp-2.yaml",
50+
PostFunction: postFLP2Deploy,
4751
},
4852
e2e.ManifestDeployDefinition{
49-
YamlFile: "flp.yaml",
50-
PostFunction: postFLPDeploy,
53+
YamlFile: "flp-1.yaml",
54+
PostFunction: postFLP1Deploy,
5155
},
5256
}
5357

0 commit comments

Comments
 (0)