Skip to content

Commit d3dd7a1

Browse files
zohar7chzohar7ch
committed
Add kafka e2e test
1 parent fd6d6aa commit d3dd7a1

File tree

3 files changed

+167
-4
lines changed

3 files changed

+167
-4
lines changed

‎.github/workflows/e2e-test-setpup-action.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ inputs:
1313
sniffer-image:
1414
required: true
1515
type: string
16+
install-extra-flags:
17+
required: false
18+
type: string
19+
default: ""
1620
registry:
1721
required: true
1822
type: string
@@ -97,7 +101,7 @@ runs:
97101
SNIFFER_FLAGS="--set-string sniffer.repository=${{ env.REGISTRY }} --set-string sniffer.image=${{ inputs.sniffer-image }} --set-string sniffer.tag=${{ inputs.sniffer-tag }} --set-string sniffer.pullPolicy=Never"
98102
TELEMETRY_FLAG="--set global.telemetry.enabled=false"
99103
helm dep up ./helm-charts/network-mapper
100-
helm install otterize ./helm-charts/network-mapper -n otterize-system --create-namespace --set debug=true $MAPPER_FLAGS $SNIFFER_FLAGS $TELEMETRY_FLAG
104+
helm install otterize ./helm-charts/network-mapper -n otterize-system --create-namespace --set debug=true $MAPPER_FLAGS $SNIFFER_FLAGS $TELEMETRY_FLAG ${{inputs.install-extra-flags}}
101105
102106
- name: Install CLI
103107
run: |-

‎.github/workflows/e2e-test.yaml‎

Lines changed: 78 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ env:
3030
REGISTRY: ${{ inputs.registry }}
3131

3232
jobs:
33-
e2e-test:
33+
e2e-test-policy-export:
3434
timeout-minutes: 5
3535
strategy:
3636
matrix:
@@ -77,15 +77,90 @@ jobs:
7777
sleep 10 ;
7878
fi
7979
done
80-
80+
8181
echo Outputting all logs
8282
echo _SNIFFER LOGS_
8383
kubectl logs -n otterize-system -l app=otterize-network-sniffer --tail=-1
8484
echo _MAPPER LOGS_
8585
kubectl logs -n otterize-system -l app=otterize-network-mapper --tail=-1
86-
86+
8787
echo "export intents and compare to expected file"
8888
otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-mapper --format=json | jq 'sort_by(.metadata.namespace + .metadata.name)' > /tmp/intents.json
8989
diff .github/workflows/tests-expected-results/simple-tutorial-intents.json /tmp/intents.json
9090
echo "expected" && cat .github/workflows/tests-expected-results/simple-tutorial-intents.json
9191
echo "actual" && cat /tmp/intents.json
92+
93+
e2e-test-kafka:
94+
timeout-minutes: 8
95+
runs-on: ubuntu-latest
96+
steps:
97+
- name: Checkout
98+
uses: actions/checkout@v2
99+
with:
100+
submodules: recursive
101+
102+
- name: Test setup
103+
uses: ./.github/workflows/e2e-test-setpup-action.yaml
104+
with:
105+
mapper-tag: ${{ inputs.mapper-tag }}
106+
sniffer-tag: ${{ inputs.sniffer-tag }}
107+
mapper-image: ${{ inputs.mapper-image }}
108+
sniffer-image: ${{ inputs.sniffer-image }}
109+
registry: ${{ inputs.registry }}
110+
install-extra-flags: " --set kafkawatcher.enable=true --set kafkawatcher.kafkaServers={\"kafka-0.kafka\"} "
111+
112+
- name: Deploy Kafka
113+
run: |-
114+
helm repo add otterize https://helm.otterize.com
115+
helm repo update
116+
helm install --create-namespace -n kafka -f https://docs.otterize.com/code-examples/kafka-mapping/helm/values.yaml kafka otterize/kafka --version 21.4.4
117+
118+
- name: Deploy Kafka Tutorial services
119+
run: |-
120+
kubectl apply -n otterize-tutorial-kafka-mapping -f https://docs.otterize.com/code-examples/kafka-mapping/all.yaml
121+
122+
- name: Wait for Kafka & Tutorial services
123+
run: |-
124+
kubectl wait pods -n kafka -l app.kubernetes.io/component=kafka --for condition=Ready --timeout=90s
125+
kubectl wait pods -n kafka -l app.kubernetes.io/component=zookeeper --for condition=Ready --timeout=90s
126+
kubectl wait pods -n otterize-system -l app=otterize-kafka-watcher --for condition=Ready --timeout=90s
127+
kubectl wait pods -n otterize-tutorial-kafka-mapping -l app=client --for condition=Ready --timeout=90s
128+
kubectl wait pods -n otterize-tutorial-kafka-mapping -l app=client-2 --for condition=Ready --timeout=90s
129+
130+
- name: Test Policy Export
131+
run: |-
132+
# wait for 2 intents to be discovered with timeout of 30 seconds.
133+
# sleeps 10 because this is the report interval from the watcher to the mapper
134+
for i in {1..5}
135+
do
136+
OUTPUT_JSON=`otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-kafka-mapping --format=json`
137+
if [ `echo "$OUTPUT_JSON" | jq ". | length"` != 2 ] || [ `echo "$OUTPUT_JSON" | jq '[.[] | select(.spec.targets[] | has("kafka"))] | length'` != 2 ] ; then
138+
echo "wait for discovered intents";
139+
echo _SNIFFER LOGS_
140+
kubectl logs --since=15s -n otterize-system -l app=otterize-network-sniffer
141+
echo _MAPPER LOGS_
142+
kubectl logs --since=15s -n otterize-system -l app=otterize-network-mapper
143+
sleep 10 ;
144+
fi
145+
done
146+
147+
echo Outputting all logs
148+
echo _SNIFFER LOGS_
149+
kubectl logs -n otterize-system -l app=otterize-network-sniffer --tail=-1
150+
echo _MAPPER LOGS_
151+
kubectl logs -n otterize-system -l app=otterize-network-mapper --tail=-1
152+
153+
echo "export intents and compare to expected file"
154+
otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-kafka-mapping --format=json | jq 'sort_by(.metadata.namespace + .metadata.name)' > /tmp/intents.json
155+
diff .github/workflows/tests-expected-results/kafka-tutorial-intents.json /tmp/intents.json
156+
echo "expected" && cat .github/workflows/tests-expected-results/kafka-tutorial-intents.json
157+
echo "actual" && cat /tmp/intents.json
158+
159+
e2e-test:
160+
needs:
161+
- e2e-test-policy-export
162+
- e2e-test-kafka
163+
runs-on: ubuntu-latest
164+
steps:
165+
- run: |-
166+
echo Success! This step is only here to depend on the tests.

‎.github/workflows/tests-expected-results/kafka-tutorial-intents.json‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
[
2+
{
3+
"kind": "ClientIntents",
4+
"apiVersion": "k8s.otterize.com/v2beta1",
5+
"metadata": {
6+
"name": "client",
7+
"namespace": "otterize-tutorial-kafka-mapping",
8+
"creationTimestamp": null
9+
},
10+
"spec": {
11+
"workload": {
12+
"name": "client",
13+
"kind": "Deployment"
14+
},
15+
"targets": [
16+
{
17+
"kubernetes": {
18+
"name": "kafka.kafka",
19+
"kind": "StatefulSet"
20+
}
21+
},
22+
{
23+
"kafka": {
24+
"name": "kafka.kafka",
25+
"topics": [
26+
{
27+
"name": "mytopic",
28+
"operations": [
29+
"describe",
30+
"consume"
31+
]
32+
}
33+
]
34+
}
35+
}
36+
]
37+
},
38+
"status": {
39+
"upToDate": false,
40+
"observedGeneration": 0
41+
}
42+
},
43+
{
44+
"kind": "ClientIntents",
45+
"apiVersion": "k8s.otterize.com/v2beta1",
46+
"metadata": {
47+
"name": "client-2",
48+
"namespace": "otterize-tutorial-kafka-mapping",
49+
"creationTimestamp": null
50+
},
51+
"spec": {
52+
"workload": {
53+
"name": "client-2",
54+
"kind": "Deployment"
55+
},
56+
"targets": [
57+
{
58+
"kubernetes": {
59+
"name": "kafka.kafka",
60+
"kind": "StatefulSet"
61+
}
62+
},
63+
{
64+
"kafka": {
65+
"name": "kafka.kafka",
66+
"topics": [
67+
{
68+
"name": "mytopic",
69+
"operations": [
70+
"describe",
71+
"produce"
72+
]
73+
}
74+
]
75+
}
76+
}
77+
]
78+
},
79+
"status": {
80+
"upToDate": false,
81+
"observedGeneration": 0
82+
}
83+
}
84+
]

0 commit comments

Comments
 (0)