Skip to content

Commit a66ac9d

Browse files
zohar7chzohar7ch
committed
Add kafka e2e test
1 parent 0dc03f6 commit a66ac9d

File tree

3 files changed

+181
-4
lines changed

3 files changed

+181
-4
lines changed

‎.github/actions/e2e-test-setup-action/action.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ inputs:
1313
sniffer-image:
1414
required: true
1515
type: string
16+
install-extra-flags:
17+
required: false
18+
type: string
19+
default: ""
1620
registry:
1721
required: true
1822
type: string
@@ -90,7 +94,7 @@ runs:
9094
SNIFFER_FLAGS="--set-string sniffer.repository=${{ env.REGISTRY }} --set-string sniffer.image=${{ inputs.sniffer-image }} --set-string sniffer.tag=${{ inputs.sniffer-tag }} --set-string sniffer.pullPolicy=Never"
9195
TELEMETRY_FLAG="--set global.telemetry.enabled=false"
9296
helm dep up ./helm-charts/network-mapper
93-
helm install otterize ./helm-charts/network-mapper -n otterize-system --create-namespace --set debug=true $MAPPER_FLAGS $SNIFFER_FLAGS $TELEMETRY_FLAG
97+
helm install otterize ./helm-charts/network-mapper -n otterize-system --create-namespace --set debug=true $MAPPER_FLAGS $SNIFFER_FLAGS $TELEMETRY_FLAG ${{inputs.install-extra-flags}}
9498
shell: bash
9599

96100
- name: Wait for Otterize

‎.github/workflows/e2e-test.yaml‎

Lines changed: 92 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ env:
3030
REGISTRY: ${{ inputs.registry }}
3131

3232
jobs:
33-
e2e-test:
33+
e2e-test-network-policy:
3434
timeout-minutes: 5
3535
strategy:
3636
matrix:
@@ -91,15 +91,104 @@ jobs:
9191
sleep 10 ;
9292
fi
9393
done
94-
94+
9595
echo Outputting all logs
9696
echo _SNIFFER LOGS_
9797
kubectl logs -n otterize-system -l app=otterize-network-sniffer --tail=-1
9898
echo _MAPPER LOGS_
9999
kubectl logs -n otterize-system -l app=otterize-network-mapper --tail=-1
100-
100+
101101
echo "export intents and compare to expected file"
102102
otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-mapper --format=json | jq 'sort_by(.metadata.namespace + .metadata.name)' > /tmp/intents.json
103103
diff .github/workflows/tests-expected-results/simple-tutorial-intents.json /tmp/intents.json
104104
echo "expected" && cat .github/workflows/tests-expected-results/simple-tutorial-intents.json
105105
echo "actual" && cat /tmp/intents.json
106+
107+
e2e-test-kafka:
108+
timeout-minutes: 8
109+
runs-on: ubuntu-latest
110+
steps:
111+
- name: Checkout
112+
uses: actions/checkout@v2
113+
with:
114+
submodules: recursive
115+
116+
- name: Login to GCR # We don't want this to happen in the setup, since we don't want to pass the SA account secret to the setup
117+
if: (github.event_name == 'push' && github.repository == 'otterize/network-mapper') || github.event.pull_request.head.repo.full_name == 'otterize/network-mapper'
118+
uses: docker/login-action@v2
119+
with:
120+
registry: ${{ env.REGISTRY }}
121+
username: _json_key_base64
122+
password: ${{ secrets.B64_GCLOUD_SERVICE_ACCOUNT_JSON }}
123+
124+
- name: Test setup
125+
uses: ./.github/actions/e2e-test-setup-action
126+
with:
127+
mapper-tag: ${{ inputs.mapper-tag }}
128+
sniffer-tag: ${{ inputs.sniffer-tag }}
129+
mapper-image: ${{ inputs.mapper-image }}
130+
sniffer-image: ${{ inputs.sniffer-image }}
131+
registry: ${{ inputs.registry }}
132+
install-extra-flags: " --set kafkawatcher.enable=true --set kafkawatcher.kafkaServers={\"kafka-0.kafka\"} "
133+
134+
- name: Install CLI
135+
run: |-
136+
wget --header="X-Otterize-Test: true" https://get.otterize.com/otterize-cli/v2.0.3/otterize_linux_x86_64.tar.gz
137+
tar xf otterize_linux_x86_64.tar.gz
138+
sudo cp otterize /usr/local/bin
139+
140+
- name: Deploy Kafka
141+
run: |-
142+
helm repo add otterize https://helm.otterize.com
143+
helm repo update
144+
helm install --create-namespace -n kafka -f https://docs.otterize.com/code-examples/kafka-mapping/helm/values.yaml kafka otterize/kafka --version 21.4.4
145+
146+
- name: Deploy Kafka Tutorial services
147+
run: |-
148+
kubectl apply -n otterize-tutorial-kafka-mapping -f https://docs.otterize.com/code-examples/kafka-mapping/all.yaml
149+
150+
- name: Wait for Kafka & Tutorial services
151+
run: |-
152+
kubectl wait pods -n kafka -l app.kubernetes.io/component=kafka --for condition=Ready --timeout=90s
153+
kubectl wait pods -n kafka -l app.kubernetes.io/component=zookeeper --for condition=Ready --timeout=90s
154+
kubectl wait pods -n otterize-system -l app=otterize-kafka-watcher --for condition=Ready --timeout=90s
155+
kubectl wait pods -n otterize-tutorial-kafka-mapping -l app=client --for condition=Ready --timeout=90s
156+
kubectl wait pods -n otterize-tutorial-kafka-mapping -l app=client-2 --for condition=Ready --timeout=90s
157+
158+
- name: Test Policy Export
159+
run: |-
160+
# wait for 2 intents to be discovered with timeout of 30 seconds.
161+
# sleeps 10 because this is the report interval from the watcher to the mapper
162+
for i in {1..5}
163+
do
164+
OUTPUT_JSON=`otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-kafka-mapping --format=json`
165+
if [ `echo "$OUTPUT_JSON" | jq ". | length"` != 2 ] || [ `echo "$OUTPUT_JSON" | jq '[.[] | select(.spec.targets[] | has("kafka"))] | length'` != 2 ] ; then
166+
echo "wait for discovered intents";
167+
echo _SNIFFER LOGS_
168+
kubectl logs --since=15s -n otterize-system -l app=otterize-network-sniffer
169+
echo _MAPPER LOGS_
170+
kubectl logs --since=15s -n otterize-system -l app=otterize-network-mapper
171+
sleep 10 ;
172+
fi
173+
done
174+
175+
echo Outputting all logs
176+
echo _SNIFFER LOGS_
177+
kubectl logs -n otterize-system -l app=otterize-network-sniffer --tail=-1
178+
echo _MAPPER LOGS_
179+
kubectl logs -n otterize-system -l app=otterize-network-mapper --tail=-1
180+
181+
echo "export intents and compare to expected file"
182+
otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-kafka-mapping --format=json | jq 'sort_by(.metadata.namespace + .metadata.name) | map(.spec.targets |= (sort_by(keys_unsorted[0]) | map(if .kafka? then .kafka.topics |= map(.operations |= sort) else . end)))' > /tmp/intents.json
183+
diff .github/workflows/tests-expected-results/kafka-tutorial-intents.json /tmp/intents.json
184+
echo "expected" && cat .github/workflows/tests-expected-results/kafka-tutorial-intents.json
185+
echo "actual" && cat /tmp/intents.json
186+
187+
e2e-test:
188+
needs:
189+
- e2e-test-policy-export
190+
- e2e-test-kafka
191+
runs-on: ubuntu-latest
192+
steps:
193+
- run: |-
194+
echo Success! This step is only here to depend on the tests.

‎.github/workflows/tests-expected-results/kafka-tutorial-intents.json‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
[
2+
{
3+
"kind": "ClientIntents",
4+
"apiVersion": "k8s.otterize.com/v2beta1",
5+
"metadata": {
6+
"name": "client",
7+
"namespace": "otterize-tutorial-kafka-mapping",
8+
"creationTimestamp": null
9+
},
10+
"spec": {
11+
"workload": {
12+
"name": "client",
13+
"kind": "Deployment"
14+
},
15+
"targets": [
16+
{
17+
"kafka": {
18+
"name": "kafka.kafka",
19+
"topics": [
20+
{
21+
"name": "mytopic",
22+
"operations": [
23+
"consume",
24+
"describe"
25+
]
26+
}
27+
]
28+
}
29+
},
30+
{
31+
"kubernetes": {
32+
"name": "kafka.kafka",
33+
"kind": "StatefulSet"
34+
}
35+
}
36+
]
37+
},
38+
"status": {
39+
"upToDate": false,
40+
"observedGeneration": 0
41+
}
42+
},
43+
{
44+
"kind": "ClientIntents",
45+
"apiVersion": "k8s.otterize.com/v2beta1",
46+
"metadata": {
47+
"name": "client-2",
48+
"namespace": "otterize-tutorial-kafka-mapping",
49+
"creationTimestamp": null
50+
},
51+
"spec": {
52+
"workload": {
53+
"name": "client-2",
54+
"kind": "Deployment"
55+
},
56+
"targets": [
57+
{
58+
"kafka": {
59+
"name": "kafka.kafka",
60+
"topics": [
61+
{
62+
"name": "mytopic",
63+
"operations": [
64+
"describe",
65+
"produce"
66+
]
67+
}
68+
]
69+
}
70+
},
71+
{
72+
"kubernetes": {
73+
"name": "kafka.kafka",
74+
"kind": "StatefulSet"
75+
}
76+
}
77+
]
78+
},
79+
"status": {
80+
"upToDate": false,
81+
"observedGeneration": 0
82+
}
83+
}
84+
]

0 commit comments

Comments
 (0)