Added e2e test for kafka

.github/actions/e2e-test-setup-action/action.yaml

@@ -0,0 +1,111 @@
+name: E2E Setup
+description: This workflow is used to set up the environment for the E2E tests.
+inputs:
+    mapper-tag:
+        required: true
+        type: string
+    sniffer-tag:
+        required: true
+        type: string
+    mapper-image:
+        required: true
+        type: string
+    sniffer-image:
+        required: true
+        type: string
+    install-extra-flags:
+        required: false
+        type: string
+        default: ""
+    registry:
+        required: true
+        type: string
+        default: us-central1-docker.pkg.dev/main-383408/otterize
+
+runs:
+  using: "composite"
+  steps:
+    - name: Start minikube
+      uses: medyagh/setup-minikube@master
+      with:
+        start-args: "--network-plugin=cni --cni=calico"
+
+    - name: Load images from GitHub Artifacts
+      if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.REGISTRY }}_${{ github.actor }}_mapper_${{ github.sha }}.tar
+
+    - name: Load Docker image
+      if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
+      run: |-
+        docker image load -i mapper.tar
+        minikube image load ${{ env.REGISTRY }}/${{ github.actor }}/mapper:${{ github.sha }}
+      shell: bash
+
+    - name: Load images from GitHub Artifacts
+      if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.REGISTRY }}_${{ github.actor }}_sniffer_${{ github.sha }}.tar
+
+    - name: Load Docker image
+      if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
+      run: |-
+        docker image load -i sniffer.tar
+        minikube image load ${{ env.REGISTRY }}/${{ github.actor }}/sniffer:${{ github.sha }}
+      shell: bash
+
+    - name: Load images from GitHub Artifacts
+      if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.REGISTRY }}_${{ github.actor }}_kafka-watcher_${{ github.sha }}.tar
+
+    - name: Load Docker image
+      if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
+      run: |-
+        docker image load -i kafka-watcher.tar
+        minikube image load ${{ env.REGISTRY }}/${{ github.actor }}/kafka-watcher:${{ github.sha }}
+      shell: bash
+
+    - name: Load Docker images from GCR
+      if: (github.event_name == 'push' && github.repository == 'otterize/network-mapper') || github.event.pull_request.head.repo.full_name == 'otterize/network-mapper'
+      run: |-
+        docker pull ${{ env.REGISTRY }}/mapper:${{ inputs.mapper-tag }}	
+        minikube image load ${{ env.REGISTRY }}/mapper:${{ inputs.mapper-tag }}	
+        docker pull ${{ env.REGISTRY }}/sniffer:${{ inputs.sniffer-tag }}	
+        minikube image load  ${{ env.REGISTRY }}/sniffer:${{ inputs.sniffer-tag }}
+      shell: bash
+
+    - name: Set up Helm
+      uses: azure/setup-helm@v3
+
+    - name: Wait for Calico startup
+      run: |-
+        kubectl wait pods -n kube-system -l k8s-app=calico-kube-controllers --for condition=Ready --timeout=90s
+        kubectl wait pods -n kube-system -l k8s-app=calico-node --for condition=Ready --timeout=90s
+        kubectl wait pods -n kube-system -l k8s-app=calico-kube-controllers --for condition=Ready --timeout=90s
+      shell: bash
+
+    - name: Deploy Network Mapper
+      run: |-
+        MAPPER_FLAGS="--set-string mapper.repository=${{ env.REGISTRY }} --set-string mapper.image=${{ inputs.mapper-image }} --set-string mapper.tag=${{ inputs.mapper-tag }} --set-string mapper.pullPolicy=Never"
+        SNIFFER_FLAGS="--set-string sniffer.repository=${{ env.REGISTRY }} --set-string sniffer.image=${{ inputs.sniffer-image }} --set-string sniffer.tag=${{ inputs.sniffer-tag }} --set-string sniffer.pullPolicy=Never"
+        TELEMETRY_FLAG="--set global.telemetry.enabled=false"
+        helm dep up ./helm-charts/network-mapper
+        helm install otterize ./helm-charts/network-mapper -n otterize-system --create-namespace --set debug=true $MAPPER_FLAGS $SNIFFER_FLAGS $TELEMETRY_FLAG ${{inputs.install-extra-flags}}
+      shell: bash
+
+    - name: Install CLI
+      run: |-
+        wget --header="X-Otterize-Test: true" https://get.otterize.com/otterize-cli/v2.0.3/otterize_linux_x86_64.tar.gz
+        tar xf otterize_linux_x86_64.tar.gz
+        sudo cp otterize /usr/local/bin
+      shell: bash
+
+    - name: Wait for Otterize
+      run: |-
+        kubectl wait pods -n otterize-system -l app=otterize-network-sniffer --for condition=Ready --timeout=90s
+        kubectl wait pods -n otterize-system -l app=otterize-network-mapper --for condition=Ready --timeout=90s
+      shell: bash

.github/workflows/e2e-test.yaml

@@ -30,7 +30,7 @@ env:
   REGISTRY: ${{ inputs.registry }}
 
 jobs:
-  e2e-test:
+  e2e-test-network-policy:
     timeout-minutes: 5
     strategy:
       matrix:
@@ -42,96 +42,27 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Start minikube
-        uses: medyagh/setup-minikube@master
-        with:
-          start-args: "--network-plugin=cni --cni=calico"
-
-      - name: Load images from GitHub Artifacts
-        if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ env.REGISTRY }}_${{ github.actor }}_mapper_${{ github.sha }}.tar
-
-      - name: Load Docker image
-        if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
-        run: |-
-          docker image load -i mapper.tar
-          minikube image load ${{ env.REGISTRY }}/${{ github.actor }}/mapper:${{ github.sha }}
-
-      - name: Load images from GitHub Artifacts
-        if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ env.REGISTRY }}_${{ github.actor }}_sniffer_${{ github.sha }}.tar
-
-      - name: Load Docker image
-        if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
-        run: |-
-          docker image load -i sniffer.tar
-          minikube image load ${{ env.REGISTRY }}/${{ github.actor }}/sniffer:${{ github.sha }}
-
-      - name: Load images from GitHub Artifacts
-        if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ env.REGISTRY }}_${{ github.actor }}_kafka-watcher_${{ github.sha }}.tar
-
-      - name: Load Docker image
-        if: github.repository != 'otterize/network-mapper' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'otterize/network-mapper')
-        run: |-
-          docker image load -i kafka-watcher.tar
-          minikube image load ${{ env.REGISTRY }}/${{ github.actor }}/kafka-watcher:${{ github.sha }}
-
-
-      - name: Login to GCR
+      - name: Login to GCR # We don't want this to happen in the setup, since we don't want to pass the SA account secret to the setupGCR
         if: (github.event_name == 'push' && github.repository == 'otterize/network-mapper') || github.event.pull_request.head.repo.full_name == 'otterize/network-mapper'
         uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: _json_key_base64
           password: ${{ secrets.B64_GCLOUD_SERVICE_ACCOUNT_JSON }}
 
-      - name: Load Docker images from GCR
-        if: (github.event_name == 'push' && github.repository == 'otterize/network-mapper') || github.event.pull_request.head.repo.full_name == 'otterize/network-mapper'
-        run: |-
-          docker pull ${{ env.REGISTRY }}/mapper:${{ inputs.mapper-tag }}
-          minikube image load ${{ env.REGISTRY }}/mapper:${{ inputs.mapper-tag }}
-          docker pull ${{ env.REGISTRY }}/sniffer:${{ inputs.sniffer-tag }}
-          minikube image load  ${{ env.REGISTRY }}/sniffer:${{ inputs.sniffer-tag }}
-
-      - name: Set up Helm
-        uses: azure/setup-helm@v3
-
-      - name: Wait for Calico startup
-        run: |-
-          kubectl wait pods -n kube-system -l k8s-app=calico-kube-controllers --for condition=Ready --timeout=90s
-          kubectl wait pods -n kube-system -l k8s-app=calico-node --for condition=Ready --timeout=90s
-          kubectl wait pods -n kube-system -l k8s-app=calico-kube-controllers --for condition=Ready --timeout=90s
-
-      - name: Deploy Network Mapper
-        run: |-
-          MAPPER_FLAGS="--set-string mapper.repository=${{ env.REGISTRY }} --set-string mapper.image=${{ inputs.mapper-image }} --set-string mapper.tag=${{ inputs.mapper-tag }} --set-string mapper.pullPolicy=Never"
-          SNIFFER_FLAGS="--set-string sniffer.repository=${{ env.REGISTRY }} --set-string sniffer.image=${{ inputs.sniffer-image }} --set-string sniffer.tag=${{ inputs.sniffer-tag }} --set-string sniffer.pullPolicy=Never"
-          TELEMETRY_FLAG="--set global.telemetry.enabled=false"
-          helm dep up ./helm-charts/network-mapper
-          helm install otterize ./helm-charts/network-mapper -n otterize-system --create-namespace --set debug=true $MAPPER_FLAGS $SNIFFER_FLAGS $TELEMETRY_FLAG
-
-      - name: Install CLI
-        run: |-
-          wget --header="X-Otterize-Test: true" https://get.otterize.com/otterize-cli/v0.1.30/otterize_linux_x86_64.tar.gz
-          tar xf otterize_linux_x86_64.tar.gz
-          sudo cp otterize /usr/local/bin
+      - name: Test setup
+        uses: ./.github/actions/e2e-test-setup-action
+        with:
+          mapper-tag: ${{ inputs.mapper-tag }}
+          sniffer-tag: ${{ inputs.sniffer-tag }}
+          mapper-image: ${{ inputs.mapper-image }}
+          sniffer-image: ${{ inputs.sniffer-image }}
+          registry: ${{ inputs.registry }}
 
       - name: Deploy Tutorial services
         run: |-
           kubectl apply -n otterize-tutorial-mapper -f https://docs.otterize.com/code-examples/network-mapper/all.yaml
 
-      - name: Wait for Otterize
-        run: |-
-          kubectl wait pods -n otterize-system -l app=otterize-network-sniffer --for condition=Ready --timeout=90s
-          kubectl wait pods -n otterize-system -l app=otterize-network-mapper --for condition=Ready --timeout=90s
-
       - name: Wait for Tutorial services
         run: |-
           kubectl wait pods -n otterize-tutorial-mapper -l app=client --for condition=Ready --timeout=90s
@@ -154,19 +85,101 @@ jobs:
           sleep 10 ;
           fi
           done
-          
+
           echo Outputting all logs
           echo _SNIFFER LOGS_
           kubectl logs -n otterize-system -l app=otterize-network-sniffer --tail=-1
           echo _MAPPER LOGS_
           kubectl logs -n otterize-system -l app=otterize-network-mapper --tail=-1
-          
+
           echo "export intents and compare to expected file" 
           otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-mapper --format=json | jq 'sort_by(.metadata.namespace + .metadata.name)' > /tmp/intents.json
           diff .github/workflows/tests-expected-results/simple-tutorial-intents.json /tmp/intents.json 
           echo "expected" && cat .github/workflows/tests-expected-results/simple-tutorial-intents.json
           echo "actual" && cat /tmp/intents.json
-
-
 
-
+  e2e-test-kafka:
+    timeout-minutes: 8
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Login to GCR # We don't want this to happen in the setup, since we don't want to pass the SA account secret to the setup
+        if: (github.event_name == 'push' && github.repository == 'otterize/network-mapper') || github.event.pull_request.head.repo.full_name == 'otterize/network-mapper'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: _json_key_base64
+          password: ${{ secrets.B64_GCLOUD_SERVICE_ACCOUNT_JSON }}
+
+      - name: Test setup
+        uses: ./.github/actions/e2e-test-setup-action
+        with:
+          mapper-tag: ${{ inputs.mapper-tag }}
+          sniffer-tag: ${{ inputs.sniffer-tag }}
+          mapper-image: ${{ inputs.mapper-image }}
+          sniffer-image: ${{ inputs.sniffer-image }}
+          registry: ${{ inputs.registry }}
+          install-extra-flags: " --set kafkawatcher.enable=true --set kafkawatcher.kafkaServers={\"kafka-0.kafka\"} "
+
+      - name: Deploy Kafka
+        run: |-
+          helm repo add otterize https://helm.otterize.com
+          helm repo update
+          helm install --create-namespace -n kafka -f https://docs.otterize.com/code-examples/kafka-mapping/helm/values.yaml kafka otterize/kafka --version 21.4.4
+
+      - name: Deploy Kafka Tutorial services
+        run: |-
+          kubectl apply -n otterize-tutorial-kafka-mapping -f https://docs.otterize.com/code-examples/kafka-mapping/all.yaml
+
+      - name: Wait for Kafka & Tutorial services
+        run: |-
+          kubectl wait pods -n kafka -l app.kubernetes.io/component=kafka --for condition=Ready --timeout=90s
+          kubectl wait pods -n kafka -l app.kubernetes.io/component=zookeeper --for condition=Ready --timeout=90s
+          kubectl wait pods -n otterize-system -l app=otterize-kafka-watcher --for condition=Ready --timeout=90s
+          kubectl wait pods -n otterize-tutorial-kafka-mapping -l app=client --for condition=Ready --timeout=90s
+          kubectl wait pods -n otterize-tutorial-kafka-mapping -l app=client-2 --for condition=Ready --timeout=90s
+
+      - name: Test Policy Export
+        run: |-
+          # wait for 2 intents to be discovered with timeout of 30 seconds. 
+          # sleeps 10 because this is the report interval from the watcher to the mapper
+          for i in {1..5}
+          do
+            OUTPUT_JSON=`otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-kafka-mapping --format=json`
+            if [ `echo "$OUTPUT_JSON" | jq ". | length"` != 2 ] || [ `echo "$OUTPUT_JSON" | jq '[.[] | select(.spec.targets[] | has("kafka"))] | length'` != 2 ] ; then
+              echo "wait for discovered intents";
+              echo _SNIFFER LOGS_
+              kubectl logs --since=15s -n otterize-system -l app=otterize-network-sniffer
+              echo _MAPPER LOGS_
+              kubectl logs --since=15s -n otterize-system -l app=otterize-network-mapper
+              sleep 10 ;
+            fi
+          done
+
+          echo Outputting all logs
+          echo _SNIFFER LOGS_
+          kubectl logs -n otterize-system -l app=otterize-network-sniffer --tail=-1
+          echo _MAPPER LOGS_
+          kubectl logs -n otterize-system -l app=otterize-network-mapper --tail=-1
+
+          echo "export intents and compare to expected file" 
+          INTENTS_JSON=`otterize network-mapper export --telemetry-enabled=false -n otterize-tutorial-kafka-mapping --format=json`
+          INTENTS_JSON_NO_KIND=`echo "$INTENTS_JSON" | jq 'map(del(.spec.workload.kind))'`
+          INTENTS_JSON_NO_KIND_AND_SORTED=`echo "$INTENTS_JSON_NO_KIND" | jq 'sort_by(.metadata.namespace + .metadata.name) | map(.spec.targets |= (sort_by(keys_unsorted[0]) | map(if .kafka? then .kafka.topics |= map(.operations |= sort) else . end)))'`
+          echo "$INTENTS_JSON_NO_KIND_AND_SORTED" > /tmp/intents.json 
+          echo "expected" && cat .github/workflows/tests-expected-results/kafka-tutorial-intents.json
+          echo "actual" && cat /tmp/intents.json
+          diff .github/workflows/tests-expected-results/kafka-tutorial-intents.json /tmp/intents.json
+
+  e2e-test:
+    needs:
+      - e2e-test-network-policy
+      - e2e-test-kafka
+    runs-on: ubuntu-latest
+    steps:
+      - run: |-
+          echo Success! This step is only here to depend on the tests.