add NIC+WAFv5 tests (#6456)

vepatel · web-flow · commit 0ca75be24238 · 2024-09-23T10:34:46.000+01:00
diff --git a/.github/actions/smoke-tests/action.yaml b/.github/actions/smoke-tests/action.yaml
@@ -31,6 +31,9 @@ inputs:
   azure-ad-secret:
     description: Azure Active Directory secret for JWKs
     required: false
+  registry-token:
+    description: JWT token for accessing container registry
+    required: false
 
 outputs:
   test-results-name:
@@ -76,13 +79,17 @@ runs:
         docker run --rm \
         --name test-runner-${{ github.run_id }} \
         --network=kind \
+        -v "/var/run/docker.sock:/var/run/docker.sock" \
+        -v ~/.docker:/root/.docker \
         -v ${{ github.workspace }}/tests:/workspace/tests \
         -v ${{ github.workspace }}/deployments:/workspace/deployments \
         -v ${{ github.workspace }}/charts:/workspace/charts \
         -v ${{ github.workspace }}/config:/workspace/config \
         -v ${{ github.workspace }}/pyproject.toml:/workspace/pyproject.toml \
         -v ${{ steps.k8s.outputs.test_output_path }}:${{ steps.k8s.outputs.test_output_path }} \
         -v ~/.kube/kind/config:/root/.kube/config ${{ inputs.test-image }} \
+        --docker-registry-user=oauth2accesstoken \
+        --docker-registry-token=${{ inputs.registry-token }} \
         --context=kind-${{ github.run_id }} \
         --image=${{ inputs.image-name }}:${{ inputs.tag }} \
         --image-pull-policy=Never \
diff --git a/.github/data/matrix-smoke-nap.json b/.github/data/matrix-smoke-nap.json
@@ -32,6 +32,14 @@
       "marker": "'appprotect_watch or appprotect_batch or appprotect_integration or appprotect_waf_policies_vsr'",
       "platforms": "linux/amd64"
     },
+    {
+      "label": "AP_WAF_V5 1/1",
+      "image": "debian-plus-nap-v5",
+      "type": "plus",
+      "nap_modules": "waf",
+      "marker": "appprotect_waf_v5",
+      "platforms": "linux/amd64"
+    },
     {
       "label": "AP_DOS 1/3",
       "image": "debian-plus-nap",
diff --git a/.github/workflows/build-test-image.yml b/.github/workflows/build-test-image.yml
@@ -55,7 +55,7 @@ jobs:
           context: "."
           cache-from: type=gha,scope=test-runner
           tags: |
-            gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') }}
+            gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') }}
             gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:latest
           pull: true
           push: true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -548,7 +548,7 @@ jobs:
       - name: Check if test image exists
         id: check-image
         run: |
-          docker pull gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}
+          docker pull gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}
         shell: bash
         continue-on-error: true
         if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}
@@ -559,7 +559,7 @@ jobs:
           file: tests/Dockerfile
           context: "."
           cache-from: type=gha,scope=test-runner
-          tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+          tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
           pull: true
           push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
           load: false
diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
@@ -271,7 +271,8 @@ jobs:
           k8s-version: ${{ matrix.k8s }}
           label: ${{ matrix.images.label }}
           azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
-          test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+          registry-token: ${{ steps.auth.outputs.access_token }}
+          test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
 
       - name: Upload Test Results
         uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
diff --git a/.github/workflows/setup-smoke.yml b/.github/workflows/setup-smoke.yml
@@ -54,7 +54,7 @@ jobs:
       - name: Set image variables
         id: image_details
         run: |
-          echo "name=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic${{ contains(inputs.nap-modules, 'dos') && '-dos' || '' }}${{ contains(inputs.nap-modules, 'waf') && '-nap' || '' }}/nginx${{ contains(inputs.image, 'plus') && '-plus' || '' }}-ingress" >> $GITHUB_OUTPUT
+          echo "name=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic${{ contains(inputs.nap-modules, 'dos') && '-dos' || '' }}${{ contains(inputs.nap-modules, 'waf') && '-nap' || '' }}${{ contains(inputs.image, 'v5') && '-v5' || '' }}/nginx${{ contains(inputs.image, 'plus') && '-plus' || '' }}-ingress" >> $GITHUB_OUTPUT
           echo "build_tag=${{ inputs.build-tag }}${{ contains(inputs.image, 'ubi-9') && '-ubi' || '' }}${{ contains(inputs.image, 'ubi-8') && '-ubi8' || '' }}${{ contains(inputs.image, 'alpine') && '-alpine' || '' }}${{ contains(inputs.target, 'aws') && '-mktpl' || '' }}${{ contains(inputs.image, 'fips') && '-fips' || ''}}" >> $GITHUB_OUTPUT
           echo "stable_tag=${{ inputs.stable-tag }}${{ contains(inputs.image, 'ubi-9') && '-ubi' || '' }}${{ contains(inputs.image, 'ubi-8') && '-ubi8' || '' }}${{ contains(inputs.image, 'alpine') && '-alpine' || '' }}${{ contains(inputs.target, 'aws') && '-mktpl' || '' }}${{ contains(inputs.image, 'fips') && '-fips' || ''}}" >> $GITHUB_OUTPUT
 
@@ -108,7 +108,7 @@ jobs:
       - name: Check if test image exists
         id: check-image
         run: |
-          docker manifest inspect "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+          docker manifest inspect "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
         shell: bash
         continue-on-error: true
         if: ${{ inputs.authenticated  }}
@@ -119,7 +119,7 @@ jobs:
           file: tests/Dockerfile
           context: "."
           cache-from: type=gha,scope=test-runner
-          tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+          tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
           pull: true
           push: ${{ inputs.authenticated }}
           load: ${{ !inputs.authenticated }}
@@ -147,6 +147,11 @@ jobs:
             ${{ contains(inputs.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }}
         if: ${{ !inputs.authenticated }}
 
+      - name: Generate WAF v5 tgz from JSON
+        run: |
+          docker run --rm --user root -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/tests/data/ap-waf-v5:/data gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/nap/waf-compiler:5.2.0 -p /data/wafv5.json -o /data/wafv5.tgz
+        if: ${{ contains(inputs.image, 'nap-v5')}}
+
       - name: Run Smoke Tests
         id: smoke-tests
         uses: ./.github/actions/smoke-tests
@@ -158,7 +163,8 @@ jobs:
           label: ${{ inputs.label }}
           k8s-version: ${{ inputs.k8s-version }}
           azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
-          test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+          registry-token: ${{ steps.auth.outputs.access_token }}
+          test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
         if: ${{ steps.stable_exists.outputs.exists != 'true'  }}
 
       - name: Upload Test Results
diff --git a/.github/workflows/single-image-regression.yml b/.github/workflows/single-image-regression.yml
@@ -107,4 +107,5 @@ jobs:
           label: "${{ inputs.image }} regression"
           k8s-version: ${{ inputs.k8s-version }}
           azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
+          registry-token: ${{ steps.auth.outputs.access_token }}
           test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ inputs.test-image-tag }}"
diff --git a/pyproject.toml b/pyproject.toml
@@ -32,6 +32,7 @@ markers =[
     "appprotect_waf_policies_block",
     "appprotect_waf_policies_grpc",
     "appprotect_waf_policies_vsr",
+    "appprotect_waf_v5",
     "appprotect_watch",
     "appprotect_batch",
     "basic_auth",
diff --git a/tests/Dockerfile b/tests/Dockerfile
@@ -23,6 +23,9 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s
 	&& install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
 	&& apt-get update && apt-get install -y apache2-utils
 
+RUN apt update -y \
+	&& curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && mv /tmp/docker/docker /usr/bin/docker
+
 COPY --link tests /workspace/tests
 
 COPY --link pyproject.toml /workspace/
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -126,6 +126,18 @@ def pytest_addoption(parser) -> None:
         default="1",
         help="Number of resources to deploy for upgrade tests",
     )
+    parser.addoption(
+        "--docker-registry-user",
+        action="store",
+        default="",
+        help="Docker registry username",
+    )
+    parser.addoption(
+        "--docker-registry-token",
+        action="store",
+        default="",
+        help="Docker registry token",
+    )
 
 
 # import fixtures into pytest global namespace
diff --git a/tests/data/ap-waf-v5/policies/waf.yaml b/tests/data/ap-waf-v5/policies/waf.yaml
@@ -0,0 +1,8 @@
+apiVersion: k8s.nginx.org/v1
+kind: Policy
+metadata:
+  name: waf-policy
+spec:
+  waf:
+    enable: true
+    apBundle: "wafv5.tgz"
diff --git a/tests/data/ap-waf-v5/standard/virtual-server.yaml b/tests/data/ap-waf-v5/standard/virtual-server.yaml
@@ -0,0 +1,20 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+  name: virtual-server
+spec:
+  host: virtual-server.example.com
+  upstreams:
+  - name: backend2
+    service: backend2-svc
+    port: 80
+  - name: backend1
+    service: backend1-svc
+    port: 80
+  routes:
+  - path: "/backend1"
+    action:
+      pass: backend1
+  - path: "/backend2"
+    action:
+      pass: backend2
diff --git a/tests/data/ap-waf-v5/virtual-server-route-waf-subroute.yaml b/tests/data/ap-waf-v5/virtual-server-route-waf-subroute.yaml
@@ -0,0 +1,22 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServerRoute
+metadata:
+  name: backends
+spec:
+  host: virtual-server-route.example.com
+  upstreams:
+  - name: backend1
+    service: backend1-svc
+    port: 80
+  - name: backend3
+    service: backend3-svc
+    port: 80
+  subroutes:
+  - path: "/backends/backend1"
+    policies:
+    - name: waf-policy
+    action:
+      pass: backend1
+  - path: "/backends/backend3"
+    action:
+      pass: backend3
diff --git a/tests/data/ap-waf-v5/virtual-server-waf-route.yaml b/tests/data/ap-waf-v5/virtual-server-waf-route.yaml
@@ -0,0 +1,22 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+  name: virtual-server
+spec:
+  host: virtual-server.example.com
+  upstreams:
+  - name: backend2
+    service: backend2-svc
+    port: 80
+  - name: backend1
+    service: backend1-svc
+    port: 80
+  routes:
+  - path: "/backend1"
+    policies:
+    - name: waf-policy
+    action:
+      pass: backend1
+  - path: "/backend2"
+    action:
+      pass: backend2
diff --git a/tests/data/ap-waf-v5/virtual-server-waf-spec.yaml b/tests/data/ap-waf-v5/virtual-server-waf-spec.yaml
@@ -0,0 +1,22 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+  name: virtual-server
+spec:
+  host: virtual-server.example.com
+  policies:
+  - name: waf-policy
+  upstreams:
+  - name: backend2
+    service: backend2-svc
+    port: 80
+  - name: backend1
+    service: backend1-svc
+    port: 80
+  routes:
+  - path: "/backend1"
+    action:
+      pass: backend1
+  - path: "/backend2"
+    action:
+      pass: backend2
diff --git a/tests/data/ap-waf-v5/wafv5.json b/tests/data/ap-waf-v5/wafv5.json
@@ -0,0 +1,112 @@
+{
+  "policy": {
+    "name": "app_protect_api_security_policy",
+    "description": "NGINX App Protect API Security Policy. The policy is intended to be used with an OpenAPI file",
+    "template": {
+      "name": "POLICY_TEMPLATE_NGINX_BASE"
+    },
+    "open-api-files": [],
+    "blocking-settings": {
+      "violations": [
+        {
+          "block": true,
+          "description": "Mandatory request body is missing",
+          "name": "VIOL_MANDATORY_REQUEST_BODY"
+        },
+        {
+          "block": true,
+          "description": "Illegal parameter location",
+          "name": "VIOL_PARAMETER_LOCATION"
+        },
+        {
+          "block": true,
+          "description": "Mandatory parameter is missing",
+          "name": "VIOL_MANDATORY_PARAMETER"
+        },
+        {
+          "block": true,
+          "description": "JSON data does not comply with JSON schema",
+          "name": "VIOL_JSON_SCHEMA"
+        },
+        {
+          "block": true,
+          "description": "Illegal parameter array value",
+          "name": "VIOL_PARAMETER_ARRAY_VALUE"
+        },
+        {
+          "block": true,
+          "description": "Illegal Base64 value",
+          "name": "VIOL_PARAMETER_VALUE_BASE64"
+        },
+        {
+          "block": true,
+          "description": "Illegal request content type",
+          "name": "VIOL_URL_CONTENT_TYPE"
+        },
+        {
+          "block": true,
+          "description": "Illegal static parameter value",
+          "name": "VIOL_PARAMETER_STATIC_VALUE"
+        },
+        {
+          "block": true,
+          "description": "Illegal parameter value length",
+          "name": "VIOL_PARAMETER_VALUE_LENGTH"
+        },
+        {
+          "block": true,
+          "description": "Illegal parameter data type",
+          "name": "VIOL_PARAMETER_DATA_TYPE"
+        },
+        {
+          "block": true,
+          "description": "Illegal parameter numeric value",
+          "name": "VIOL_PARAMETER_NUMERIC_VALUE"
+        },
+        {
+          "block": true,
+          "description": "Parameter value does not comply with regular expression",
+          "name": "VIOL_PARAMETER_VALUE_REGEXP"
+        },
+        {
+          "block": true,
+          "description": "Illegal URL",
+          "name": "VIOL_URL"
+        },
+        {
+          "block": true,
+          "description": "Illegal parameter",
+          "name": "VIOL_PARAMETER"
+        },
+        {
+          "block": true,
+          "description": "Illegal empty parameter value",
+          "name": "VIOL_PARAMETER_EMPTY_VALUE"
+        },
+        {
+          "block": true,
+          "description": "Illegal repeated parameter name",
+          "name": "VIOL_PARAMETER_REPEATED"
+        },
+        {
+          "block": true,
+          "description": "Illegal method",
+          "name": "VIOL_METHOD"
+        },
+        {
+          "block": true,
+          "description": "Illegal gRPC method",
+          "name": "VIOL_GRPC_METHOD"
+        }
+      ]
+    },
+    "xml-profiles": [
+      {
+        "name": "Default",
+        "defenseAttributes": {
+          "maximumNameLength": 1024
+        }
+      }
+    ]
+  }
+}
diff --git a/tests/settings.py b/tests/settings.py
@@ -30,3 +30,7 @@
 BATCH_RELOAD_NUMBER = 2
 # Number of namespaces to deploy to measure Pod performance, used in test_multiple_ns_perf.py
 NS_COUNT = 0
+# Nginx registry address to pull waf components from
+NGX_REG = "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr"
+# WAF component version to pull from above registry
+WAF_V5_VERSION = "5.2.0"
diff --git a/tests/suite/fixtures/ic_fixtures.py b/tests/suite/fixtures/ic_fixtures.py
diff --git a/tests/suite/test_app_protect_wafv5_integration.py b/tests/suite/test_app_protect_wafv5_integration.py
diff --git a/tests/suite/utils/resources_utils.py b/tests/suite/utils/resources_utils.py
