Include latest tests improvements.

Add pytest hook to update test report. Change condition back to 'ContainersReady'. Increase time of waiting fot Deployment removal. Set strict versions to python libraries.

Author: tellet

deployments/custom-resource-definitions/virtualserver.yaml

@@ -8,9 +8,11 @@ NODE_IP =
 TAG = latest
 PREFIX = test-runner
 KUBE_CONFIG_FOLDER = $${HOME}/.kube
+SHOW_IC_LOGS = no
+PYTEST_ARGS =
 
 build:
 	docker build -t $(PREFIX):$(TAG) -f docker/Dockerfile ..
 
 run-tests:
-	docker run --rm -v $(KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) --context=$(CONTEXT) --image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) --deployment-type=$(DEPLOYMENT_TYPE) --ic-type=$(IC_TYPE) --service=$(SERVICE) --node-ip=$(NODE_IP)
+	docker run --rm -v $(KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) --context=$(CONTEXT) --image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) --deployment-type=$(DEPLOYMENT_TYPE) --ic-type=$(IC_TYPE) --service=$(SERVICE) --node-ip=$(NODE_IP) --show-ic-logs=$(SHOW_IC_LOGS) $(PYTEST_ARGS)

tests/Makefile

@@ -52,5 +52,7 @@ The table below shows various configuration options for the tests. If you use Py
 | `--node-ip` | `NODE_IP` | The public IP of a cluster node. Not required if you use the loadbalancer service (see --service argument). | `""` |
 | `--kubeconfig` | `N/A` | An absolute path to a kubeconfig file. | `~/.kube/config` or the value of the `KUBECONFIG` env variable |
 | `N/A` | `KUBE_CONFIG_FOLDER` | A path to a folder with a kubeconfig file. | `~/.kube/` |
+| `--show-ic-logs` | `SHOW_IC_LOGS` | A flag to control accumulating IC logs in stdout. | `no` |
+| `N/A` | `PYTEST_ARGS` | Any additional pytest command-line arguments (i.e `-m "smoke"`) | `""` |
 
 If you would like to use an IDE (such as PyCharm) to run the tests, use the [pytest.ini](pytest.ini) file to set the command-line arguments.

tests/README.md

@@ -5,6 +5,7 @@
 
 from kubernetes.config.kube_config import KUBE_CONFIG_DEFAULT_LOCATION
 from settings import DEFAULT_IMAGE, DEFAULT_PULL_POLICY, DEFAULT_IC_TYPE, DEFAULT_SERVICE, DEFAULT_DEPLOYMENT_TYPE
+from suite.resources_utils import get_first_pod_name
 
 
 def pytest_addoption(parser) -> None:
@@ -33,6 +34,7 @@ def pytest_addoption(parser) -> None:
                      action="store",
                      default=os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION),
                      help="An absolute path to a kubeconfig file.")
+    parser.addoption("--show-ic-logs", action="store", default="no", help="Show IC logs in stdout on test failure")
 
 
 # import fixtures into pytest global namespace
@@ -54,3 +56,32 @@ def pytest_collection_modifyitems(config, items) -> None:
         for item in items:
             if "skip_for_nginx_oss" in item.keywords:
                 item.add_marker(skip_for_nginx_oss)
+    if config.getoption("--ic-type") == "nginx-plus-ingress":
+        skip_for_nginx_plus = pytest.mark.skip(reason="Skip a test for Nginx Plus")
+        for item in items:
+            if "skip_for_nginx_plus" in item.keywords:
+                item.add_marker(skip_for_nginx_plus)
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item) -> None:
+    """
+    Print out IC Pod logs on test failure.
+
+    Only look at actual failing test calls, not setup/teardown.
+    Only show the logs if commandline argument `--show-ic-logs` is set to 'yes'
+
+    :param item:
+    :return:
+    """
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # we only look at actual failing test calls, not setup/teardown
+    if rep.when == "call" and rep.failed and item.config.getoption("--show-ic-logs") == "yes":
+        pod_namespace = item.funcargs['ingress_controller_prerequisites'].namespace
+        pod_name = get_first_pod_name(item.funcargs['kube_apis'].v1, pod_namespace)
+        print("\n===================== IC Logs Start =====================")
+        print(item.funcargs['kube_apis'].v1.read_namespaced_pod_log(pod_name, pod_namespace))
+        print("\n===================== IC Logs End =====================")

tests/conftest.py

@@ -1,9 +1,9 @@
-PyYAML
-requests
-forcediphttpsadapter
+PyYAML==5.1.2
+requests==2.22.0
+forcediphttpsadapter==1.0.1
 kubernetes==8.0.1
-pytest
-ipaddress>=1.0.17
-pyopenssl
-certifi
-urllib3
+pytest==4.4.1
+ipaddress==1.0.22 # >= 1.0.17
+pyopenssl==19.0.0
+certifi==2019.6.16
+urllib3==1.25.3

tests/requirements.txt

@@ -157,21 +157,21 @@ def create_daemon_set(extensions_v1_beta1: ExtensionsV1beta1Api, namespace, body
 
 def wait_until_all_pods_are_ready(v1: CoreV1Api, namespace) -> None:
     """
-    Wait for all the pods to be 'Ready'.
+    Wait for all the pods to be 'ContainersReady'.
 
     :param v1: CoreV1Api
     :param namespace: namespace of a pod
     :return:
     """
-    print("Start waiting for all pods in a namespace to be Ready")
+    print("Start waiting for all pods in a namespace to be ContainersReady")
     counter = 0
     while not are_all_pods_in_ready_state(v1, namespace) and counter < 20:
-        print("There are pods that are not running. Wait for 4 sec...")
+        print("There are pods that are not ContainersReady. Wait for 4 sec...")
         time.sleep(4)
         counter = counter + 1
     if counter >= 20:
-        pytest.fail("After several seconds the pods aren't Ready. Exiting...")
-    print("All pods are Ready")
+        pytest.fail("After several seconds the pods aren't ContainersReady. Exiting...")
+    print("All pods are ContainersReady")
 
 
 def get_first_pod_name(v1: CoreV1Api, namespace) -> str:
@@ -188,7 +188,7 @@ def get_first_pod_name(v1: CoreV1Api, namespace) -> str:
 
 def are_all_pods_in_ready_state(v1: CoreV1Api, namespace) -> bool:
     """
-    Check if all the pods have Ready condition.
+    Check if all the pods have ContainersReady condition.
 
     :param v1: CoreV1Api
     :param namespace: namespace
@@ -202,8 +202,8 @@ def are_all_pods_in_ready_state(v1: CoreV1Api, namespace) -> bool:
         if pod.status.conditions is None:
             return False
         for condition in pod.status.conditions:
-            # wait for 'Ready' state instead of 'ContainersReady' for backwards compatibility with k8s 1.10
-            if condition.type == 'Ready' and condition.status == 'True':
+            # wait for 'ContainersReady' no backwards compatibility with k8s 1.10
+            if condition.type == 'ContainersReady' and condition.status == 'True':
                 pod_ready_amount = pod_ready_amount + 1
                 break
     return pod_ready_amount == len(pods.items)
@@ -385,12 +385,12 @@ def ensure_item_removal(get_item, *args, **kwargs) -> None:
     """
     try:
         counter = 0
-        while counter < 20:
+        while counter < 30:
             time.sleep(1)
             get_item(*args, **kwargs)
             counter = counter + 1
-        if counter >= 20:
-            pytest.fail("Failed to remove the item after 20 seconds")
+        if counter >= 30:
+            pytest.fail("Failed to remove the item after 30 seconds")
     except ApiException as ex:
         if ex.status == 404:
             print("Item was removed")
@@ -938,9 +938,10 @@ def ensure_response_from_backend(req_url, host) -> None:
     :param host:
     :return:
     """
-    for _ in range(5):
+    for _ in range(10):
         resp = requests.get(req_url, headers={"host": host}, verify=False)
         if resp.status_code != 502:
+            print(f"At last after {_ * 2} seconds got 200. Continue...")
             return
         time.sleep(2)
-    pytest.fail(f"Keep getting 502 from {req_url} after 10 seconds. Exiting...")
+    pytest.fail(f"Keep getting 502 from {req_url} after 20 seconds. Exiting...")

tests/suite/resources_utils.py

@@ -190,6 +190,7 @@ def assert_event_and_get_count(event_text, events_list) -> int:
     pytest.fail(f"Failed to find the event \"{event_text}\" in the list. Exiting...")
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize('crd_ingress_controller',
                          [({"type": "complete", "extra_args": [f"-enable-custom-resources"]})],
                          indirect=True)

tests/suite/test_v_s_route.py

@@ -9,6 +9,7 @@
 from suite.yaml_utils import get_paths_from_vs_yaml, get_first_vs_host_from_yaml, get_names_from_yaml
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize('crd_ingress_controller, virtual_server_setup',
                          [({"type": "complete", "extra_args": [f"-enable-custom-resources"]}, {"example": "virtual-server", "app_type": "simple"})],
                          indirect=True)
@@ -179,6 +180,24 @@ def test_responses_after_crd_removal_on_the_fly(self, kube_apis, crd_ingress_con
         assert resp.status_code == 200
 
 
+def wait_and_assert_status_code(code, req_url, host) -> None:
+    """
+    Wait for a specific response status code.
+
+    :param  code: status_code
+    :param  req_url: request url
+    :param  host: request headers if any
+    :return:
+    """
+    counter = 0
+    resp = requests.get(req_url, headers={"host": host})
+    while not resp.status_code == code and counter <= 4:
+        wait_before_test(1)
+        counter = counter + 1
+        resp = requests.get(req_url, headers={"host": host})
+    assert resp.status_code == code, f"After a few seconds the status_code is not {code}"
+
+
 @pytest.mark.parametrize('crd_ingress_controller, virtual_server_setup',
                          [({"type": "rbac-without-vs", "extra_args": [f"-enable-custom-resources"]},
                            {"example": "virtual-server", "app_type": "simple"})],
@@ -195,10 +214,5 @@ def test_responses_after_rbac_misconfiguration(self, kube_apis, crd_ingress_cont
 
         print("Step 2: configure RBAC and check")
         patch_rbac(kube_apis.rbac_v1_beta1, f"{DEPLOYMENTS}/rbac/rbac.yaml")
-        wait_before_test(1)
-        resp = requests.get(virtual_server_setup.backend_1_url,
-                            headers={"host": virtual_server_setup.vs_host})
-        assert resp.status_code == 200
-        resp = requests.get(virtual_server_setup.backend_2_url,
-                            headers={"host": virtual_server_setup.vs_host})
-        assert resp.status_code == 200
+        wait_and_assert_status_code(200, virtual_server_setup.backend_1_url, virtual_server_setup.vs_host)
+        wait_and_assert_status_code(200, virtual_server_setup.backend_2_url, virtual_server_setup.vs_host)

tests/suite/test_virtual_server.py

@@ -16,6 +16,7 @@ def execute_assertions(resp_1, resp_2, resp_3):
     assert "Server name: backend4-" in resp_3.text
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize('crd_ingress_controller, virtual_server_setup',
                          [({"type": "complete", "extra_args": [f"-enable-custom-resources"]},
                            {"example": "virtual-server-advanced-routing", "app_type": "advanced-routing"})],

tests/suite/test_virtual_server_advanced_routing.py

@@ -38,6 +38,7 @@ def get_upstreams_of_splitting(file) -> []:
     return upstreams
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize('crd_ingress_controller, virtual_server_setup',
                          [({"type": "complete", "extra_args": [f"-enable-custom-resources"]},
                            {"example": "virtual-server-split-traffic", "app_type": "split"})],