Added additional test cases with running binary nuzzle (#226)

Author: mwang2016

.circleci/config.yml

@@ -92,22 +92,6 @@ jobs:
           name: Executing tests
           command: |
             .circleci/functional-test.sh
-
-  teardown:
-    docker:
-      - image: circleci/golang:1.12
-    working_directory: /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
-    steps:
-      - attach_workspace:
-          at: /tmp
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            curl https://glide.sh/get | sh
-            go get -t ./...
-            cp -R /tmp/splunk-firehose-nozzle .
-            .circleci/pre-req.sh
       - run:
           name: Teardown
           command: |
@@ -146,13 +130,3 @@ workflows:
               only:
                 - develop
                 - master
-      - teardown:
-          requires:
-            - build
-            - deploy-nozzle
-            - execute_tests
-          filters:
-            branches:
-              only:
-                - develop
-                - master

.circleci/functional-test.sh

@@ -2,4 +2,4 @@
 
 cd testing/integration
 . venv/bin/activate
-pytest -rfps --durations=10
+pytest

testing/integration/conftest.py

@@ -5,12 +5,15 @@
 import pytest
 import configparser
 from os.path import join
+import subprocess
+import time
 
 
 from lib.helper import get_config_folder, get_project_folder
 
 _current_dir = os.path.dirname(os.path.realpath(__file__))
 sys.path.insert(0, get_project_folder())
+_env_var = os.environ
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -32,17 +35,30 @@ def pytest_addoption(parser):
     """
     This function is sued to add command line parameters to test suite
     """
-    env_var = os.environ
     parser.addoption("--splunk-url", help="splunk url used to send test data to.",
-                     default=env_var.get('SPLUNK_URL'))
+                     default=_env_var.get('SPLUNK_URL'))
     parser.addoption("--splunk-user", help="splunk username",
-                     default=env_var.get('SPLUNK_USER'))
+                     default=_env_var.get('SPLUNK_USER'))
     parser.addoption("--splunk-password", help="splunk user password",
-                     default=env_var.get('SPLUNK_PASSWORD'))
+                     default=_env_var.get('SPLUNK_PASSWORD'))
     parser.addoption("--api-endpoint", help="pleasanton cf api endpoint.",
-                     default=env_var.get('API_ENDPOINT'))
+                     default=_env_var.get('API_ENDPOINT'))
     parser.addoption("--splunk-index", help="splunk index on hec setting.",
-                     default=env_var.get('SPLUNK_INDEX'))
+                     default=_env_var.get('SPLUNK_INDEX'))
+
+
+@pytest.fixture(scope="class")
+def test_setup(request, nozzle_logger):
+    nozzle_logger.info("Stopping nozzle...")
+    cmd = "cf login --skip-ssl-validation -a {0} -u {1} -p {2} -o system -s system; " \
+          "cf stop splunk-firehose-nozzle".format(_env_var['API_ENDPOINT'],
+                                                  _env_var['API_USER'],
+                                                  _env_var['API_PASSWORD'])
+
+    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    output, error = proc.communicate()
+    time.sleep(5)
 
 
 @pytest.fixture(scope="class")

testing/integration/lib/helper.py

@@ -3,9 +3,9 @@
 import os
 
 
-def get_project_folder():
+def get_integration_folder():
     """
-    returns the project root folder
+    returns the integration test folder
     """
     return os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
 
@@ -14,7 +14,14 @@ def get_config_folder():
     """
     returns the config folder
     """
-    return os.path.join(get_project_folder(), "config")
+    return os.path.join(get_integration_folder(), "config")
+
+
+def get_project_folder():
+    """
+    returns the project root folder
+    """
+    return get_integration_folder().replace("/testing/integration", "/")
 
 
 

testing/integration/lib/update_env.py

@@ -0,0 +1,34 @@
+#! /usr/bin/env python
+
+from .helper import *
+_env_var = os.environ
+import time
+_path = get_project_folder()
+
+
+def update_environment_variables(input_dict=None):
+    default_dict = \
+        {
+            'ADD_APP_INFO': 'true',
+            'EVENTS': 'ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric',
+            'SPLUNK_TOKEN': _env_var.get('SPLUNK_TOKEN'),
+            'SPLUNK_HOST': _env_var.get('SPLUNK_HOST'),
+            'SPLUNK_INDEX': _env_var.get('SPLUNK_INDEX'),
+            'FIREHOSE_SUBSCRIPTION_ID': 'splunk-ci',
+            'CLIENT_ID': _env_var.get('CLIENT_ID'),
+            'CLIENT_SECRET': _env_var.get('CLIENT_SECRET'),
+            'ENABLE_EVENT_TRACING': 'true',
+            'SKIP_SSL_VALIDATION_CF': 'true',
+            'SKIP_SSL_VALIDATION_SPLUNK': 'true',
+            'EXTRA_FIELDS': 'name:update-ci-test'
+        }
+
+    if input_dict:
+        default_dict.update(input_dict)
+
+    path = os.path.join(get_project_folder(), "env.sh")
+    with open(path, 'w') as file:
+        file.write('''#! /bin/bash''')
+        for key, value in default_dict.items():
+            file.write("\nexport {0}={1}".format(key, value))
+    time.sleep(2)

testing/integration/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --durations=10
+addopts = -rfps --durations=10 --disable-pytest-warnings --continue-on-collection-errors
 
 markers =
     Critical: mark a test as a critical test case.

testing/integration/scripts/start_nozzle.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+FILE_PATH=$1
+TIME_INTERVAL=$2
+
+cd $FILE_PATH
+source env.sh
+./splunk-firehose-nozzle & sleep $TIME_INTERVAL ; kill $!

testing/integration/testcases/test_nozzle_configurations.py

@@ -48,24 +48,10 @@ def test_search_by_incorrect_splunk_index(self, test_env, splunk_logger, query_i
 
     @pytest.mark.Critical
     @pytest.mark.parametrize("query_input", [
-        "index={} cf_app_name=data_gen",  # when add_app_info is true, cf_app_name is searchable
-        "index={} cf_org_name=splunk-ci-org",  # when add_app_info is true, cf_org_name is searchable
-        "index={} cf_space_name=splunk-ci-space"  # when add_app_info is true, cf_space_name is searchable
-    ])
-    def test_add_app_info_is_true(self, test_env, splunk_logger, query_input):
-        self.splunk_api = SplunkApi(test_env, splunk_logger)
-
-        search_results = self.splunk_api.check_events_from_splunk(
-            query=query_input.format(test_env['splunk_index']),
-            start_time="-15m@m")
-        assert len(search_results) > 0, \
-            '\nNumber of events from Splunk should not be {}, however the result is {}'.format(0, len(search_results))
-
-    @pytest.mark.Critical
-    @pytest.mark.parametrize("query_input", [
-        "index={} cf_app_name=data_gen",  # when add_app_info is true, cf_app_name is searchable
-        "index={} cf_org_name=splunk-ci-org",  # when add_app_info is true, cf_org_name is searchable
-        "index={} cf_space_name=splunk-ci-space"  # when add_app_info is true, cf_space_name is searchable
+        "index={0} cf_space_id=*",  # when cf_space_id is true, cf_org_name is searchable
+        "index={0} cf_org_id=*",  # cf_org_id add_app_info is true, cf_org_name is searchable
+        "index={0} cf_org_name=*",  # when add_app_info is true, cf_org_name is searchable
+        "index={0} cf_space_name=*"  # when add_app_info is true, cf_space_name is searchable
     ])
     def test_add_app_info_is_true(self, test_env, splunk_logger, query_input):
         self.splunk_api = SplunkApi(test_env, splunk_logger)

testing/integration/testcases/test_nozzle_configurations_local.py

@@ -0,0 +1,97 @@
+import pytest
+from lib.update_env import *
+from lib.splunk_api import *
+import string
+import random
+_tag = ''.join(random.choices(string.ascii_uppercase + string.digits, k=6))
+
+
+class TestSplunkNozzleLocal():
+    @pytest.fixture(scope='class', autouse=True)
+    def setup_class(self, test_env, splunk_logger, nozzle_logger, test_setup):
+        update_environment_variables(input_dict={'EVENTS': 'LogMessage',
+                                                 'EXTRA_FIELDS': 'test_tag:{},test2.0:nozzle2.0'.format(_tag),
+                                                 'ENABLE_EVENT_TRACING': False,
+                                                 'ADD_APP_INFO': False}
+                                     )
+
+        path = os.path.join(get_integration_folder(), "scripts")
+        env_path = get_project_folder()
+        time_interval = 20
+        cmd = "cd {0}; ./start_nozzle.sh {1} {2}".format(path, env_path, time_interval)
+        proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                                stderr=subprocess.STDOUT)
+        proc.communicate()
+
+    @pytest.mark.Critical
+    @pytest.mark.parametrize("query_input, is_result_empty", [
+        ("index={0} test_tag::{1} event_type=ValueMetric", True),
+        ("index={0} test_tag::{1} event_type=CounterEvent", True),
+        ("index={0} test_tag::{1} event_type=LogMessage", False),
+        ("index={0} test_tag::{1} event_type=HttpStartStop", True),
+        ("index={0} test_tag::{1} event_type=ContainerMetric", True)
+    ])
+    def test_search_event_on_splunk_is_not_empty(self, query_input, is_result_empty, test_env, splunk_logger, nozzle_logger):
+        self.splunk_api = SplunkApi(test_env, splunk_logger)
+        search_results = self.splunk_api.check_events_from_splunk(
+            query=query_input.format(test_env['splunk_index'], _tag),
+            start_time="-10m@m")
+
+        if is_result_empty:
+            assert len(search_results) == 0, \
+                '\nNumber of events from Splunk should be {}, however the result is {}'.format(0, len(search_results))
+        else:
+            assert len(search_results) > 0, \
+                '\nNumber of events from Splunk should not be {}, however the result is {}'\
+                    .format(0, len(search_results))
+
+    @pytest.mark.Critical
+    @pytest.mark.parametrize("query_input, is_result_empty", [
+        ("index={0} test_tag::{1} test2.0::nozzle2.0", False),
+        ("index={0} test_tag::{1} test::nozzle2.0", True),
+        ("index={0} test_tag::{1} test2.0::nozzle", True)
+    ])
+    def test_search_by_extra_fields(self, query_input, is_result_empty, test_env, splunk_logger):
+        self.splunk_api = SplunkApi(test_env, splunk_logger)
+        search_results = self.splunk_api.check_events_from_splunk(
+            query=query_input.format(test_env['splunk_index'], _tag),
+            start_time="-10m@m")
+
+        if is_result_empty:
+            assert len(search_results) == 0, \
+                '\nNumber of events from Splunk should be {}, however the result is {}'.format(0, len(search_results))
+        else:
+            assert len(search_results) > 0, \
+                '\nNumber of events from Splunk should not be {}, however the result is {}'\
+                    .format(0, len(search_results))
+
+    @pytest.mark.Critical
+    @pytest.mark.parametrize("query_input", [
+        "index={0} test_tag::{1} nozzle-event-counter>0",  # nozzle-event-counter should not be searchable
+        "index={0} test_tag::{1} subscription-id::splunk-ci",  # subscription-id should not be searchable
+        "index={0} test_tag::{1} uuid::*"  # uuid should not be searchable
+    ])
+    def test_enable_event_tracing_is_false(self, test_env, query_input, splunk_logger):
+        self.splunk_api = SplunkApi(test_env, splunk_logger)
+        search_results = self.splunk_api.check_events_from_splunk(
+            query=query_input.format(test_env['splunk_index'], _tag),
+            start_time="-10m@m")
+
+        assert len(search_results) == 0, \
+            '\nNumber of events from Splunk should be {}, however the result is {}'.format(0, len(search_results))
+
+    @pytest.mark.Critical
+    @pytest.mark.parametrize("query_input", [
+        "index={0} test_tag::{1} cf_space_id=*",  # when cf_space_id is false, cf_org_name is not searchable
+        "index={0} test_tag::{1} cf_org_id=*",  # cf_org_id add_app_info is false, cf_org_name is not searchable
+        "index={0} test_tag::{1} cf_org_name=*",  # when add_app_info is false, cf_org_name is not searchable
+        "index={0} test_tag::{1} cf_space_name=*"  # when add_app_info is false, cf_space_name is not searchable
+    ])
+    def test_add_app_info_is_false(self, test_env, query_input, splunk_logger):
+        self.splunk_api = SplunkApi(test_env, splunk_logger)
+        search_results = self.splunk_api.check_events_from_splunk(
+            query=query_input.format(test_env['splunk_index'], _tag),
+            start_time="-10m@m")
+
+        assert len(search_results) == 0, \
+            '\nNumber of events from Splunk should be {}, however the result is {}'.format(0, len(search_results))