create integration test folder (#222)

Automation code base for functional testing
Added more tests and functions for testing framework
Set up execute_tests job for circleci workflow
Update .gitignore
Update config.yml
Update default.ini

Author: mwang2016

.circleci/config.yml

@@ -15,7 +15,7 @@ jobs:
           name: Builder
           command: make build
       - run:
-          name: Run tests
+          name: Run unit tests
           command: |
             make testall
             cp splunk-firehose-nozzle /tmp
@@ -44,13 +44,6 @@ jobs:
             .circleci/update_manifest.sh
             .circleci/pre-req.sh
             cf push -f .circleci/ci_nozzle_manifest.yml -u process --random-route
-      - run:
-          name: Teardown deployment env
-          command: |
-            sleep 10
-            echo "Teardown deployment env"
-            cf delete splunk-firehose-nozzle -f
-            cf delete-org splunk-ci -f
 
   tile-builder:
     docker:
@@ -71,6 +64,60 @@ jobs:
             cp -R /tmp/splunk-firehose-nozzle .
             .circleci/push_tile.sh
 
+  execute_tests:
+    docker:
+      - image: circleci/golang:1.12
+    working_directory: /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
+    steps:
+      - attach_workspace:
+          at: /tmp
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            curl https://glide.sh/get | sh
+            go get -t ./...
+            cp -R /tmp/splunk-firehose-nozzle .
+      - run:
+          name: Deploy data-gen
+          command: |
+            .circleci/pre-req.sh
+            cf push -f .circleci/data_gen_manifest.yml -u process -p tools/data_gen --random-route
+            sleep 10
+      - run:
+          name: Prepare test environment
+          command: |
+            .circleci/pre-functional-test.sh
+      - run:
+          name: Executing tests
+          command: |
+            .circleci/functional-test.sh
+
+  teardown:
+    docker:
+      - image: circleci/golang:1.12
+    working_directory: /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
+    steps:
+      - attach_workspace:
+          at: /tmp
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            curl https://glide.sh/get | sh
+            go get -t ./...
+            cp -R /tmp/splunk-firehose-nozzle .
+            .circleci/pre-req.sh
+      - run:
+          name: Teardown
+          command: |
+            echo "Teardown deployment env"
+            cf delete splunk-firehose-nozzle -f
+            cf delete data_gen -f
+            cf delete-org splunk-ci-org -f
+          when:
+            always
+
 workflows:
   version: 2
   build-and-deploy-nozzle:
@@ -89,4 +136,23 @@ workflows:
             - deploy-nozzle
           filters:
             branches:
-              only: master
+              only: master
+      - execute_tests:
+          requires:
+            - build
+            - deploy-nozzle
+          filters:
+            branches:
+              only:
+                - develop
+                - master
+      - teardown:
+          requires:
+            - build
+            - deploy-nozzle
+            - execute_tests
+          filters:
+            branches:
+              only:
+                - develop
+                - master

.circleci/functional-test.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cd testing/integration
+. venv/bin/activate
+pytest --durations=10

.circleci/pre-functional-test.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+sudo apt-get install python3.7
+sudo apt-get install python3-pip
+cd testing/integration
+pip3 install virtualenv
+virtualenv venv
+source venv/bin/activate
+pip3 install -r requirements.txt

.circleci/pre-req.sh

@@ -12,13 +12,13 @@ sudo apt-get install cf-cli
 #CF Login
 cf login --skip-ssl-validation -a $API_ENDPOINT -u $API_USER -p $API_PASSWORD -o system -s system
 #Create splunk-ci org and space
-if [  "`cf o | grep "splunk-ci"`" == "splunk-ci" ]; then
-   echo "splunk-ci org already exists"
-   cf target -o "splunk-ci" -s "splunk-ci"
+if [  "`cf o | grep "splunk-ci-org"`" == "splunk-ci-org" ]; then
+   echo "splunk-ci-org org already exists"
+   cf target -o "splunk-ci-org" -s "splunk-ci-space"
 else
-   echo "creating splunk-ci org and space"
-   cf create-org splunk-ci
-   cf target -o splunk-ci
-   cf create-space splunk-ci
-   cf target -o "splunk-ci" -s "splunk-ci"
+   echo "creating splunk-ci-org org and space"
+   cf create-org splunk-ci-org
+   cf target -o splunk-ci-org
+   cf create-space splunk-ci-space
+   cf target -o "splunk-ci-org" -s "splunk-ci-space"
 fi

.gitignore

@@ -14,3 +14,6 @@ data_gen
 manifest*.yml
 splunk-firehose-nozzle.iml
 vendor/github.com/cloudfoundry/sonde-go/definitions/
+testing/integration/venv/
+testing/integration/config/local.ini
+*.pyc

testing/integration/README.md

@@ -0,0 +1,4 @@
+## PCL Integration Tests Environment Setup
+
+## Using remote environment
+

testing/integration/config/config.ini

@@ -0,0 +1,25 @@
+[DEFAULT]
+splunk_url =
+splunk_user =
+splunk_password =
+go_package_name = main
+api_endpoint =
+client_id =
+client_secret =
+splunk_host =
+splunk_token =
+splunk_index =
+job_name = splunk-nozzle
+add_app_info = true
+bolted_path = cache.db
+events = ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric
+extra_fields = name:update-ci-test
+subscription_id = splunk-ci
+enable_event_tracing = true
+
+max_retries = 5
+
+
+
+
+

testing/integration/conftest.py

@@ -0,0 +1,75 @@
+import logging
+import logging.config
+import os
+import sys
+import pytest
+import configparser
+from os.path import join
+
+
+from lib.helper import get_config_folder, get_project_folder
+
+_current_dir = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, get_project_folder())
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_logging():
+    logging.config.fileConfig(os.path.join(_current_dir, "logging.conf"))
+
+
+@pytest.fixture(scope="session", autouse=True)
+def nozzle_logger(setup_logging):
+    return logging.getLogger("nozzle")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def splunk_logger(setup_logging):
+    return logging.getLogger("splunk")
+
+
+def pytest_addoption(parser):
+    """
+    This function is sued to add command line parameters to test suite
+    """
+    env_var = os.environ
+    parser.addoption("--splunk-url", help="splunk url used to send test data to.",
+                     default=env_var.get('SPLUNK_URL'))
+    parser.addoption("--splunk-user", help="splunk username",
+                     default=env_var.get('SPLUNK_USER'))
+    parser.addoption("--splunk-password", help="splunk user password",
+                     default=env_var.get('SPLUNK_PASSWORD'))
+    parser.addoption("--api-endpoint", help="pleasanton cf api endpoint.",
+                     default=env_var.get('API_ENDPOINT'))
+    parser.addoption("--splunk-index", help="splunk index on hec setting.",
+                     default=env_var.get('SPLUNK_INDEX'))
+
+
+@pytest.fixture(scope="class")
+def test_env(request):
+    """
+    provides the config dict based on default and local properties
+
+    local properties over ride default properties. sensitive information should
+    be placed in ``config/local.ini`` and this file should not be version
+    controlled.
+    """
+    config_folder = get_config_folder()
+    parser = configparser.ConfigParser()
+
+    if os.path.exists(join(config_folder, 'local.ini')):
+        parser.read([join(config_folder, 'config.ini'),
+                     join(config_folder, 'local.ini')])
+    else:
+        parser.read(join(config_folder, 'config.ini'))
+
+    cfg = parser.items('DEFAULT')
+    conf = dict(cfg)
+
+    conf["splunk_url"] = request.config.getoption("--splunk-url")
+    conf["splunk_user"] = request.config.getoption("--splunk-user")
+    conf["splunk_password"] = request.config.getoption("--splunk-password")
+    conf["api_endpoint"] = request.config.getoption("--api-endpoint")
+    conf["splunk_index"] = request.config.getoption("--splunk-index")
+
+    return conf

testing/integration/lib/helper.py

@@ -0,0 +1,20 @@
+# Common functions used in this project
+
+import os
+
+
+def get_project_folder():
+    """
+    returns the project root folder
+    """
+    return os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+
+
+def get_config_folder():
+    """
+    returns the config folder
+    """
+    return os.path.join(get_project_folder(), "config")
+
+
+

testing/integration/lib/json_assert.py

@@ -0,0 +1,50 @@
+from itertools import zip_longest
+from json_delta import udiff
+
+from functools import partial
+import json
+dumps = partial(json.dumps, ensure_ascii=False, indent=2)
+
+
+def assert_json_contains(expected, actual, msg="JSON mismatch"):
+    """
+    This method asserts that `expected` is a subset of `actual`
+    parameters:
+        @expected:
+        @actual:
+        @msg:
+    """
+    __tracebackhide__ = True
+
+    subset = _extract_subset(actual, expected)
+    if subset != expected:
+        diff = '\n'.join(udiff(expected, subset, indent=2))
+        raise AssertionError("{msg}:\n{diff}".format(msg=msg, diff=diff))
+
+
+def _extract_subset(source, mask):
+    if isinstance(mask, dict) and isinstance(source, dict):
+        return _extract_dict_subset(source, mask)
+    elif isinstance(mask, list) and isinstance(source, list):
+        return _extract_list_subset(source, mask)
+    else:
+        return source
+
+
+def _extract_dict_subset(source, mask):
+    target = {}
+    for key in mask:
+        if key in source:
+            target[key] = _extract_subset(source[key], mask[key])
+    return target
+
+
+MISSING = object()
+def _extract_list_subset(source, mask):
+    target = []
+    for source_item, mask_item in zip_longest(source, mask, fillvalue=MISSING):
+        if mask_item is MISSING:
+            target.append(source_item)
+        elif source_item is not MISSING:
+            target.append(_extract_subset(source_item, mask_item))
+    return target