Merge pull request #40 from meganlkm/test-harness

aboe76 · web-flow · commit ae373afd8bef · 2019-01-12T23:17:36.000+01:00
implement test harness
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,10 @@
+*.egg
+*.egg-info/
+*.py[cod]
+.env
+.pytest_cache/
+__pycache__/
+Dockerfile.*_*
+ignore/
+.venv/
+tmp/
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,24 @@
+env:
+  matrix:
+    - OS_ID: centos_master_2017.7.2
+    - OS_ID: debian_master_2017.7.2
+    - OS_ID: opensuse_master_2017.7.2
+    - OS_ID: ubuntu_master_2016.11.3
+    - OS_ID: ubuntu_master_2017.7.2
+
+sudo: required
+
+language: python
+
+services:
+- docker
+
+before_install:
+- pip install Jinja2
+- python ${TRAVIS_BUILD_DIR}/tools/filltmpl.py node ${OS_ID}
+
+install:
+- docker build --force-rm -t "node:salt-testing-${OS_ID}" -f "Dockerfile.${OS_ID}" .
+
+script:
+- ./tools/run-tests.sh node ${OS_ID}
diff --git a/Makefile b/Makefile
@@ -0,0 +1,80 @@
+FORMULA_NAME = "node"
+PWD = $(shell pwd)
+
+# ---------------------------------------------------------------
+define render_dockerfile
+	python $(PWD)/tools/filltmpl.py $(FORMULA_NAME) $(1)
+endef
+
+define docker_build
+	docker build --force-rm -t $(FORMULA_NAME):salt-testing-$(1) -f Dockerfile.$(1) .
+endef
+
+define docker_run_local
+	docker run --rm -v $(PWD):/opt/$(FORMULA_NAME)-formula --env=STAGE=TEST -h salt-testing-$(1) --name salt-testing-$(1) -it $(FORMULA_NAME):salt-testing-$(1) /bin/bash
+endef
+
+define run_tests
+	./tools/run-tests.sh $(FORMULA_NAME) $(1)
+endef
+
+# --- convenience functions -------------------------------------
+define build_thing
+	$(call render_dockerfile,$(1)) && $(call docker_build,$(1))
+endef
+
+define run_local_tests
+	$(call build_thing,$(1)) && $(call run_tests,$(1))
+endef
+
+define run_local
+	$(call build_thing,$(1)) && $(call docker_run_local,$(1))
+endef
+
+# ---------------------------------------------------------------
+setup:
+	pip install Jinja2
+
+clean:
+	find . -name '*.pyc' -exec rm '{}' ';'
+	rm -rf Dockerfile.*
+	# delete pytest caches...
+	# rm -rf tests/pytests/*/.pytest_cache
+	# rm -rf tests/pytests/*/__pycache__
+	rm -rf tests/pytests/apply-all-tests/.pytest_cache
+	rm -rf tests/pytests/apply-all-tests/__pycache__
+
+# --- centos_master_2017.7.2 ------------------------------------
+test-centos_master_2017.7.2: clean
+	$(call run_local_tests,centos_master_2017.7.2)
+
+local-centos_master_2017.7.2: clean
+	$(call run_local,centos_master_2017.7.2)
+
+# --- debian_master_2017.7.2 ------------------------------------
+test-debian_master_2017.7.2: clean
+	$(call run_local_tests,debian_master_2017.7.2)
+
+local-debian_master_2017.7.2: clean
+	$(call run_local,debian_master_2017.7.2)
+
+# --- opensuse_master_2017.7.2 ------------------------------------
+test-opensuse_master_2017.7.2: clean
+	$(call run_local_tests,opensuse_master_2017.7.2)
+
+local-opensuse_master_2017.7.2: clean
+	$(call run_local,opensuse_master_2017.7.2)
+
+# --- ubuntu_master_2016.11.3 ------------------------------------
+test-ubuntu_master_2016.11.3: clean
+	$(call run_local_tests,ubuntu_master_2016.11.3)
+
+local-ubuntu_master_2016.11.3: clean
+	$(call run_local,ubuntu_master_2016.11.3)
+
+# --- ubuntu_master_2017.7.2 ------------------------------------
+test-ubuntu_master_2017.7.2: clean
+	$(call run_local_tests,ubuntu_master_2017.7.2)
+
+local-ubuntu_master_2017.7.2: clean
+	$(call run_local,ubuntu_master_2017.7.2)
diff --git a/README.md b/README.md
@@ -41,10 +41,54 @@ The checksum for the *node-v…-linux-x64.tar.gz* file has to be provided.
 When building from sources, only newer node versions with npm included are supported (Versions 0.9.x and newer should
 be safe). Also, the executable will be called `node`.
 
-If you need to install older versions, where the executable is called `nodejs`, use 
+If you need to install older versions, where the executable is called `nodejs`, use
 [previous commits](https://github.com/saltstack-formulas/node-formula/commit/bcc649588c162686c4dbde486da840ccd060edf6)
 of this formula.
 
 ## Installing from packages
 
 See the [example pillar file](https://github.com/saltstack-formulas/node-formula/blob/master/pillar.example).
+
+
+## <a name='testing'></a> Running Tests
+
+This test runner was implemented using the [formula-test-harness](https://github.com/intuitivetechnologygroup/formula-test-harness) project.
+
+
+Tests will be run on the following base images:
+
+* `simplyadrian/allsalt:centos_master_2017.7.2`
+* `simplyadrian/allsalt:debian_master_2017.7.2`
+* `simplyadrian/allsalt:opensuse_master_2017.7.2`
+* `simplyadrian/allsalt:ubuntu_master_2016.11.3`
+* `simplyadrian/allsalt:ubuntu_master_2017.7.2`
+
+##### Start a virtualenv
+
+```bash
+pip install -U virtualenv
+virtualenv .venv
+source .venv/bin/activate
+```
+
+##### Install local requirements
+
+```bash
+make setup
+```
+
+##### Run tests
+
+* `make test-centos_master_2017.7.2`
+* `make test-debian_master_2017.7.2`
+* `make test-opensuse_master_2017.7.2`
+* `make test-ubuntu_master_2016.11.3`
+* `make test-ubuntu_master_2017.7.2`
+
+### <a name='run-containers'></a> Run Containers
+
+* `make local-centos_master_2017.7.2`
+* `make local-debian_master_2017.7.2`
+* `make local-opensuse_master_2017.7.2`
+* `make local-ubuntu_master_2016.11.3`
+* `make local-ubuntu_master_2017.7.2`
diff --git a/tests/pytests/apply-all-tests/__init__.py b/tests/pytests/apply-all-tests/__init__.py
diff --git a/tests/pytests/apply-all-tests/test_000_apply_state.py b/tests/pytests/apply-all-tests/test_000_apply_state.py
@@ -0,0 +1,23 @@
+from subprocess import check_output
+from unittest import TestCase
+
+
+class ApplyStateTest(TestCase):
+
+    def test_000_apply(self):
+        state_apply_response = check_output(["salt-call", "--local", "state.apply"])
+        print('')
+        print('-' * 50)
+        print('state_apply_response:')
+        print(state_apply_response)
+        print('-' * 50)
+        print('')
+
+        state_apply_response = state_apply_response.split('\n')
+        summary = state_apply_response[-8:]
+        failed = 0
+        for line in summary:
+            if line.startswith('Failed:'):
+                failed = int(line.split(':').pop().strip())
+
+        self.assertEqual(failed, 0)
diff --git a/tests/srv/salt/top.sls b/tests/srv/salt/top.sls
@@ -0,0 +1,3 @@
+base:
+  '*':
+    - node
diff --git a/tools/filltmpl.py b/tools/filltmpl.py
@@ -0,0 +1,27 @@
+import os
+import sys
+
+from jinja2 import Template
+
+# base/tests
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+# base
+base_path = os.path.dirname(dir_path)
+
+
+if __name__ == '__main__':
+    formula_name = sys.argv[1]
+    image_tag = sys.argv[2]
+
+    template = Template(
+        open(os.path.join(dir_path, 'templates', 'Dockerfile.j2')).read()
+    )
+
+    dockerfile = template.render({
+        'formula_name': formula_name,
+        'image_tag': image_tag
+    })
+
+    with open(os.path.join(base_path, 'Dockerfile.{}'.format(image_tag)), 'w') as fh:
+        fh.write(dockerfile)
diff --git a/tools/run-tests.sh b/tools/run-tests.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -ev
+
+test -z $2 && echo "Usage: ${0} FORMULA_NAME OS_ID" && exit 1
+export FORMULA_NAME=$1
+export OS_ID=$2
+
+
+function docker-run-pytest() {
+    docker run --rm \
+        -v "$@":/opt/tests \
+        --env=STAGE=TEST \
+        -h "salt-testing-${OS_ID}" \
+        --name "salt-testing-${OS_ID}" \
+        -it ${FORMULA_NAME}:"salt-testing-${OS_ID}" \
+        pytest -sv /opt/tests
+}
+
+for i in $(find $PWD/tests/pytests/* -maxdepth 0 -type d); do
+    docker-run-pytest $i;
+done
diff --git a/tools/templates/Dockerfile.j2 b/tools/templates/Dockerfile.j2
@@ -0,0 +1,14 @@
+FROM simplyadrian/allsalt:{{ image_tag }}
+
+{% if 'debian' in image_tag or 'ubuntu' in image_tag -%}
+RUN apt-get update && \
+    apt-get install -y python-pip
+{% endif %}
+
+RUN pip install pytest && \
+    sed -i "s/#master: salt/master: localhost/g" /etc/salt/minion
+
+ADD tests/srv /srv
+ADD {{ formula_name }} /srv/salt/{{ formula_name }}
+
+WORKDIR /srv/salt
