Merge branch 'release/1.3.5'

Author: mblaschke

CHANGELOG.md

@@ -2,6 +2,10 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [WebDevOps.io Dockerfile](https://github.com/webdevops/Dockerfile).
 
+## [1.3.5] - 2017-02-12
+- Improve and fix parallel run of serverspec
+- Reduced python subprocess wait loop cpu time
+
 ## [1.3.4] - 2017-02-03
 - Add serverspec wrapper (do not run tests if Dockerfile isn't set or available)
 

Makefile

@@ -36,7 +36,7 @@ requirements:
 	cd tests/serverspec && bundle install --path=vendor
 
 test:
-	python bin/console test:serverspec --threads=auto/2 -v
+	python bin/console test:serverspec --threads=auto -v
 
 baselayout:
 	python bin/console generate:provision --baselayout

bin/webdevops/Command.py

@@ -18,7 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os, subprocess, tempfile
+import os, subprocess, tempfile, copy, time
 
 def execute(cmd, cwd=False, env=None):
     """
@@ -27,9 +27,20 @@ def execute(cmd, cwd=False, env=None):
 
     print 'Execute: %s' % ' '.join(cmd)
 
-    # remove _ from env (prevent errors)
-    if env is not None and '_' in env:
-        del env['_']
+    if env is not None:
+        env = copy.deepcopy(env)
+        env['PWD'] = os.environ['PWD']
+        env['LC_CTYPE'] = os.environ['LC_CTYPE']
+        env['SHELL'] = os.environ['SHELL']
+
+        if '_' in env:
+            del env['_']
+
+        # add system env vars
+        system_env_vars = ['PWD', 'PATH', 'LC_CTYPE', 'SHELL', 'DOCKER_HOST', 'DOCKER_CERT_PATH', 'DOCKER_MACHINE_NAME', 'DOCKER_TLS_VERIFY']
+        for var_name in system_env_vars:
+            if not var_name in env and var_name in os.environ:
+                env[var_name] = os.environ[var_name]
 
     # set current working directory
     path_current = os.getcwd()
@@ -51,7 +62,7 @@ def execute(cmd, cwd=False, env=None):
 
     # wait for process end
     while proc.poll() is None:
-        pass
+        time.sleep(1)
 
     # output stdout
     with open(file_stdout.name, 'r') as f:

bin/webdevops/command/DoitCommand.py

@@ -28,7 +28,7 @@ def run_doit(self, task_loader, configuration):
         extra_configuration = {}
 
         if 'threads' in configuration and configuration.get('threads') > 1:
-            arguments.extend(['-n', str(configuration.get('threads'))])
+            arguments.extend(['-n', str(configuration.get('threads')), '--parallel-type', 'process'])
 
         if 'doitConfig' in configuration:
             extra_configuration = configuration.get('doitConfig')

bin/webdevops/taskloader/DockerBuildTaskLoader.py

@@ -39,7 +39,7 @@ def generate_task_list(self, dockerfileList):
             }
 
             if dockerfile['dependency']:
-                task['task_dep'].append('DockerBuild|%s' % dockerfile['dependency']);
+                task['task_dep'].append('DockerBuild|%s' % dockerfile['dependency'])
 
             tasklist.append(task)
 

bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py

@@ -18,7 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os, re, tempfile, json
+import os, re, tempfile, json, base64
 from webdevops import Command
 from .BaseDockerTaskLoader import BaseDockerTaskLoader
 from .BaseTaskLoader import BaseTaskLoader
@@ -40,6 +40,10 @@ def generate_task_list(self, dockerfile_list):
                 'actions': [(BaseTaskLoader.task_runner, [DockerTestServerspecTaskLoader.task_run, [dockerfile, self.configuration]])],
                 'task_dep': []
             }
+
+            if dockerfile['dependency']:
+                task['task_dep'].append('DockerTestServerspec|%s' % dockerfile['dependency'])
+
             tasklist.append(task)
 
         # task = {
@@ -71,18 +75,18 @@ def task_run(dockerfile, configuration, task):
         tmp_suffix = '.%s_%s_%s.tmp' % (dockerfile['image']['repository'], dockerfile['image']['imageName'], dockerfile['image']['tag'])
         test_dockerfile = tempfile.NamedTemporaryFile(prefix='Dockerfile.', suffix=tmp_suffix, dir=configuration.get('serverspecPath'), bufsize=0, delete=False)
 
-        # serverspec options
-        serverspec_opts = []
-        serverspec_opts.extend(['--pattern', spec_path])
-
-        # serverspec env
-        serverspec_env = DockerTestServerspecTaskLoader.generate_serverspec_environment(
+        # serverspec conf
+        serverspec_conf = DockerTestServerspecTaskLoader.generate_serverspec_configuration(
             path=os.path.basename(test_dockerfile.name),
             dockerfile=dockerfile,
             configuration=configuration,
             is_toolimage=is_toolimage
         )
 
+        # serverspec options
+        serverspec_opts = []
+        serverspec_opts.extend([spec_path, dockerfile['image']['fullname'], base64.b64encode(json.dumps(serverspec_conf)), os.path.basename(test_dockerfile.name)])
+
         # dockerfile content
         dockerfile_content = DockerTestServerspecTaskLoader.generate_dockerfile(
             dockerfile=dockerfile,
@@ -99,15 +103,13 @@ def task_run(dockerfile, configuration, task):
             print '          path: %s' % (spec_path)
             print '          args: %s' % (' '.join(serverspec_opts))
             print ''
-            print 'environment:'
-            print '------------'
-            print json.dumps(serverspec_env, indent=4, sort_keys=True)
+            print 'spec configuration:'
+            print '-------------------'
+            print json.dumps(serverspec_conf, indent=4, sort_keys=True)
             print ''
             print 'Dockerfile:'
             print '-----------'
             print dockerfile_content
-
-            os.remove(test_dockerfile.name)
             return True
 
         # check if we have any tests
@@ -119,10 +121,6 @@ def task_run(dockerfile, configuration, task):
         cmd = ['bash', 'serverspec.sh']
         cmd.extend(serverspec_opts)
 
-        # Set environment variables
-        env = os.environ.copy()
-        env.update(serverspec_env)
-
         # create Dockerfile
         with open(test_dockerfile.name, mode='w', buffering=0) as f:
             f.write(dockerfile_content)
@@ -133,7 +131,7 @@ def task_run(dockerfile, configuration, task):
         test_status = False
         for retry_count in range(0, configuration.get('retry')):
             try:
-                test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath'), env=env)
+                test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath'))
             except Exception as e:
                 print e
                 pass
@@ -145,25 +143,24 @@ def task_run(dockerfile, configuration, task):
             else:
                 print '    failed, giving up'
 
-        os.remove(test_dockerfile.name)
         return test_status
 
     @staticmethod
-    def generate_serverspec_environment(path, dockerfile, configuration, is_toolimage=False):
+    def generate_serverspec_configuration(path, dockerfile, configuration, is_toolimage=False):
         """
-        Generate serverspec environment dict
+        Generate serverspec configuration dict
         """
         ret = {}
 
         # add default vars
-        default_env_list = configuration.get('dockerTest.environment.default', False)
+        default_env_list = configuration.get('dockerTest.configuration.default', False)
         if default_env_list:
-            ret = default_env_list.to_dict()
+            ret = default_env_list.to_dict().copy()
 
         # add docker image specific vars
-        image_env_list = configuration.get('dockerTest.environment.image')
+        image_env_list = configuration.get('dockerTest.configuration.image')
         if image_env_list:
-            image_env_list = image_env_list.to_dict()
+            image_env_list = image_env_list.to_dict().copy()
             for term in image_env_list:
                 if term in dockerfile['image']['fullname']:
                     for key in image_env_list[term]:
@@ -187,6 +184,7 @@ def generate_dockerfile(dockerfile, configuration, is_toolimage=False):
 
         ret.append('FROM %s' % dockerfile['image']['fullname'])
         ret.append('COPY conf/ /')
+        ret.append('RUN echo "%s" > /DOCKER.IMAGENAME' % dockerfile['image']['fullname'])
 
         if is_toolimage:
             ret.append('RUN chmod +x /loop-entrypoint.sh')

bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py

@@ -40,6 +40,10 @@ def generate_task_list(self, dockerfile_list):
                 'actions': [(BaseTaskLoader.task_runner, [DockerTestTestinfraTaskLoader.task_run, [dockerfile, self.configuration]])],
                 'task_dep': []
             }
+
+            if dockerfile['dependency']:
+                task['task_dep'].append('DockerTestTestinfra|%s' % dockerfile['dependency'])
+
             tasklist.append(task)
 
         # task = {

conf/console.yml

@@ -7,6 +7,7 @@ imagePath: 'documentation/docs/resources/images/'
 baselayoutPath: 'baselayout'
 testinfraPath: 'tests/testinfra'
 serverspecPath: 'tests/serverspec'
+testDockerfilePath: 'tests/dockerfile'
 blacklistFile: 'BLACKLIST'
 
 docker:
@@ -35,7 +36,7 @@ dockerTest:
     '/varnish':
       - 'ENV VARNISH_BACKEND_HOST webdevops.io'
 
-  environment:
+  configuration:
     default:
       OS_FAMILY: 'ubuntu'
       OS_VERSION: '16.04'

docker/ansible/alpine-3/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:alpine-3
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/centos-7/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:centos-7
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5