Merge pull request #206 from RickyCook/fix-dockerfile-cache

Fix dockerfile cache

Author: RickyCook

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Add shields.io for projects #197
 - Redirect to new project on creation #202
 - Command output is a string, rather than a Python array dump #204
+- Set mtime of files with ADD directive in a Dockerfile #206
 
 ### v0.0.4-1
 - Fix issue browsing anonymously #187

dockci/models/job.py

@@ -33,6 +33,7 @@
                                                 )
 from dockci.models.job_meta.stages_prepare import (GitChangesStage,
                                                    GitInfoStage,
+                                                   GitMtimeStage,
                                                    ProvisionStage,
                                                    TagVersionStage,
                                                    WorkdirStage,
@@ -295,6 +296,7 @@ def _run_now(self):
 
                 prepare = (stage() for stage in (
                     lambda: GitChangesStage(self, workdir).run(0),
+                    lambda: GitMtimeStage(self, workdir).run(None),
                     lambda: TagVersionStage(self, workdir).run(None),
                     lambda: ProvisionStage(self).run(0),
                     lambda: BuildStage(self, workdir).run(0),

dockci/models/job_meta/stages_prepare.py

@@ -2,17 +2,23 @@
 Preparation for the main job stages
 """
 
+import glob
+import json
 import re
 import subprocess
 
+from datetime import datetime
+from itertools import chain
+
 import docker
 import docker.errors
+import py.error  # pylint:disable=import-error
 
 from dockci.models.project import Project
 from dockci.models.job_meta.config import JobConfig
 from dockci.models.job_meta.stages import JobStageBase, CommandJobStage
 from dockci.server import CONFIG
-from dockci.util import docker_ensure_image, FauxDockerLog
+from dockci.util import docker_ensure_image, FauxDockerLog, write_all
 
 
 class WorkdirStage(CommandJobStage):
@@ -151,6 +157,169 @@ def runnable(self, handle):
         return True
 
 
+def recursive_mtime(path, timestamp):
+    """
+    Recursively set mtime on the given path, returning the number of
+    additional files or directories changed
+    """
+    path.setmtime(timestamp)
+    extra = 0
+    if path.isdir():
+        for subpath in path.visit():
+            try:
+                subpath.setmtime(timestamp)
+                extra += 1
+            except py.error.ENOENT:
+                pass
+
+    return extra
+
+
+class GitMtimeStage(JobStageBase):
+    """
+    Change the modified time to the commit time for any files in an ADD
+    directive of a Dockerfile
+    """
+
+    slug = 'git_mtime'
+
+    def __init__(self, job, workdir):
+        super(GitMtimeStage, self).__init__(job)
+        self.workdir = workdir
+
+    def dockerfile_globs(self):
+        """ Get all glob patterns from the Dockerfile """
+        dockerfile = self.workdir.join('Dockerfile')
+        with dockerfile.open() as handle:
+            for line in handle:
+                if line[:4] == 'ADD ':
+                    add_value = line[4:]
+                    try:
+                        for path in json.loads(add_value)[:-1]:
+                            yield path
+
+                    except ValueError:
+                        add_file, _ = add_value.split(' ', 1)
+                        yield add_file
+
+        yield 'Dockerfile'
+        yield '.dockerignore'
+
+    def sorted_dockerfile_globs(self, reverse=False):
+        """
+        Sorted globs from the Dockerfile. Paths are sorted based on depth
+        """
+        def keyfunc(glob_str):
+            """ Compare paths, ranking higher level dirs lower """
+            path = self.workdir.join(glob_str)
+            try:
+                if path.samefile(self.workdir):
+                    return -1
+            except py.error.ENOENT:
+                pass
+
+            return len(path.parts())
+        return sorted(self.dockerfile_globs(), key=keyfunc, reverse=reverse)
+
+    def timestamp_for(self, path):
+        """ Get the timestamp for the given path """
+        if path.samefile(self.workdir):
+            git_cmd = [
+                'git', 'log', '-1', '--format=format:%ct',
+            ]
+        else:
+            git_cmd = [
+                'git', 'log', '-1', '--format=format:%ct', '--', path.strpath,
+            ]
+
+        # Get the timestamp
+        return int(subprocess.check_output(
+            git_cmd,
+            stderr=subprocess.STDOUT,
+            cwd=self.workdir.strpath,
+        ))
+
+    def path_mtime(self, handle, path):
+        """
+        Set the mtime on the given path, writitng messages to the file handle
+        given as necessary
+        """
+        # Ensure path is inside workdir
+        if not path.common(self.workdir).samefile(self.workdir):
+            write_all(handle,
+                      "%s not in the workdir; failing" % path.strpath)
+            return False
+
+        if not path.check():
+            return True
+
+        # Show the file, relative to workdir
+        relpath = self.workdir.bestrelpath(path)
+        write_all(handle, "%s: " % relpath)
+
+        try:
+            timestamp = self.timestamp_for(path)
+
+        except subprocess.CalledProcessError as ex:
+            # Something happened with the git command
+            write_all(handle, [
+                "Could not retrieve commit time from git. Exit "
+                "code %d:\n" % ex.returncode,
+
+                ex.output,
+            ])
+            return False
+
+        except ValueError as ex:
+            # A non-int value returned
+            write_all(handle,
+                      "Unexpected output from git: %s\n" % ex.args[0])
+            return False
+
+        # User output
+        mtime = datetime.fromtimestamp(timestamp)
+        write_all(handle, "%s... " % mtime.strftime('%Y-%m-%d %H:%M:%S'))
+
+        # Set the time!
+        extra = recursive_mtime(path, timestamp)
+
+        extra_txt = ("(and %d more) " % extra) if extra > 0 else ""
+        handle.write("{}DONE!\n".format(extra_txt).encode())
+        if path.samefile(self.workdir):
+            write_all(
+                handle,
+                "** Note: Performance benefits may be gained by adding "
+                "only necessary files, rather than the whole source tree "
+                "**\n",
+            )
+
+        return True
+
+    def runnable(self, handle):
+        """ Scrape the Dockerfile, update any ``mtime``s """
+        try:
+            globs = self.sorted_dockerfile_globs()
+
+        except py.error.ENOENT:
+            write_all(handle, "No Dockerfile! Can not continue")
+            return 1
+
+        # Join with workdir, unglob, and turn into py.path.local
+        all_files = chain(*(
+            (
+                py.path.local(path)
+                for path in glob.iglob(self.workdir.join(repo_glob).strpath)
+            )
+            for repo_glob in globs
+        ))
+
+        success = True
+        for path in all_files:
+            success &= self.path_mtime(handle, path)
+
+        return 0 if success else 1
+
+
 class TagVersionStage(CommandJobStage):
     """
     Try and add a version to the job, based on git tag

dockci/util.py

@@ -433,3 +433,18 @@ def validate_auth_token(secret, form_data, user, model):
 
     return hmac.compare_digest(req_auth_token,
                                form_data.get('auth_token', None))
+
+
+def write_all(handle, lines, flush=True):
+    """ Encode, write, then flush the line """
+    if isinstance(lines, (tuple, list)):
+        for line in lines:
+            write_all(handle, line, False)
+    else:
+        if isinstance(lines, bytes):
+            handle.write(lines)
+        else:
+            handle.write(str(lines).encode())
+
+        if flush:
+            handle.flush()