Add support for recursive wildcard pattern in .dockerignore

Signed-off-by: Joffrey F <[email protected]>

Author: shin-

docker/utils/__init__.py

@@ -1,12 +1,13 @@
 # flake8: noqa
+from .build import tar, exclude_paths
+from .decorators import check_resource, minimum_version, update_headers
 from .utils import (
     compare_version, convert_port_bindings, convert_volume_binds,
-    mkbuildcontext, tar, exclude_paths, parse_repository_tag, parse_host,
+    mkbuildcontext, parse_repository_tag, parse_host,
     kwargs_from_env, convert_filters, datetime_to_timestamp,
     create_host_config, parse_bytes, ping_registry, parse_env_file, version_lt,
     version_gte, decode_json_header, split_command, create_ipam_config,
     create_ipam_pool, parse_devices, normalize_links, convert_service_networks,
     format_environment, create_archive
 )
 
-from .decorators import check_resource, minimum_version, update_headers

docker/utils/build.py

@@ -0,0 +1,138 @@
+import os
+
+from .fnmatch import fnmatch
+from .utils import create_archive
+
+
+def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False):
+    root = os.path.abspath(path)
+    exclude = exclude or []
+
+    return create_archive(
+        files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)),
+        root=root, fileobj=fileobj, gzip=gzip
+    )
+
+
+def exclude_paths(root, patterns, dockerfile=None):
+    """
+    Given a root directory path and a list of .dockerignore patterns, return
+    an iterator of all paths (both regular files and directories) in the root
+    directory that do *not* match any of the patterns.
+
+    All paths returned are relative to the root.
+    """
+    if dockerfile is None:
+        dockerfile = 'Dockerfile'
+
+    exceptions = [p for p in patterns if p.startswith('!')]
+
+    include_patterns = [p[1:] for p in exceptions]
+    include_patterns += [dockerfile, '.dockerignore']
+
+    exclude_patterns = list(set(patterns) - set(exceptions))
+
+    paths = get_paths(root, exclude_patterns, include_patterns,
+                      has_exceptions=len(exceptions) > 0)
+
+    return set(paths).union(
+        # If the Dockerfile is in a subdirectory that is excluded, get_paths
+        # will not descend into it and the file will be skipped. This ensures
+        # it doesn't happen.
+        set([dockerfile])
+        if os.path.exists(os.path.join(root, dockerfile)) else set()
+    )
+
+
+def should_include(path, exclude_patterns, include_patterns):
+    """
+    Given a path, a list of exclude patterns, and a list of inclusion patterns:
+
+    1. Returns True if the path doesn't match any exclusion pattern
+    2. Returns False if the path matches an exclusion pattern and doesn't match
+       an inclusion pattern
+    3. Returns true if the path matches an exclusion pattern and matches an
+       inclusion pattern
+    """
+    for pattern in exclude_patterns:
+        if match_path(path, pattern):
+            for pattern in include_patterns:
+                if match_path(path, pattern):
+                    return True
+            return False
+    return True
+
+
+def should_check_directory(directory_path, exclude_patterns, include_patterns):
+    """
+    Given a directory path, a list of exclude patterns, and a list of inclusion
+    patterns:
+
+    1. Returns True if the directory path should be included according to
+       should_include.
+    2. Returns True if the directory path is the prefix for an inclusion
+       pattern
+    3. Returns False otherwise
+    """
+
+    # To account for exception rules, check directories if their path is a
+    # a prefix to an inclusion pattern. This logic conforms with the current
+    # docker logic (2016-10-27):
+    # https://github.com/docker/docker/blob/bc52939b0455116ab8e0da67869ec81c1a1c3e2c/pkg/archive/archive.go#L640-L671
+
+    def normalize_path(path):
+        return path.replace(os.path.sep, '/')
+
+    path_with_slash = normalize_path(directory_path) + '/'
+    possible_child_patterns = [
+        pattern for pattern in map(normalize_path, include_patterns)
+        if (pattern + '/').startswith(path_with_slash)
+    ]
+    directory_included = should_include(
+        directory_path, exclude_patterns, include_patterns
+    )
+    return directory_included or len(possible_child_patterns) > 0
+
+
+def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False):
+    paths = []
+
+    for parent, dirs, files in os.walk(root, topdown=True, followlinks=False):
+        parent = os.path.relpath(parent, root)
+        if parent == '.':
+            parent = ''
+
+        # Remove excluded patterns from the list of directories to traverse
+        # by mutating the dirs we're iterating over.
+        # This looks strange, but is considered the correct way to skip
+        # traversal. See https://docs.python.org/2/library/os.html#os.walk
+        dirs[:] = [
+            d for d in dirs if should_check_directory(
+                os.path.join(parent, d), exclude_patterns, include_patterns
+            )
+        ]
+
+        for path in dirs:
+            if should_include(os.path.join(parent, path),
+                              exclude_patterns, include_patterns):
+                paths.append(os.path.join(parent, path))
+
+        for path in files:
+            if should_include(os.path.join(parent, path),
+                              exclude_patterns, include_patterns):
+                paths.append(os.path.join(parent, path))
+
+    return paths
+
+
+def match_path(path, pattern):
+    pattern = pattern.rstrip('/' + os.path.sep)
+    if pattern:
+        pattern = os.path.relpath(pattern)
+
+    if '**' not in pattern:
+        pattern_components = pattern.split(os.path.sep)
+        path_components = path.split(os.path.sep)[:len(pattern_components)]
+    else:
+        path_components = path.split(os.path.sep)
+    return fnmatch('/'.join(path_components), pattern)

docker/utils/fnmatch.py

@@ -0,0 +1,106 @@
+"""Filename matching with shell patterns.
+
+fnmatch(FILENAME, PATTERN) matches according to the local convention.
+fnmatchcase(FILENAME, PATTERN) always takes case in account.
+
+The functions operate by translating the pattern into a regular
+expression.  They cache the compiled regular expressions for speed.
+
+The function translate(PATTERN) returns a regular expression
+corresponding to PATTERN.  (It does not compile it.)
+"""
+
+import re
+
+__all__ = ["fnmatch", "fnmatchcase", "translate"]
+
+_cache = {}
+_MAXCACHE = 100
+
+
+def _purge():
+    """Clear the pattern cache"""
+    _cache.clear()
+
+
+def fnmatch(name, pat):
+    """Test whether FILENAME matches PATTERN.
+
+    Patterns are Unix shell style:
+
+    *       matches everything
+    ?       matches any single character
+    [seq]   matches any character in seq
+    [!seq]  matches any char not in seq
+
+    An initial period in FILENAME is not special.
+    Both FILENAME and PATTERN are first case-normalized
+    if the operating system requires it.
+    If you don't want this, use fnmatchcase(FILENAME, PATTERN).
+    """
+
+    import os
+    name = os.path.normcase(name)
+    pat = os.path.normcase(pat)
+    return fnmatchcase(name, pat)
+
+
+def fnmatchcase(name, pat):
+    """Test whether FILENAME matches PATTERN, including case.
+
+    This is a version of fnmatch() which doesn't case-normalize
+    its arguments.
+    """
+
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
+        res = translate(pat)
+        if len(_cache) >= _MAXCACHE:
+            _cache.clear()
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
+
+
+def translate(pat):
+    """Translate a shell PATTERN to a regular expression.
+
+    There is no way to quote meta-characters.
+    """
+
+    recursive_mode = False
+    i, n = 0, len(pat)
+    res = ''
+    while i < n:
+        c = pat[i]
+        i = i + 1
+        if c == '*':
+            if i < n and pat[i] == '*':
+                recursive_mode = True
+                i = i + 1
+            res = res + '.*'
+        elif c == '?':
+            res = res + '.'
+        elif c == '[':
+            j = i
+            if j < n and pat[j] == '!':
+                j = j + 1
+            if j < n and pat[j] == ']':
+                j = j + 1
+            while j < n and pat[j] != ']':
+                j = j + 1
+            if j >= n:
+                res = res + '\\['
+            else:
+                stuff = pat[i:j].replace('\\', '\\\\')
+                i = j + 1
+                if stuff[0] == '!':
+                    stuff = '^' + stuff[1:]
+                elif stuff[0] == '^':
+                    stuff = '\\' + stuff
+                res = '%s[%s]' % (res, stuff)
+        elif recursive_mode and c == '/':
+            res = res + '/?'
+        else:
+            res = res + re.escape(c)
+    return res + '\Z(?ms)'