Merge pull request #252 from acsone/issue-251-sbi

File finders spec compliance, work in subdirectories too
fixes #251 
fixes #248

Author: RonnyPfannschmidt

setup.py

@@ -81,8 +81,8 @@ def parse(root):
         pip-egg-info = setuptools_scm.hacks:parse_pip_egg_info
 
         [setuptools_scm.files_command]
-        .hg = setuptools_scm.hg:FILES_COMMAND
-        .git = setuptools_scm.git_file_finder:find_files
+        .hg = setuptools_scm.file_finder_hg:hg_find_files
+        .git = setuptools_scm.file_finder_git:git_find_files
 
         [setuptools_scm.version_scheme]
         guess-next-dev = setuptools_scm.version:guess_next_dev_version

setuptools_scm/file_finder.py

@@ -0,0 +1,33 @@
+import os
+
+
+def scm_find_files(path, scm_files, scm_dirs):
+    """ setuptools compatible file finder that follows symlinks
+
+    - path: the root directory from which to search
+    - scm_files: set of scm controlled files
+    - scm_files: set of scm controlled directories
+
+    scm_files and scm_dirs must be absolute with symlinks resolved (realpath),
+    with normalized case (normcase)
+
+    Spec here: http://setuptools.readthedocs.io/en/latest/setuptools.html#\
+        adding-support-for-revision-control-systems
+    """
+    realpath = os.path.normcase(os.path.realpath(path))
+    seen = set()
+    res = []
+    for dirpath, dirnames, filenames in os.walk(realpath, followlinks=True):
+        # dirpath with symlinks resolved
+        realdirpath = os.path.normcase(os.path.realpath(dirpath))
+        if realdirpath not in scm_dirs or realdirpath in seen:
+            dirnames[:] = []
+            continue
+        for filename in filenames:
+            # dirpath + filename with symlinks preserved
+            fullfilename = os.path.join(dirpath, filename)
+            if os.path.normcase(os.path.realpath(fullfilename)) in scm_files:
+                res.append(
+                    os.path.join(path, os.path.relpath(fullfilename, path)))
+        seen.add(realdirpath)
+    return res

setuptools_scm/file_finder_git.py

@@ -0,0 +1,45 @@
+import os
+import subprocess
+import tarfile
+
+from .file_finder import scm_find_files
+
+
+def _git_toplevel(path):
+    try:
+        with open(os.devnull, 'wb') as devnull:
+            out = subprocess.check_output([
+                'git', 'rev-parse', '--show-toplevel',
+            ], cwd=(path or '.'), universal_newlines=True, stderr=devnull)
+        return os.path.normcase(os.path.realpath(out.strip()))
+    except subprocess.CalledProcessError:
+        # git returned error, we are not in a git repo
+        return None
+    except OSError:
+        # git command not found, probably
+        return None
+
+
+def _git_ls_files_and_dirs(toplevel):
+    # use git archive instead of git ls-file to honor
+    # export-ignore git attribute
+    cmd = ['git', 'archive', '--prefix', toplevel + os.path.sep, 'HEAD']
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=toplevel)
+    tf = tarfile.open(fileobj=proc.stdout, mode='r|*')
+    git_files = set()
+    git_dirs = set([toplevel])
+    for member in tf.getmembers():
+        name = os.path.normcase(member.name).replace('/', os.path.sep)
+        if member.type == tarfile.DIRTYPE:
+            git_dirs.add(name)
+        else:
+            git_files.add(name)
+    return git_files, git_dirs
+
+
+def git_find_files(path=''):
+    toplevel = _git_toplevel(path)
+    if not toplevel:
+        return []
+    git_files, git_dirs = _git_ls_files_and_dirs(toplevel)
+    return scm_find_files(path, git_files, git_dirs)

setuptools_scm/file_finder_hg.py

@@ -0,0 +1,44 @@
+import os
+import subprocess
+
+from .file_finder import scm_find_files
+
+
+def _hg_toplevel(path):
+    try:
+        with open(os.devnull, 'wb') as devnull:
+            out = subprocess.check_output([
+                'hg', 'root',
+            ], cwd=(path or '.'), universal_newlines=True, stderr=devnull)
+        return os.path.normcase(os.path.realpath(out.strip()))
+    except subprocess.CalledProcessError:
+        # hg returned error, we are not in a mercurial repo
+        return None
+    except OSError:
+        # hg command not found, probably
+        return None
+
+
+def _hg_ls_files_and_dirs(toplevel):
+    hg_files = set()
+    hg_dirs = set([toplevel])
+    out = subprocess.check_output([
+        'hg', 'files',
+    ], cwd=toplevel, universal_newlines=True)
+    for name in out.splitlines():
+        name = os.path.normcase(name).replace('/', os.path.sep)
+        fullname = os.path.join(toplevel, name)
+        hg_files.add(fullname)
+        dirname = os.path.dirname(fullname)
+        while len(dirname) > len(toplevel) and dirname not in hg_dirs:
+            hg_dirs.add(dirname)
+            dirname = os.path.dirname(dirname)
+    return hg_files, hg_dirs
+
+
+def hg_find_files(path=''):
+    toplevel = _hg_toplevel(path)
+    if not toplevel:
+        return []
+    hg_files, hg_dirs = _hg_ls_files_and_dirs(toplevel)
+    return scm_find_files(path, hg_files, hg_dirs)

setuptools_scm/git_file_finder.py

@@ -2,8 +2,6 @@
 from .utils import do, trace, data_from_mime, has_command
 from .version import meta, tags_to_versions
 
-FILES_COMMAND = 'hg locate -I .'
-
 
 def _hg_tagdist_normalize_tagcommit(root, tag, dist, node, branch):
     dirty = node.endswith('+')

setuptools_scm/hg.py

@@ -1,4 +1,4 @@
-import os
+from pkg_resources import iter_entry_points
 
 from .version import _warn_if_setuptools_outdated
 from .utils import do
@@ -22,22 +22,14 @@ def version_keyword(dist, keyword, value):
     dist.metadata.version = get_version(**value)
 
 
-def find_files(path='.'):
-    if not path:
-        path = '.'
-    abs = os.path.abspath(path)
-    ep = next(iter_matching_entrypoints(
-        abs, 'setuptools_scm.files_command'), None)
-    if ep:
+def find_files(path=''):
+    for ep in iter_entry_points('setuptools_scm.files_command'):
         command = ep.load()
-        try:
-            if isinstance(command, str):
-                return do(ep.load(), path).splitlines()
-            else:
-                return command(path)
-        except Exception:
-            print("File Finder Failed for %s" % ep)
-            raise
-
-    else:
-        return []
+        if isinstance(command, str):
+            # this technique is deprecated
+            res = do(ep.load(), path or '.').splitlines()
+        else:
+            res = command(path)
+        if res:
+            return res
+    return []

setuptools_scm/integration.py

@@ -3,16 +3,21 @@
 
 import pytest
 
-from setuptools_scm.git_file_finder import find_files
-
-
-@pytest.fixture
-def inwd(wd):
-    wd('git init')
-    wd('git config user.email [email protected]')
-    wd('git config user.name "a test"')
-    wd.add_command = 'git add .'
-    wd.commit_command = 'git commit -m test-{reason}'
+from setuptools_scm.integration import find_files
+
+
+@pytest.fixture(params=['git', 'hg'])
+def inwd(request, wd):
+    if request.param == 'git':
+        wd('git init')
+        wd('git config user.email [email protected]')
+        wd('git config user.name "a test"')
+        wd.add_command = 'git add .'
+        wd.commit_command = 'git commit -m test-{reason}'
+    elif request.param == 'hg':
+        wd('hg init')
+        wd.add_command = 'hg add .'
+        wd.commit_command = 'hg commit -m test-{reason} -u test -d "0 0"'
     (wd.cwd / 'file1').ensure(file=True)
     adir = (wd.cwd / 'adir').ensure(dir=True)
     (adir / 'filea').ensure(file=True)
@@ -46,6 +51,15 @@ def test_basic(inwd):
     })
 
 
+def test_whitespace(inwd):
+    (inwd.cwd / 'adir' / 'space file').ensure(file=True)
+    inwd.add_and_commit()
+    assert set(find_files('adir')) == _sep({
+        'adir/space file',
+        'adir/filea',
+    })
+
+
 def test_case(inwd):
     (inwd.cwd / 'CamelFile').ensure(file=True)
     (inwd.cwd / 'file2').ensure(file=True)

testing/test_git_file_finder.py renamed to testing/test_file_finder.py

@@ -138,6 +138,16 @@ def test_git_archive_subdirectory(wd):
         assert integration.find_files('.') == [opj('.', 'foobar', 'test1.txt')]
 
 
+@pytest.mark.issue(251)
+def test_git_archive_run_from_subdirectory(wd):
+    wd('mkdir foobar')
+    wd.write('foobar/test1.txt', 'test')
+    wd('git add foobar')
+    wd.commit()
+    with (wd.cwd / 'foobar').as_cwd():
+        assert integration.find_files('.') == [opj('.', 'test1.txt')]
+
+
 def test_git_feature_branch_increments_major(wd):
     wd.commit_testfile()
     wd("git tag 1.0.0")

testing/test_git.py

@@ -41,7 +41,12 @@ def test_archival_to_version(expected, data):
 def test_find_files_stop_at_root_hg(wd):
     wd.commit_testfile()
     wd.cwd.ensure('project/setup.cfg')
+    # setup.cfg has not been committed
     assert integration.find_files(str(wd.cwd / 'project')) == []
+    # issue 251
+    wd.add_and_commit()
+    with (wd.cwd / 'project').as_cwd():
+        assert integration.find_files() == ['setup.cfg']
 
 
 # XXX: better tests for tag prefixes