Add 'Rule' to unasync for per-directory config (#55)

* Add 'Rule' to unasync for per-directory config

* Switch to unasync.cmdclass_build_py(), update docs

Author: sethmlarson

README.rst

@@ -44,12 +44,34 @@ And then in your :code:`setup.py` place the following code.
 
     setuptools.setup(
         ...
-        cmdclass={'build_py': unasync.build_py},
+        cmdclass={'build_py': unasync.cmdclass_build_py()},
         ...
     )
 
 And when you will build your package you will get your synchronous code in **_sync** folder.
 
+If you'd like to customize where certain rules are applied you can pass
+customized :code:`unasync.Rule` instances to :code:`unasync.cmdclass_build_py()`
+
+.. code-block:: python
+
+    import unasync
+
+    setuptools.setup(
+        ...
+        cmdclass={'build_py': unasync.cmdclass_build_py(rules=[
+            # This rule transforms files within 'ahip' -> 'hip'
+            # instead of the default '_async' -> '_sync'.
+            unasync.Rule("/ahip/", "/hip/"),
+
+            # This rule's 'fromdir' is more specific so will take precedent
+            # over the above rule if the path is within /ahip/tests/...
+            # This rule adds an additional token replacement over the default replacements.
+            unasync.Rule("/ahip/tests/", "/hip/tests/", replacements={"ahip", "hip"}),
+        ])},
+        ...
+    )
+
 Documentation
 =============
 

docs/source/index.rst

@@ -1,9 +1,3 @@
-.. documentation master file, created by
-   sphinx-quickstart on Sat Jan 21 19:11:14 2017.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-
 =======
 unasync
 =======
@@ -56,7 +50,7 @@ And then in your :code:`setup.py` place the following code.
 
     setuptools.setup(
         ...
-        cmdclass={'build_py': unasync.build_py},
+        cmdclass={'build_py': unasync.cmdclass_build_py()},
         ...
     )
 
@@ -70,6 +64,28 @@ Then create a file **pyproject.toml** in the root of your project and mention **
 
 And when you will build your package you will get your synchronous code in **_sync** folder.
 
+If you'd like to customize where certain rules are applied you can pass
+customized :code:`unasync.Rule` instances to :code:`unasync.cmdclass_build_py()`
+
+.. code-block:: python
+
+    import unasync
+
+    setuptools.setup(
+        ...
+        cmdclass={'build_py': unasync.cmdclass_build_py(rules=[
+            # This rule transforms files within 'ahip' -> 'hip'
+            # instead of the default '_async' -> '_sync'.
+            unasync.Rule("/ahip/", "/hip/"),
+
+            # This rule's 'fromdir' is more specific so will take precedent
+            # over the above rule if the path is within /ahip/tests/...
+            # This rule adds an additional token replacement over the default replacements.
+            unasync.Rule("/ahip/tests/", "/hip/tests/", replacements={"ahip", "hip"}),
+        ])},
+        ...
+    )
+
 
 .. toctree::
    :maxdepth: 2

src/unasync/__init__.py

@@ -12,7 +12,13 @@
 
 from ._version import __version__  # NOQA
 
-ASYNC_TO_SYNC = {
+__all__ = [
+    "Rule",
+    "cmdclass_build_py",
+]
+
+
+_ASYNC_TO_SYNC = {
     "__aenter__": "__enter__",
     "__aexit__": "__exit__",
     "__aiter__": "__iter__",
@@ -27,6 +33,81 @@
     "StopAsyncIteration": "StopIteration",
 }
 
+
+class Rule:
+    """A single set of rules for 'unasync'ing file(s)"""
+
+    def __init__(self, fromdir, todir, replacements=None):
+        self.fromdir = fromdir.replace("/", os.sep)
+        self.todir = todir.replace("/", os.sep)
+
+        # Add any additional user-defined token replacements to our list.
+        self.token_replacements = _ASYNC_TO_SYNC.copy()
+        for key, val in (replacements or {}).items():
+            self.token_replacements[key] = val
+
+    def match(self, filepath):
+        """Determines if a Rule matches a given filepath and if so
+        returns a higher comparable value if the match is more specific.
+        """
+        file_segments = [x for x in filepath.split(os.sep) if x]
+        from_segments = [x for x in self.fromdir.split(os.sep) if x]
+        len_from_segments = len(from_segments)
+
+        if len_from_segments > len(file_segments):
+            return False
+
+        for i in range(len(file_segments) - len_from_segments + 1):
+            if file_segments[i : i + len_from_segments] == from_segments:
+                return len_from_segments, i
+
+        return False
+
+    def unasync_file(self, filepath):
+        with open(filepath, "rb") as f:
+            write_kwargs = {}
+            if sys.version_info[0] >= 3:
+                encoding, _ = std_tokenize.detect_encoding(f.readline)
+                write_kwargs["encoding"] = encoding
+                f.seek(0)
+            tokens = tokenize(f)
+            tokens = self.unasync_tokens(tokens)
+            result = untokenize(tokens)
+            outfilepath = filepath.replace(self.fromdir, self.todir)
+            makedirs_existok(os.path.dirname(outfilepath))
+            with open(outfilepath, "w", **write_kwargs) as f:
+                print(result, file=f, end="")
+
+    def unasync_tokens(self, tokens):
+        # TODO __await__, ...?
+        used_space = None
+        for space, toknum, tokval in tokens:
+            if tokval in ["async", "await"]:
+                # When removing async or await, we want to use the whitespace that
+                # was before async/await before the next token so that
+                # `print(await stuff)` becomes `print(stuff)` and not
+                # `print( stuff)`
+                used_space = space
+            else:
+                if toknum == std_tokenize.NAME:
+                    tokval = self.unasync_name(tokval)
+                elif toknum == std_tokenize.STRING:
+                    left_quote, name, right_quote = tokval[0], tokval[1:-1], tokval[-1]
+                    tokval = left_quote + self.unasync_name(name) + right_quote
+                if used_space is None:
+                    used_space = space
+                yield (used_space, tokval)
+                used_space = None
+
+    def unasync_name(self, name):
+        if name in self.token_replacements:
+            return self.token_replacements[name]
+        # Convert classes prefixed with 'Async' into 'Sync'
+        elif len(name) > 5 and name.startswith("Async") and name[5].isupper():
+            return "Sync" + name[5:]
+        return name
+
+
 Token = collections.namedtuple("Token", ["type", "string", "start", "end", "line"])
 
 
@@ -60,37 +141,6 @@ def tokenize(f):
             last_end = (tok.end[0] + 1, 0)
 
 
-def unasync_name(name):
-    if name in ASYNC_TO_SYNC:
-        return ASYNC_TO_SYNC[name]
-    # Convert classes prefixed with 'Async' into 'Sync'
-    elif len(name) > 5 and name.startswith("Async") and name[5].isupper():
-        return "Sync" + name[5:]
-    return name
-
-
-def unasync_tokens(tokens):
-    # TODO __await__, ...?
-    used_space = None
-    for space, toknum, tokval in tokens:
-        if tokval in ["async", "await"]:
-            # When removing async or await, we want to use the whitespace that
-            # was before async/await before the next token so that
-            # `print(await stuff)` becomes `print(stuff)` and not
-            # `print( stuff)`
-            used_space = space
-        else:
-            if toknum == std_tokenize.NAME:
-                tokval = unasync_name(tokval)
-            elif toknum == std_tokenize.STRING:
-                left_quote, name, right_quote = tokval[0], tokval[1:-1], tokval[-1]
-                tokval = left_quote + unasync_name(name) + right_quote
-            if used_space is None:
-                used_space = space
-            yield (used_space, tokval)
-            used_space = None
-
-
 def untokenize(tokens):
     return "".join(space + tokval for space, tokval in tokens)
 
@@ -103,34 +153,21 @@ def makedirs_existok(dir):
             raise
 
 
-def unasync_file(filepath, fromdir, todir):
-    with open(filepath, "rb") as f:
-        write_kwargs = {}
-        if sys.version_info[0] >= 3:
-            encoding, _ = std_tokenize.detect_encoding(f.readline)
-            write_kwargs["encoding"] = encoding
-            f.seek(0)
-        tokens = tokenize(f)
-        tokens = unasync_tokens(tokens)
-        result = untokenize(tokens)
-        outfilepath = filepath.replace(fromdir, todir)
-        makedirs_existok(os.path.dirname(outfilepath))
-        with open(outfilepath, "w", **write_kwargs) as f:
-            print(result, file=f, end="")
+_DEFAULT_RULE = Rule(fromdir="/_async/", todir="/_sync/")
 
 
-class build_py(orig.build_py):
+class _build_py(orig.build_py):
     """
     Subclass build_py from setuptools to modify its behavior.
 
     Convert files in _async dir from being asynchronous to synchronous
     and saves them in _sync dir.
     """
 
-    RENAME_DIR_FROM_TO = ("_async", "_sync")  # Controls what directory will be renamed.
+    UNASYNC_RULES = (_DEFAULT_RULE,)
 
     def run(self):
-        dir_from, dir_to = self.RENAME_DIR_FROM_TO
+        rules = self.UNASYNC_RULES
 
         self._updated_files = []
 
@@ -143,8 +180,17 @@ def run(self):
 
         # Our modification!
         for f in self._updated_files:
-            if os.sep + dir_from + os.sep in f:
-                unasync_file(f, dir_from, dir_to)
+            found_rule = None
+            found_weight = None
+
+            for rule in rules:
+                weight = rule.match(f)
+                if weight and (found_weight is None or weight > found_weight):
+                    found_rule = rule
+                    found_weight = weight
+
+            if found_rule:
+                found_rule.unasync_file(f)
 
         # Remaining base class code
         self.byte_compile(self.get_outputs(include_bytecode=0))
@@ -156,8 +202,10 @@ def build_module(self, module, module_file, package):
         return outfile, copied
 
 
-def customize_build_py(rename_dir_from_to=("_async", "_sync")):
-    class _build_py(build_py):
-        RENAME_DIR_FROM_TO = rename_dir_from_to
+def cmdclass_build_py(rules=(_DEFAULT_RULE,)):
+    """Creates a 'build_py' class for use within 'cmdclass={"build_py": ...}'"""
+
+    class _custom_build_py(_build_py):
+        UNASYNC_RULES = rules
 
-    return _build_py
+    return _custom_build_py

tests/data/example_custom_pkg/src/__init__.py renamed to tests/data/example_custom_pkg/__init__.py

@@ -9,9 +9,18 @@
     author_email="[email protected]",
     description="A package used to test customized unasync",
     url="https://github.com/pypa/sampleproject",
-    packages=["ahip", "ahip.some_dir"],
+    packages=["ahip", "ahip.some_dir", "ahip.tests"],
     cmdclass={
-        "build_py": unasync.customize_build_py(rename_dir_from_to=("ahip", "hip"))
+        "build_py": unasync.cmdclass_build_py(
+            rules=[
+                unasync.Rule(fromdir="/ahip/", todir="/hip/"),
+                unasync.Rule(
+                    fromdir="/ahip/tests/",
+                    todir="/hip/tests/",
+                    replacements={"ahip": "hip"},
+                ),
+            ]
+        )
     },
     package_dir={"": "src"},
 )

tests/data/example_custom_pkg/setup.py

@@ -0,0 +1,8 @@
+import ahip
+
+
+async def test_connection():
+    async def f():
+        return None
+
+    x = await f()

tests/data/example_custom_pkg/src/ahip/tests/__init__.py

@@ -10,5 +10,5 @@
     description="A package used to test unasync",
     url="https://github.com/pypa/sampleproject",
     py_modules=["_async.some_file"],
-    cmdclass={"build_py": unasync.build_py},
+    cmdclass={"build_py": unasync.cmdclass_build_py()},
 )

tests/data/example_custom_pkg/src/ahip/tests/test_conn.py

@@ -10,6 +10,6 @@
     description="A package used to test unasync",
     url="https://github.com/pypa/sampleproject",
     packages=["example_pkg", "example_pkg._async", "example_pkg._async.some_dir"],
-    cmdclass={"build_py": unasync.build_py},
+    cmdclass={"build_py": unasync.cmdclass_build_py()},
     package_dir={"": "src"},
 )

tests/data/example_mod/setup.py

@@ -28,12 +28,16 @@ def list_files(startpath):
     return output
 
 
+def test_rule_on_short_path():
+    rule = unasync.Rule("/ahip/tests/", "/hip/tests/")
+    assert rule.match("/ahip/") is False
+
+
 @pytest.mark.parametrize("source_file", TEST_FILES)
 def test_unasync(tmpdir, source_file):
 
-    unasync.unasync_file(
-        os.path.join(ASYNC_DIR, source_file), fromdir=ASYNC_DIR, todir=str(tmpdir)
-    )
+    rule = unasync.Rule(fromdir=ASYNC_DIR, todir=str(tmpdir))
+    rule.unasync_file(os.path.join(ASYNC_DIR, source_file))
 
     encoding = "latin-1" if "encoding" in source_file else "utf-8"
     with io.open(os.path.join(SYNC_DIR, source_file), encoding=encoding) as f:
@@ -109,6 +113,10 @@ def test_project_structure_after_customized_build_py_packages(tmpdir):
     subprocess.check_call(["python", "setup.py", "build"], cwd=pkg_dir, env=env)
 
     _async_dir_tree = list_files(os.path.join(source_pkg_dir, "src/ahip/."))
-    unasynced_dir_tree = list_files(os.path.join(pkg_dir, "build/lib/hip/."))
+    unasynced_dir_path = os.path.join(pkg_dir, "build/lib/hip/.")
+    unasynced_dir_tree = list_files(unasynced_dir_path)
 
     assert _async_dir_tree == unasynced_dir_tree
+
+    with open(os.path.join(unasynced_dir_path, "tests/test_conn.py")) as f:
+        assert "import hip\n" in f.read()