service: dev: Create fresh archive instead of cleaning

Signed-off-by: John Andersen <[email protected]>

Author: pdxjohnny

CHANGELOG.md

@@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Change Core to Official to clarify who maintains each plugin
 - Name of output of unsupervised model from "Prediction" to "cluster"
 - Test scikit LR documentation examples in CI
+- Create a fresh archive of the git repo for release instead of cleaning
+  existing repo with `git clean` for development service release command.
 
 ## [0.3.4] - 2020-02-28
 ### Added

dffml/service/dev.py

@@ -3,9 +3,11 @@
 import ast
 import json
 import pydoc
+import shutil
 import asyncio
 import pathlib
 import getpass
+import tempfile
 import importlib
 import contextlib
 import configparser
@@ -423,17 +425,36 @@ async def run(self):
                 if package_json["info"]["version"] == version:
                     print(f"Version {version} of {name} already on PyPi")
                     return
-            # Upload if not present
-            for cmd in [
-                ["git", "clean", "-fdx"],
-                [sys.executable, "setup.py", "sdist"],
-                [sys.executable, "-m", "twine", "upload", "dist/*"],
-            ]:
-                print(f"$ {' '.join(cmd)}")
-                proc = await asyncio.create_subprocess_exec(*cmd)
-                await proc.wait()
-                if proc.returncode != 0:
-                    raise RuntimeError
+            # Create a fresh copy of the codebase to upload
+            with tempfile.TemporaryDirectory() as tempdir:
+                # The directory where the fresh copy will live
+                clean_dir = pathlib.Path(tempdir, "clean")
+                clean_dir.mkdir()
+                archive_file = pathlib.Path(tempdir, "archive.tar")
+                # Create the archive
+                with open(archive_file, "wb") as archive:
+                    cmd = ["git", "archive", "--format=tar", "HEAD"]
+                    print(f"$ {' '.join(cmd)}")
+                    proc = await asyncio.create_subprocess_exec(
+                        *cmd, stdout=archive
+                    )
+                    await proc.wait()
+                    if proc.returncode != 0:
+                        raise RuntimeError
+                # Change directory into the clean copy
+                with chdir(clean_dir):
+                    # Extract the archive
+                    shutil.unpack_archive(archive_file)
+                    # Upload if not present
+                    for cmd in [
+                        [sys.executable, "setup.py", "sdist"],
+                        [sys.executable, "-m", "twine", "upload", "dist/*"],
+                    ]:
+                        print(f"$ {' '.join(cmd)}")
+                        proc = await asyncio.create_subprocess_exec(*cmd)
+                        await proc.wait()
+                        if proc.returncode != 0:
+                            raise RuntimeError
 
 
 class BumpMain(CMD):

tests/service/test_dev.py

@@ -4,12 +4,13 @@
 import json
 import glob
 import inspect
+import tarfile
 import tempfile
 import unittest
 import contextlib
 import dataclasses
 import unittest.mock
-from typing import Type
+from typing import Type, List, BinaryIO
 
 from dffml.version import VERSION
 from dffml.df.types import DataFlow
@@ -156,20 +157,39 @@ async def test_run(self):
 
 @dataclasses.dataclass
 class FakeProcess:
+    cmd: List[str] = None
     returncode: int = 0
+    stdout: BinaryIO = None
+
+    def __post_init__(self):
+        if self.cmd is None:
+            self.cmd = []
 
     async def communicate(self):
         return b"", b""
 
     async def wait(self):
-        return
+        if not "archive" in self.cmd:
+            return
+        with contextlib.ExitStack() as stack:
+            # Create the bytes objects to build the tarfile in memory
+            tar_fileobj = stack.enter_context(io.BytesIO())
+            hello_txt_fileobj = stack.enter_context(io.BytesIO(b"world"))
+            # Create the TarInfo objects
+            hello_txt_tarinfo = tarfile.TarInfo(name="somedir/hello.txt")
+            hello_txt_tarinfo.size = len(hello_txt_fileobj.getvalue())
+            # Create the archive using the bytes objects
+            with tarfile.open(mode="w|", fileobj=tar_fileobj) as archive:
+                archive.addfile(hello_txt_tarinfo, fileobj=hello_txt_fileobj)
+            # Write out the contents of the tar to the client
+            self.stdout.write(tar_fileobj.getvalue())
 
 
 def mkexec(proc_cls: Type[FakeProcess] = FakeProcess):
     async def fake_create_subprocess_exec(
         *args, stdin=None, stdout=None, stderr=None
     ):
-        return proc_cls()
+        return proc_cls(cmd=args, stdout=stdout)
 
     return fake_create_subprocess_exec
 
@@ -227,7 +247,7 @@ async def test_okay(self):
             stdout.getvalue().strip(),
             inspect.cleandoc(
                 f"""
-                $ git clean -fdx
+                $ git archive --format=tar HEAD
                 $ {sys.executable} setup.py sdist
                 $ {sys.executable} -m twine upload dist/*
                 """