Add new --standalone option to create redistributable Python environments

README.md

@@ -58,6 +58,11 @@ Pack an environment located at an explicit path into my_env.zip
 venv-pack -p explicit\path\to\env -o env.zip 
 ```
 
+Pack a standalone package that doesn't require the base Python environment
+```
+venv-pack --standalone -o env.zip 
+```
+
 ### On the target machine
 
 Unpack environment into directory `my_env`
@@ -72,5 +77,5 @@ All features of venv should keep working from my_env folder.
 
 This tool is new, and has a few caveats.
 
-1. Python is not packaged with the environment, but rather symlinked in the environment. On Windows python venv does so in a pyvenv.cfg file. This is useful for deployment situations where Python is already installed on the machine, but the required library dependencies may not be.
+1. Python is not packaged with the environment unless the '--standalone' option is specified when packing. By default Python is symlinked in the environment. On Windows python venv does so in a pyvenv.cfg file. This is useful for deployment situations where Python is already installed on the machine, but the required library dependencies may not be.
 2. The os type where the environment was built must match the os type of the target. This means that environments built on windows can’t be relocated to linux.

venv_pack/__main__.py

@@ -84,6 +84,9 @@ def build_parser():
     parser.add_argument("--force", "-f",
                         action="store_true",
                         help="Overwrite any existing archive at the output path.")
+    parser.add_argument("--standalone",
+                        action="store_true",
+                        help="Include the entire base Python distribution in the package.")
     parser.add_argument("--quiet", "-q",
                         action="store_true",
                         help="Do not report progress")
@@ -114,7 +117,8 @@ def main(args=None, pack=pack):
              zip_symlinks=args.zip_symlinks,
              zip_64=not args.no_zip_64,
              verbose=not args.quiet,
-             filters=args.filters)
+             filters=args.filters,
+             standalone=args.standalone)
     except VenvPackException as e:
         fail("VenvPackError: %s" % e)
     except KeyboardInterrupt:  # pragma: nocover

venv_pack/core.py

@@ -86,14 +86,15 @@ class Env(object):
     >>> Env().pack(output="environment.tar.gz")
     "/full/path/to/environment.tar.gz"
     """
-    __slots__ = ('_context', 'files', '_excluded_files')
+    __slots__ = ('_context', 'files', '_excluded_files', '_base_env')
 
     def __init__(self, prefix=None):
         context, files = load_environment(prefix)
 
         self._context = context
         self.files = files
         self._excluded_files = []
+        self._base_env = None
 
     def _copy_with_files(self, files, excluded_files):
         out = object.__new__(Env)
@@ -102,6 +103,13 @@ def _copy_with_files(self, files, excluded_files):
         out._excluded_files = excluded_files
         return out
 
+    @property
+    def base_env(self):
+        if self._base_env is None \
+        and self.kind in ("virtualenv", "venv"):
+            self._base_env = Env(self.orig_prefix)
+        return self._base_env
+
     @property
     def prefix(self):
         return self._context.prefix
@@ -224,7 +232,7 @@ def _output_and_format(self, output=None, format='infer'):
 
     def pack(self, output=None, format='infer', python_prefix=None,
              verbose=False, force=False, compress_level=4, zip_symlinks=False,
-             zip_64=True):
+             zip_64=True, standalone=False):
         """Package the virtual environment into an archive file.
 
         Parameters
@@ -259,6 +267,10 @@ def pack(self, output=None, format='infer', python_prefix=None,
             symlinks*. Default is False. Ignored if format isn't ``zip``.
         zip_64 : bool, optional
             Whether to enable ZIP64 extensions. Default is True.
+        standalone : bool, optional
+            If True include the files from the base Python environment in the
+            package and create a standalone Python environment that does not
+            need the base environment when unpacked.
 
         Returns
         -------
@@ -274,6 +286,10 @@ def pack(self, output=None, format='infer', python_prefix=None,
         if verbose:
             print("Packing environment at %r to %r" % (self.prefix, output))
 
+        all_files = list(self.files)
+        if standalone:
+            all_files = combine_env_files(self, self.base_env)
+
         fd, temp_path = tempfile.mkstemp()
 
         try:
@@ -282,8 +298,8 @@ def pack(self, output=None, format='infer', python_prefix=None,
                              compress_level=compress_level,
                              zip_symlinks=zip_symlinks,
                              zip_64=zip_64) as arc:
-                    packer = Packer(self._context, arc, python_prefix)
-                    with progressbar(self.files, enabled=verbose) as files:
+                    packer = Packer(self._context, arc, python_prefix, standalone=standalone)
+                    with progressbar(all_files, enabled=verbose) as files:
                         try:
                             for f in files:
                                 packer.add(f)
@@ -323,7 +339,7 @@ class File(namedtuple('File', ('source', 'target'))):
 
 def pack(prefix=None, output=None, format='infer', python_prefix=None,
          verbose=False, force=False, compress_level=4, zip_symlinks=False,
-         zip_64=True, filters=None):
+         zip_64=True, filters=None, standalone=False):
     """Package an existing virtual environment into an archive file.
 
     Parameters
@@ -365,6 +381,10 @@ def pack(prefix=None, output=None, format='infer', python_prefix=None,
         ``(kind, pattern)``, where ``kind`` is either ``'exclude'`` or
         ``'include'`` and ``pattern`` is a file pattern. Filters are applied in
         the order specified.
+    standalone : bool, optional
+        If True include the files from the base Python environment in the
+        package and create a standalone Python environment that does not
+        need the base environment when unpacked.
 
     Returns
     -------
@@ -389,7 +409,8 @@ def pack(prefix=None, output=None, format='infer', python_prefix=None,
                     python_prefix=python_prefix,
                     verbose=verbose, force=force,
                     compress_level=compress_level,
-                    zip_symlinks=zip_symlinks, zip_64=zip_64)
+                    zip_symlinks=zip_symlinks, zip_64=zip_64,
+                    standalone=standalone)
 
 
 def check_prefix(prefix=None):
@@ -404,7 +425,7 @@ def check_prefix(prefix=None):
     if not os.path.exists(prefix):
         raise VenvPackException("Environment path %r doesn't exist" % prefix)
 
-    for check in [check_venv, check_virtualenv]:
+    for check in [check_venv, check_virtualenv, check_baseenv]:
         try:
             return check(prefix)
         except VenvPackException:
@@ -422,7 +443,7 @@ def check_venv(prefix):
         for line in fil:
             key, val = line.split('=')
             if key.strip().lower() == 'home':
-                orig_prefix = os.path.dirname(val.strip())
+                orig_prefix = val.strip()
                 break
         else:  # pragma: nocover
             raise VenvPackException("%r is not a valid virtual "
@@ -460,6 +481,22 @@ def check_virtualenv(prefix):
 
     return context
 
+def check_baseenv(prefix):
+    python_lib, python_include = find_python_lib_include(prefix)
+
+    if not os.path.exists(os.path.join(prefix, python_lib, "os.py")):
+        raise VenvPackException("%r is not a valid Python environment" % prefix)
+
+
+    context = AttrDict()
+
+    context.kind = 'base'
+    context.prefix = prefix
+    context.orig_prefix = None
+    context.py_lib = python_lib
+    context.py_include = python_include
+
+    return context
 
 def find_python_lib_include(prefix):
     if on_win:
@@ -512,13 +549,12 @@ def load_environment(prefix):
     check_no_editable_packages(context)
 
     # Files to ignore
-    remove = {join(BIN_DIR, f) for f in ['activate', 'activate.csh',
-                                         'activate.fish']}
+    remove = {join(BIN_DIR, "activate" + suffix) for suffix in ('', '.csh', '.fish', '.bat', '.ps1')}
 
     if context.kind == 'virtualenv':
-        remove.add(join(context.prefix, context.py_lib, 'orig-prefix.txt'))
+        remove.add(join(context.py_lib, 'orig-prefix.txt'))
     else:
-        remove.add(join(context.prefix, 'pyvenv.cfg'))
+        remove.add('pyvenv.cfg')
 
     res = []
 
@@ -546,7 +582,7 @@ def load_environment(prefix):
 
     files = [File(os.path.join(prefix, p), p)
              for p in res
-             if not (p in remove or p.endswith('~') or p.endswith('.DS_STORE'))]
+             if not (p.lower() in remove or p.endswith('~') or p.endswith('.DS_STORE'))]
 
     return context, files
 
@@ -616,7 +652,7 @@ def _rewrite_shebang(data, target, prefix):
 
         shebang, executable, options = shebang_match.groups()
 
-        if executable.startswith(prefix):
+        if executable.lower().startswith(prefix.lower()):
             # shebang points inside environment, rewrite
             new_shebang = (b'#!%s'
                            if on_win else
@@ -660,11 +696,44 @@ def check_python_prefix(python_prefix, context):
     return python_prefix, rewrites
 
 
+def combine_env_files(venv, base_env):
+    """Combines the files from a virtual environment and it's base environment.
+    Returns a new list of files.
+    """
+    all_files = {}
+    exe_suffix = '.exe' if on_win else ''
+
+    # Don't include the Python executables from the virtual environment
+    exclude = {
+        os.path.join(venv.prefix, BIN_DIR, 'python' + exe_suffix).lower(),
+        os.path.join(venv.prefix, BIN_DIR, 'pythonw' + exe_suffix).lower()
+    }
+
+    # Copy the base Python excectuables and shared libraries into BIN_DIR as the
+    # other scripts need them to be there for them to work.
+    for file in os.listdir(base_env.prefix):
+        source = os.path.join(base_env.prefix, file)
+        if os.path.isfile(source):
+            _, ext = os.path.splitext(file)
+            if ext.lower() in ("", ".dll", ".so", ".exe", ".pdb"):
+                target = os.path.join(BIN_DIR, file)
+                all_files[target] = File(source, target)
+                exclude.add(source.lower())
+
+    # Include all files from the base env, and for any files that exist in both use the
+    # file from the virtual env.
+    all_files.update({f.target: f for f in base_env.files if f.source.lower() not in exclude})
+    all_files.update({f.target: f for f in venv.files if f.source.lower() not in exclude})
+
+    return all_files.values()
+
+
 class Packer(object):
-    def __init__(self, context, archive, python_prefix):
+    def __init__(self, context, archive, python_prefix, standalone=False):
         self.context = context
         self.prefix = context.prefix
         self.archive = archive
+        self.standalone = standalone
 
         python_prefix, rewrites = check_python_prefix(python_prefix, context)
         self.python_prefix = python_prefix
@@ -690,34 +759,35 @@ def add(self, file):
             self.archive.add(file.source, file.target)
 
     def finish(self):
-        script_dirs = ['common', 'nt']
-
-        for d in script_dirs:
-            dirpath = os.path.join(SCRIPTS, d)
-            for f in os.listdir(dirpath):
-                source = os.path.join(dirpath, f)
-                target = os.path.join(BIN_DIR, f)
-                self.archive.add(source, target)
-
-        if self.context.kind == 'venv':
-            pyvenv_cfg = os.path.join(self.prefix, 'pyvenv.cfg')
-            if self.python_prefix is None:
-                self.archive.add(pyvenv_cfg, 'pyvenv.cfg')
-            else:
-                with open(pyvenv_cfg) as fil:
-                    data = fil.read()
-                data = data.replace(self.context.orig_prefix,
-                                    self.python_prefix)
-                self.archive.add_bytes(pyvenv_cfg, data.encode(), 'pyvenv.cfg')
-        else:
-            origprefix_txt = os.path.join(self.context.prefix,
-                                          self.context.py_lib,
-                                          'orig-prefix.txt')
-            target = os.path.relpath(origprefix_txt, self.prefix)
-
-            if self.python_prefix is None:
-                self.archive.add(origprefix_txt, target)
+        if not self.standalone:
+            script_dirs = ['common', 'nt']
+
+            for d in script_dirs:
+                dirpath = os.path.join(SCRIPTS, d)
+                for f in os.listdir(dirpath):
+                    source = os.path.join(dirpath, f)
+                    target = os.path.join(BIN_DIR, f)
+                    self.archive.add(source, target)
+
+            if self.context.kind == 'venv':
+                pyvenv_cfg = os.path.join(self.prefix, 'pyvenv.cfg')
+                if self.python_prefix is None:
+                    self.archive.add(pyvenv_cfg, 'pyvenv.cfg')
+                else:
+                    with open(pyvenv_cfg) as fil:
+                        data = fil.read()
+                    data = data.replace(self.context.orig_prefix,
+                                        self.python_prefix)
+                    self.archive.add_bytes(pyvenv_cfg, data.encode(), 'pyvenv.cfg')
             else:
-                self.archive.add_bytes(origprefix_txt,
-                                       self.python_prefix.encode(),
-                                       target)
+                origprefix_txt = os.path.join(self.context.prefix,
+                                            self.context.py_lib,
+                                            'orig-prefix.txt')
+                target = os.path.relpath(origprefix_txt, self.prefix)
+
+                if self.python_prefix is None:
+                    self.archive.add(origprefix_txt, target)
+                else:
+                    self.archive.add_bytes(origprefix_txt,
+                                        self.python_prefix.encode(),
+                                        target)