improvements to the installation and upgrade processes (#3186)

- Moved all postinstallation config file and model munging code out of
the CLI and into a separate script named `invokeai-postinstall`

- Fixed two calls to `shutil.copytree()` so that they don't try to
preserve the file mode of the copied files. This is necessary to run
correctly in a Nix environment (see thread at
https://discord.com/channels/1020123559063990373/1091716696965918732/1095662756738371615)

- Update the installer so that an existing virtual environment will be
updated, not overwritten.

- Pin npyscreen version to see if this fixes issues people have had with
installing this module.

Author: lstein

installer/lib/installer.py

@@ -132,12 +132,13 @@ def app_venv(self, path: str = None):
 
         # Prefer to copy python executables
         # so that updates to system python don't break InvokeAI
-        try:
-            venv.create(venv_dir, with_pip=True)
-        # If installing over an existing environment previously created with symlinks,
-        # the executables will fail to copy. Keep symlinks in that case
-        except shutil.SameFileError:
-            venv.create(venv_dir, with_pip=True, symlinks=True)
+        if not venv_dir.exists():
+            try:
+                venv.create(venv_dir, with_pip=True)
+            # If installing over an existing environment previously created with symlinks,
+            # the executables will fail to copy. Keep symlinks in that case
+            except shutil.SameFileError:
+                venv.create(venv_dir, with_pip=True, symlinks=True)
 
         # upgrade pip in Python 3.9 environments
         if int(platform.python_version_tuple()[1]) == 9:

ldm/invoke/CLI.py

@@ -4,7 +4,6 @@
 import sys
 import traceback
 from argparse import Namespace
-from packaging import version
 from pathlib import Path
 from typing import Union
 
@@ -26,7 +25,6 @@
 from .globals import Globals, global_config_dir
 from .image_util import make_grid
 from .log import write_log
-from .model_manager import ModelManager
 from .pngwriter import PngWriter, retrieve_metadata, write_metadata
 from .readline import Completer, get_completer
 from ..util import url_attachment_name
@@ -66,9 +64,6 @@ def main():
     Globals.sequential_guidance = args.sequential_guidance
     Globals.ckpt_convert = True  # always true as of 2.3.4 for LoRA support
 
-    # run any post-install patches needed
-    run_patches()
-
     print(f">> Internet connectivity is {Globals.internet_available}")
 
     if not args.conf:
@@ -113,9 +108,6 @@ def main():
     if opt.lora_path:
         Globals.lora_models_dir = opt.lora_path
 
-    # migrate legacy models
-    ModelManager.migrate_models()
-
     # load the infile as a list of lines
     if opt.infile:
         try:
@@ -1299,62 +1291,6 @@ def retrieve_last_used_model()->str:
     with open(model_file_path,'r') as f:
         return f.readline()
 
-# This routine performs any patch-ups needed after installation
-def run_patches():
-    install_missing_config_files()
-    version_file = Path(Globals.root,'.version')
-    if version_file.exists():
-        with open(version_file,'r') as f:
-            root_version = version.parse(f.readline() or 'v2.3.2')
-    else:
-        root_version = version.parse('v2.3.2')
-    app_version = version.parse(ldm.invoke.__version__)
-    if root_version < app_version:
-        try:
-            do_version_update(root_version, ldm.invoke.__version__)
-            with open(version_file,'w') as f:
-                f.write(ldm.invoke.__version__)
-        except:
-            print("** Update failed. Will try again on next launch")
-
-def install_missing_config_files():
-    """
-    install ckpt configuration files that may have been added to the
-    distro after original root directory configuration
-    """
-    import invokeai.configs as conf
-    from shutil import copyfile
-    
-    root_configs = Path(global_config_dir(), 'stable-diffusion')
-    repo_configs = Path(conf.__path__[0], 'stable-diffusion')
-    for src in repo_configs.iterdir():
-        dest = root_configs / src.name
-        if not dest.exists():
-            copyfile(src,dest)
-    
-def do_version_update(root_version: version.Version, app_version: Union[str, version.Version]):
-    """
-    Make any updates to the launcher .sh and .bat scripts that may be needed
-    from release to release. This is not an elegant solution. Instead, the 
-    launcher should be moved into the source tree and installed using pip.
-    """
-    if root_version < version.Version('v2.3.4'):
-        dest = Path(Globals.root,'loras')
-        dest.mkdir(exist_ok=True)
-    if root_version < version.Version('v2.3.3'):
-        if sys.platform == "linux":
-            print('>> Downloading new version of launcher script and its config file')
-            from ldm.util import download_with_progress_bar
-            url_base = f'https://raw.githubusercontent.com/invoke-ai/InvokeAI/v{str(app_version)}/installer/templates/'
-
-            dest = Path(Globals.root,'invoke.sh.in')
-            assert download_with_progress_bar(url_base+'invoke.sh.in',dest)
-            dest.replace(Path(Globals.root,'invoke.sh'))
-            os.chmod(Path(Globals.root,'invoke.sh'), 0o0755)
-
-            dest = Path(Globals.root,'dialogrc')
-            assert download_with_progress_bar(url_base+'dialogrc',dest)
-            dest.replace(Path(Globals.root,'.dialogrc'))
 
 if __name__ == '__main__':
     main()

ldm/invoke/config/invokeai_configure.py

@@ -21,7 +21,6 @@
 from shutil import get_terminal_size
 
 import npyscreen
-import torch
 import transformers
 from diffusers import AutoencoderKL
 from huggingface_hub import HfFolder
@@ -664,8 +663,19 @@ def initialize_rootdir(root: str, yes_to_all: bool = False):
     configs_src = Path(configs.__path__[0])
     configs_dest = Path(root) / "configs"
     if not os.path.samefile(configs_src, configs_dest):
-        shutil.copytree(configs_src, configs_dest, dirs_exist_ok=True)
-
+        shutil.copytree(configs_src,
+                        configs_dest,
+                        dirs_exist_ok=True,
+                        copy_function=shutil.copyfile,
+                        )
+    # Fix up directory permissions so that they are writable
+    # This can happen when running under Nix environment which
+    # makes the runtime directory template immutable.
+    for root,dirs,files in os.walk(os.path.join(root,name)):
+        for d in dirs:
+            Path(root,d).chmod(0o775)
+        for f in files:
+            Path(root,d).chmod(0o644)
 
 # -------------------------------------
 def run_console_ui(

ldm/invoke/config/invokeai_update.py

@@ -42,8 +42,18 @@ def invokeai_is_running()->bool:
         except psutil.AccessDenied:
             continue
     return False
+
+def do_post_install():
+    '''
+    Run postinstallation script.
+    '''
+    print("Looking for postinstallation script to run on this version...")
+    try:
+        from ldm.invoke.config.post_install.py import post_install
+        post_install()
+    except:
+        print("Postinstallation script not available for this version of InvokeAI")
 
-    
 def welcome(versions: dict):
 
     @group()
@@ -107,6 +117,7 @@ def main():
         print(f':heavy_check_mark: Upgrade successful')
     else:
         print(f':exclamation: [bold red]Upgrade failed[/red bold]')
+    do_post_install()
 
 if __name__ == "__main__":
     try:

ldm/invoke/config/model_install_backend.py

@@ -388,7 +388,19 @@ def update_config_file(successfully_downloaded: dict, config_file: Path):
     if config_file is default_config_file() and not config_file.parent.exists():
         configs_src = Dataset_path.parent
         configs_dest = default_config_file().parent
-        shutil.copytree(configs_src, configs_dest, dirs_exist_ok=True)
+        shutil.copytree(configs_src,
+                        configs_dest,
+                        dirs_exist_ok=True,
+                        copy_function=shutil.copyfile,
+                        )
+    # Fix up directory permissions so that they are writable
+    # This can happen when running under Nix environment which
+    # makes the runtime directory template immutable.
+    for root,dirs,files in os.walk(default_config_file().parent):
+        for d in dirs:
+            Path(root,d).chmod(0o775)
+        for f in files:
+            Path(root,d).chmod(0o644)
 
     yaml = new_config_file_contents(successfully_downloaded, config_file)
 

ldm/invoke/config/post_install.py

@@ -0,0 +1,168 @@
+'''ldm.invoke.config.post_install
+
+This defines a single exportable function, post_install(), which does
+post-installation stuff like migrating models directories, adding new
+config files, etc.
+
+From the command line, its entry point is invokeai-postinstall.
+'''
+
+import os
+import sys
+from packaging import version
+from pathlib import Path
+from shutil import move,rmtree,copyfile
+from typing import Union
+
+import invokeai.configs as conf
+import ldm.invoke
+from ..globals import Globals, global_cache_dir, global_config_dir
+
+def post_install():
+    '''
+    Do version and model updates, etc.
+    Should be called once after every version update.
+    '''
+    _migrate_models()
+    _run_patches()
+
+
+def _migrate_models():
+    """
+    Migrate the ~/invokeai/models directory from the legacy format used through 2.2.5
+    to the 2.3.0 "diffusers" version. This should be a one-time operation, called at
+    script startup time.
+    """
+    # Three transformer models to check: bert, clip and safety checker, and
+    # the diffusers as well
+    models_dir = Path(Globals.root, "models")
+    legacy_locations = [
+        Path(
+            models_dir,
+            "CompVis/stable-diffusion-safety-checker/models--CompVis--stable-diffusion-safety-checker",
+        ),
+        Path("bert-base-uncased/models--bert-base-uncased"),
+        Path(
+            "openai/clip-vit-large-patch14/models--openai--clip-vit-large-patch14"
+        ),
+    ]
+    legacy_locations.extend(list(global_cache_dir("diffusers").glob("*")))
+    legacy_layout = False
+    for model in legacy_locations:
+        legacy_layout = legacy_layout or model.exists()
+    if not legacy_layout:
+        return
+
+    print(
+        """
+>> ALERT:
+>> The location of your previously-installed diffusers models needs to move from
+>> invokeai/models/diffusers to invokeai/models/hub due to a change introduced by
+>> diffusers version 0.14. InvokeAI will now move all models from the "diffusers" directory
+>> into "hub" and then remove the diffusers directory. This is a quick, safe, one-time
+>> operation. However if you have customized either of these directories and need to
+>> make adjustments, please press ctrl-C now to abort and relaunch InvokeAI when you are ready.
+>> Otherwise press <enter> to continue."""
+    )
+    print("** This is a quick one-time operation.")
+    input("continue> ")
+
+    # transformer files get moved into the hub directory
+    if _is_huggingface_hub_directory_present():
+        hub = global_cache_dir("hub")
+    else:
+        hub = models_dir / "hub"
+
+    os.makedirs(hub, exist_ok=True)
+    for model in legacy_locations:
+        source = models_dir / model
+        dest = hub / model.stem
+        if dest.exists() and not source.exists():
+            continue
+        print(f"** {source} => {dest}")
+        if source.exists():
+            if dest.is_symlink():
+                print(f"** Found symlink at {dest.name}. Not migrating.")
+            elif dest.exists():
+                if source.is_dir():
+                    rmtree(source)
+                else:
+                    source.unlink()
+            else:
+                move(source, dest)
+
+    # now clean up by removing any empty directories
+    empty = [
+        root
+        for root, dirs, files, in os.walk(models_dir)
+        if not len(dirs) and not len(files)
+    ]
+    for d in empty:
+        os.rmdir(d)
+    print("** Migration is done. Continuing...")
+
+
+def _is_huggingface_hub_directory_present() -> bool:
+    return (
+        os.getenv("HF_HOME") is not None or os.getenv("XDG_CACHE_HOME") is not None
+    )
+
+# This routine performs any patch-ups needed after installation
+def _run_patches():
+    _install_missing_config_files()
+    version_file = Path(Globals.root,'.version')
+    if version_file.exists():
+        with open(version_file,'r') as f:
+            root_version = version.parse(f.readline() or 'v2.3.2')
+    else:
+        root_version = version.parse('v2.3.2')
+    app_version = version.parse(ldm.invoke.__version__)
+    if root_version < app_version:
+        try:
+            _do_version_update(root_version, ldm.invoke.__version__)
+            with open(version_file,'w') as f:
+                f.write(ldm.invoke.__version__)
+        except:
+            print("** Version patching failed. Please try invokeai-postinstall later.")
+
+def _install_missing_config_files():
+    """
+    install ckpt configuration files that may have been added to the
+    distro after original root directory configuration
+    """
+    root_configs = Path(global_config_dir(), 'stable-diffusion')
+    repo_configs = None
+    for f in conf.__path__:
+        if Path(f, 'stable-diffusion', 'v1-inference.yaml').exists():
+            repo_configs = Path(f, 'stable-diffusion')
+            break
+    if not repo_configs:
+        return
+    for src in repo_configs.iterdir():
+        dest = root_configs / src.name
+        if not dest.exists():
+            copyfile(src,dest)
+    
+def _do_version_update(root_version: version.Version, app_version: Union[str, version.Version]):
+    """
+    Make any updates to the launcher .sh and .bat scripts that may be needed
+    from release to release. This is not an elegant solution. Instead, the 
+    launcher should be moved into the source tree and installed using pip.
+    """
+    if root_version < version.Version('v2.3.4'):
+        dest = Path(Globals.root,'loras')
+        dest.mkdir(exist_ok=True)
+    if root_version < version.Version('v2.3.3'):
+        if sys.platform == "linux":
+            print('>> Downloading new version of launcher script and its config file')
+            from ldm.util import download_with_progress_bar
+            url_base = f'https://raw.githubusercontent.com/invoke-ai/InvokeAI/v{str(app_version)}/installer/templates/'
+
+            dest = Path(Globals.root,'invoke.sh.in')
+            assert download_with_progress_bar(url_base+'invoke.sh.in',dest)
+            dest.replace(Path(Globals.root,'invoke.sh'))
+            os.chmod(Path(Globals.root,'invoke.sh'), 0o0755)
+
+            dest = Path(Globals.root,'dialogrc')
+            assert download_with_progress_bar(url_base+'dialogrc',dest)
+            dest.replace(Path(Globals.root,'.dialogrc'))