Detach process (#243)

* working on toy

* subprocess.Popen works

* port to app

* copy before detaching

* print message in main process

* quiet server logging

* fix regex warning

* update message and notes

* Apply suggestions from code review

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: danvk

README.md

@@ -144,7 +144,7 @@ To iterate on the PyPI package, run:
 
     pip3 uninstall webdiff
     poetry build
-    pip3 install dist/webdiff-?.?.?.tar.gz
+    pip3 install dist/webdiff-(latest).tar.gz
 
 To publish to pypitest:
 
@@ -155,6 +155,8 @@ And to the real pypi:
 
     poetry publish
 
+You can publish pre-release versions to pypi by adding "bN" to the version number.
+
 See [pypirc][] and [poetry][] docs for details on setting up tokens for pypi.
 
 Publication checklist. Do these from _outside_ the webdiff directory:
@@ -184,7 +186,10 @@ When you run `git webdiff (args)`, it runs:
 
 This tells `git` to set up two directories and invoke `webdiff leftdir rightdir`.
 
-There's one complication involving symlinks. `git difftool -d` may fill one of the sides (typically the right) with symlinks. This is faster than copying files, but unfortunately `git diff --no-index` does not resolve these symlinks. To make this work, if a directory contains symlinks, webdiff makes a copy of it before diffing. For file diffs, it resolves the symlink before passing it to `git diff --no-index`. The upshot is that you can run `git webdiff`, edit a file, reload the browser window and see the changes.
+There are two wrinkles here:
+
+- `git difftool -d` may fill one of the sides (typically the right) with symlinks. This is faster than copying files, but unfortunately `git diff --no-index` does not resolve these symlinks. To make this work, if a directory contains symlinks, webdiff makes a copy of it before diffing. For file diffs, it resolves the symlink before passing it to `git diff --no-index`. The upshot is that you can run `git webdiff`, edit a file, reload the browser window and see the changes.
+- `git difftool` cleans up its temporary directories when the main webdiff process terminates. Since webdiff detaches to give you back your terminal, it has to make another copy of the directories (this time without resolving symlinks) to make sure they're still there for the child process.
 
 [pypirc]: https://packaging.python.org/specifications/pypirc/
 [Homebrew]: https://brew.sh/

webdiff/app.py

@@ -13,6 +13,7 @@
 import platform
 import signal
 import socket
+import subprocess
 import sys
 import threading
 import time
@@ -24,6 +25,7 @@
 from binaryornot.check import is_binary
 
 from webdiff import argparser, diff, options, util
+from webdiff.dirdiff import make_resolved_dir
 
 VERSION = importlib.metadata.version('webdiff')
 
@@ -46,6 +48,7 @@ def determine_path():
 PORT = None
 HOSTNAME = 'localhost'
 DEBUG = os.environ.get('DEBUG')
+DEBUG_DETACH = os.environ.get('DEBUG_DETACH')
 WEBDIFF_DIR = determine_path()
 
 if DEBUG:
@@ -273,15 +276,8 @@ def pick_a_port(args, webdiff_config):
 
 
 def run_http():
-    sys.stderr.write(
-        """Serving diffs on http://%s:%s
-Close the browser tab or hit Ctrl-C when you're done.
-"""
-        % (HOSTNAME, PORT)
-    )
     threading.Timer(0.1, open_browser).start()
-
-    web.run_app(app, host=HOSTNAME, port=PORT)
+    web.run_app(app, host=HOSTNAME, port=PORT, print=print if DEBUG else None)
     logging.debug('http server shut down')
 
 
@@ -291,7 +287,7 @@ def maybe_shutdown():
 
     def shutdown():
         if LAST_REQUEST_MS <= last_ms:  # subsequent requests abort shutdown
-            sys.stderr.write('Shutting down...\n')
+            logging.debug('Shutting down...')
             signal.raise_signal(signal.SIGINT)
         else:
             logging.debug('Received subsequent request; shutdown aborted.')
@@ -334,7 +330,36 @@ def run():
         else:
             HOSTNAME = _hostname
 
-    run_http()
+    run_in_process = os.environ.get('WEBDIFF_RUN_IN_PROCESS') or (
+        DEBUG and not DEBUG_DETACH
+    )
+
+    if not os.environ.get('WEBDIFF_LOGGED_MESSAGE'):
+        # Printing this in the main process gives you your prompt back more cleanly.
+        print(
+            """Serving diffs on http://%s:%s
+Close the browser tab when you're done to terminate the process."""
+            % (HOSTNAME, PORT)
+        )
+        os.environ['WEBDIFF_LOGGED_MESSAGE'] = '1'
+
+    if run_in_process:
+        run_http()
+    else:
+        os.environ['WEBDIFF_RUN_IN_PROCESS'] = '1'
+        os.environ['WEBDIFF_PORT'] = str(PORT)
+        if os.environ.get('WEBDIFF_FROM_GIT_DIFFTOOL'):
+            # git difftool will clean up these directories when we detach.
+            # To make them accessible to the child process, we make a (shallow) copy.
+            assert 'dirs' in parsed_args
+            dir_a, dir_b = parsed_args['dirs']
+            copied_dir_a = make_resolved_dir(dir_a)
+            copied_dir_b = make_resolved_dir(dir_b)
+            os.environ['WEBDIFF_DIR_A'] = copied_dir_a
+            os.environ['WEBDIFF_DIR_B'] = copied_dir_b
+            logging.debug(f'Copied {dir_a} -> {copied_dir_a} before detaching')
+            logging.debug(f'Copied {dir_b} -> {copied_dir_b} before detaching')
+        subprocess.Popen((sys.executable, *sys.argv))
 
 
 if __name__ == '__main__':

webdiff/argparser.py

@@ -4,9 +4,7 @@
 import os
 import re
 
-from webdiff import dirdiff
-from webdiff import githubdiff
-from webdiff import github_fetcher
+from webdiff import dirdiff, github_fetcher, githubdiff
 from webdiff.localfilediff import LocalFileDiff
 
 
@@ -81,6 +79,12 @@ def parse(args, version=None):
 
     else:
         a, b = args.dirs
+        if os.environ.get('WEBDIFF_DIR_A') and os.environ.get('WEBDIFF_DIR_B'):
+            # This happens when you run "git webdiff" and we have to make a copy of
+            # the temp directories before we detach and git difftool cleans them up.
+            a = os.environ.get('WEBDIFF_DIR_A')
+            b = os.environ.get('WEBDIFF_DIR_B')
+
         for x in (a, b):
             if not os.path.exists(x):
                 raise UsageError('"%s" does not exist' % x)

webdiff/dirdiff.py

@@ -1,7 +1,7 @@
 """Compute the diff between two directories on local disk."""
 
-import os
 import logging
+import os
 import shutil
 import subprocess
 import tempfile
@@ -27,7 +27,7 @@ def contains_symlinks(dir: str):
     return False
 
 
-def make_resolved_dir(dir: str) -> str:
+def make_resolved_dir(dir: str, follow_symlinks=False) -> str:
     # TODO: clean up this directory
     temp_dir = tempfile.mkdtemp(prefix='webdiff')
     for root, dirs, files in os.walk(dir):
@@ -38,7 +38,7 @@ def make_resolved_dir(dir: str) -> str:
             src_file = os.path.join(root, file_name)
             rel = os.path.relpath(src_file, dir)
             dst_file = os.path.join(temp_dir, rel)
-            shutil.copy(src_file, dst_file, follow_symlinks=True)
+            shutil.copy(src_file, dst_file, follow_symlinks=follow_symlinks)
     return temp_dir
 
 
@@ -49,11 +49,11 @@ def gitdiff(a_dir: str, b_dir: str, webdiff_config):
         cmd += ' ' + extra_args
     a_dir_nosym = a_dir
     if contains_symlinks(a_dir):
-        a_dir_nosym = make_resolved_dir(a_dir)
+        a_dir_nosym = make_resolved_dir(a_dir, follow_symlinks=True)
         logging.debug(f'Inlined symlinks in left directory {a_dir} -> {a_dir_nosym}')
     b_dir_nosym = b_dir
     if contains_symlinks(b_dir):
-        b_dir_nosym = make_resolved_dir(b_dir)
+        b_dir_nosym = make_resolved_dir(b_dir, follow_symlinks=True)
         logging.debug(f'Inlined symlinks in right directory {b_dir} -> {b_dir_nosym}')
     args = cmd.split(' ') + [a_dir_nosym, b_dir_nosym]
     logging.debug('Running git command: %s', args)

webdiff/github_fetcher.py

@@ -6,11 +6,11 @@
 # Use this PR for testing to see all four types of change at once:
 # https://github.com/danvk/test-repo/pull/2/
 
-from collections import OrderedDict
 import os
 import re
 import subprocess
 import sys
+from collections import OrderedDict
 
 from github import Github, UnknownObjectException
 
@@ -138,7 +138,7 @@ def parse(remote):
 
 # e.g. 'origin	[email protected]:danvk/expandable-image-grid.git (push)'
 ssh_push_re = re.compile(
-    '(?P<name>[^\s]+)\s+((?P<user>[^@]+)@)?(?P<host>[^:]+)(?::(?P<path>[^\s]+))?\s\(push\)'
+    r'(?P<name>[^\s]+)\s+((?P<user>[^@]+)@)?(?P<host>[^:]+)(?::(?P<path>[^\s]+))?\s\(push\)'
 )
 
 # e.g. 'origin	https://github.com/danvk/git-helpers.git (push)'

webdiff/gitwebdiff.py

@@ -21,6 +21,7 @@ def run(argv=sys.argv):
             if not os.environ.get('DEBUG')
             else os.path.join(os.path.curdir, 'test.sh')
         )
+        os.environ['WEBDIFF_FROM_GIT_DIFFTOOL'] = '1'
         subprocess.call(f'git difftool -d -x {cmd}'.split(' ') + argv[1:])
     except KeyboardInterrupt:
         # Don't raise an exception to the user when sigint is received

webdiff/toy.py

@@ -0,0 +1,26 @@
+# Trying to make a server that detaches
+import os
+import subprocess
+import sys
+import time
+
+
+def run():
+    print('running server...')
+    print(f'{sys.argv=}')
+    print(f'{os.getpid()=}')
+    time.sleep(3)
+    print('shutting down.')
+
+
+def main():
+    if os.environ.get('SUB'):
+        run()
+    else:
+        os.environ['SUB'] = '1'
+        subprocess.Popen((sys.executable, *sys.argv))
+        print('terminating parent process')
+
+
+if __name__ == '__main__':
+    main()