Redesign sync command with delegation, TUI, dry-run, and parallelism

- Add SyncOutcome enum and PkgSyncResult dataclass (pkg_sync.py) mirroring
  the status command's result types
- Add SyncProgressListener callback interface for live parallel progress
- Implement PackageGit.sync() with full logic: tag-pinned, read-only,
  detached HEAD, dirty detection, fetch, ahead/behind, dry-run variants,
  real merge with conflict detection + abort, submodule update
- Implement ProjectOps.sync() with asyncio.gather + Semaphore parallelism;
  progress events fired from async wrapper (clean separation from pkg classes)
- Add RichSyncTUI (live spinner + final Rich table) and TranscriptSyncTUI
  (plain-text >> / << progress + final table); both implement SyncProgressListener
- Replace 139-line CmdSync monolith with 38-line delegator that wires TUI
  as progress listener and calls tui.start()/stop() around sync
- Add -n/--dry-run, -j/--jobs, -p/--packages, --no-rich to sync subcommand
- Add patch_lock_after_sync() and _write_lock_dict() to package_lock.py;
  fix SHA256 handling in lock rewrite
- Add 21 unit tests in test_sync_ops.py covering all outcomes, dry-run,
  parallel execution, and progress callbacks (all local git repos, no network)
- Exit code: only SyncOutcome.ERROR causes sys.exit(1); CONFLICT/DIRTY/AHEAD
  are informational (exit 0)
- Add sync-design.md design document at repo root

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: mballance

src/ivpm/__main__.py

@@ -220,6 +220,13 @@ def get_parser(parser_ext : List = None, options_ext : List = None):
     sync_cmd = subparser.add_parser("sync",
         help="Synchronizes dependent packages with an upstream source (if available)")
     sync_cmd.set_defaults(func=CmdSync())
+    sync_cmd.add_argument("-p", "--project-dir", dest="project_dir", default=None)
+    sync_cmd.add_argument("-n", "--dry-run", dest="dry_run", action="store_true",
+        default=False, help="Fetch and report sync-ability without merging")
+    sync_cmd.add_argument("-j", "--jobs", dest="jobs", type=int, default=0,
+        help="Number of parallel sync operations (default: CPU count)")
+    sync_cmd.add_argument("--no-rich", action="store_true", default=False,
+        help="Plain-text output without Rich formatting")
 
     status_cmd = subparser.add_parser("status",
         help="Checks the status of sub-dependencies such as git repositories")

src/ivpm/cmds/cmd_sync.py

@@ -4,135 +4,39 @@
 @author: mballance
 '''
 import os
-import subprocess
 import sys
-import stat
 
-from ivpm.arg_utils import ensure_have_project_dir
-from ivpm.msg import fatal, note
 from ..project_ops import ProjectOps
-from ..proj_info import ProjInfo
-from ..package_lock import write_lock, read_lock
-
+from ..pkg_sync import SyncOutcome
+from ..sync_tui import create_sync_tui
 
 
 class CmdSync(object):
-    
+
     def __init__(self):
         pass
-    
+
     def __call__(self, args):
         if args.project_dir is None:
             args.project_dir = os.getcwd()
-    
-        proj_info = ProjInfo.mkFromProj(args.project_dir)
-
-        if proj_info is None:
-            fatal("Failed to locate IVPM meta-data (eg ivpm.yaml)")
-            
-        packages_dir = os.path.join(args.project_dir, proj_info.deps_dir)
-    
-        # After that check, go ahead and just check directories
-        for dir in os.listdir(packages_dir):
-            pkg_path = os.path.join(packages_dir, dir)
-            git_dir = os.path.join(pkg_path, ".git")
-            
-            if os.path.isdir(git_dir):
-                # Check if the package is editable by testing if it's writable
-                # Cached (non-editable) packages are made read-only
-                try:
-                    mode = os.stat(pkg_path).st_mode
-                    is_writable = bool(mode & stat.S_IWUSR)
-                except Exception:
-                    is_writable = False
-                
-                if not is_writable:
-                    print("Note: skipping cached (read-only) package \"%s\"" % dir)
-                    continue
-                
-                print("Package: " + dir)
-                cwd = os.getcwd()
-                os.chdir(pkg_path)
-                try:
-                    branch = subprocess.check_output(["git", "branch"])
-                except Exception as e:
-                    print("Note: Failed to get branch of package \"" + dir + "\"")
-                    continue
-
-                branch = branch.strip()
-                if len(branch) == 0:
-                    raise Exception("Error: branch is empty")
 
-                branch_lines = branch.decode().splitlines()
-                branch = None
-                for bl in branch_lines:
-                    if bl[0] == "*":
-                        branch = bl[1:].strip()
-                        break
-                if branch is None:
-                    raise Exception("Failed to identify branch")
+        dry_run = getattr(args, "dry_run", False)
+        tui = create_sync_tui(args)
 
-                status = subprocess.run(["git", "fetch"])
-                if status.returncode != 0:
-                    fatal("Failed to run git fetch on package %s" % dir)
-                status = subprocess.run(["git", "merge", "origin/" + branch])
-                if status.returncode != 0:
-                    fatal("Failed to run git merge origin/%s on package %s" % (branch, dir))
-                os.chdir(cwd)
-            elif os.path.isdir(pkg_path):
-                print("Note: skipping non-Git package \"" + dir + "\"")
-                sys.stdout.flush()
-
-        # Update package-lock.json to reflect the new state of all packages.
-        # We re-read the current commit hashes by patching the existing lock
-        # entries — this preserves all non-git package entries unchanged.
-        self._update_lock_after_sync(packages_dir)
-
-    def _update_lock_after_sync(self, packages_dir: str):
-        """Regenerate package-lock.json after sync by reading current HEAD commits."""
-        lock_path = os.path.join(packages_dir, "package-lock.json")
-        if not os.path.isfile(lock_path):
-            return  # No lock file to update
+        # Wire the TUI as the live progress listener so it receives
+        # per-package notifications during the parallel sync.
+        args._sync_progress = tui
 
+        tui.start()
         try:
-            lock = read_lock(lock_path)
-        except Exception as e:
-            note("Warning: could not read package-lock.json for update: %s" % e)
-            return
-
-        packages = lock.get("packages", {})
-
-        for name, entry in packages.items():
-            if entry.get("src") != "git":
-                continue
-            pkg_path = os.path.join(packages_dir, name)
-            if not os.path.isdir(os.path.join(pkg_path, ".git")):
-                continue
-            try:
-                result = subprocess.run(
-                    ["git", "rev-parse", "HEAD"],
-                    capture_output=True, text=True, cwd=pkg_path, timeout=10
-                )
-                if result.returncode == 0:
-                    entry["commit_resolved"] = result.stdout.strip()
-            except Exception:
-                pass
-
-        # Re-use write_lock's atomic write by building a minimal fake pkgs dict.
-        # Simpler: just write the patched lock dict directly (atomically).
-        import json, hashlib
-        from datetime import datetime, timezone
+            results = ProjectOps(args.project_dir).sync(args=args)
+        finally:
+            tui.stop()
 
-        lock["generated"] = datetime.now(timezone.utc).isoformat()
-        lock.pop("sha256", None)
-        body = json.dumps(lock, indent=2, sort_keys=True)
-        checksum = hashlib.sha256(body.encode()).hexdigest()
-        lock["sha256"] = checksum
+        tui.render(results, dry_run=dry_run)
 
-        tmp_path = lock_path + ".tmp"
-        with open(tmp_path, "w") as f:
-            json.dump(lock, indent=2, sort_keys=True, fp=f)
-            f.write("\n")
-        os.replace(tmp_path, lock_path)
-        note("Updated package-lock.json after sync")
+        # Only exit non-zero on true fatal errors (network failure, git crash, etc.).
+        # CONFLICT, DIRTY, and AHEAD are informational — the tool completed successfully.
+        if any(r.outcome == SyncOutcome.ERROR for r in results):
+            sys.exit(1)
 

src/ivpm/package.py

@@ -112,8 +112,15 @@ def build(self, pkgs_info):
     def status(self, pkgs_info):
         pass
 
-    def sync(self, pkgs_info):
-        pass
+    def sync(self, sync_info):
+        from .pkg_sync import PkgSyncResult, SyncOutcome
+        return PkgSyncResult(
+            name=self.name,
+            src_type=str(getattr(self, "src_type", "") or ""),
+            path=os.path.join(sync_info.deps_dir, self.name),
+            outcome=SyncOutcome.SKIPPED,
+            skipped_reason="not a VCS package",
+        )
 
     def update(self, update_info : ProjectUpdateInfo) -> 'ProjInfo':
         from .proj_info import ProjInfo

src/ivpm/package_lock.py

@@ -173,10 +173,73 @@ def write_lock(
     os.replace(tmp_path, lock_path)
     _logger.info("Wrote package-lock.json (%d packages)", len(packages))
 
+def _write_lock_dict(lock_path: str, lock: dict) -> None:
+    """Atomically (re-)write a lock dict to *lock_path*.
+
+    Strips any stale ``sha256`` key, refreshes the ``generated`` timestamp,
+    recomputes the checksum over the canonical body (without the checksum
+    key), then inserts it before the final write.  This is the single place
+    where the lock-file integrity stamp is managed.
+    """
+    lock = dict(lock)       # shallow copy — don't mutate caller's dict
+    lock.pop("sha256", None)
+    lock["generated"] = datetime.now(timezone.utc).isoformat()
+
+    body = json.dumps(lock, indent=2, sort_keys=True)
+    lock["sha256"] = hashlib.sha256(body.encode()).hexdigest()
+
+    tmp_path = lock_path + ".tmp"
+    with open(tmp_path, "w") as f:
+        json.dump(lock, indent=2, sort_keys=True, fp=f)
+        f.write("\n")
+    os.replace(tmp_path, lock_path)
+
+
+def patch_lock_after_sync(lock_path: str, sync_results) -> None:
+    """Update ``commit_resolved`` for synced packages and re-write the lock.
+
+    Called by ``ProjectOps.sync()`` after a successful (non-dry-run) sync.
+    Only packages whose outcome is SYNCED are updated; all others are left
+    unchanged.
+    """
+    from .pkg_sync import SyncOutcome
+
+    if not os.path.isfile(lock_path):
+        return
+
+    try:
+        lock = read_lock(lock_path)
+    except Exception as e:
+        _logger.warning("Could not read package-lock.json for sync update: %s", e)
+        return
+
+    packages = lock.get("packages", {})
+    changed = False
+    for result in sync_results:
+        if result.outcome != SyncOutcome.SYNCED:
+            continue
+        if result.name not in packages:
+            continue
+        entry = packages[result.name]
+        if entry.get("src") != "git":
+            continue
+        # Re-read full commit hash from the working directory
+        try:
+            r = subprocess.run(
+                ["git", "rev-parse", "HEAD"],
+                capture_output=True, text=True, cwd=result.path, timeout=10,
+            )
+            if r.returncode == 0:
+                entry["commit_resolved"] = r.stdout.strip()
+                changed = True
+        except Exception:
+            pass
+
+    if changed:
+        _write_lock_dict(lock_path, lock)
+        _logger.info("Updated package-lock.json after sync")
+
 
-# ---------------------------------------------------------------------------
-# Read
-# ---------------------------------------------------------------------------
 
 def read_lock(lock_path: str) -> dict:
     """Read and validate a lock file.  Returns the parsed dict."""

src/ivpm/pkg_sync.py

@@ -0,0 +1,73 @@
+#****************************************************************************
+#* pkg_sync.py
+#*
+#* Copyright 2024 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may
+#* not use this file except in compliance with the License.
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software
+#* distributed under the License is distributed on an "AS IS" BASIS,
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#* See the License for the specific language governing permissions and
+#* limitations under the License.
+#*
+#****************************************************************************
+"""
+pkg_sync.py — result types for `ivpm sync`.
+
+Mirrors pkg_status.py; returned by Package.sync() implementations.
+"""
+import dataclasses as dc
+import enum
+from typing import List, Optional
+
+
+class SyncOutcome(str, enum.Enum):
+    UP_TO_DATE          = "up-to-date"      # already at latest
+    SYNCED              = "synced"          # fast-forward merge succeeded
+    CONFLICT            = "conflict"        # merge conflict — user must resolve
+    DIRTY               = "dirty"           # uncommitted changes block merge
+    AHEAD               = "ahead"           # local commits not on origin
+    ERROR               = "error"           # network / git command failure
+    SKIPPED             = "skipped"         # read-only / non-git / tag-pinned
+
+    # dry-run variants (--dry-run / -n)
+    DRY_WOULD_SYNC      = "dry:sync"        # would fast-forward cleanly
+    DRY_WOULD_CONFLICT  = "dry:conflict"    # diverged — would produce conflicts
+    DRY_DIRTY           = "dry:dirty"       # dirty tree would block merge
+
+
+class SyncProgressListener:
+    """Callback interface for live sync progress.
+
+    Implement and pass as ``ProjectSyncInfo.progress`` to receive
+    per-package notifications during a (potentially parallel) sync.
+    """
+    def on_pkg_start(self, name: str) -> None:
+        """Called just before a package's sync work begins."""
+
+    def on_pkg_result(self, result: 'PkgSyncResult') -> None:
+        """Called immediately after a package's sync work finishes."""
+
+
+@dc.dataclass
+class PkgSyncResult:
+    """Per-package sync result returned by Package.sync()."""
+    name: str
+    src_type: str
+    path: str
+    outcome: SyncOutcome
+    branch: Optional[str] = None
+    old_commit: Optional[str] = None       # short commit before merge
+    new_commit: Optional[str] = None       # short commit after merge (or would-be)
+    commits_behind: Optional[int] = None   # upstream commits pulled / to be pulled
+    commits_ahead: Optional[int] = None    # local commits not on origin
+    conflict_files: List[str] = dc.field(default_factory=list)
+    dirty_files: List[str] = dc.field(default_factory=list)
+    next_steps: List[str] = dc.field(default_factory=list)  # shell commands to show
+    error: Optional[str] = None
+    skipped_reason: Optional[str] = None