snapshot: implement lazy parent recovery for proxy mode (#700)

* snapshot: implement lazy parent recovery for proxy mode

Implement lazy parent recovery to handle the desynchronization scenarios
where nydus-snapshotter's BoltDB is missing snapshot entries that
containerd has a reference to it. This can happen due to:
- BoltDB file corruption
- Storage issues affecting only nydus data
- Backup/restore asymmetry
- Upgrade failures

When createSnapshot fails with 'missing parent bucket: not found', the
new recovery mechanism:
1. Detects the error pattern
2. For proxy mode (Kata Containers guest pulling), creates a placeholder
   committed snapshot in the local BoltDB
3. Retries the original operation

The placeholder snapshot is marked with labels:
- nydus.recovered=true
- nydus.recovered.at=<timestamp>

This allows the system to self-heal and continue operating rather than
requiring manual intervention to clean up stale containerd references.

Fixes: #680

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>

* filesystem: fix proxy mode's Unmount()

Umount() for proxy mode (FsDriverProxy) did nothing, leaving stale
RafsInstance entries in nydus.db. When the same snapshot ID was reused
(e.g., pod restart), Mount() would fail with 'already exists' when
trying to add a new RafsInstance.

The fix adds fsManager.RemoveRafsInstance(snapshotID) to the proxy mode
case in Umount(), ensuring DB entries are cleaned up properly.

Note: The error is logged but not returned since the entry might not
exist (e.g., if Mount() failed before creating the DB entry).

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>

---------

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>

Author: fidencio

pkg/filesystem/fs.go

@@ -539,7 +539,12 @@ func (fs *Filesystem) Umount(_ context.Context, snapshotID string) error {
 			return errors.Wrapf(err, "remove snapshot %s", snapshotID)
 		}
 	case config.FsDriverNodev, config.FsDriverProxy:
-		// Nothing to do
+		// For proxy mode, we still need to clean up the DB entry to prevent
+		// "already exists" errors on subsequent Mount() calls.
+		if err := fsManager.RemoveRafsInstance(snapshotID); err != nil {
+			// Log but don't fail - the entry might not exist
+			log.L.Debugf("remove instance %s from DB: %v", snapshotID, err)
+		}
 	default:
 		return errors.Errorf("unknown filesystem driver %s for snapshot %s", fsDriver, snapshotID)
 	}

snapshot/snapshot.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -729,6 +730,15 @@ func (o *snapshotter) findMetaLayer(ctx context.Context, key string) (string, sn
 }
 
 func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (info *snapshots.Info, _ storage.Snapshot, err error) {
+	return o.createSnapshotWithRecovery(ctx, kind, key, parent, opts, false)
+}
+
+// createSnapshotWithRecovery attempts to create a snapshot and recovers from
+// "missing parent" errors by querying containerd for the parent's metadata and
+// recreating it in the local BoltDB. This handles the desynchronization
+// scenarios where nydus-snapshotter's BoltDB is missing entries that
+// containerd knows about.
+func (o *snapshotter) createSnapshotWithRecovery(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt, isRecovery bool) (info *snapshots.Info, _ storage.Snapshot, err error) {
 	ctx, t, err := o.ms.TransactionContext(ctx, true)
 	if err != nil {
 		return nil, storage.Snapshot{}, err
@@ -774,6 +784,37 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
 
 	s, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...)
 	if err != nil {
+		// Check if this is a "missing parent" error and we can attempt recovery
+		if !isRecovery && parent != "" && o.isMissingParentError(err) {
+			log.G(ctx).WithError(err).Warnf("Missing parent %q in local BoltDB, attempting lazy recovery from containerd", parent)
+
+			// Rollback current transaction before recovery
+			rollback = true
+			if rerr := t.Rollback(); rerr != nil {
+				log.G(ctx).WithError(rerr).Warn("failed to rollback transaction before recovery")
+			}
+			rollback = false // Don't rollback again in defer
+
+			// Clean up the temp directory we created
+			if td != "" {
+				if err1 := o.cleanupSnapshotDirectory(ctx, td); err1 != nil {
+					log.G(ctx).WithError(err1).Warn("failed to clean up temp snapshot directory during recovery")
+				}
+				td = ""
+			}
+
+			// Attempt to recover the parent
+			// The recovery function uses a fresh context internally
+			if recoverErr := o.recoverParentFromContainerd(ctx, parent); recoverErr != nil {
+				log.G(ctx).WithError(recoverErr).Errorf("Failed to recover parent %q from containerd", parent)
+				return nil, storage.Snapshot{}, errors.Wrapf(err, "create snapshot (recovery failed: %v)", recoverErr)
+			}
+
+			log.G(ctx).Infof("Successfully recovered parent %q from containerd, retrying snapshot creation", parent)
+			// Retry with recovery flag set to prevent infinite recursion
+			// Use the original context for the retry
+			return o.createSnapshotWithRecovery(ctx, kind, key, parent, opts, true)
+		}
 		return nil, storage.Snapshot{}, errors.Wrap(err, "create snapshot")
 	}
 
@@ -804,6 +845,123 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
 	return &base, s, nil
 }
 
+// isMissingParentError checks if the error is a "missing parent bucket" error
+// from the storage layer, which indicates the parent snapshot exists in containerd
+// but not in our local BoltDB.
+func (o *snapshotter) isMissingParentError(err error) bool {
+	if err == nil {
+		return false
+	}
+	errStr := err.Error()
+	return strings.Contains(errStr, "missing parent") && strings.Contains(errStr, "bucket")
+}
+
+// recoverParentFromContainerd attempts to recover a missing parent snapshot.
+// This handles the desyncrhonization scenario where nydus-snapshotter's BoltDB
+// was wiped but containerd still has metadata references.
+//
+// For proxy mode (used by Kata Containers for guest image pulling), we create
+// a minimal placeholder snapshot since there's no actual filesystem data stored.
+// The real data will be pulled fresh by the guest.
+func (o *snapshotter) recoverParentFromContainerd(ctx context.Context, parent string) error {
+	log.G(ctx).Infof("Attempting lazy recovery for missing parent: %q", parent)
+
+	// For proxy mode (Kata guest pulling), create a minimal placeholder
+	// since there's no actual filesystem content to restore
+	fsDriver := config.GetFsDriver()
+	if fsDriver == config.FsDriverProxy {
+		log.G(ctx).Infof("Proxy mode detected - creating placeholder snapshot for %q", parent)
+		// Use the full parent key as-is since the storage layer uses this exact key
+		return o.createPlaceholderSnapshot(ctx, parent)
+	}
+
+	// For other modes (fusedev, fscache), we would need the actual parent metadata
+	// from containerd. Unfortunately, containerd's snapshot service for remote
+	// snapshotters calls back to us, creating a circular dependency when our DB is empty.
+	return errors.Errorf(
+		"lazy parent recovery not supported for fs_driver=%q. "+
+			"Please clean up containerd's stale snapshot references with: "+
+			"ctr snapshot --snapshotter nydus rm %s",
+		fsDriver, parent)
+}
+
+// createPlaceholderSnapshot creates a minimal committed snapshot entry in the local
+// BoltDB for recovery purposes. This is used in proxy mode where no actual filesystem
+// content is stored locally.
+func (o *snapshotter) createPlaceholderSnapshot(ctx context.Context, key string) error {
+	// Use a fresh context to avoid any transaction context pollution
+	cleanCtx := context.Background()
+
+	// First check if the snapshot already exists (from a previous partial recovery)
+	// nolint:dogsled
+	_, _, _, existErr := snapshot.GetSnapshotInfo(cleanCtx, o.ms, key)
+	if existErr == nil {
+		log.G(ctx).Infof("Placeholder snapshot %q already exists, skipping creation", key)
+		return nil
+	}
+
+	txCtx, t, err := o.ms.TransactionContext(cleanCtx, true)
+	if err != nil {
+		return errors.Wrap(err, "begin transaction for placeholder snapshot")
+	}
+	rollback := true
+	defer func() {
+		if rollback {
+			if rerr := t.Rollback(); rerr != nil {
+				log.G(ctx).WithError(rerr).Warn("failed to rollback placeholder transaction")
+			}
+		}
+	}()
+
+	// Prepare the snapshot directory
+	td, err := o.prepareDirectory(o.snapshotRoot(), snapshots.KindCommitted)
+	if err != nil {
+		return errors.Wrap(err, "prepare directory for placeholder snapshot")
+	}
+
+	// Create a placeholder with minimal labels indicating it was recovered
+	opts := []snapshots.Opt{
+		snapshots.WithLabels(map[string]string{
+			"nydus.recovered":    "true",
+			"nydus.recovered.at": fmt.Sprintf("%d", time.Now().Unix()),
+		}),
+	}
+
+	// Create as active first (no parent)
+	// Use a unique key to avoid conflicts with retries
+	activeKey := fmt.Sprintf("recovery-%d-%s", time.Now().UnixNano(), key)
+	s, err := storage.CreateSnapshot(txCtx, snapshots.KindActive, activeKey, "", opts...)
+	if err != nil {
+		if err1 := o.cleanupSnapshotDirectory(cleanCtx, td); err1 != nil {
+			log.G(ctx).WithError(err1).Warn("failed to clean up temp directory")
+		}
+		return errors.Wrapf(err, "create placeholder snapshot entry for %q", key)
+	}
+
+	// Move temp directory to final location
+	path := o.snapshotDir(s.ID)
+	if err = os.Rename(td, path); err != nil {
+		return errors.Wrap(err, "rename placeholder snapshot directory")
+	}
+
+	// Commit the active snapshot to create the final committed snapshot
+	// CommitActive(ctx, key, name) commits active snapshot `key` as committed snapshot `name`
+	if _, err := storage.CommitActive(txCtx, activeKey, key, snapshots.Usage{}, opts...); err != nil {
+		if err1 := o.cleanupSnapshotDirectory(cleanCtx, path); err1 != nil {
+			log.G(ctx).WithError(err1).Warn("failed to clean up snapshot directory")
+		}
+		return errors.Wrapf(err, "commit placeholder snapshot %q", key)
+	}
+
+	rollback = false
+	if err = t.Commit(); err != nil {
+		return errors.Wrap(err, "commit placeholder transaction")
+	}
+
+	log.G(ctx).Infof("Successfully created placeholder snapshot %q for lazy recovery", key)
+	return nil
+}
+
 func (o *snapshotter) mergeTarfs(ctx context.Context, s storage.Snapshot, pID string, pInfo snapshots.Info) error {
 	if err := o.fs.MergeTarfsLayers(s, func(id string) string { return o.upperPath(id) }); err != nil {
 		return errors.Wrapf(err, "tarfs merge fail %s", pID)