Add automatic garbage collection when GC threshold is exceeded

cmd/init.go

@@ -332,6 +332,8 @@ func runInit(cmd *cobra.Command) error {
 		dedupMaxChunkSize,
 		dedupGCThreshold,
 		true, // index enabled
+		// Default automatic GC settings
+		true, "1h", 1000, true, ".sietch/logs/gc.log", 5000, "",
 	)
 
 	// Initialize RSA config if not present

cmd/root.go

@@ -4,10 +4,17 @@ Copyright © 2025 SubstantialCattle5, nilaysharan.com
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
 	"github.com/spf13/cobra"
+
+	"github.com/substantialcattle5/sietch/internal/fs"
+	"github.com/substantialcattle5/sietch/internal/gc"
 )
 
 // rootCmd represents the base command when called without any subcommands
@@ -24,12 +31,66 @@ encrypted data across machines, even with limited connectivity.`,
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
+	// Create context that can be cancelled
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Setup signal handling for graceful shutdown
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	// Start GC manager in background if in a vault
+	var gcCancel context.CancelFunc
+	if shouldStartGC() {
+		gcCtx, gcCancelFunc := context.WithCancel(ctx)
+		gcCancel = gcCancelFunc
+
+		go func() {
+			if err := gc.StartGlobalGC(gcCtx); err != nil {
+				// Log error but don't fail execution
+				fmt.Printf("Warning: Failed to start automatic GC: %v\n", err)
+			}
+		}()
+	}
+
+	// Handle shutdown
+	go func() {
+		sig := <-sigChan
+		fmt.Printf("\nReceived signal %v, shutting down...\n", sig)
+
+		// Cancel GC manager
+		if gcCancel != nil {
+			gcCancel()
+			time.Sleep(100 * time.Millisecond) // Give GC time to stop
+		}
+
+		cancel()
+		os.Exit(0)
+	}()
+
+	// Execute the command
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 }
 
+// shouldStartGC determines if GC should be started
+func shouldStartGC() bool {
+	// Check if we're in a vault directory
+	vaultRoot, err := fs.FindVaultRoot()
+	if err != nil {
+		return false
+	}
+
+	// Check if vault is initialized
+	if !fs.IsVaultInitialized(vaultRoot) {
+		return false
+	}
+
+	return true
+}
+
 func init() {
 	// Here you will define your flags and configuration settings.
 	// Cobra supports persistent flags, which, if defined here,

cmd/scaffold.go

@@ -132,6 +132,8 @@ func runScaffold(templateName, name, path string, force bool) error {
 		cfg.DedupMaxSize,
 		cfg.DedupGCThreshold,
 		cfg.DedupIndexEnabled,
+		// Default automatic GC settings
+		true, "1h", 1000, true, ".sietch/logs/gc.log", 5000, "",
 	)
 
 	// Initialize RSA config if not present

internal/config/vault.go

@@ -92,6 +92,20 @@ type DeduplicationConfig struct {
 	GCThreshold  int    `yaml:"gc_threshold"`   // Unreferenced chunk count before GC suggestion
 	IndexEnabled bool   `yaml:"index_enabled"`  // Enable chunk index for faster lookups
 	// CrossFileDedup bool   `yaml:"cross_file_dedup"` // Enable deduplication across different files
+
+	// Automatic GC settings
+	AutoGC AutoGCConfig `yaml:"auto_gc,omitempty"` // Automatic garbage collection settings
+}
+
+// AutoGCConfig contains settings for automatic garbage collection
+type AutoGCConfig struct {
+	Enabled         bool   `yaml:"enabled"`           // Enable automatic GC
+	CheckInterval   string `yaml:"check_interval"`    // How often to check (e.g., "1h", "30m")
+	AutoGCThreshold int    `yaml:"auto_gc_threshold"` // Threshold for automatic GC (overrides vault default)
+	EnableLogging   bool   `yaml:"enable_logging"`    // Enable GC logging
+	LogFile         string `yaml:"log_file"`          // Log file path
+	AlertThreshold  int    `yaml:"alert_threshold"`   // Alert when unreferenced chunks exceed this
+	AlertWebhook    string `yaml:"alert_webhook"`     // Webhook URL for alerts
 }
 
 // SyncConfig contains synchronization settings
@@ -193,6 +207,8 @@ func BuildVaultConfig(
 		keyConfig,
 		// Default deduplication settings
 		true, "content", "1KB", "64MB", 1000, true,
+		// Default automatic GC settings
+		true, "1h", 1000, true, ".sietch/logs/gc.log", 5000, "",
 	)
 }
 
@@ -207,6 +223,8 @@ func BuildVaultConfigWithDeduplication(
 	// Deduplication parameters
 	enableDedup bool, dedupStrategy, dedupMinSize, dedupMaxSize string,
 	dedupGCThreshold int, dedupIndexEnabled bool,
+	autoGCEnabled bool, autoGCCheckInterval string, autoGCThreshold int,
+	autoGCLogging bool, autoGCLogFile string, autoGCAlertThreshold int, autoGCWebhook string,
 ) VaultConfig {
 	config := VaultConfig{
 		VaultID:       vaultID,
@@ -237,6 +255,15 @@ func BuildVaultConfigWithDeduplication(
 	config.Deduplication.GCThreshold = dedupGCThreshold
 	config.Deduplication.IndexEnabled = dedupIndexEnabled
 
+	// Set automatic GC configuration
+	config.Deduplication.AutoGC.Enabled = autoGCEnabled
+	config.Deduplication.AutoGC.CheckInterval = autoGCCheckInterval
+	config.Deduplication.AutoGC.AutoGCThreshold = autoGCThreshold
+	config.Deduplication.AutoGC.EnableLogging = autoGCLogging
+	config.Deduplication.AutoGC.LogFile = autoGCLogFile
+	config.Deduplication.AutoGC.AlertThreshold = autoGCAlertThreshold
+	config.Deduplication.AutoGC.AlertWebhook = autoGCWebhook
+
 	// Set sync configuration
 	config.Sync.Mode = syncMode
 	config.Sync.KnownPeers = []string{} // Initialize as empty array

internal/gc/manager.go

@@ -0,0 +1,168 @@
+package gc
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"github.com/substantialcattle5/sietch/internal/config"
+	"github.com/substantialcattle5/sietch/internal/deduplication"
+	"github.com/substantialcattle5/sietch/internal/fs"
+)
+
+// Manager manages automatic garbage collection for Sietch vaults
+type Manager struct {
+	vaultRoot string
+	monitor   *Monitor
+	config    config.VaultConfig
+	started   bool
+}
+
+// NewManager creates a new GC manager for a vault
+func NewManager(vaultRoot string) (*Manager, error) {
+	// Check if vault is initialized
+	if !fs.IsVaultInitialized(vaultRoot) {
+		return nil, fmt.Errorf("vault not initialized")
+	}
+
+	// Load vault configuration
+	vaultConfig, err := config.LoadVaultConfig(vaultRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load vault configuration: %w", err)
+	}
+
+	return &Manager{
+		vaultRoot: vaultRoot,
+		config:    *vaultConfig,
+		started:   false,
+	}, nil
+}
+
+// Start begins automatic GC monitoring for the vault
+func (m *Manager) Start(ctx context.Context) error {
+	if m.started {
+		return fmt.Errorf("GC manager already started")
+	}
+
+	// Check if automatic GC is enabled
+	if !m.config.Deduplication.AutoGC.Enabled {
+		return fmt.Errorf("automatic GC is disabled in vault configuration")
+	}
+
+	// Parse check interval
+	checkInterval, err := time.ParseDuration(m.config.Deduplication.AutoGC.CheckInterval)
+	if err != nil {
+		return fmt.Errorf("invalid check interval '%s': %w", m.config.Deduplication.AutoGC.CheckInterval, err)
+	}
+
+	// Create monitor configuration
+	monitorConfig := MonitorConfig{
+		Enabled:         m.config.Deduplication.AutoGC.Enabled,
+		CheckInterval:   checkInterval,
+		AutoGCThreshold: m.getEffectiveGCThreshold(),
+		EnableLogging:   m.config.Deduplication.AutoGC.EnableLogging,
+		LogFile:         m.getEffectiveLogFile(),
+		AlertThreshold:  m.config.Deduplication.AutoGC.AlertThreshold,
+		AlertWebhook:    m.config.Deduplication.AutoGC.AlertWebhook,
+	}
+
+	// Create and start monitor
+	monitor, err := NewMonitor(m.vaultRoot, monitorConfig, m.config.Deduplication)
+	if err != nil {
+		return fmt.Errorf("failed to create GC monitor: %w", err)
+	}
+
+	if err := monitor.Start(ctx); err != nil {
+		return fmt.Errorf("failed to start GC monitor: %w", err)
+	}
+
+	m.monitor = monitor
+	m.started = true
+
+	return nil
+}
+
+// Stop halts automatic GC monitoring
+func (m *Manager) Stop() error {
+	if !m.started {
+		return fmt.Errorf("GC manager not started")
+	}
+
+	err := m.monitor.Stop()
+	m.started = false
+	return err
+}
+
+// IsRunning returns whether the GC manager is currently running
+func (m *Manager) IsRunning() bool {
+	return m.started && m.monitor.IsRunning()
+}
+
+// GetStats returns current deduplication statistics
+func (m *Manager) GetStats() (deduplication.DeduplicationStats, error) {
+	if !m.started {
+		return deduplication.DeduplicationStats{}, fmt.Errorf("GC manager not started")
+	}
+	return m.monitor.GetStats(), nil
+}
+
+// getEffectiveGCThreshold returns the effective GC threshold to use
+func (m *Manager) getEffectiveGCThreshold() int {
+	if m.config.Deduplication.AutoGC.AutoGCThreshold > 0 {
+		return m.config.Deduplication.AutoGC.AutoGCThreshold
+	}
+	return m.config.Deduplication.GCThreshold
+}
+
+// getEffectiveLogFile returns the effective log file path
+func (m *Manager) getEffectiveLogFile() string {
+	if m.config.Deduplication.AutoGC.LogFile != "" {
+		// Convert relative path to absolute path within vault
+		if !filepath.IsAbs(m.config.Deduplication.AutoGC.LogFile) {
+			return filepath.Join(m.vaultRoot, m.config.Deduplication.AutoGC.LogFile)
+		}
+		return m.config.Deduplication.AutoGC.LogFile
+	}
+	return filepath.Join(m.vaultRoot, ".sietch", "logs", "gc.log")
+}
+
+// Global variable to hold the active GC manager
+var activeManager *Manager
+
+// StartGlobalGC starts automatic GC for the vault in the current directory
+func StartGlobalGC(ctx context.Context) error {
+	vaultRoot, err := fs.FindVaultRoot()
+	if err != nil {
+		// Not in a vault, silently ignore
+		return nil
+	}
+
+	manager, err := NewManager(vaultRoot)
+	if err != nil {
+		return fmt.Errorf("failed to create GC manager: %w", err)
+	}
+
+	if err := manager.Start(ctx); err != nil {
+		return fmt.Errorf("failed to start GC manager: %w", err)
+	}
+
+	activeManager = manager
+	return nil
+}
+
+// StopGlobalGC stops the global GC manager
+func StopGlobalGC() error {
+	if activeManager == nil {
+		return nil
+	}
+
+	err := activeManager.Stop()
+	activeManager = nil
+	return err
+}
+
+// GetGlobalGCManager returns the active GC manager
+func GetGlobalGCManager() *Manager {
+	return activeManager
+}