diff --git a/config/config.go b/config/config.go index 1fb9036c..7fcd3e4c 100644 --- a/config/config.go +++ b/config/config.go @@ -38,6 +38,7 @@ type Config struct { Slack *SlackConfig `yaml:"slack,omitempty"` Scheduler *SchedulerConfig `yaml:"scheduler,omitempty"` Tasks *TasksConfig `yaml:"tasks,omitempty"` + Backup *BackupConfig `yaml:"backup,omitempty"` } func (c *Config) ResolvedMode() Mode { @@ -207,6 +208,25 @@ type TasksConfig struct { Storage StorageConfig `yaml:"storage,omitempty"` } +type BackupConfig struct { + Enabled bool `yaml:"enabled,omitempty"` + Schedule string `yaml:"schedule,omitempty"` // "6h", "12h", "24h", or "" (manual) + Destination string `yaml:"destination,omitempty"` // "r2" or "gdrive" + R2 *R2Config `yaml:"r2,omitempty"` + GDrive *GDriveConfig `yaml:"gdrive,omitempty"` +} + +type R2Config struct { + Bucket string `yaml:"bucket,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` + AccessKeyRef string `yaml:"access_key_ref,omitempty"` // e.g. "keychain:obk/r2-access-key" + SecretKeyRef string `yaml:"secret_key_ref,omitempty"` // e.g. "keychain:obk/r2-secret-key" +} + +type GDriveConfig struct { + FolderID string `yaml:"folder_id,omitempty"` +} + type StorageConfig struct { Driver string `yaml:"driver,omitempty"` // "sqlite" or "postgres" DSN string `yaml:"dsn,omitempty"` diff --git a/config/paths.go b/config/paths.go index 6b65b992..6a3401e3 100644 --- a/config/paths.go +++ b/config/paths.go @@ -90,3 +90,15 @@ func CleanScratch(sessionID string) error { func LearningsDir() string { return filepath.Join(Dir(), "learnings") } + +func BackupDir() string { + return filepath.Join(Dir(), "backup") +} + +func BackupStagingDir() string { + return filepath.Join(BackupDir(), "staging") +} + +func BackupLastManifestPath() string { + return filepath.Join(BackupDir(), "last_manifest.json") +} diff --git a/daemon/jobs/backup.go b/daemon/jobs/backup.go new file mode 100644 index 00000000..e0fd89a2 --- /dev/null +++ b/daemon/jobs/backup.go @@ -0,0 +1,87 @@ +package jobs + +import ( + "context" + "fmt" + "log/slog" + "net/http" + + "github.com/riverqueue/river" + + "github.com/73ai/openbotkit/config" + "github.com/73ai/openbotkit/oauth/google" + "github.com/73ai/openbotkit/provider" + backupsvc "github.com/73ai/openbotkit/service/backup" +) + +type BackupArgs struct{} + +func (BackupArgs) Kind() string { return "backup" } + +type BackupWorker struct { + river.WorkerDefaults[BackupArgs] + Cfg *config.Config +} + +func (w *BackupWorker) Work(ctx context.Context, job *river.Job[BackupArgs]) error { + if w.Cfg.Backup == nil || !w.Cfg.Backup.Enabled { + slog.Info("backup: not enabled, skipping") + return nil + } + + if !config.IsSourceLinked("backup") { + slog.Info("backup: not linked, skipping") + return nil + } + + slog.Info("starting backup job") + + backend, err := backupsvc.ResolveBackend(ctx, backendOpts(w.Cfg)) + if err != nil { + return fmt.Errorf("resolve backend: %w", err) + } + + svc := backupsvc.New(backend, config.Dir()) + result, err := svc.Run(ctx) + if err != nil { + return fmt.Errorf("backup: %w", err) + } + + slog.Info("backup complete", + "changed", result.Changed, + "skipped", result.Skipped, + "uploaded", result.Uploaded, + "duration", result.Duration, + ) + return nil +} + +func backendOpts(cfg *config.Config) backupsvc.ResolveBackendOpts { + opts := backupsvc.ResolveBackendOpts{ + ResolveCred: provider.ResolveAPIKey, + BackupDest: cfg.Backup.Destination, + GoogleClient: func(ctx context.Context, gcfg backupsvc.GoogleClientConfig) (*http.Client, error) { + gp := google.New(google.Config{ + CredentialsFile: cfg.GoogleCredentialsFile(), + TokenDBPath: cfg.GoogleTokenDBPath(), + }) + accounts, err := gp.Accounts(ctx) + if err != nil || len(accounts) == 0 { + return nil, fmt.Errorf("no Google account found") + } + return gp.Client(ctx, accounts[0], gcfg.Scopes) + }, + } + if cfg.Backup.R2 != nil { + opts.R2Bucket = cfg.Backup.R2.Bucket + opts.R2Endpoint = cfg.Backup.R2.Endpoint + opts.R2AccessRef = cfg.Backup.R2.AccessKeyRef + opts.R2SecretRef = cfg.Backup.R2.SecretKeyRef + } + if cfg.Backup.GDrive != nil { + opts.GDriveFolderID = cfg.Backup.GDrive.FolderID + } + return opts +} + +var _ river.Worker[BackupArgs] = (*BackupWorker)(nil) diff --git a/daemon/river.go b/daemon/river.go index a84305c5..56e8d593 100644 --- a/daemon/river.go +++ b/daemon/river.go @@ -42,26 +42,42 @@ func newRiverClient(ctx context.Context, cfg *config.Config, notifier *SyncNotif }) river.AddWorker(workers, &jobs.ReminderWorker{}) river.AddWorker(workers, &jobs.ScheduledTaskWorker{Cfg: cfg}) + river.AddWorker(workers, &jobs.BackupWorker{Cfg: cfg}) period, err := time.ParseDuration(cfg.Daemon.GmailSyncPeriod) if err != nil { period = 15 * time.Minute } + periodicJobs := []*river.PeriodicJob{ + river.NewPeriodicJob( + river.PeriodicInterval(period), + func() (river.JobArgs, *river.InsertOpts) { + return jobs.GmailSyncArgs{}, nil + }, + &river.PeriodicJobOpts{RunOnStart: true}, + ), + } + + if cfg.Backup != nil && cfg.Backup.Enabled && cfg.Backup.Schedule != "" { + backupPeriod, err := time.ParseDuration(cfg.Backup.Schedule) + if err == nil { + periodicJobs = append(periodicJobs, river.NewPeriodicJob( + river.PeriodicInterval(backupPeriod), + func() (river.JobArgs, *river.InsertOpts) { + return jobs.BackupArgs{}, nil + }, + &river.PeriodicJobOpts{RunOnStart: false}, + )) + } + } + riverCfg := &river.Config{ Queues: map[string]river.QueueConfig{ river.QueueDefault: {MaxWorkers: 5}, }, - Workers: workers, - PeriodicJobs: []*river.PeriodicJob{ - river.NewPeriodicJob( - river.PeriodicInterval(period), - func() (river.JobArgs, *river.InsertOpts) { - return jobs.GmailSyncArgs{}, nil - }, - &river.PeriodicJobOpts{RunOnStart: true}, - ), - }, + Workers: workers, + PeriodicJobs: periodicJobs, } client, err := river.NewClient(driver, riverCfg) diff --git a/docs/architecture.excalidraw b/docs/architecture.excalidraw index f65fde2c..e194fc81 100644 --- a/docs/architecture.excalidraw +++ b/docs/architecture.excalidraw @@ -1,92 +1,63 @@ { "type": "excalidraw", "version": 2, - "source": "openbotkit", "elements": [ { - "id": "you_rect", - "type": "rectangle", - "x": 370, - "y": 20, - "width": 120, - "height": 50, - "angle": 0, + "id": "user-ellipse", + "type": "ellipse", + "x": 535, + "y": 15, + "width": 130, + "height": 55, "strokeColor": "#1e1e1e", - "backgroundColor": "#ffc9c9", + "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 1001, + "seed": 1000, "version": 1, - "versionNonce": 1001, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "you_text" + "id": "user-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "you_text", + "id": "user-text", "type": "text", - "x": 407, - "y": 30, - "width": 46, - "height": 30, - "angle": 0, + "x": 565, + "y": 25, + "width": 70, + "height": 35, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 1002, - "version": 1, - "versionNonce": 1002, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, "text": "You", - "fontSize": 24, + "fontSize": 28, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "you_rect", - "originalText": "You" + "roughness": 1, + "seed": 1001, + "version": 1, + "isDeleted": false, + "originalText": "You", + "containerId": "user-ellipse" }, { - "id": "arrow_you_channels", + "id": "arrow-1", "type": "arrow", - "x": 430, + "x": 600, "y": 70, "width": 0, - "height": 40, - "angle": 0, + "height": 55, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, - "opacity": 100, - "seed": 1003, + "seed": 1002, "version": 1, - "versionNonce": 1003, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, "points": [ [ 0, @@ -94,241 +65,136 @@ ], [ 0, - 40 + 55 ] ], - "startArrowhead": "arrow", + "startArrowhead": null, "endArrowhead": "arrow" }, { - "id": "channels_bg", - "type": "rectangle", - "x": 285, - "y": 112, - "width": 290, - "height": 100, - "angle": 0, - "strokeColor": "#1971c2", - "backgroundColor": "#d0ebff", - "fillStyle": "solid", - "strokeWidth": 2, - "roughness": 1, - "opacity": 100, - "seed": 2000, - "version": 1, - "versionNonce": 2000, - "isDeleted": false, - "boundElements": [], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } - }, - { - "id": "channels_label", + "id": "channels-label", "type": "text", - "x": 380, - "y": 116, - "width": 100, + "x": 555, + "y": 128, + "width": 90, "height": 20, - "angle": 0, "strokeColor": "#1971c2", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 2001, - "version": 1, - "versionNonce": 2001, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, "text": "Channels", "fontSize": 16, "fontFamily": 1, - "textAlign": "center", + "textAlign": "left", "verticalAlign": "top", - "containerId": null, + "roughness": 1, + "seed": 1003, + "version": 1, + "isDeleted": false, "originalText": "Channels" }, { - "id": "tg_rect", + "id": "telegram-box", "type": "rectangle", - "x": 302, - "y": 148, - "width": 120, - "height": 50, - "angle": 0, - "strokeColor": "#1971c2", + "x": 330, + "y": 152, + "width": 240, + "height": 48, + "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 2010, + "roundness": { + "type": 3 + }, + "seed": 1004, "version": 1, - "versionNonce": 2010, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "tg_text" + "id": "telegram-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "tg_text", + "id": "telegram-text", "type": "text", - "x": 318, - "y": 160, - "width": 88, + "x": 384, + "y": 163.5, + "width": 132, "height": 25, - "angle": 0, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 2011, - "version": 1, - "versionNonce": 2011, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Telegram", - "fontSize": 18, + "text": "Telegram Bot", + "fontSize": 20, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "tg_rect", - "originalText": "Telegram" + "roughness": 1, + "seed": 1005, + "version": 1, + "isDeleted": false, + "originalText": "Telegram Bot", + "containerId": "telegram-box" }, { - "id": "cli_rect", + "id": "cli-box", "type": "rectangle", - "x": 438, - "y": 148, - "width": 120, - "height": 50, - "angle": 0, - "strokeColor": "#1971c2", + "x": 630, + "y": 152, + "width": 240, + "height": 48, + "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 2020, + "roundness": { + "type": 3 + }, + "seed": 1006, "version": 1, - "versionNonce": 2020, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "cli_text" + "id": "cli-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "cli_text", + "id": "cli-text", "type": "text", - "x": 480, - "y": 160, - "width": 36, + "x": 733.5, + "y": 163.5, + "width": 33, "height": 25, - "angle": 0, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 2021, - "version": 1, - "versionNonce": 2021, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, "text": "CLI", - "fontSize": 18, + "fontSize": 20, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "cli_rect", - "originalText": "CLI" - }, - { - "id": "approve_note", - "type": "text", - "x": 600, - "y": 118, - "width": 150, - "height": 50, - "angle": 0, - "strokeColor": "#e03131", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, "roughness": 1, - "opacity": 100, - "seed": 2030, + "seed": 1007, "version": 1, - "versionNonce": 2030, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "approve / deny\n\u2190 every action", - "fontSize": 18, - "fontFamily": 1, - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "approve / deny\n\u2190 every action" + "originalText": "CLI", + "containerId": "cli-box" }, { - "id": "arrow_channels_agent", + "id": "arrow-2", "type": "arrow", - "x": 430, - "y": 214, + "x": 600, + "y": 200, "width": 0, - "height": 36, - "angle": 0, + "height": 44, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, - "opacity": 100, - "seed": 3000, + "seed": 1008, "version": 1, - "versionNonce": 3000, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, "points": [ [ 0, @@ -336,223 +202,182 @@ ], [ 0, - 36 + 44 ] ], - "startArrowhead": "arrow", + "startArrowhead": null, "endArrowhead": "arrow" }, { - "id": "agent_rect", + "id": "modes-label", + "type": "text", + "x": 570, + "y": 247, + "width": 60, + "height": 20, + "strokeColor": "#087f5b", + "text": "Modes", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "roughness": 1, + "seed": 1009, + "version": 1, + "isDeleted": false, + "originalText": "Modes" + }, + { + "id": "local-box", "type": "rectangle", - "x": 285, - "y": 252, - "width": 290, - "height": 65, - "angle": 0, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", + "x": 230, + "y": 270, + "width": 210, + "height": 48, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 3010, + "roundness": { + "type": 3 + }, + "seed": 1010, "version": 1, - "versionNonce": 3010, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "agent_text" + "id": "local-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "agent_text", + "id": "local-text", "type": "text", - "x": 290, - "y": 260, - "width": 280, - "height": 44, - "angle": 0, - "strokeColor": "#2f9e44", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 3011, - "version": 1, - "versionNonce": 3011, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Agent Loop\nClaude \u00b7 Gemini \u00b7 OpenAI \u00b7 Groq", - "fontSize": 14, + "x": 307.5, + "y": 281.5, + "width": 55.00000000000001, + "height": 25, + "strokeColor": "#1e1e1e", + "text": "Local", + "fontSize": 20, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "agent_rect", - "originalText": "Agent Loop\nClaude \u00b7 Gemini \u00b7 OpenAI \u00b7 Groq" + "roughness": 1, + "seed": 1011, + "version": 1, + "isDeleted": false, + "originalText": "Local", + "containerId": "local-box" }, { - "id": "agent_sub", - "type": "text", - "x": 310, - "y": 286, - "width": 240, - "height": 20, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", + "id": "remote-box", + "type": "rectangle", + "x": 495, + "y": 270, + "width": 210, + "height": 48, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 3012, - "version": 1, - "versionNonce": 3012, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Claude \u00b7 Gemini \u00b7 OpenAI \u00b7 Groq", - "fontSize": 14, + "roundness": { + "type": 3 + }, + "seed": 1012, + "version": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "remote-text", + "type": "text" + } + ] + }, + { + "id": "remote-text", + "type": "text", + "x": 567, + "y": 281.5, + "width": 66, + "height": 25, + "strokeColor": "#1e1e1e", + "text": "Remote", + "fontSize": 20, "fontFamily": 1, "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "Claude \u00b7 Gemini \u00b7 OpenAI \u00b7 Groq" + "verticalAlign": "middle", + "roughness": 1, + "seed": 1013, + "version": 1, + "isDeleted": false, + "originalText": "Remote", + "containerId": "remote-box" }, { - "id": "llm_rect", + "id": "server-box", "type": "rectangle", - "x": 640, - "y": 260, - "width": 150, - "height": 50, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "#e9ecef", + "x": 760, + "y": 270, + "width": 210, + "height": 48, + "strokeColor": "#1e1e1e", + "backgroundColor": "#96f2d7", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 3020, + "roundness": { + "type": 3 + }, + "seed": 1014, "version": 1, - "versionNonce": 3020, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "llm_text" + "id": "server-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "llm_text", + "id": "server-text", "type": "text", - "x": 665, - "y": 268, - "width": 100, - "height": 30, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 3021, - "version": 1, - "versionNonce": 3021, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "LLM API\n(cloud)", - "fontSize": 14, + "x": 793.5, + "y": 281.5, + "width": 143, + "height": 25, + "strokeColor": "#1e1e1e", + "text": "Server (HTTP)", + "fontSize": 20, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "llm_rect", - "originalText": "LLM API\n(cloud)" - }, - { - "id": "arrow_agent_llm", - "type": "arrow", - "x": 577, - "y": 285, - "width": 60, - "height": 0, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, "roughness": 1, - "opacity": 100, - "seed": 3030, + "seed": 1015, "version": 1, - "versionNonce": 3030, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "points": [ - [ - 0, - 0 - ], - [ - 60, - 0 - ] - ], - "startArrowhead": "arrow", - "endArrowhead": "arrow" + "originalText": "Server (HTTP)", + "containerId": "server-box" }, { - "id": "arrow_agent_safety", + "id": "arrow-3", "type": "arrow", - "x": 430, - "y": 319, + "x": 600, + "y": 318, "width": 0, - "height": 31, - "angle": 0, + "height": 44, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, - "opacity": 100, - "seed": 4000, + "seed": 1016, "version": 1, - "versionNonce": 4000, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, "points": [ [ 0, @@ -560,631 +385,576 @@ ], [ 0, - 31 + 44 ] ], "startArrowhead": null, "endArrowhead": "arrow" }, { - "id": "safety_bg", + "id": "agent-container", "type": "rectangle", - "x": 60, - "y": 352, - "width": 740, - "height": 210, - "angle": 0, - "strokeColor": "#e03131", - "backgroundColor": "#fff5f5", + "x": 150, + "y": 362, + "width": 900, + "height": 255, + "strokeColor": "#2b8a3e", + "backgroundColor": "#ebfbee", "fillStyle": "solid", - "strokeWidth": 3, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4001, - "version": 1, - "versionNonce": 4001, - "isDeleted": false, - "boundElements": [], - "groupIds": [], - "frameId": null, "roundness": { "type": 3 - } + }, + "seed": 1017, + "version": 1, + "isDeleted": false, + "boundElements": [] }, { - "id": "safety_title", + "id": "agent-label", "type": "text", - "x": 330, - "y": 358, - "width": 200, - "height": 30, - "angle": 0, - "strokeColor": "#e03131", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, + "x": 530, + "y": 366, + "width": 110, + "height": 20, + "strokeColor": "#2b8a3e", + "text": "Agent Core", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", "roughness": 1, - "opacity": 100, - "seed": 4002, + "seed": 1018, "version": 1, - "versionNonce": 4002, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Safety Layer", - "fontSize": 24, - "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "Safety Layer" + "originalText": "Agent Core" }, { - "id": "gate_rect", + "id": "orch-box", "type": "rectangle", - "x": 80, - "y": 398, - "width": 160, - "height": 65, - "angle": 0, - "strokeColor": "#e03131", - "backgroundColor": "#ffc9c9", + "x": 200, + "y": 392, + "width": 800, + "height": 48, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4010, + "roundness": { + "type": 3 + }, + "seed": 1019, "version": 1, - "versionNonce": 4010, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "gate_text" + "id": "orch-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "gate_text", + "id": "orch-text", "type": "text", - "x": 85, - "y": 403, - "width": 150, - "height": 48, - "angle": 0, - "strokeColor": "#c92a2a", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 4011, - "version": 1, - "versionNonce": 4011, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Approval Gates\nevery write action\nneeds your OK", - "fontSize": 11, + "x": 501, + "y": 403.5, + "width": 198.00000000000003, + "height": 25, + "strokeColor": "#1e1e1e", + "text": "Agent Orchestrator", + "fontSize": 20, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "gate_rect", - "originalText": "Approval Gates\nevery write action\nneeds your OK" - }, - { - "id": "gate_sub", - "type": "text", - "x": 62, - "y": 422, - "width": 180, - "height": 30, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, "roughness": 1, - "opacity": 100, - "seed": 4012, - "version": 1, - "versionNonce": 4012, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "every write action\nneeds your OK", - "fontSize": 11, - "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "every write action\nneeds your OK" + "seed": 1020, + "version": 1, + "isDeleted": false, + "originalText": "Agent Orchestrator", + "containerId": "orch-box" }, { - "id": "inject_rect", + "id": "safety-container", "type": "rectangle", - "x": 260, - "y": 398, - "width": 160, - "height": 65, - "angle": 0, - "strokeColor": "#e03131", + "x": 200, + "y": 452, + "width": 800, + "height": 152, + "strokeColor": "#c92a2a", "backgroundColor": "#ffc9c9", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4020, - "version": 1, - "versionNonce": 4020, - "isDeleted": false, - "boundElements": [ - { - "type": "text", - "id": "inject_text" - } - ], - "groupIds": [], - "frameId": null, "roundness": { "type": 3 - } + }, + "seed": 1021, + "version": 1, + "isDeleted": false, + "boundElements": [] }, { - "id": "inject_text", + "id": "safety-label", "type": "text", - "x": 265, - "y": 403, - "width": 150, - "height": 48, - "angle": 0, + "x": 380, + "y": 456, + "width": 240, + "height": 19, "strokeColor": "#c92a2a", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, + "text": "Safety & Approval Layer", + "fontSize": 15, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", "roughness": 1, - "opacity": 100, - "seed": 4021, + "seed": 1022, "version": 1, - "versionNonce": 4021, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Injection Defense\nboundary markers\nbase64 \u00b7 homoglyph", - "fontSize": 11, - "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "inject_rect", - "originalText": "Injection Defense\nboundary markers\nbase64 \u00b7 homoglyph" + "originalText": "Safety & Approval Layer" }, { - "id": "inject_sub", + "id": "safety-text", "type": "text", - "x": 254, - "y": 422, - "width": 180, - "height": 30, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, + "x": 220, + "y": 480, + "width": 420, + "height": 108, + "strokeColor": "#1e1e1e", + "text": "• System prompt hardening + context boundaries\n• Boundary markers + injection scanning\n• Tiered approval + auto-approve patterns\n• Session-scoped auto-approve + rubber-stamp detection\n• Bash command filter\n• Token budget enforcement", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", "roughness": 1, - "opacity": 100, - "seed": 4022, - "version": 1, - "versionNonce": 4022, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "boundary markers\nbase64 \u00b7 homoglyph", - "fontSize": 11, + "seed": 1023, + "version": 1, + "isDeleted": false, + "originalText": "• System prompt hardening + context boundaries\n• Boundary markers + injection scanning\n• Tiered approval + auto-approve patterns\n• Session-scoped auto-approve + rubber-stamp detection\n• Bash command filter\n• Token budget enforcement" + }, + { + "id": "arrow-4", + "type": "arrow", + "x": 600, + "y": 617, + "width": 0, + "height": 44, + "strokeColor": "#1e1e1e", + "strokeWidth": 2, + "roughness": 1, + "seed": 1024, + "version": 1, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 44 + ] + ], + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "providers-label", + "type": "text", + "x": 500, + "y": 664, + "width": 130, + "height": 20, + "strokeColor": "#1971c2", + "text": "LLM Providers", + "fontSize": 16, "fontFamily": 1, - "textAlign": "center", + "textAlign": "left", "verticalAlign": "top", - "containerId": null, - "originalText": "boundary markers\nbase64 \u00b7 homoglyph" + "roughness": 1, + "seed": 1025, + "version": 1, + "isDeleted": false, + "originalText": "LLM Providers" }, { - "id": "cmd_rect", + "id": "prov-claude-box", "type": "rectangle", - "x": 440, - "y": 398, - "width": 160, - "height": 65, - "angle": 0, - "strokeColor": "#e03131", - "backgroundColor": "#ffc9c9", + "x": 117.5, + "y": 688, + "width": 145, + "height": 45, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4030, + "roundness": { + "type": 3 + }, + "seed": 1026, "version": 1, - "versionNonce": 4030, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "cmd_text" + "id": "prov-claude-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "cmd_text", + "id": "prov-claude-text", "type": "text", - "x": 445, - "y": 403, - "width": 150, - "height": 48, - "angle": 0, - "strokeColor": "#c92a2a", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 4031, - "version": 1, - "versionNonce": 4031, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Cmd Filtering\nblocklist \u00b7 allowlist\nscheduled restrictions", - "fontSize": 11, + "x": 160.3, + "y": 699.25, + "width": 59.400000000000006, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Claude", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "cmd_rect", - "originalText": "Cmd Filtering\nblocklist \u00b7 allowlist\nscheduled restrictions" + "roughness": 1, + "seed": 1027, + "version": 1, + "isDeleted": false, + "originalText": "Claude", + "containerId": "prov-claude-box" }, { - "id": "cmd_sub", - "type": "text", - "x": 446, - "y": 422, - "width": 180, - "height": 30, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", + "id": "prov-gpt-box", + "type": "rectangle", + "x": 281.5, + "y": 688, + "width": 145, + "height": 45, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4032, - "version": 1, - "versionNonce": 4032, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "blocklist \u00b7 allowlist\nscheduled restrictions", - "fontSize": 11, + "roundness": { + "type": 3 + }, + "seed": 1028, + "version": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "prov-gpt-text", + "type": "text" + } + ] + }, + { + "id": "prov-gpt-text", + "type": "text", + "x": 339.15, + "y": 699.25, + "width": 29.700000000000003, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "GPT", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "blocklist \u00b7 allowlist\nscheduled restrictions" + "verticalAlign": "middle", + "roughness": 1, + "seed": 1029, + "version": 1, + "isDeleted": false, + "originalText": "GPT", + "containerId": "prov-gpt-box" }, { - "id": "audit_rect", + "id": "prov-gemini-box", "type": "rectangle", - "x": 620, - "y": 398, - "width": 160, - "height": 65, - "angle": 0, - "strokeColor": "#e03131", - "backgroundColor": "#ffc9c9", + "x": 445.5, + "y": 688, + "width": 145, + "height": 45, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4040, + "roundness": { + "type": 3 + }, + "seed": 1030, "version": 1, - "versionNonce": 4040, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "audit_text" + "id": "prov-gemini-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "audit_text", + "id": "prov-gemini-text", "type": "text", - "x": 625, - "y": 403, - "width": 150, - "height": 48, - "angle": 0, - "strokeColor": "#c92a2a", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 4041, - "version": 1, - "versionNonce": 4041, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Audit Log\nevery tool execution\nlogged to SQLite", - "fontSize": 11, + "x": 488.3, + "y": 699.25, + "width": 59.400000000000006, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Gemini", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "audit_rect", - "originalText": "Audit Log\nevery tool execution\nlogged to SQLite" + "roughness": 1, + "seed": 1031, + "version": 1, + "isDeleted": false, + "originalText": "Gemini", + "containerId": "prov-gemini-box" }, { - "id": "audit_sub", - "type": "text", - "x": 638, - "y": 422, - "width": 180, - "height": 30, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", + "id": "prov-groq-box", + "type": "rectangle", + "x": 609.5, + "y": 688, + "width": 145, + "height": 45, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4042, - "version": 1, - "versionNonce": 4042, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "every tool execution\nlogged to SQLite", - "fontSize": 11, + "roundness": { + "type": 3 + }, + "seed": 1032, + "version": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "prov-groq-text", + "type": "text" + } + ] + }, + { + "id": "prov-groq-text", + "type": "text", + "x": 662.2, + "y": 699.25, + "width": 39.6, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Groq", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "every tool execution\nlogged to SQLite" + "verticalAlign": "middle", + "roughness": 1, + "seed": 1033, + "version": 1, + "isDeleted": false, + "originalText": "Groq", + "containerId": "prov-groq-box" }, { - "id": "risk_rect", + "id": "prov-openrouter-box", "type": "rectangle", - "x": 80, - "y": 480, - "width": 340, - "height": 60, - "angle": 0, - "strokeColor": "#e03131", - "backgroundColor": "#ffe3e3", + "x": 773.5, + "y": 688, + "width": 145, + "height": 45, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4050, + "roundness": { + "type": 3 + }, + "seed": 1034, "version": 1, - "versionNonce": 4050, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "risk_text1" + "id": "prov-openrouter-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "risk_text1", + "id": "prov-openrouter-text", "type": "text", - "x": 85, - "y": 485, - "width": 330, - "height": 36, - "angle": 0, - "strokeColor": "#c92a2a", - "backgroundColor": "transparent", + "x": 796.5, + "y": 699.25, + "width": 99.00000000000001, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "OpenRouter", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "roughness": 1, + "seed": 1035, + "version": 1, + "isDeleted": false, + "originalText": "OpenRouter", + "containerId": "prov-openrouter-box" + }, + { + "id": "prov-cerebras-box", + "type": "rectangle", + "x": 937.5, + "y": 688, + "width": 145, + "height": 45, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4051, + "roundness": { + "type": 3 + }, + "seed": 1036, "version": 1, - "versionNonce": 4051, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Low\u2192notify Med\u2192approve High\u2192preview\nRubber-stamp: 5+ in 30s \u2192 warning", - "fontSize": 10, + "boundElements": [ + { + "id": "prov-cerebras-text", + "type": "text" + } + ] + }, + { + "id": "prov-cerebras-text", + "type": "text", + "x": 970.4, + "y": 699.25, + "width": 79.2, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Cerebras", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "risk_rect", - "originalText": "Low\u2192notify Med\u2192approve High\u2192preview\nRubber-stamp: 5+ in 30s \u2192 warning" + "roughness": 1, + "seed": 1037, + "version": 1, + "isDeleted": false, + "originalText": "Cerebras", + "containerId": "prov-cerebras-box" }, { - "id": "risk_text2", - "type": "text", - "x": 60, - "y": 510, - "width": 375, - "height": 18, - "angle": 0, - "strokeColor": "#c92a2a", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, + "id": "arrow-5", + "type": "arrow", + "x": 600, + "y": 733, + "width": 0, + "height": 44, + "strokeColor": "#1e1e1e", + "strokeWidth": 2, "roughness": 1, - "opacity": 100, - "seed": 4052, - "version": 1, - "versionNonce": 4052, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Rubber-stamp: 5+ in 30s \u2192 warning", - "fontSize": 11, - "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "Rubber-stamp: 5+ in 30s \u2192 warning" + "seed": 1038, + "version": 1, + "isDeleted": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 44 + ] + ], + "startArrowhead": null, + "endArrowhead": "arrow" }, { - "id": "auto_rect", + "id": "skills-container", "type": "rectangle", - "x": 440, - "y": 480, - "width": 340, - "height": 60, - "angle": 0, - "strokeColor": "#e03131", - "backgroundColor": "#ffe3e3", + "x": 200, + "y": 777, + "width": 800, + "height": 84, + "strokeColor": "#9c36b5", + "backgroundColor": "#f3d9fa", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 4060, - "version": 1, - "versionNonce": 4060, - "isDeleted": false, - "boundElements": [ - { - "type": "text", - "id": "auto_text1" - } - ], - "groupIds": [], - "frameId": null, "roundness": { "type": 3 - } + }, + "seed": 1039, + "version": 1, + "isDeleted": false, + "boundElements": [] }, { - "id": "auto_text1", + "id": "skills-label", "type": "text", - "x": 445, - "y": 485, - "width": 330, - "height": 36, - "angle": 0, - "strokeColor": "#c92a2a", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, + "x": 310, + "y": 786, + "width": 380, + "height": 25, + "strokeColor": "#862e9c", + "text": "100+ Skills · 80+ Agent Tools", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", "roughness": 1, - "opacity": 100, - "seed": 4061, + "seed": 1040, "version": 1, - "versionNonce": 4061, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Auto-approve: after 3 same-pattern OKs\nScheduled: restricted registry, no writes", - "fontSize": 10, - "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "auto_rect", - "originalText": "Auto-approve: after 3 same-pattern OKs\nScheduled: restricted registry, no writes" + "originalText": "100+ Skills · 80+ Agent Tools" }, { - "id": "auto_text2", + "id": "skills-detail", "type": "text", - "x": 445, - "y": 510, - "width": 375, + "x": 240, + "y": 816, + "width": 520, "height": 18, - "angle": 0, - "strokeColor": "#c92a2a", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 4062, - "version": 1, - "versionNonce": 4062, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Scheduled: restricted registry, no writes", - "fontSize": 11, + "strokeColor": "#495057", + "text": "Google Workspace (50+) · Data Sources · Personas · Recipes", + "fontSize": 14, "fontFamily": 1, - "textAlign": "center", + "textAlign": "left", "verticalAlign": "top", - "containerId": null, - "originalText": "Scheduled: restricted registry, no writes" + "roughness": 1, + "seed": 1041, + "version": 1, + "isDeleted": false, + "originalText": "Google Workspace (50+) · Data Sources · Personas · Recipes" }, { - "id": "arrow_safety_tools", + "id": "arrow-6", "type": "arrow", - "x": 430, - "y": 564, + "x": 600, + "y": 861, "width": 0, - "height": 30, - "angle": 0, + "height": 44, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, - "opacity": 100, - "seed": 5000, + "seed": 1042, "version": 1, - "versionNonce": 5000, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, "points": [ [ 0, @@ -1192,283 +962,320 @@ ], [ 0, - 30 + 44 ] ], "startArrowhead": null, "endArrowhead": "arrow" }, { - "id": "tools_rect", + "id": "services-label", + "type": "text", + "x": 550, + "y": 908, + "width": 80, + "height": 20, + "strokeColor": "#e67700", + "text": "Services", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "roughness": 1, + "seed": 1043, + "version": 1, + "isDeleted": false, + "originalText": "Services" + }, + { + "id": "svc-memory-box", "type": "rectangle", - "x": 285, - "y": 596, - "width": 290, - "height": 60, - "angle": 0, - "strokeColor": "#862e9c", - "backgroundColor": "#e5dbff", + "x": 110, + "y": 932, + "width": 150, + "height": 44, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 5010, + "roundness": { + "type": 3 + }, + "seed": 1044, "version": 1, - "versionNonce": 5010, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "tools_text" + "id": "svc-memory-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "tools_text", + "id": "svc-memory-text", "type": "text", - "x": 290, - "y": 604, - "width": 280, + "x": 155.3, + "y": 942.75, + "width": 59.400000000000006, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Memory", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "roughness": 1, + "seed": 1045, + "version": 1, + "isDeleted": false, + "originalText": "Memory", + "containerId": "svc-memory-box" + }, + { + "id": "svc-contacts-box", + "type": "rectangle", + "x": 276, + "y": 932, + "width": 150, "height": 44, - "angle": 0, - "strokeColor": "#862e9c", - "backgroundColor": "transparent", + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 5011, + "roundness": { + "type": 3 + }, + "seed": 1046, "version": 1, - "versionNonce": 5011, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Tool Executor\nbash \u00b7 file_read \u00b7 load_skills", - "fontSize": 13, - "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "tools_rect", - "originalText": "Tool Executor\nbash \u00b7 file_read \u00b7 load_skills" + "boundElements": [ + { + "id": "svc-contacts-text", + "type": "text" + } + ] }, { - "id": "tools_sub", + "id": "svc-contacts-text", "type": "text", - "x": 320, - "y": 630, - "width": 220, - "height": 18, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 5012, - "version": 1, - "versionNonce": 5012, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "bash \u00b7 file_read \u00b7 load_skills", - "fontSize": 13, + "x": 311.4, + "y": 942.75, + "width": 79.2, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Contacts", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "bash \u00b7 file_read \u00b7 load_skills" + "verticalAlign": "middle", + "roughness": 1, + "seed": 1047, + "version": 1, + "isDeleted": false, + "originalText": "Contacts", + "containerId": "svc-contacts-box" }, { - "id": "skills_rect", + "id": "svc-history-box", "type": "rectangle", - "x": 640, - "y": 596, + "x": 442, + "y": 932, "width": 150, - "height": 60, - "angle": 0, - "strokeColor": "#862e9c", - "backgroundColor": "#f3d9fa", + "height": 44, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 5020, + "roundness": { + "type": 3 + }, + "seed": 1048, "version": 1, - "versionNonce": 5020, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "skills_text" + "id": "svc-history-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "skills_text", + "id": "svc-history-text", "type": "text", - "x": 660, - "y": 608, - "width": 110, - "height": 35, - "angle": 0, - "strokeColor": "#862e9c", - "backgroundColor": "transparent", + "x": 482.35, + "y": 942.75, + "width": 69.30000000000001, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "History", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "roughness": 1, + "seed": 1049, + "version": 1, + "isDeleted": false, + "originalText": "History", + "containerId": "svc-history-box" + }, + { + "id": "svc-scheduler-box", + "type": "rectangle", + "x": 608, + "y": 932, + "width": 150, + "height": 44, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 5021, + "roundness": { + "type": 3 + }, + "seed": 1050, "version": 1, - "versionNonce": 5021, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "100+ Skills\n(plain text)", - "fontSize": 14, + "boundElements": [ + { + "id": "svc-scheduler-text", + "type": "text" + } + ] + }, + { + "id": "svc-scheduler-text", + "type": "text", + "x": 638.45, + "y": 942.75, + "width": 89.10000000000001, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Scheduler", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "skills_rect", - "originalText": "100+ Skills\n(plain text)" - }, - { - "id": "arrow_tools_skills", - "type": "arrow", - "x": 577, - "y": 626, - "width": 60, - "height": 0, - "angle": 0, - "strokeColor": "#862e9c", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, "roughness": 1, - "opacity": 100, - "seed": 5030, + "seed": 1051, "version": 1, - "versionNonce": 5030, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "points": [ - [ - 0, - 0 - ], - [ - 60, - 0 - ] - ], - "startArrowhead": null, - "endArrowhead": "arrow" + "originalText": "Scheduler", + "containerId": "svc-scheduler-box" }, { - "id": "daemon_rect", + "id": "svc-usage-box", "type": "rectangle", - "x": 60, - "y": 596, - "width": 160, - "height": 60, - "angle": 0, - "strokeColor": "#5c940d", - "backgroundColor": "#d8f5a2", + "x": 774, + "y": 932, + "width": 150, + "height": 44, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 5040, + "roundness": { + "type": 3 + }, + "seed": 1052, "version": 1, - "versionNonce": 5040, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "daemon_text" + "id": "svc-usage-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "daemon_text", + "id": "svc-usage-text", "type": "text", - "x": 80, - "y": 608, - "width": 120, - "height": 35, - "angle": 0, - "strokeColor": "#5c940d", - "backgroundColor": "transparent", + "x": 824.25, + "y": 942.75, + "width": 49.50000000000001, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Usage", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "roughness": 1, + "seed": 1053, + "version": 1, + "isDeleted": false, + "originalText": "Usage", + "containerId": "svc-usage-box" + }, + { + "id": "svc-backup-box", + "type": "rectangle", + "x": 940, + "y": 932, + "width": 150, + "height": 44, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 5041, + "roundness": { + "type": 3 + }, + "seed": 1054, "version": 1, - "versionNonce": 5041, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Sync Daemon\n(background)", - "fontSize": 14, + "boundElements": [ + { + "id": "svc-backup-text", + "type": "text" + } + ] + }, + { + "id": "svc-backup-text", + "type": "text", + "x": 985.3, + "y": 942.75, + "width": 59.400000000000006, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Backup", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "daemon_rect", - "originalText": "Sync Daemon\n(background)" + "roughness": 1, + "seed": 1055, + "version": 1, + "isDeleted": false, + "originalText": "Backup", + "containerId": "svc-backup-box" }, { - "id": "arrow_tools_sources", + "id": "arrow-7", "type": "arrow", - "x": 430, - "y": 658, + "x": 600, + "y": 976, "width": 0, - "height": 32, - "angle": 0, + "height": 44, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, - "opacity": 100, - "seed": 6000, + "seed": 1056, "version": 1, - "versionNonce": 6000, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, "points": [ [ 0, @@ -1476,548 +1283,497 @@ ], [ 0, - 32 + 44 ] ], "startArrowhead": null, "endArrowhead": "arrow" }, { - "id": "arrow_daemon_sources", - "type": "arrow", - "x": 140, - "y": 658, - "width": 0, - "height": 32, - "angle": 0, - "strokeColor": "#5c940d", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, + "id": "sources-label", + "type": "text", + "x": 510, + "y": 1023, + "width": 120, + "height": 20, + "strokeColor": "#d9480f", + "text": "Data Sources", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", "roughness": 1, - "opacity": 100, - "seed": 6001, + "seed": 1057, "version": 1, - "versionNonce": 6001, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "points": [ - [ - 0, - 0 - ], - [ - 0, - 32 - ] - ], - "startArrowhead": null, - "endArrowhead": "arrow" + "originalText": "Data Sources" }, { - "id": "sources_bg", + "id": "src-gmail-box", "type": "rectangle", - "x": 60, - "y": 692, - "width": 740, - "height": 100, - "angle": 0, - "strokeColor": "#e8590c", - "backgroundColor": "#fff4e6", + "x": 180, + "y": 1047, + "width": 195, + "height": 44, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6010, - "version": 1, - "versionNonce": 6010, - "isDeleted": false, - "boundElements": [], - "groupIds": [], - "frameId": null, "roundness": { "type": 3 - } + }, + "seed": 1058, + "version": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "src-gmail-text", + "type": "text" + } + ] }, { - "id": "sources_title", + "id": "src-gmail-text", "type": "text", - "x": 350, - "y": 698, - "width": 160, - "height": 20, - "angle": 0, - "strokeColor": "#e8590c", - "backgroundColor": "transparent", + "x": 252.75, + "y": 1057.75, + "width": 49.50000000000001, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "Gmail", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "roughness": 1, + "seed": 1059, + "version": 1, + "isDeleted": false, + "originalText": "Gmail", + "containerId": "src-gmail-box" + }, + { + "id": "src-whatsapp-box", + "type": "rectangle", + "x": 395, + "y": 1047, + "width": 195, + "height": 44, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd8a8", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6011, + "roundness": { + "type": 3 + }, + "seed": 1060, "version": 1, - "versionNonce": 6011, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Data Sources", - "fontSize": 16, + "boundElements": [ + { + "id": "src-whatsapp-text", + "type": "text" + } + ] + }, + { + "id": "src-whatsapp-text", + "type": "text", + "x": 452.9, + "y": 1057.75, + "width": 79.2, + "height": 22.5, + "strokeColor": "#1e1e1e", + "text": "WhatsApp", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "Data Sources" + "verticalAlign": "middle", + "roughness": 1, + "seed": 1061, + "version": 1, + "isDeleted": false, + "originalText": "WhatsApp", + "containerId": "src-whatsapp-box" }, { - "id": "s1", + "id": "src-slack-box", "type": "rectangle", - "x": 76, - "y": 728, - "width": 88, + "x": 610, + "y": 1047, + "width": 195, "height": 44, - "angle": 0, - "strokeColor": "#e8590c", + "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6100, + "roundness": { + "type": 3 + }, + "seed": 1062, "version": 1, - "versionNonce": 6100, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "s1t" + "id": "src-slack-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "s1t", + "id": "src-slack-text", "type": "text", - "x": 86, - "y": 738, - "width": 68, - "height": 25, - "angle": 0, + "x": 682.75, + "y": 1057.75, + "width": 49.50000000000001, + "height": 22.5, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 6101, - "version": 1, - "versionNonce": 6101, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Gmail", - "fontSize": 16, + "text": "Slack", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "s1", - "originalText": "Gmail" + "roughness": 1, + "seed": 1063, + "version": 1, + "isDeleted": false, + "originalText": "Slack", + "containerId": "src-slack-box" }, { - "id": "s2", + "id": "src-imessage-box", "type": "rectangle", - "x": 174, - "y": 728, - "width": 98, + "x": 825, + "y": 1047, + "width": 195, "height": 44, - "angle": 0, - "strokeColor": "#e8590c", + "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6110, + "roundness": { + "type": 3 + }, + "seed": 1064, "version": 1, - "versionNonce": 6110, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "s2t" + "id": "src-imessage-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "s2t", + "id": "src-imessage-text", "type": "text", - "x": 184, - "y": 738, - "width": 78, - "height": 25, - "angle": 0, + "x": 882.9, + "y": 1057.75, + "width": 79.2, + "height": 22.5, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 6111, - "version": 1, - "versionNonce": 6111, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "WhatsApp", - "fontSize": 16, + "text": "iMessage", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "s2", - "originalText": "WhatsApp" + "roughness": 1, + "seed": 1065, + "version": 1, + "isDeleted": false, + "originalText": "iMessage", + "containerId": "src-imessage-box" }, { - "id": "s3", + "id": "src2-applenotes-box", "type": "rectangle", - "x": 282, - "y": 728, - "width": 78, + "x": 180, + "y": 1101, + "width": 195, "height": 44, - "angle": 0, - "strokeColor": "#e8590c", + "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6120, + "roundness": { + "type": 3 + }, + "seed": 1066, "version": 1, - "versionNonce": 6120, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "s3t" + "id": "src2-applenotes-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "s3t", + "id": "src2-applenotes-text", "type": "text", - "x": 292, - "y": 738, - "width": 58, - "height": 25, - "angle": 0, + "x": 223.05, + "y": 1111.75, + "width": 108.9, + "height": 22.5, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 6121, - "version": 1, - "versionNonce": 6121, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Slack", - "fontSize": 16, + "text": "Apple Notes", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "s3", - "originalText": "Slack" + "roughness": 1, + "seed": 1067, + "version": 1, + "isDeleted": false, + "originalText": "Apple Notes", + "containerId": "src2-applenotes-box" }, { - "id": "s4", + "id": "src2-websearch-box", "type": "rectangle", - "x": 370, - "y": 728, - "width": 98, + "x": 395, + "y": 1101, + "width": 195, "height": 44, - "angle": 0, - "strokeColor": "#e8590c", + "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6130, + "roundness": { + "type": 3 + }, + "seed": 1068, "version": 1, - "versionNonce": 6130, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "s4t" + "id": "src2-websearch-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "s4t", + "id": "src2-websearch-text", "type": "text", - "x": 380, - "y": 738, - "width": 78, - "height": 25, - "angle": 0, + "x": 443, + "y": 1111.75, + "width": 99.00000000000001, + "height": 22.5, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 6131, - "version": 1, - "versionNonce": 6131, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "iMessage", - "fontSize": 16, + "text": "Web Search", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "s4", - "originalText": "iMessage" + "roughness": 1, + "seed": 1069, + "version": 1, + "isDeleted": false, + "originalText": "Web Search", + "containerId": "src2-websearch-box" }, { - "id": "s5", + "id": "src2-finance-box", "type": "rectangle", - "x": 478, - "y": 728, - "width": 98, + "x": 610, + "y": 1101, + "width": 195, "height": 44, - "angle": 0, - "strokeColor": "#e8590c", + "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6140, + "roundness": { + "type": 3 + }, + "seed": 1070, "version": 1, - "versionNonce": 6140, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "s5t" + "id": "src2-finance-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "s5t", + "id": "src2-finance-text", "type": "text", - "x": 488, - "y": 738, - "width": 78, - "height": 25, - "angle": 0, + "x": 672.85, + "y": 1111.75, + "width": 69.30000000000001, + "height": 22.5, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 6141, - "version": 1, - "versionNonce": 6141, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Contacts", - "fontSize": 16, + "text": "Finance", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "s5", - "originalText": "Contacts" + "roughness": 1, + "seed": 1071, + "version": 1, + "isDeleted": false, + "originalText": "Finance", + "containerId": "src2-finance-box" }, { - "id": "s6", + "id": "src2-contacts-box", "type": "rectangle", - "x": 586, - "y": 728, - "width": 68, + "x": 825, + "y": 1101, + "width": 195, "height": 44, - "angle": 0, - "strokeColor": "#e8590c", + "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6150, + "roundness": { + "type": 3 + }, + "seed": 1072, "version": 1, - "versionNonce": 6150, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "s6t" + "id": "src2-contacts-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "s6t", + "id": "src2-contacts-text", "type": "text", - "x": 596, - "y": 738, - "width": 48, - "height": 25, - "angle": 0, + "x": 882.9, + "y": 1111.75, + "width": 79.2, + "height": 22.5, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 1, - "roughness": 1, - "opacity": 100, - "seed": 6151, - "version": 1, - "versionNonce": 6151, - "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Web", - "fontSize": 16, + "text": "Contacts", + "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "s6", - "originalText": "Web" + "roughness": 1, + "seed": 1073, + "version": 1, + "isDeleted": false, + "originalText": "Contacts", + "containerId": "src2-contacts-box" }, { - "id": "s7", - "type": "rectangle", - "x": 664, - "y": 728, - "width": 120, + "id": "arrow-8", + "type": "arrow", + "x": 600, + "y": 1145, + "width": 0, "height": 44, - "angle": 0, - "strokeColor": "#e8590c", - "backgroundColor": "#ffd8a8", - "fillStyle": "solid", - "strokeWidth": 1, + "strokeColor": "#1e1e1e", + "strokeWidth": 2, "roughness": 1, - "opacity": 100, - "seed": 6160, + "seed": 1074, "version": 1, - "versionNonce": 6160, "isDeleted": false, - "boundElements": [ - { - "type": "text", - "id": "s7t" - } + "points": [ + [ + 0, + 0 + ], + [ + 0, + 44 + ] ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + "startArrowhead": null, + "endArrowhead": "arrow" }, { - "id": "s7t", - "type": "text", - "x": 674, - "y": 738, - "width": 100, - "height": 25, - "angle": 0, - "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", + "id": "daemon-container", + "type": "rectangle", + "x": 200, + "y": 1189, + "width": 800, + "height": 80, + "strokeColor": "#868e96", + "backgroundColor": "#f1f3f5", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 6161, + "roundness": { + "type": 3 + }, + "seed": 1075, "version": 1, - "versionNonce": 6161, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Notes +3", + "boundElements": [] + }, + { + "id": "daemon-label", + "type": "text", + "x": 400, + "y": 1194, + "width": 200, + "height": 20, + "strokeColor": "#495057", + "text": "Background Daemon", "fontSize": 16, "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "s7", - "originalText": "Notes +3" + "textAlign": "left", + "verticalAlign": "top", + "roughness": 1, + "seed": 1076, + "version": 1, + "isDeleted": false, + "originalText": "Background Daemon" + }, + { + "id": "daemon-detail", + "type": "text", + "x": 215, + "y": 1220, + "width": 570, + "height": 16, + "strokeColor": "#495057", + "text": "Job Queue (River) · Gmail Sync · Tasks · Reminders · Backup · launchd/systemd", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "roughness": 1, + "seed": 1077, + "version": 1, + "isDeleted": false, + "originalText": "Job Queue (River) · Gmail Sync · Tasks · Reminders · Backup · launchd/systemd" }, { - "id": "arrow_sources_db", + "id": "arrow-9", "type": "arrow", - "x": 430, - "y": 794, + "x": 600, + "y": 1269, "width": 0, - "height": 30, - "angle": 0, + "height": 44, "strokeColor": "#1e1e1e", - "backgroundColor": "transparent", - "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, - "opacity": 100, - "seed": 7000, + "seed": 1078, "version": 1, - "versionNonce": 7000, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, "points": [ [ 0, @@ -2025,104 +1781,172 @@ ], [ 0, - 30 + 44 ] ], "startArrowhead": null, "endArrowhead": "arrow" }, { - "id": "db_rect", + "id": "storage-label", + "type": "text", + "x": 555, + "y": 1316, + "width": 72, + "height": 20, + "strokeColor": "#495057", + "text": "Storage", + "fontSize": 16, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "roughness": 1, + "seed": 1079, + "version": 1, + "isDeleted": false, + "originalText": "Storage" + }, + { + "id": "st-sqlite-box", "type": "rectangle", - "x": 260, - "y": 826, - "width": 340, - "height": 55, - "angle": 0, - "strokeColor": "#2b8a3e", - "backgroundColor": "#d3f9d8", + "x": 195, + "y": 1340, + "width": 250, + "height": 48, + "strokeColor": "#1e1e1e", + "backgroundColor": "#dee2e6", "fillStyle": "solid", "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 7010, + "roundness": { + "type": 3 + }, + "seed": 1080, "version": 1, - "versionNonce": 7010, "isDeleted": false, "boundElements": [ { - "type": "text", - "id": "db_text" + "id": "st-sqlite-text", + "type": "text" } - ], - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - } + ] }, { - "id": "db_text", + "id": "st-sqlite-text", "type": "text", - "x": 265, - "y": 832, - "width": 330, - "height": 40, - "angle": 0, - "strokeColor": "#2b8a3e", - "backgroundColor": "transparent", + "x": 229.25, + "y": 1354.625, + "width": 181.50000000000003, + "height": 18.75, + "strokeColor": "#1e1e1e", + "text": "Local SQLite (~/.obk/)", + "fontSize": 15, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "roughness": 1, + "seed": 1081, + "version": 1, + "isDeleted": false, + "originalText": "Local SQLite (~/.obk/)", + "containerId": "st-sqlite-box" + }, + { + "id": "st-pg-box", + "type": "rectangle", + "x": 475, + "y": 1340, + "width": 250, + "height": 48, + "strokeColor": "#1e1e1e", + "backgroundColor": "#dee2e6", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 7011, + "roundness": { + "type": 3 + }, + "seed": 1082, "version": 1, - "versionNonce": 7011, "isDeleted": false, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "Local SQLite (~/.obk/)\nall data stays on your machine", - "fontSize": 13, + "boundElements": [ + { + "id": "st-pg-text", + "type": "text" + } + ] + }, + { + "id": "st-pg-text", + "type": "text", + "x": 513.375, + "y": 1354.625, + "width": 173.25, + "height": 18.75, + "strokeColor": "#1e1e1e", + "text": "PostgreSQL (optional)", + "fontSize": 15, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", - "containerId": "db_rect", - "originalText": "Local SQLite (~/.obk/)\nall data stays on your machine" + "roughness": 1, + "seed": 1083, + "version": 1, + "isDeleted": false, + "originalText": "PostgreSQL (optional)", + "containerId": "st-pg-box" }, { - "id": "db_sub", - "type": "text", - "x": 310, - "y": 856, - "width": 240, - "height": 18, - "angle": 0, - "strokeColor": "#868e96", - "backgroundColor": "transparent", + "id": "st-kr-box", + "type": "rectangle", + "x": 755, + "y": 1340, + "width": 250, + "height": 48, + "strokeColor": "#1e1e1e", + "backgroundColor": "#dee2e6", "fillStyle": "solid", - "strokeWidth": 1, + "strokeWidth": 2, "roughness": 1, "opacity": 100, - "seed": 7012, - "version": 1, - "versionNonce": 7012, - "isDeleted": true, - "boundElements": null, - "groupIds": [], - "frameId": null, - "roundness": null, - "text": "all data stays on your machine", - "fontSize": 14, + "roundness": { + "type": 3 + }, + "seed": 1084, + "version": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "st-kr-text", + "type": "text" + } + ] + }, + { + "id": "st-kr-text", + "type": "text", + "x": 838.75, + "y": 1354.625, + "width": 82.5, + "height": 18.75, + "strokeColor": "#1e1e1e", + "text": "OS Keyring", + "fontSize": 15, "fontFamily": 1, "textAlign": "center", - "verticalAlign": "top", - "containerId": null, - "originalText": "all data stays on your machine" + "verticalAlign": "middle", + "roughness": 1, + "seed": 1085, + "version": 1, + "isDeleted": false, + "originalText": "OS Keyring", + "containerId": "st-kr-box" } ], "appState": { "viewBackgroundColor": "#ffffff" - } + }, + "files": {} } \ No newline at end of file diff --git a/docs/architecture.png b/docs/architecture.png index 604dd798..d717c58f 100644 Binary files a/docs/architecture.png and b/docs/architecture.png differ diff --git a/go.mod b/go.mod index 0c73ec5f..b0635ad7 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/google/uuid v1.6.0 github.com/lib/pq v1.11.2 github.com/mattn/go-sqlite3 v1.14.34 + github.com/minio/minio-go/v7 v7.0.99 github.com/refraction-networking/utls v1.8.2 github.com/riverqueue/river v0.31.0 github.com/riverqueue/river/riverdriver/riversqlite v0.31.0 @@ -34,6 +35,18 @@ require ( modernc.org/sqlite v1.46.1 ) +require ( + github.com/go-ini/ini v1.67.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect +) + require ( al.essio.dev/pkg/shellescape v1.5.1 // indirect cloud.google.com/go/auth v0.18.2 // indirect @@ -88,7 +101,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.8.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect diff --git a/go.sum b/go.sum index 750bc3a6..b97f696d 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -167,8 +169,13 @@ github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -194,6 +201,12 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -233,6 +246,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 h1:KPpdlQLZcHfTMQRi6bFQ7ogNO0ltFT4PmtwTLW4W+14= github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -262,6 +277,7 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -300,6 +316,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -343,6 +361,8 @@ go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpu go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= diff --git a/internal/cli/backup/backup.go b/internal/cli/backup/backup.go new file mode 100644 index 00000000..ccd87b5c --- /dev/null +++ b/internal/cli/backup/backup.go @@ -0,0 +1,15 @@ +package backup + +import "github.com/spf13/cobra" + +var Cmd = &cobra.Command{ + Use: "backup", + Short: "Manage backups", +} + +func init() { + Cmd.AddCommand(nowCmd) + Cmd.AddCommand(listCmd) + Cmd.AddCommand(statusCmd) + Cmd.AddCommand(restoreCmd) +} diff --git a/internal/cli/backup/list.go b/internal/cli/backup/list.go new file mode 100644 index 00000000..3f8bb641 --- /dev/null +++ b/internal/cli/backup/list.go @@ -0,0 +1,57 @@ +package backup + +import ( + "fmt" + "strings" + "time" + + "github.com/73ai/openbotkit/config" + backupsvc "github.com/73ai/openbotkit/service/backup" + "github.com/spf13/cobra" +) + +var listCmd = &cobra.Command{ + Use: "list", + Short: "List available backup snapshots", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + ctx := cmd.Context() + backend, err := resolveBackend(ctx, cfg) + if err != nil { + return err + } + + svc := backupsvc.New(backend, config.Dir()) + snapshots, err := svc.ListSnapshots(ctx) + if err != nil { + return fmt.Errorf("list snapshots: %w", err) + } + + if len(snapshots) == 0 { + fmt.Println("No snapshots found.") + return nil + } + + for _, id := range snapshots { + fmt.Printf(" %s %s\n", formatSnapshotDate(id), id) + } + return nil + }, +} + +func formatSnapshotDate(id string) string { + // ID format: 20060102T150405Z- + ts := id + if idx := strings.Index(id, "-"); idx > 0 { + ts = id[:idx] + } + t, err := time.Parse("20060102T150405Z", ts) + if err != nil { + return " " + } + return t.Local().Format("2006-01-02 15:04:05") +} diff --git a/internal/cli/backup/list_test.go b/internal/cli/backup/list_test.go new file mode 100644 index 00000000..7207d3f6 --- /dev/null +++ b/internal/cli/backup/list_test.go @@ -0,0 +1,26 @@ +package backup + +import ( + "strings" + "testing" +) + +func TestFormatSnapshotDate(t *testing.T) { + // Valid ID with random suffix. + got := formatSnapshotDate("20260321T150405Z-abcd1234") + if !strings.Contains(got, "2026") || !strings.Contains(got, "03") { + t.Errorf("expected formatted date, got %q", got) + } + + // Invalid ID returns blank padding. + got = formatSnapshotDate("invalid") + if strings.TrimSpace(got) != "" { + t.Errorf("expected blank for invalid ID, got %q", got) + } + + // Old format ID without suffix (backwards compat). + got = formatSnapshotDate("20260321T150405Z") + if !strings.Contains(got, "2026") { + t.Errorf("expected formatted date for old format, got %q", got) + } +} diff --git a/internal/cli/backup/now.go b/internal/cli/backup/now.go new file mode 100644 index 00000000..947f8159 --- /dev/null +++ b/internal/cli/backup/now.go @@ -0,0 +1,49 @@ +package backup + +import ( + "fmt" + + "github.com/73ai/openbotkit/config" + backupsvc "github.com/73ai/openbotkit/service/backup" + "github.com/spf13/cobra" +) + +var nowCmd = &cobra.Command{ + Use: "now", + Short: "Run a backup immediately", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + ctx := cmd.Context() + backend, err := resolveBackend(ctx, cfg) + if err != nil { + return err + } + + svc := backupsvc.New(backend, config.Dir()) + result, err := svc.Run(ctx) + if err != nil { + return fmt.Errorf("backup failed: %w", err) + } + + fmt.Printf("Backup complete: %d changed, %d unchanged, %s uploaded in %s\n", + result.Changed, result.Skipped, formatBytes(result.Uploaded), result.Duration.Round(100*1e6)) + return nil + }, +} + +func formatBytes(b int64) string { + switch { + case b >= 1<<30: + return fmt.Sprintf("%.1f GB", float64(b)/(1<<30)) + case b >= 1<<20: + return fmt.Sprintf("%.1f MB", float64(b)/(1<<20)) + case b >= 1<<10: + return fmt.Sprintf("%.1f KB", float64(b)/(1<<10)) + default: + return fmt.Sprintf("%d B", b) + } +} diff --git a/internal/cli/backup/now_test.go b/internal/cli/backup/now_test.go new file mode 100644 index 00000000..bff7fff9 --- /dev/null +++ b/internal/cli/backup/now_test.go @@ -0,0 +1,26 @@ +package backup + +import "testing" + +func TestFormatBytes(t *testing.T) { + tests := []struct { + input int64 + want string + }{ + {0, "0 B"}, + {512, "512 B"}, + {1023, "1023 B"}, + {1024, "1.0 KB"}, + {1536, "1.5 KB"}, + {1048576, "1.0 MB"}, + {1572864, "1.5 MB"}, + {1073741824, "1.0 GB"}, + {1610612736, "1.5 GB"}, + } + for _, tt := range tests { + got := formatBytes(tt.input) + if got != tt.want { + t.Errorf("formatBytes(%d) = %q, want %q", tt.input, got, tt.want) + } + } +} diff --git a/internal/cli/backup/resolve.go b/internal/cli/backup/resolve.go new file mode 100644 index 00000000..ffcb4d50 --- /dev/null +++ b/internal/cli/backup/resolve.go @@ -0,0 +1,52 @@ +package backup + +import ( + "context" + "fmt" + "net/http" + + "github.com/73ai/openbotkit/config" + "github.com/73ai/openbotkit/oauth/google" + "github.com/73ai/openbotkit/provider" + backupsvc "github.com/73ai/openbotkit/service/backup" +) + +func resolveBackend(ctx context.Context, cfg *config.Config) (backupsvc.Backend, error) { + if cfg.Backup == nil || !cfg.Backup.Enabled { + return nil, fmt.Errorf("backup not configured — run 'obk setup' and select Backup") + } + + return backupsvc.ResolveBackend(ctx, backendOpts(cfg)) +} + +func backendOpts(cfg *config.Config) backupsvc.ResolveBackendOpts { + opts := backupsvc.ResolveBackendOpts{ + ResolveCred: provider.ResolveAPIKey, + BackupDest: cfg.Backup.Destination, + GoogleClient: makeGoogleClient(cfg), + } + if cfg.Backup.R2 != nil { + opts.R2Bucket = cfg.Backup.R2.Bucket + opts.R2Endpoint = cfg.Backup.R2.Endpoint + opts.R2AccessRef = cfg.Backup.R2.AccessKeyRef + opts.R2SecretRef = cfg.Backup.R2.SecretKeyRef + } + if cfg.Backup.GDrive != nil { + opts.GDriveFolderID = cfg.Backup.GDrive.FolderID + } + return opts +} + +func makeGoogleClient(cfg *config.Config) backupsvc.GoogleClientFactory { + return func(ctx context.Context, gcfg backupsvc.GoogleClientConfig) (*http.Client, error) { + gp := google.New(google.Config{ + CredentialsFile: cfg.GoogleCredentialsFile(), + TokenDBPath: cfg.GoogleTokenDBPath(), + }) + accounts, err := gp.Accounts(ctx) + if err != nil || len(accounts) == 0 { + return nil, fmt.Errorf("no Google account found — run 'obk setup'") + } + return gp.Client(ctx, accounts[0], gcfg.Scopes) + } +} diff --git a/internal/cli/backup/restore.go b/internal/cli/backup/restore.go new file mode 100644 index 00000000..0d899ee1 --- /dev/null +++ b/internal/cli/backup/restore.go @@ -0,0 +1,59 @@ +package backup + +import ( + "fmt" + + "github.com/73ai/openbotkit/config" + backupsvc "github.com/73ai/openbotkit/service/backup" + "github.com/spf13/cobra" +) + +var restoreCmd = &cobra.Command{ + Use: "restore ", + Short: "Restore from a backup snapshot", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + snapshotID := args[0] + + cfg, err := config.Load() + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + ctx := cmd.Context() + backend, err := resolveBackend(ctx, cfg) + if err != nil { + return err + } + + svc := backupsvc.New(backend, config.Dir()) + + force, _ := cmd.Flags().GetBool("force") + if !force { + manifest, err := svc.GetManifest(ctx, snapshotID) + if err != nil { + return fmt.Errorf("fetch manifest: %w", err) + } + fmt.Printf("This will overwrite %d files in %s\n", len(manifest.Files), config.Dir()) + fmt.Print("Continue? [y/N] ") + var confirm string + fmt.Scanln(&confirm) + if confirm != "y" && confirm != "Y" { + fmt.Println("Cancelled.") + return nil + } + } + + result, err := svc.Restore(ctx, snapshotID) + if err != nil { + return fmt.Errorf("restore failed: %w", err) + } + + fmt.Printf("Restored %d files from snapshot %s\n", result.Restored, snapshotID) + return nil + }, +} + +func init() { + restoreCmd.Flags().Bool("force", false, "Skip confirmation prompt") +} diff --git a/internal/cli/backup/status.go b/internal/cli/backup/status.go new file mode 100644 index 00000000..443ed72d --- /dev/null +++ b/internal/cli/backup/status.go @@ -0,0 +1,40 @@ +package backup + +import ( + "fmt" + "time" + + "github.com/73ai/openbotkit/config" + backupsvc "github.com/73ai/openbotkit/service/backup" + "github.com/spf13/cobra" +) + +var statusCmd = &cobra.Command{ + Use: "status", + Short: "Show last backup info", + RunE: func(cmd *cobra.Command, args []string) error { + manifest, err := backupsvc.LoadManifest(config.BackupLastManifestPath()) + if err != nil { + return fmt.Errorf("load last manifest: %w", err) + } + + if manifest.ID == "" { + fmt.Println("No backup has been run yet.") + fmt.Println("Run: obk backup now") + return nil + } + + ago := time.Since(manifest.Timestamp).Truncate(time.Second) + fmt.Printf("Last backup: %s (%s ago)\n", manifest.Timestamp.Local().Format("2006-01-02 15:04:05"), ago) + fmt.Printf("Snapshot: %s\n", manifest.ID) + fmt.Printf("Files: %d\n", len(manifest.Files)) + + var totalSize, totalCompressed int64 + for _, f := range manifest.Files { + totalSize += f.Size + totalCompressed += f.CompressedSize + } + fmt.Printf("Total size: %s (compressed: %s)\n", formatBytes(totalSize), formatBytes(totalCompressed)) + return nil + }, +} diff --git a/internal/cli/root.go b/internal/cli/root.go index ebf6318e..11913b3d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -5,6 +5,7 @@ import ( "os" applenotescli "github.com/73ai/openbotkit/internal/cli/applenotes" + backupcli "github.com/73ai/openbotkit/internal/cli/backup" contactscli "github.com/73ai/openbotkit/internal/cli/contacts" financecli "github.com/73ai/openbotkit/internal/cli/finance" "github.com/73ai/openbotkit/internal/cli/gmail" @@ -37,6 +38,7 @@ var versionCmd = &cobra.Command{ func init() { rootCmd.AddCommand(versionCmd) rootCmd.AddCommand(applenotescli.Cmd) + rootCmd.AddCommand(backupcli.Cmd) rootCmd.AddCommand(contactscli.Cmd) rootCmd.AddCommand(financecli.Cmd) rootCmd.AddCommand(imessagecli.Cmd) diff --git a/internal/cli/settings_cmd.go b/internal/cli/settings_cmd.go index b1570371..34e0ac19 100644 --- a/internal/cli/settings_cmd.go +++ b/internal/cli/settings_cmd.go @@ -3,10 +3,12 @@ package cli import ( "context" "fmt" + "net/http" "time" "github.com/73ai/openbotkit/config" settingstui "github.com/73ai/openbotkit/internal/settings/tui" + "github.com/73ai/openbotkit/oauth/google" "github.com/73ai/openbotkit/provider" _ "github.com/73ai/openbotkit/provider/anthropic" _ "github.com/73ai/openbotkit/provider/gemini" @@ -14,6 +16,7 @@ import ( _ "github.com/73ai/openbotkit/provider/openai" _ "github.com/73ai/openbotkit/provider/openrouter" _ "github.com/73ai/openbotkit/provider/zai" + backupsvc "github.com/73ai/openbotkit/service/backup" "github.com/73ai/openbotkit/settings" "github.com/spf13/cobra" ) @@ -30,6 +33,9 @@ var settingsCmd = &cobra.Command{ settings.WithStoreCred(provider.StoreCredential), settings.WithLoadCred(provider.LoadCredential), settings.WithVerifyProvider(verifyProviderKey), + settings.WithVerifyBackup(verifyBackupDest), + settings.WithSetupGDrive(setupGDriveBackup), + settings.WithTriggerBackup(triggerBackupNow), ) return settingstui.Run(svc) }, @@ -71,6 +77,104 @@ func verifyProviderKey(name string, pcfg config.ModelProviderConfig) error { return nil } +func verifyBackupDest(dest string, cfg *config.Config) error { + if dest != "r2" { + return nil + } + if cfg.Backup == nil || cfg.Backup.R2 == nil { + return fmt.Errorf("R2 not configured") + } + r2 := cfg.Backup.R2 + accessKey, err := provider.LoadCredential(r2.AccessKeyRef) + if err != nil { + return fmt.Errorf("load access key: %w", err) + } + secretKey, err := provider.LoadCredential(r2.SecretKeyRef) + if err != nil { + return fmt.Errorf("load secret key: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return backupsvc.ValidateR2(ctx, r2.Endpoint, accessKey, secretKey, r2.Bucket) +} + +func setupGDriveBackup(cfg *config.Config, folderName string) (string, error) { + gp := google.New(google.Config{ + CredentialsFile: cfg.GoogleCredentialsFile(), + TokenDBPath: cfg.GoogleTokenDBPath(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + accounts, _ := gp.Accounts(ctx) + var account string + if len(accounts) > 0 { + account = accounts[0] + } + + scopes := []string{"https://www.googleapis.com/auth/drive.file"} + if _, err := gp.GrantScopes(ctx, account, scopes); err != nil { + return "", fmt.Errorf("Google auth: %w", err) + } + + httpClient, err := gp.Client(ctx, account, scopes) + if err != nil { + return "", fmt.Errorf("get Drive client: %w", err) + } + + folderID, err := backupsvc.FindOrCreateDriveFolder(ctx, httpClient, folderName) + if err != nil { + return "", fmt.Errorf("create Drive folder: %w", err) + } + + return folderID, nil +} + +func triggerBackupNow(cfg *config.Config) error { + if cfg.Backup == nil || !cfg.Backup.Enabled { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + opts := backupsvc.ResolveBackendOpts{ + ResolveCred: provider.ResolveAPIKey, + BackupDest: cfg.Backup.Destination, + GoogleClient: func(ctx context.Context, gcfg backupsvc.GoogleClientConfig) (*http.Client, error) { + gp := google.New(google.Config{ + CredentialsFile: cfg.GoogleCredentialsFile(), + TokenDBPath: cfg.GoogleTokenDBPath(), + }) + accounts, _ := gp.Accounts(ctx) + if len(accounts) == 0 { + return nil, fmt.Errorf("no Google account found") + } + return gp.Client(ctx, accounts[0], gcfg.Scopes) + }, + } + if cfg.Backup.R2 != nil { + opts.R2Bucket = cfg.Backup.R2.Bucket + opts.R2Endpoint = cfg.Backup.R2.Endpoint + opts.R2AccessRef = cfg.Backup.R2.AccessKeyRef + opts.R2SecretRef = cfg.Backup.R2.SecretKeyRef + } + if cfg.Backup.GDrive != nil { + opts.GDriveFolderID = cfg.Backup.GDrive.FolderID + } + + backend, err := backupsvc.ResolveBackend(ctx, opts) + if err != nil { + return err + } + + svc := backupsvc.New(backend, config.Dir()) + _, err = svc.Run(ctx) + return err +} + func init() { rootCmd.AddCommand(settingsCmd) } diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 0b2e86fd..78db40f3 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -11,6 +11,7 @@ import ( "github.com/charmbracelet/huh" "github.com/73ai/openbotkit/config" + backupsvc "github.com/73ai/openbotkit/service/backup" "github.com/73ai/openbotkit/internal/skills" "github.com/73ai/openbotkit/internal/tty" "github.com/73ai/openbotkit/oauth/google" @@ -72,6 +73,7 @@ var setupCmd = &cobra.Command{ huh.NewOption("Google Contacts", "people"), } sourceOptions = append(sourceOptions, huh.NewOption("Slack", "slack")) + sourceOptions = append(sourceOptions, huh.NewOption("Backup (Cloudflare R2 or Google Drive)", "backup")) if runtime.GOOS == "darwin" { sourceOptions = append(sourceOptions, huh.NewOption("Apple Notes", "applenotes")) sourceOptions = append(sourceOptions, huh.NewOption("Apple Contacts", "applecontacts")) @@ -181,6 +183,10 @@ var setupCmd = &cobra.Command{ if err := setupSlack(cfg); err != nil { return err } + case "backup": + if err := setupBackup(cfg); err != nil { + return err + } } } @@ -248,6 +254,8 @@ var setupCmd = &cobra.Command{ fmt.Println(" - Slack is ready! Try: obk slack channels") case "telegram": fmt.Println(" - Telegram bot is ready! Send it a message.") + case "backup": + fmt.Println(" - Backup configured! Run `obk backup now` for your first backup.") } } return nil @@ -716,6 +724,189 @@ func setupSlack(cfg *config.Config) error { return nil } +func setupBackup(cfg *config.Config) error { + fmt.Println("\n -- Backup Setup --") + + var destination string + err := huh.NewForm( + huh.NewGroup( + huh.NewSelect[string](). + Title("Where would you like to back up to?"). + Options( + huh.NewOption("Cloudflare R2 (S3-compatible)", "r2"), + huh.NewOption("Google Drive", "gdrive"), + ). + Value(&destination), + ), + ).Run() + if err != nil { + return err + } + + if cfg.Backup == nil { + cfg.Backup = &config.BackupConfig{} + } + cfg.Backup.Destination = destination + + switch destination { + case "r2": + if err := setupBackupR2(cfg); err != nil { + return err + } + case "gdrive": + if err := setupBackupGDrive(cfg); err != nil { + return err + } + } + + cfg.Backup.Enabled = true + cfg.Backup.Schedule = "6h" + + if err := config.EnsureSourceDir("backup"); err != nil { + return fmt.Errorf("create backup dir: %w", err) + } + + if err := config.LinkSource("backup"); err != nil { + return fmt.Errorf("link source: %w", err) + } + + fmt.Println(" Backup configured!") + return nil +} + +func setupBackupR2(cfg *config.Config) error { + var bucket, endpoint, accessKey, secretKey string + err := huh.NewForm( + huh.NewGroup( + huh.NewInput(). + Title("R2 Bucket name"). + Description("Cloudflare Dashboard → R2 Object Storage → your bucket"). + Value(&bucket), + huh.NewInput(). + Title("R2 Endpoint"). + Description("Bucket → Settings → S3 API → copy the endpoint URL"). + Placeholder("https://.r2.cloudflarestorage.com"). + Value(&endpoint), + huh.NewInput(). + Title("Access Key ID"). + Description("R2 → Manage R2 API Tokens → Create API Token"). + Value(&accessKey), + huh.NewInput(). + Title("Secret Access Key"). + Description("Shown once when you create the API token above"). + EchoMode(huh.EchoModePassword). + Value(&secretKey), + ), + ).Run() + if err != nil { + return err + } + + bucket = strings.TrimSpace(bucket) + endpoint = strings.TrimSpace(endpoint) + accessKey = strings.TrimSpace(accessKey) + secretKey = strings.TrimSpace(secretKey) + + if bucket == "" || endpoint == "" || accessKey == "" || secretKey == "" { + return fmt.Errorf("all R2 fields are required") + } + + fmt.Print(" Validating R2 connection... ") + ctx := context.Background() + if err := backupsvc.ValidateR2(ctx, endpoint, accessKey, secretKey, bucket); err != nil { + fmt.Println("failed!") + return fmt.Errorf("R2 validation: %w", err) + } + fmt.Println("ok!") + + accessKeyRef := "keychain:obk/r2-access-key" + secretKeyRef := "keychain:obk/r2-secret-key" + + if err := provider.StoreCredential(accessKeyRef, accessKey); err != nil { + return fmt.Errorf("store R2 access key: %w", err) + } + if err := provider.StoreCredential(secretKeyRef, secretKey); err != nil { + return fmt.Errorf("store R2 secret key: %w", err) + } + + if cfg.Backup.R2 == nil { + cfg.Backup.R2 = &config.R2Config{} + } + cfg.Backup.R2.Bucket = bucket + cfg.Backup.R2.Endpoint = endpoint + cfg.Backup.R2.AccessKeyRef = accessKeyRef + cfg.Backup.R2.SecretKeyRef = secretKeyRef + + return nil +} + +func setupBackupGDrive(cfg *config.Config) error { + gp := google.New(google.Config{ + CredentialsFile: cfg.GoogleCredentialsFile(), + TokenDBPath: cfg.GoogleTokenDBPath(), + }) + + ctx := context.Background() + + accounts, _ := gp.Accounts(ctx) + var account string + if len(accounts) > 0 { + account = accounts[0] + } + + scopes := []string{"https://www.googleapis.com/auth/drive.file"} + + if account != "" { + granted, err := gp.GrantedScopes(ctx, account) + if err == nil { + hasDriveScope := false + for _, s := range granted { + if s == "https://www.googleapis.com/auth/drive.file" { + hasDriveScope = true + break + } + } + if !hasDriveScope { + fmt.Println(" Granting Google Drive file scope...") + _, err := gp.GrantScopes(ctx, account, scopes) + if err != nil { + return fmt.Errorf("grant Drive scope: %w", err) + } + } + } + } else { + fmt.Println(" No Google account found. Starting OAuth flow...") + email, err := gp.GrantScopes(ctx, "", scopes) + if err != nil { + return fmt.Errorf("Google auth: %w", err) + } + account = email + fmt.Printf(" Authenticated as %s\n", email) + } + + folderName := "obk-backup" + + httpClient, err := gp.Client(ctx, account, scopes) + if err != nil { + return fmt.Errorf("get Drive client: %w", err) + } + + fmt.Print(" Finding or creating folder... ") + folderID, err := backupsvc.FindOrCreateDriveFolder(ctx, httpClient, folderName) + if err != nil { + fmt.Println("failed!") + return fmt.Errorf("create Drive folder: %w", err) + } + fmt.Printf("ok! (ID: %s)\n", folderID) + + if cfg.Backup.GDrive == nil { + cfg.Backup.GDrive = &config.GDriveConfig{} + } + cfg.Backup.GDrive.FolderID = folderID + + return nil +} + func isGWSService(s string) bool { for _, svc := range gwsServices { if s == svc { diff --git a/internal/settings/tui/backup_wizard.go b/internal/settings/tui/backup_wizard.go new file mode 100644 index 00000000..3c021821 --- /dev/null +++ b/internal/settings/tui/backup_wizard.go @@ -0,0 +1,372 @@ +package tui + +import ( + "fmt" + "strings" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/huh" + "github.com/73ai/openbotkit/config" + backupsvc "github.com/73ai/openbotkit/service/backup" + "github.com/73ai/openbotkit/settings" +) + +// --- Backup wizard: destination → credentials → verify → save --- + +// enterBackupWizard starts the first-time wizard (destination not yet configured). +func (m model) enterBackupWizard() (model, tea.Cmd) { + m.wizardBackupSnapshot = cloneBackupConfig(m.svc.Config().Backup) + + cfg := m.svc.Config() + dest := settings.BackupDest(cfg) + if dest != "" && !m.svc.IsBackupDestConfigured(dest) { + // Destination set but not authenticated — skip picker, go to auth. + d := dest + m.wizardBackupDest = &d + return m.enterBackupAuth(dest) + } + return m.enterBackupDest() +} + +// enterDestinationChange handles transactional destination change from settings tree. +func (m model) enterDestinationChange() (model, tea.Cmd) { + m.wizardBackupSnapshot = cloneBackupConfig(m.svc.Config().Backup) + return m.enterBackupDest() +} + +func (m model) enterBackupDest() (model, tea.Cmd) { + m.state = stateBackupDest + m.wizardError = "" + + dest := "" + cfg := m.svc.Config() + if cfg.Backup != nil && cfg.Backup.Destination != "" { + dest = cfg.Backup.Destination + } + m.wizardBackupDest = &dest + + m.form = huh.NewForm( + huh.NewGroup( + huh.NewSelect[string](). + Title("Where would you like to back up to?"). + Options( + huh.NewOption("Cloudflare R2 (S3-compatible)", "r2"), + huh.NewOption("Google Drive", "gdrive"), + ). + Value(m.wizardBackupDest), + ), + ) + return m, m.form.Init() +} + +func (m model) updateBackupDest(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + if msg.String() == "esc" { + return m.rollbackBackup() + } + } + + form, cmd := m.form.Update(msg) + if f, ok := form.(*huh.Form); ok { + m.form = f + } + + if m.form.State == huh.StateCompleted { + newDest := *m.wizardBackupDest + + // If destination is already authenticated, commit immediately. + if m.svc.IsBackupDestConfigured(newDest) { + return m.commitDestinationSwap(newDest) + } + + // Otherwise, start auth flow for the new destination. + return m.enterBackupAuth(newDest) + } + + return m, cmd +} + +// enterBackupAuth routes to the correct auth flow for a destination. +func (m model) enterBackupAuth(dest string) (model, tea.Cmd) { + switch dest { + case "r2": + return m.enterBackupR2Creds() + case "gdrive": + return m.enterBackupGDriveCreds() + } + return m.rollbackBackup() +} + +// commitDestinationSwap saves the destination change and triggers backup if stale. +func (m model) commitDestinationSwap(dest string) (model, tea.Cmd) { + cfg := m.svc.Config() + ensureBackup(cfg) + cfg.Backup.Destination = dest + cfg.Backup.Enabled = true + if cfg.Backup.Schedule == "" { + cfg.Backup.Schedule = "6h" + } + + if err := m.svc.Save(); err != nil { + m.flash = fmt.Sprintf("Error saving: %v", err) + return m.rollbackBackup() + } + + m.flash = "Destination updated!" + m.wizardBackupSnapshot = nil + m.state = stateBrowse + m.form = nil + m.wizardBackupDest = nil + m.svc.RebuildTree() + m.rebuildRows() + m.viewport.SetContent(m.renderTree()) + return m, tea.Batch( + tea.Tick(2*time.Second, func(time.Time) tea.Msg { + return flashMsg{} + }), + triggerBackupIfStaleCmd(m.svc), + ) +} + +func (m model) enterBackupR2Creds() (model, tea.Cmd) { + m.state = stateBackupCreds + m.wizardError = "" + + bucket := "" + endpoint := "" + ak := "" + sk := "" + + cfg := m.svc.Config() + if cfg.Backup != nil && cfg.Backup.R2 != nil { + bucket = cfg.Backup.R2.Bucket + endpoint = cfg.Backup.R2.Endpoint + } + + m.wizardBackupBucket = &bucket + m.wizardBackupEndpoint = &endpoint + m.wizardBackupAK = &ak + m.wizardBackupSK = &sk + + m.form = huh.NewForm( + huh.NewGroup( + huh.NewInput(). + Title("R2 Bucket name"). + Description("Cloudflare Dashboard → R2 Object Storage → your bucket"). + Value(m.wizardBackupBucket), + huh.NewInput(). + Title("R2 Endpoint"). + Description("Bucket → Settings → S3 API → copy the endpoint URL"). + Placeholder("https://.r2.cloudflarestorage.com"). + Value(m.wizardBackupEndpoint), + huh.NewInput(). + Title("Access Key ID"). + Description("R2 → Manage R2 API Tokens → Create API Token"). + Value(m.wizardBackupAK), + huh.NewInput(). + Title("Secret Access Key"). + Description("Shown once when you create the API token above"). + EchoMode(huh.EchoModePassword). + Value(m.wizardBackupSK), + ), + ) + return m, m.form.Init() +} + +func (m model) enterBackupGDriveCreds() (model, tea.Cmd) { + cfg := m.svc.Config() + ensureBackup(cfg) + cfg.Backup.Destination = "gdrive" + + m.state = stateVerifying + m.wizardError = "" + return m, tea.Batch( + m.wizardSpinner.Tick, + setupGDriveCmd(m.svc, "obk-backup"), + ) +} + +func (m model) updateBackupCreds(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + if msg.String() == "esc" { + return m.rollbackBackup() + } + } + + form, cmd := m.form.Update(msg) + if f, ok := form.(*huh.Form); ok { + m.form = f + } + + if m.form.State == huh.StateCompleted { + return m.handleR2CredsComplete() + } + + return m, cmd +} + +func (m model) handleR2CredsComplete() (model, tea.Cmd) { + bucket := strings.TrimSpace(*m.wizardBackupBucket) + endpoint := strings.TrimSpace(*m.wizardBackupEndpoint) + ak := strings.TrimSpace(*m.wizardBackupAK) + sk := strings.TrimSpace(*m.wizardBackupSK) + + if bucket == "" || endpoint == "" || ak == "" || sk == "" { + m.wizardError = "All R2 fields are required" + return m.enterBackupR2Creds() + } + + akRef := "keychain:obk/r2-access-key" + skRef := "keychain:obk/r2-secret-key" + + if err := m.svc.StoreCredential(akRef, ak); err != nil { + m.wizardError = fmt.Sprintf("Store access key: %v", err) + return m.enterBackupR2Creds() + } + if err := m.svc.StoreCredential(skRef, sk); err != nil { + m.wizardError = fmt.Sprintf("Store secret key: %v", err) + return m.enterBackupR2Creds() + } + + cfg := m.svc.Config() + ensureBackup(cfg) + cfg.Backup.Destination = "r2" + if cfg.Backup.R2 == nil { + cfg.Backup.R2 = &config.R2Config{} + } + cfg.Backup.R2.Bucket = bucket + cfg.Backup.R2.Endpoint = endpoint + cfg.Backup.R2.AccessKeyRef = akRef + cfg.Backup.R2.SecretKeyRef = skRef + + m.state = stateVerifying + m.wizardError = "" + return m, tea.Batch( + m.wizardSpinner.Tick, + verifyBackupCmd(m.svc, "r2"), + ) +} + +// saveBackup completes the wizard: enables backup, saves, triggers if stale. +func (m model) saveBackup() (model, tea.Cmd) { + cfg := m.svc.Config() + cfg.Backup.Enabled = true + if cfg.Backup.Schedule == "" { + cfg.Backup.Schedule = "6h" + } + + if err := m.svc.Save(); err != nil { + m.flash = fmt.Sprintf("Error saving: %v", err) + return m.rollbackBackup() + } + + m.flash = "Backup configured and enabled!" + m.wizardBackupSnapshot = nil + m.state = stateBrowse + m.form = nil + m.wizardBackupDest = nil + m.svc.RebuildTree() + m.rebuildRows() + m.viewport.SetContent(m.renderTree()) + return m, tea.Batch( + tea.Tick(2*time.Second, func(time.Time) tea.Msg { + return flashMsg{} + }), + triggerBackupIfStaleCmd(m.svc), + ) +} + +// rollbackBackup reverts config to the snapshot taken before the wizard started. +func (m model) rollbackBackup() (model, tea.Cmd) { + m.svc.Config().Backup = m.wizardBackupSnapshot + m.wizardBackupSnapshot = nil + m.state = stateBrowse + m.form = nil + m.wizardBackupDest = nil + m.svc.RebuildTree() + m.rebuildRows() + m.viewport.SetContent(m.renderTree()) + return m, nil +} + +func verifyBackupCmd(svc *settings.Service, dest string) tea.Cmd { + return func() tea.Msg { + err := svc.VerifyBackup(dest, svc.Config()) + return backupVerifyResultMsg{err: err} + } +} + +func setupGDriveCmd(svc *settings.Service, folderName string) tea.Cmd { + return func() tea.Msg { + folderID, err := svc.SetupGDrive(svc.Config(), folderName) + if err != nil { + return backupVerifyResultMsg{err: err} + } + return backupVerifyResultMsg{folderID: folderID} + } +} + +// triggerBackupIfStaleCmd triggers a backup if the last one is older than the schedule. +func triggerBackupIfStaleCmd(svc *settings.Service) tea.Cmd { + return func() tea.Msg { + cfg := svc.Config() + if cfg.Backup == nil || !cfg.Backup.Enabled { + return nil + } + + schedule := parseSchedule(cfg.Backup.Schedule) + if schedule == 0 { + return nil // manual only + } + + manifest, err := backupsvc.LoadManifest(config.BackupLastManifestPath()) + if err != nil { + return nil + } + + if manifest.ID != "" && time.Since(manifest.Timestamp) < schedule { + return nil // recent enough + } + + err = svc.TriggerBackup() + return backupTriggeredMsg{err: err} + } +} + +func parseSchedule(s string) time.Duration { + switch s { + case "6h": + return 6 * time.Hour + case "12h": + return 12 * time.Hour + case "24h": + return 24 * time.Hour + default: + return 0 + } +} + +func ensureBackup(c *config.Config) { + if c.Backup == nil { + c.Backup = &config.BackupConfig{} + } +} + +func cloneBackupConfig(b *config.BackupConfig) *config.BackupConfig { + if b == nil { + return nil + } + clone := *b + if b.R2 != nil { + r2 := *b.R2 + clone.R2 = &r2 + } + if b.GDrive != nil { + gd := *b.GDrive + clone.GDrive = &gd + } + return &clone +} diff --git a/internal/settings/tui/backup_wizard_test.go b/internal/settings/tui/backup_wizard_test.go new file mode 100644 index 00000000..d56b308d --- /dev/null +++ b/internal/settings/tui/backup_wizard_test.go @@ -0,0 +1,256 @@ +package tui + +import ( + "testing" + + "github.com/73ai/openbotkit/config" + "github.com/73ai/openbotkit/settings" +) + +func testSvc(cfg *config.Config) *settings.Service { + creds := make(map[string]string) + return settings.New(cfg, + settings.WithSaveFn(func(*config.Config) error { return nil }), + settings.WithStoreCred(func(ref, value string) error { + creds[ref] = value + return nil + }), + settings.WithLoadCred(func(ref string) (string, error) { + return creds[ref], nil + }), + ) +} + +func TestCloneBackupConfigNil(t *testing.T) { + clone := cloneBackupConfig(nil) + if clone != nil { + t.Error("cloning nil should return nil") + } +} + +func TestCloneBackupConfigDeepCopy(t *testing.T) { + original := &config.BackupConfig{ + Enabled: true, + Destination: "r2", + Schedule: "6h", + R2: &config.R2Config{ + Bucket: "my-bucket", + Endpoint: "https://endpoint", + }, + GDrive: &config.GDriveConfig{ + FolderID: "folder123", + }, + } + + clone := cloneBackupConfig(original) + + // Verify values copied. + if clone.Destination != "r2" || clone.Schedule != "6h" || !clone.Enabled { + t.Error("top-level fields not copied") + } + if clone.R2.Bucket != "my-bucket" { + t.Error("R2 fields not copied") + } + if clone.GDrive.FolderID != "folder123" { + t.Error("GDrive fields not copied") + } + + // Verify deep copy — mutating clone shouldn't affect original. + clone.Destination = "gdrive" + clone.R2.Bucket = "other-bucket" + clone.GDrive.FolderID = "other-folder" + + if original.Destination != "r2" { + t.Error("mutating clone affected original destination") + } + if original.R2.Bucket != "my-bucket" { + t.Error("mutating clone affected original R2") + } + if original.GDrive.FolderID != "folder123" { + t.Error("mutating clone affected original GDrive") + } +} + +func TestParseSchedule(t *testing.T) { + tests := []struct { + input string + hours int + }{ + {"6h", 6}, + {"12h", 12}, + {"24h", 24}, + {"", 0}, + {"invalid", 0}, + } + for _, tt := range tests { + d := parseSchedule(tt.input) + gotHours := int(d.Hours()) + if gotHours != tt.hours { + t.Errorf("parseSchedule(%q) = %d hours, want %d", tt.input, gotHours, tt.hours) + } + } +} + +func TestBackupDest(t *testing.T) { + if settings.BackupDest(config.Default()) != "" { + t.Error("should return empty for nil backup") + } + + cfg := config.Default() + cfg.Backup = &config.BackupConfig{Destination: "r2"} + if settings.BackupDest(cfg) != "r2" { + t.Error("should return r2") + } +} + +func TestEnsureBackup(t *testing.T) { + cfg := config.Default() + if cfg.Backup != nil { + t.Fatal("precondition: backup should be nil") + } + ensureBackup(cfg) + if cfg.Backup == nil { + t.Error("ensureBackup should create backup config") + } +} + +func TestRollbackRestoresSnapshot(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Enabled: true, + Destination: "r2", + Schedule: "12h", + } + svc := testSvc(cfg) + m := newModel(svc) + + // Snapshot the config. + m.wizardBackupSnapshot = cloneBackupConfig(cfg.Backup) + + // Mutate config (simulating wizard mid-flow). + cfg.Backup.Destination = "gdrive" + cfg.Backup.Enabled = false + + // Rollback. + m, _ = m.rollbackBackup() + + if cfg.Backup.Destination != "r2" { + t.Errorf("destination not reverted: got %q, want r2", cfg.Backup.Destination) + } + if !cfg.Backup.Enabled { + t.Error("enabled not reverted: got false, want true") + } + if cfg.Backup.Schedule != "12h" { + t.Errorf("schedule not reverted: got %q, want 12h", cfg.Backup.Schedule) + } + if m.state != stateBrowse { + t.Error("state should be stateBrowse after rollback") + } + if m.wizardBackupSnapshot != nil { + t.Error("snapshot should be cleared after rollback") + } +} + +func TestEnterBackupWizardSkipsDestWhenSet(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Destination: "r2", + // R2 not configured — should skip dest picker, go to auth (R2 creds). + } + svc := testSvc(cfg) + m := newModel(svc) + + m, _ = m.enterBackupWizard() + + if m.state != stateBackupCreds { + t.Errorf("should go to stateBackupCreds when dest set but not authed, got state %d", m.state) + } + if m.wizardBackupSnapshot == nil { + t.Error("snapshot should be set") + } +} + +func TestEnterBackupWizardShowsDestWhenEmpty(t *testing.T) { + cfg := config.Default() + svc := testSvc(cfg) + m := newModel(svc) + + m, _ = m.enterBackupWizard() + + if m.state != stateBackupDest { + t.Errorf("should go to stateBackupDest when no dest set, got state %d", m.state) + } +} + +func TestCommitDestinationSwapSetsConfig(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Enabled: true, + Destination: "r2", + R2: &config.R2Config{ + Bucket: "b", Endpoint: "e", + AccessKeyRef: "ak", SecretKeyRef: "sk", + }, + GDrive: &config.GDriveConfig{FolderID: "f123"}, + } + svc := testSvc(cfg) + m := newModel(svc) + m.wizardBackupSnapshot = cloneBackupConfig(cfg.Backup) + + m, _ = m.commitDestinationSwap("gdrive") + + if cfg.Backup.Destination != "gdrive" { + t.Errorf("destination should be gdrive, got %q", cfg.Backup.Destination) + } + if !cfg.Backup.Enabled { + t.Error("should stay enabled when swapping to authenticated dest") + } + if m.state != stateBrowse { + t.Error("should return to stateBrowse") + } + if m.wizardBackupSnapshot != nil { + t.Error("snapshot should be cleared on commit") + } +} + +func TestSaveBackupSetsDefaults(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Destination: "r2", + R2: &config.R2Config{ + Bucket: "b", Endpoint: "e", + AccessKeyRef: "ak", SecretKeyRef: "sk", + }, + } + svc := testSvc(cfg) + m := newModel(svc) + + m, _ = m.saveBackup() + + if !cfg.Backup.Enabled { + t.Error("saveBackup should set enabled=true") + } + if cfg.Backup.Schedule != "6h" { + t.Errorf("saveBackup should default schedule to 6h, got %q", cfg.Backup.Schedule) + } +} + +func TestSaveBackupPreservesExistingSchedule(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Destination: "r2", + Schedule: "24h", + R2: &config.R2Config{ + Bucket: "b", Endpoint: "e", + AccessKeyRef: "ak", SecretKeyRef: "sk", + }, + } + svc := testSvc(cfg) + m := newModel(svc) + + m, _ = m.saveBackup() + + if cfg.Backup.Schedule != "24h" { + t.Errorf("saveBackup should preserve existing schedule, got %q", cfg.Backup.Schedule) + } +} diff --git a/internal/settings/tui/model.go b/internal/settings/tui/model.go index 2a894c0d..3413d093 100644 --- a/internal/settings/tui/model.go +++ b/internal/settings/tui/model.go @@ -26,6 +26,8 @@ const ( stateProviderAuth stateVerifying stateModelSelect + stateBackupDest + stateBackupCreds ) type flashMsg struct{} @@ -43,6 +45,17 @@ type verifyModelResultMsg struct { err error } +// backupVerifyResultMsg is returned by async backup verification. +type backupVerifyResultMsg struct { + folderID string // set for GDrive setup + err error +} + +// backupTriggeredMsg is returned after an async backup trigger attempt. +type backupTriggeredMsg struct { + err error +} + // modelsLoadedMsg is returned when background model loading completes. type modelsLoadedMsg struct { provider string @@ -75,6 +88,14 @@ type model struct { wizardSpinner spinner.Model wizardAPIKey *string wizardError string + + // Backup wizard state + wizardBackupDest *string + wizardBackupBucket *string + wizardBackupEndpoint *string + wizardBackupAK *string + wizardBackupSK *string + wizardBackupSnapshot *config.BackupConfig // for transactional rollback } func newModel(svc *settings.Service) model { @@ -113,6 +134,10 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m.updateVerifying(msg) case stateModelSelect: return m.updateModelSelect(msg) + case stateBackupDest: + return m.updateBackupDest(msg) + case stateBackupCreds: + return m.updateBackupCreds(msg) default: return m.updateBrowse(msg) } @@ -135,6 +160,17 @@ func (m model) updateBrowse(msg tea.Msg) (tea.Model, tea.Cmd) { m.viewport.SetContent(m.renderTree()) return m, nil + case backupTriggeredMsg: + if msg.err != nil { + m.flash = fmt.Sprintf("Backup failed: %v", msg.err) + } else { + m.flash = "Backup started" + } + m.viewport.SetContent(m.renderTree()) + return m, tea.Tick(2*time.Second, func(time.Time) tea.Msg { + return flashMsg{} + }) + case modelsLoadedMsg: // Silently update cache, no UI change. if msg.err == nil && len(msg.models) > 0 { @@ -193,8 +229,18 @@ func (m model) handleEnter() (tea.Model, tea.Cmd) { if r.node.Field != nil { f := r.node.Field + // Backup wizard — only when not yet configured. + if f.Key == "backup.enabled" && f.ReadOnly != nil && f.ReadOnly(m.svc.Config()) { + return m.enterBackupWizard() + } + + // Destination change — transactional flow. + if f.Key == "backup.destination" { + return m.enterDestinationChange() + } + if f.ReadOnly != nil && f.ReadOnly(m.svc.Config()) { - m.flash = "Locked by profile — change profile to edit" + m.flash = "Locked — configure prerequisites first" m.viewport.SetContent(m.renderTree()) return m, tea.Tick(2*time.Second, func(time.Time) tea.Msg { return flashMsg{} @@ -257,14 +303,28 @@ func (m model) updateEdit(msg tea.Msg) (tea.Model, tea.Cmd) { } } + // Rebuild tree when backup config changes (shows/hides R2 category). + if strings.HasPrefix(m.editField.Key, "backup.") { + m.svc.RebuildTree() + } + + // Trigger backup when re-enabling (false → true). + var triggerCmd tea.Cmd + if m.editField.Key == "backup.enabled" && value == "true" { + triggerCmd = triggerBackupIfStaleCmd(m.svc) + } + m.state = stateBrowse m.form = nil m.editField = nil m.rebuildRows() m.viewport.SetContent(m.renderTree()) - return m, tea.Tick(2*time.Second, func(time.Time) tea.Msg { - return flashMsg{} - }) + return m, tea.Batch( + tea.Tick(2*time.Second, func(time.Time) tea.Msg { + return flashMsg{} + }), + triggerCmd, + ) } return m, cmd @@ -473,6 +533,24 @@ func (m model) updateVerifying(msg tea.Msg) (tea.Model, tea.Cmd) { m.wizardSpinner, cmd = m.wizardSpinner.Update(msg) return m, cmd + case backupVerifyResultMsg: + if msg.err != nil { + m.flash = fmt.Sprintf("Backup verification failed: %v", msg.err) + ret, _ := m.rollbackBackup() + return ret, tea.Tick(3*time.Second, func(time.Time) tea.Msg { + return flashMsg{} + }) + } + if msg.folderID != "" { + cfg := m.svc.Config() + ensureBackup(cfg) + if cfg.Backup.GDrive == nil { + cfg.Backup.GDrive = &config.GDriveConfig{} + } + cfg.Backup.GDrive.FolderID = msg.folderID + } + return m.saveBackup() + case verifyResultMsg: if msg.err != nil { m.wizardError = fmt.Sprintf("%s verification failed: %v", msg.provider, msg.err) @@ -491,6 +569,9 @@ func (m model) updateVerifying(msg tea.Msg) (tea.Model, tea.Cmd) { case tea.KeyMsg: if msg.String() == "esc" { + if m.wizardBackupSnapshot != nil { + return m.rollbackBackup() + } m.state = stateBrowse m.form = nil m.viewport.SetContent(m.renderTree()) @@ -673,7 +754,8 @@ func (m model) renderTree() string { func (m model) View() string { switch m.state { - case stateEdit, stateProfileSelect, stateProviderAuth, stateModelSelect: + case stateEdit, stateProfileSelect, stateProviderAuth, stateModelSelect, + stateBackupDest, stateBackupCreds: if m.form != nil { var b strings.Builder if m.wizardError != "" { @@ -686,6 +768,13 @@ func (m model) View() string { return b.String() } case stateVerifying: + if m.wizardBackupDest != nil && *m.wizardBackupDest != "" { + label := "Verifying backup connection..." + if *m.wizardBackupDest == "gdrive" { + label = "Setting up Google Drive (check your browser)..." + } + return fmt.Sprintf("\n %s %s\n", m.wizardSpinner.View(), label) + } provName := "" if m.wizardProvIdx < len(m.wizardProviders) { provName = m.wizardProviders[m.wizardProvIdx] diff --git a/service/backup/backend.go b/service/backup/backend.go new file mode 100644 index 00000000..cc828c8a --- /dev/null +++ b/service/backup/backend.go @@ -0,0 +1,101 @@ +package backup + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" +) + +type Backend interface { + Put(ctx context.Context, key string, r io.Reader, size int64) error + Get(ctx context.Context, key string) (io.ReadCloser, error) + Head(ctx context.Context, key string) (bool, error) + List(ctx context.Context, prefix string) ([]string, error) + Delete(ctx context.Context, key string) error +} + +type LocalBackend struct { + root string +} + +func NewLocalBackend(root string) *LocalBackend { + return &LocalBackend{root: root} +} + +func (b *LocalBackend) Put(_ context.Context, key string, r io.Reader, _ int64) error { + path := filepath.Join(b.root, key) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return fmt.Errorf("create dir for %s: %w", key, err) + } + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("create %s: %w", key, err) + } + defer f.Close() + if _, err := io.Copy(f, r); err != nil { + return fmt.Errorf("write %s: %w", key, err) + } + return nil +} + +func (b *LocalBackend) Get(_ context.Context, key string) (io.ReadCloser, error) { + path := filepath.Join(b.root, key) + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open %s: %w", key, err) + } + return f, nil +} + +func (b *LocalBackend) Head(_ context.Context, key string) (bool, error) { + path := filepath.Join(b.root, key) + _, err := os.Stat(path) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (b *LocalBackend) List(_ context.Context, prefix string) ([]string, error) { + dir := filepath.Join(b.root, prefix) + var keys []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if info.IsDir() { + return nil + } + rel, err := filepath.Rel(b.root, path) + if err != nil { + return err + } + keys = append(keys, strings.ReplaceAll(rel, string(filepath.Separator), "/")) + return nil + }) + if err != nil { + return nil, fmt.Errorf("list %s: %w", prefix, err) + } + sort.Strings(keys) + return keys, nil +} + +func (b *LocalBackend) Delete(_ context.Context, key string) error { + path := filepath.Join(b.root, key) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("delete %s: %w", key, err) + } + return nil +} + +var _ Backend = (*LocalBackend)(nil) diff --git a/service/backup/backup.go b/service/backup/backup.go new file mode 100644 index 00000000..b5fe2548 --- /dev/null +++ b/service/backup/backup.go @@ -0,0 +1,224 @@ +package backup + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/klauspost/compress/zstd" + + "github.com/73ai/openbotkit/config" +) + +type RunResult struct { + Changed int + Skipped int + Uploaded int64 + Duration time.Duration +} + +type Service struct { + backend Backend + baseDir string + manifestPath string + stagingDir string +} + +func New(backend Backend, baseDir string) *Service { + return &Service{ + backend: backend, + baseDir: baseDir, + manifestPath: config.BackupLastManifestPath(), + stagingDir: config.BackupStagingDir(), + } +} + +func NewWithPaths(backend Backend, baseDir, manifestPath, stagingDir string) *Service { + return &Service{ + backend: backend, + baseDir: baseDir, + manifestPath: manifestPath, + stagingDir: stagingDir, + } +} + +func (s *Service) Run(ctx context.Context) (*RunResult, error) { + start := time.Now() + + lastManifest, err := LoadManifest(s.manifestPath) + if err != nil { + return nil, fmt.Errorf("load last manifest: %w", err) + } + + files, err := ScanFiles(s.baseDir) + if err != nil { + return nil, fmt.Errorf("scan files: %w", err) + } + + stagingDir := s.stagingDir + if err := os.MkdirAll(stagingDir, 0700); err != nil { + return nil, fmt.Errorf("create staging dir: %w", err) + } + defer os.RemoveAll(stagingDir) + + hashes := make(map[string]string) + stagedPaths := make(map[string]string) + + for _, rel := range files { + absPath := filepath.Join(s.baseDir, rel) + var filePath string + + if strings.HasSuffix(rel, ".db") { + vacuumed, err := VacuumInto(absPath, stagingDir, rel) + if err != nil { + return nil, fmt.Errorf("vacuum %s: %w", rel, err) + } + filePath = vacuumed + } else { + filePath = absPath + } + + hash, err := hashFile(filePath) + if err != nil { + return nil, fmt.Errorf("hash %s: %w", rel, err) + } + hashes[rel] = "sha256:" + hash + stagedPaths[rel] = filePath + } + + diff := DiffManifest(lastManifest, hashes) + + hostname, _ := os.Hostname() + manifest := NewManifest(hostname) + + var result RunResult + + for _, rel := range diff.Changed { + filePath := stagedPaths[rel] + hash := hashes[rel] + + compressed, err := compressFile(filePath) + if err != nil { + return nil, fmt.Errorf("compress %s: %w", rel, err) + } + + objectKey := objectKeyFromHash(hash) + exists, err := s.backend.Head(ctx, objectKey) + if err != nil { + return nil, fmt.Errorf("check object %s: %w", objectKey, err) + } + + if !exists { + reader := bytes.NewReader(compressed) + if err := s.backend.Put(ctx, objectKey, reader, int64(len(compressed))); err != nil { + return nil, fmt.Errorf("upload %s: %w", rel, err) + } + result.Uploaded += int64(len(compressed)) + } + + info, err := os.Stat(filePath) + if err != nil { + return nil, fmt.Errorf("stat %s: %w", rel, err) + } + + manifest.Files[rel] = ManifestFile{ + Hash: hash, + Size: info.Size(), + CompressedSize: int64(len(compressed)), + } + result.Changed++ + } + + for rel, hash := range hashes { + if _, ok := manifest.Files[rel]; ok { + continue + } + prev, ok := lastManifest.Files[rel] + if ok { + manifest.Files[rel] = prev + } else { + info, _ := os.Stat(stagedPaths[rel]) + size := int64(0) + if info != nil { + size = info.Size() + } + manifest.Files[rel] = ManifestFile{ + Hash: hash, + Size: size, + } + } + result.Skipped++ + } + + manifestKey := fmt.Sprintf("snapshots/%s.json", manifest.ID) + manifestData, err := marshalManifest(manifest) + if err != nil { + return nil, fmt.Errorf("marshal manifest: %w", err) + } + if err := s.backend.Put(ctx, manifestKey, bytes.NewReader(manifestData), int64(len(manifestData))); err != nil { + return nil, fmt.Errorf("upload manifest: %w", err) + } + + if err := os.MkdirAll(filepath.Dir(s.manifestPath), 0700); err != nil { + return nil, fmt.Errorf("create backup dir: %w", err) + } + if err := SaveManifest(s.manifestPath, manifest); err != nil { + return nil, fmt.Errorf("save local manifest: %w", err) + } + + result.Duration = time.Since(start) + return &result, nil +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func compressFile(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var buf bytes.Buffer + w, err := zstd.NewWriter(&buf) + if err != nil { + return nil, err + } + if _, err := io.Copy(w, f); err != nil { + w.Close() + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func objectKeyFromHash(hash string) string { + hex := strings.TrimPrefix(hash, "sha256:") + return fmt.Sprintf("objects/%s/%s", hex[:2], hex) +} + +func marshalManifest(m *Manifest) ([]byte, error) { + return json.MarshalIndent(m, "", " ") +} diff --git a/service/backup/backup_test.go b/service/backup/backup_test.go new file mode 100644 index 00000000..a504b3a1 --- /dev/null +++ b/service/backup/backup_test.go @@ -0,0 +1,777 @@ +package backup + +import ( + "bytes" + "context" + "database/sql" + "io" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestManifestDiff(t *testing.T) { + old := &Manifest{ + Files: map[string]ManifestFile{ + "config.yaml": {Hash: "sha256:aaa"}, + "gmail/data.db": {Hash: "sha256:bbb"}, + "removed.db": {Hash: "sha256:ccc"}, + }, + } + + current := map[string]string{ + "config.yaml": "sha256:aaa", // unchanged + "gmail/data.db": "sha256:ddd", // changed + "new/data.db": "sha256:eee", // new + } + + diff := DiffManifest(old, current) + + changedSet := make(map[string]bool) + for _, c := range diff.Changed { + changedSet[c] = true + } + if !changedSet["gmail/data.db"] { + t.Error("expected gmail/data.db to be changed") + } + if !changedSet["new/data.db"] { + t.Error("expected new/data.db to be changed (new)") + } + if changedSet["config.yaml"] { + t.Error("config.yaml should not be changed") + } + + removedSet := make(map[string]bool) + for _, r := range diff.Removed { + removedSet[r] = true + } + if !removedSet["removed.db"] { + t.Error("expected removed.db to be in removed list") + } +} + +func TestManifestDiffEmpty(t *testing.T) { + old := &Manifest{Files: make(map[string]ManifestFile)} + current := map[string]string{} + diff := DiffManifest(old, current) + if len(diff.Changed) != 0 { + t.Errorf("expected 0 changed, got %d", len(diff.Changed)) + } + if len(diff.Removed) != 0 { + t.Errorf("expected 0 removed, got %d", len(diff.Removed)) + } +} + +func TestManifestLoadSave(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "manifest.json") + + m := NewManifest("testhost") + m.Files["config.yaml"] = ManifestFile{ + Hash: "sha256:abc123", + Size: 1024, + CompressedSize: 512, + } + + if err := SaveManifest(path, m); err != nil { + t.Fatalf("save: %v", err) + } + + loaded, err := LoadManifest(path) + if err != nil { + t.Fatalf("load: %v", err) + } + + if loaded.Hostname != "testhost" { + t.Errorf("hostname = %q, want testhost", loaded.Hostname) + } + if len(loaded.Files) != 1 { + t.Errorf("files = %d, want 1", len(loaded.Files)) + } + f := loaded.Files["config.yaml"] + if f.Hash != "sha256:abc123" { + t.Errorf("hash = %q, want sha256:abc123", f.Hash) + } + if f.Size != 1024 { + t.Errorf("size = %d, want 1024", f.Size) + } +} + +func TestManifestLoadMissing(t *testing.T) { + m, err := LoadManifest("/nonexistent/manifest.json") + if err != nil { + t.Fatalf("expected nil error for missing file, got: %v", err) + } + if m.Files == nil { + t.Error("expected empty but non-nil Files map") + } + if len(m.Files) != 0 { + t.Errorf("expected 0 files, got %d", len(m.Files)) + } +} + +func TestScanFiles(t *testing.T) { + dir := t.TempDir() + + // Create included files. + mkFile(t, dir, "config.yaml") + mkFile(t, dir, "gmail/data.db") + mkFile(t, dir, "whatsapp/data.db") + mkFile(t, dir, "whatsapp/session.db") + mkFile(t, dir, "learnings/topic/note.md") + mkFile(t, dir, "models/custom.json") + mkFile(t, dir, "providers/google/creds.json") + mkFile(t, dir, "skills/email/metadata.yaml") + mkFile(t, dir, "env") + mkFile(t, dir, "ngrok.yml") + mkFile(t, dir, "applenotes/config.json") + mkFile(t, dir, "applecontacts/config.json") + + // Create excluded files. + mkFile(t, dir, "gmail/data.db-wal") + mkFile(t, dir, "gmail/data.db-shm") + mkFile(t, dir, "daemon.log") + mkFile(t, dir, "server.log") + mkFile(t, dir, "jobs.db") + mkFile(t, dir, "backup/last_manifest.json") + mkFile(t, dir, "scratch/session/tmp.txt") + mkFile(t, dir, "bin/obk") + mkFile(t, dir, "something.lock") + + files, err := ScanFiles(dir) + if err != nil { + t.Fatalf("scan: %v", err) + } + + fileSet := make(map[string]bool) + for _, f := range files { + fileSet[f] = true + } + + expected := []string{ + "config.yaml", "gmail/data.db", "whatsapp/data.db", + "whatsapp/session.db", "learnings/topic/note.md", + "models/custom.json", "providers/google/creds.json", + "skills/email/metadata.yaml", "env", "ngrok.yml", + "applenotes/config.json", "applecontacts/config.json", + } + for _, e := range expected { + if !fileSet[e] { + t.Errorf("expected %q to be included", e) + } + } + + excluded := []string{ + "gmail/data.db-wal", "gmail/data.db-shm", "daemon.log", + "server.log", "jobs.db", "backup/last_manifest.json", + "scratch/session/tmp.txt", "bin/obk", "something.lock", + } + for _, e := range excluded { + if fileSet[e] { + t.Errorf("expected %q to be excluded", e) + } + } +} + +func TestScanFilesSkipsSymlinks(t *testing.T) { + dir := t.TempDir() + + mkFile(t, dir, "config.yaml") + // Create a symlink that points back to the base dir (infinite loop risk). + if err := os.Symlink(dir, filepath.Join(dir, "learnings")); err != nil { + t.Skip("symlinks not supported on this OS") + } + + files, err := ScanFiles(dir) + if err != nil { + t.Fatalf("scan: %v", err) + } + + for _, f := range files { + if f == "learnings" || filepath.Dir(f) == "learnings" { + t.Errorf("symlink target should not be followed: found %q", f) + } + } +} + +func TestLocalBackendPutGetHeadListDelete(t *testing.T) { + dir := t.TempDir() + backend := NewLocalBackend(dir) + ctx := context.Background() + + data := []byte("hello world") + if err := backend.Put(ctx, "objects/ab/test", bytes.NewReader(data), int64(len(data))); err != nil { + t.Fatalf("put: %v", err) + } + + exists, err := backend.Head(ctx, "objects/ab/test") + if err != nil { + t.Fatalf("head: %v", err) + } + if !exists { + t.Error("expected object to exist") + } + + exists, err = backend.Head(ctx, "objects/ab/missing") + if err != nil { + t.Fatalf("head missing: %v", err) + } + if exists { + t.Error("expected object to not exist") + } + + rc, err := backend.Get(ctx, "objects/ab/test") + if err != nil { + t.Fatalf("get: %v", err) + } + got, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != "hello world" { + t.Errorf("got %q, want %q", got, "hello world") + } + + keys, err := backend.List(ctx, "objects/") + if err != nil { + t.Fatalf("list: %v", err) + } + if len(keys) != 1 || keys[0] != "objects/ab/test" { + t.Errorf("list = %v, want [objects/ab/test]", keys) + } + + if err := backend.Delete(ctx, "objects/ab/test"); err != nil { + t.Fatalf("delete: %v", err) + } + exists, _ = backend.Head(ctx, "objects/ab/test") + if exists { + t.Error("expected object to be deleted") + } +} + +func TestLocalBackendListEmpty(t *testing.T) { + dir := t.TempDir() + backend := NewLocalBackend(dir) + keys, err := backend.List(context.Background(), "nonexistent/") + if err != nil { + t.Fatalf("list empty: %v", err) + } + if len(keys) != 0 { + t.Errorf("expected 0 keys, got %d", len(keys)) + } +} + +func TestLocalBackendDeleteNonExistent(t *testing.T) { + dir := t.TempDir() + backend := NewLocalBackend(dir) + if err := backend.Delete(context.Background(), "nonexistent"); err != nil { + t.Fatalf("delete nonexistent should succeed: %v", err) + } +} + +func TestObjectKeyFromHash(t *testing.T) { + key := objectKeyFromHash("sha256:abcdef0123456789") + want := "objects/ab/abcdef0123456789" + if key != want { + t.Errorf("objectKeyFromHash = %q, want %q", key, want) + } +} + +func TestCompressDecompress(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + content := "hello world! this is a test of compression." + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + + compressed, err := compressFile(path) + if err != nil { + t.Fatalf("compress: %v", err) + } + if len(compressed) == 0 { + t.Fatal("compressed output is empty") + } + + decompressed, err := decompressData(compressed) + if err != nil { + t.Fatalf("decompress: %v", err) + } + + if string(decompressed) != content { + t.Errorf("round-trip mismatch: got %q, want %q", decompressed, content) + } +} + +func TestVacuumInto(t *testing.T) { + dir := t.TempDir() + stagingDir := t.TempDir() + + // Create a real SQLite database. + dbPath := filepath.Join(dir, "gmail", "data.db") + if err := os.MkdirAll(filepath.Dir(dbPath), 0700); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE test (id INTEGER PRIMARY KEY, val TEXT)"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO test (val) VALUES ('hello')"); err != nil { + t.Fatal(err) + } + db.Close() + + vacuumed, err := VacuumInto(dbPath, stagingDir, "gmail/data.db") + if err != nil { + t.Fatalf("vacuum into: %v", err) + } + + // The vacuumed file should be at stagingDir/gmail/data.db. + expectedPath := filepath.Join(stagingDir, "gmail", "data.db") + if vacuumed != expectedPath { + t.Errorf("vacuumed path = %q, want %q", vacuumed, expectedPath) + } + + // Verify the vacuumed database is readable. + vdb, err := sql.Open("sqlite", vacuumed) + if err != nil { + t.Fatal(err) + } + defer vdb.Close() + var val string + if err := vdb.QueryRow("SELECT val FROM test WHERE id = 1").Scan(&val); err != nil { + t.Fatalf("read from vacuumed db: %v", err) + } + if val != "hello" { + t.Errorf("val = %q, want hello", val) + } +} + +func TestVacuumIntoQuoteInPath(t *testing.T) { + dir := t.TempDir() + stagingDir := t.TempDir() + + // Create a database in a directory with a single quote in the name. + dbPath := filepath.Join(dir, "it's-a-test", "data.db") + if err := os.MkdirAll(filepath.Dir(dbPath), 0700); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE test (val TEXT)"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO test (val) VALUES ('quoted')"); err != nil { + t.Fatal(err) + } + db.Close() + + vacuumed, err := VacuumInto(dbPath, stagingDir, "it's-a-test/data.db") + if err != nil { + t.Fatalf("vacuum with quote in path: %v", err) + } + + vdb, err := sql.Open("sqlite", vacuumed) + if err != nil { + t.Fatal(err) + } + defer vdb.Close() + var val string + if err := vdb.QueryRow("SELECT val FROM test").Scan(&val); err != nil { + t.Fatalf("read from vacuumed db: %v", err) + } + if val != "quoted" { + t.Errorf("val = %q, want quoted", val) + } +} + +func TestVacuumIntoNoCollision(t *testing.T) { + dir := t.TempDir() + stagingDir := t.TempDir() + + // Create two databases with the same filename in different directories. + for _, sub := range []string{"gmail", "whatsapp"} { + dbPath := filepath.Join(dir, sub, "data.db") + if err := os.MkdirAll(filepath.Dir(dbPath), 0700); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE test (source TEXT)"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO test (source) VALUES (?)", sub); err != nil { + t.Fatal(err) + } + db.Close() + } + + // Vacuum both — they should NOT collide. + v1, err := VacuumInto(filepath.Join(dir, "gmail", "data.db"), stagingDir, "gmail/data.db") + if err != nil { + t.Fatal(err) + } + v2, err := VacuumInto(filepath.Join(dir, "whatsapp", "data.db"), stagingDir, "whatsapp/data.db") + if err != nil { + t.Fatal(err) + } + + if v1 == v2 { + t.Fatalf("path collision: both vacuumed to %q", v1) + } + + // Verify each database has the correct data. + for _, tc := range []struct { + path string + source string + }{ + {v1, "gmail"}, + {v2, "whatsapp"}, + } { + db, err := sql.Open("sqlite", tc.path) + if err != nil { + t.Fatal(err) + } + var source string + if err := db.QueryRow("SELECT source FROM test").Scan(&source); err != nil { + t.Fatal(err) + } + db.Close() + if source != tc.source { + t.Errorf("source = %q, want %q (path: %s)", source, tc.source, tc.path) + } + } +} + +func TestServiceRun(t *testing.T) { + baseDir := t.TempDir() + remoteDir := t.TempDir() + manifestPath := filepath.Join(t.TempDir(), "last_manifest.json") + stagingDir := t.TempDir() + + mkFileWithContent(t, baseDir, "config.yaml", "mode: local") + mkFileWithContent(t, baseDir, "learnings/topic/note.md", "some learning") + + backend := NewLocalBackend(remoteDir) + svc := NewWithPaths(backend, baseDir, manifestPath, stagingDir) + ctx := context.Background() + + // First run: everything is new. + result, err := svc.Run(ctx) + if err != nil { + t.Fatalf("first run: %v", err) + } + if result.Changed != 2 { + t.Errorf("first run: changed = %d, want 2", result.Changed) + } + if result.Skipped != 0 { + t.Errorf("first run: skipped = %d, want 0", result.Skipped) + } + if result.Uploaded == 0 { + t.Error("first run: expected some bytes uploaded") + } + + // Verify manifest was saved. + manifest, err := LoadManifest(manifestPath) + if err != nil { + t.Fatalf("load manifest: %v", err) + } + if len(manifest.Files) != 2 { + t.Errorf("manifest files = %d, want 2", len(manifest.Files)) + } + + // Verify objects were uploaded. + objects, err := backend.List(ctx, "objects/") + if err != nil { + t.Fatalf("list objects: %v", err) + } + if len(objects) != 2 { + t.Errorf("uploaded objects = %d, want 2", len(objects)) + } + + // Verify snapshot manifest was uploaded. + snapshots, err := backend.List(ctx, "snapshots/") + if err != nil { + t.Fatalf("list snapshots: %v", err) + } + if len(snapshots) != 1 { + t.Errorf("snapshots = %d, want 1", len(snapshots)) + } + + // Second run: nothing changed. + result2, err := svc.Run(ctx) + if err != nil { + t.Fatalf("second run: %v", err) + } + if result2.Changed != 0 { + t.Errorf("second run: changed = %d, want 0", result2.Changed) + } + if result2.Skipped != 2 { + t.Errorf("second run: skipped = %d, want 2", result2.Skipped) + } + if result2.Uploaded != 0 { + t.Errorf("second run: uploaded = %d, want 0", result2.Uploaded) + } +} + +func TestServiceRunIncremental(t *testing.T) { + baseDir := t.TempDir() + remoteDir := t.TempDir() + manifestPath := filepath.Join(t.TempDir(), "last_manifest.json") + stagingDir := t.TempDir() + + mkFileWithContent(t, baseDir, "config.yaml", "mode: local") + backend := NewLocalBackend(remoteDir) + svc := NewWithPaths(backend, baseDir, manifestPath, stagingDir) + ctx := context.Background() + + // First run. + result1, err := svc.Run(ctx) + if err != nil { + t.Fatalf("first run: %v", err) + } + if result1.Changed != 1 { + t.Fatalf("first run: changed = %d, want 1", result1.Changed) + } + + // Modify the file. + mkFileWithContent(t, baseDir, "config.yaml", "mode: remote") + + // Second run: should detect the change. + result2, err := svc.Run(ctx) + if err != nil { + t.Fatalf("second run: %v", err) + } + if result2.Changed != 1 { + t.Errorf("second run: changed = %d, want 1", result2.Changed) + } + + // Should have at least 1 snapshot (2 if runs happen in different seconds). + snapshots, err := backend.List(ctx, "snapshots/") + if err != nil { + t.Fatalf("list snapshots: %v", err) + } + if len(snapshots) < 1 { + t.Errorf("snapshots = %d, want >= 1", len(snapshots)) + } + + // The manifest should reflect the latest state. + manifest, err := LoadManifest(manifestPath) + if err != nil { + t.Fatal(err) + } + if f, ok := manifest.Files["config.yaml"]; !ok { + t.Error("config.yaml missing from manifest") + } else if f.Size == 0 { + t.Error("config.yaml size should be > 0") + } +} + +func TestServiceRestore(t *testing.T) { + baseDir := t.TempDir() + remoteDir := t.TempDir() + manifestPath := filepath.Join(t.TempDir(), "last_manifest.json") + stagingDir := t.TempDir() + + mkFileWithContent(t, baseDir, "config.yaml", "mode: local") + mkFileWithContent(t, baseDir, "learnings/topic/note.md", "important note") + + backend := NewLocalBackend(remoteDir) + svc := NewWithPaths(backend, baseDir, manifestPath, stagingDir) + ctx := context.Background() + + // Run backup. + _, err := svc.Run(ctx) + if err != nil { + t.Fatalf("backup: %v", err) + } + + // Get the snapshot ID. + snapshots, err := svc.ListSnapshots(ctx) + if err != nil { + t.Fatalf("list snapshots: %v", err) + } + if len(snapshots) != 1 { + t.Fatalf("expected 1 snapshot, got %d", len(snapshots)) + } + snapshotID := snapshots[0] + + // Verify GetManifest works. + manifest, err := svc.GetManifest(ctx, snapshotID) + if err != nil { + t.Fatalf("get manifest: %v", err) + } + if len(manifest.Files) != 2 { + t.Errorf("manifest files = %d, want 2", len(manifest.Files)) + } + + // Restore to a fresh directory. + restoreDir := t.TempDir() + restoreSvc := NewWithPaths(backend, restoreDir, filepath.Join(t.TempDir(), "m.json"), t.TempDir()) + + result, err := restoreSvc.Restore(ctx, snapshotID) + if err != nil { + t.Fatalf("restore: %v", err) + } + if result.Restored != 2 { + t.Errorf("restored = %d, want 2", result.Restored) + } + + // Verify restored files. + got, err := os.ReadFile(filepath.Join(restoreDir, "config.yaml")) + if err != nil { + t.Fatalf("read restored config.yaml: %v", err) + } + if string(got) != "mode: local" { + t.Errorf("restored config.yaml = %q, want %q", got, "mode: local") + } + + got, err = os.ReadFile(filepath.Join(restoreDir, "learnings/topic/note.md")) + if err != nil { + t.Fatalf("read restored note.md: %v", err) + } + if string(got) != "important note" { + t.Errorf("restored note.md = %q, want %q", got, "important note") + } +} + +func TestServiceListSnapshotsEmpty(t *testing.T) { + remoteDir := t.TempDir() + backend := NewLocalBackend(remoteDir) + svc := NewWithPaths(backend, t.TempDir(), filepath.Join(t.TempDir(), "m.json"), t.TempDir()) + + snapshots, err := svc.ListSnapshots(context.Background()) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(snapshots) != 0 { + t.Errorf("expected 0 snapshots, got %d", len(snapshots)) + } +} + +func TestServiceRestoreMissingObject(t *testing.T) { + baseDir := t.TempDir() + remoteDir := t.TempDir() + manifestPath := filepath.Join(t.TempDir(), "last_manifest.json") + stagingDir := t.TempDir() + + mkFileWithContent(t, baseDir, "config.yaml", "mode: local") + + backend := NewLocalBackend(remoteDir) + svc := NewWithPaths(backend, baseDir, manifestPath, stagingDir) + ctx := context.Background() + + // Run backup to create a valid snapshot. + _, err := svc.Run(ctx) + if err != nil { + t.Fatalf("backup: %v", err) + } + + snapshots, err := svc.ListSnapshots(ctx) + if err != nil || len(snapshots) == 0 { + t.Fatal("expected at least one snapshot") + } + + // Delete the object files from the backend to simulate corruption. + objects, _ := backend.List(ctx, "objects/") + for _, key := range objects { + backend.Delete(ctx, key) + } + + // Restore should fail because the objects are missing. + restoreDir := t.TempDir() + restoreSvc := NewWithPaths(backend, restoreDir, filepath.Join(t.TempDir(), "m.json"), t.TempDir()) + + _, err = restoreSvc.Restore(ctx, snapshots[0]) + if err == nil { + t.Fatal("expected error when restoring with missing objects") + } +} + +func TestServiceRunWithSQLiteDB(t *testing.T) { + baseDir := t.TempDir() + remoteDir := t.TempDir() + manifestPath := filepath.Join(t.TempDir(), "last_manifest.json") + stagingDir := t.TempDir() + + // Create a real SQLite database. + dbPath := filepath.Join(baseDir, "gmail", "data.db") + if err := os.MkdirAll(filepath.Dir(dbPath), 0700); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE emails (id INTEGER PRIMARY KEY, subject TEXT)"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO emails (subject) VALUES ('test email')"); err != nil { + t.Fatal(err) + } + db.Close() + + mkFileWithContent(t, baseDir, "config.yaml", "mode: local") + + backend := NewLocalBackend(remoteDir) + svc := NewWithPaths(backend, baseDir, manifestPath, stagingDir) + ctx := context.Background() + + result, err := svc.Run(ctx) + if err != nil { + t.Fatalf("run: %v", err) + } + if result.Changed != 2 { + t.Errorf("changed = %d, want 2", result.Changed) + } + + // Restore and verify DB is intact. + restoreDir := t.TempDir() + restoreSvc := NewWithPaths(backend, restoreDir, filepath.Join(t.TempDir(), "m.json"), t.TempDir()) + + snapshots, _ := svc.ListSnapshots(ctx) + _, err = restoreSvc.Restore(ctx, snapshots[0]) + if err != nil { + t.Fatalf("restore: %v", err) + } + + restoredDB, err := sql.Open("sqlite", filepath.Join(restoreDir, "gmail", "data.db")) + if err != nil { + t.Fatal(err) + } + defer restoredDB.Close() + + var subject string + if err := restoredDB.QueryRow("SELECT subject FROM emails WHERE id = 1").Scan(&subject); err != nil { + t.Fatalf("read restored db: %v", err) + } + if subject != "test email" { + t.Errorf("subject = %q, want 'test email'", subject) + } +} + +func mkFile(t *testing.T, base, rel string) { + t.Helper() + mkFileWithContent(t, base, rel, "test content") +} + +func mkFileWithContent(t *testing.T, base, rel, content string) { + t.Helper() + path := filepath.Join(base, rel) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } +} diff --git a/service/backup/gdrive.go b/service/backup/gdrive.go new file mode 100644 index 00000000..ce4a7f16 --- /dev/null +++ b/service/backup/gdrive.go @@ -0,0 +1,254 @@ +package backup + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" +) + +var errNotFound = errors.New("not found") + +type GDriveBackend struct { + srv *drive.Service + folderID string +} + +func NewGDriveBackend(ctx context.Context, httpClient *http.Client, folderID string) (*GDriveBackend, error) { + srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient)) + if err != nil { + return nil, fmt.Errorf("create drive service: %w", err) + } + return &GDriveBackend{srv: srv, folderID: folderID}, nil +} + +func (b *GDriveBackend) Put(ctx context.Context, key string, r io.Reader, _ int64) error { + name := key + parentID := b.folderID + + parts := strings.Split(key, "/") + if len(parts) > 1 { + var err error + parentID, err = b.ensureFolderPath(ctx, b.folderID, parts[:len(parts)-1]) + if err != nil { + return fmt.Errorf("create folder path for %s: %w", key, err) + } + name = parts[len(parts)-1] + } + + existing, err := b.findFile(ctx, parentID, name) + if err != nil { + return err + } + + if existing != "" { + _, err = b.srv.Files.Update(existing, &drive.File{}). + Context(ctx). + Media(r). + Do() + } else { + _, err = b.srv.Files.Create(&drive.File{ + Name: name, + Parents: []string{parentID}, + }).Context(ctx). + Media(r). + Do() + } + if err != nil { + return fmt.Errorf("put %s: %w", key, err) + } + return nil +} + +func (b *GDriveBackend) Get(ctx context.Context, key string) (io.ReadCloser, error) { + fileID, err := b.resolveKey(ctx, key) + if err != nil { + return nil, err + } + resp, err := b.srv.Files.Get(fileID).Context(ctx).Download() + if err != nil { + return nil, fmt.Errorf("get %s: %w", key, err) + } + return resp.Body, nil +} + +func (b *GDriveBackend) Head(ctx context.Context, key string) (bool, error) { + _, err := b.resolveKey(ctx, key) + if err != nil { + if errors.Is(err, errNotFound) { + return false, nil + } + return false, err + } + return true, nil +} + +func (b *GDriveBackend) List(ctx context.Context, prefix string) ([]string, error) { + folderID := b.folderID + + parts := strings.Split(prefix, "/") + if len(parts) > 0 && prefix != "" { + var err error + folderID, err = b.resolveFolderPath(ctx, b.folderID, parts) + if err != nil { + return nil, nil + } + } + + return b.listRecursive(ctx, folderID, prefix) +} + +func (b *GDriveBackend) Delete(ctx context.Context, key string) error { + fileID, err := b.resolveKey(ctx, key) + if err != nil { + if errors.Is(err, errNotFound) { + return nil + } + return err + } + return b.srv.Files.Delete(fileID).Context(ctx).Do() +} + +func (b *GDriveBackend) resolveKey(ctx context.Context, key string) (string, error) { + parts := strings.Split(key, "/") + parentID := b.folderID + + for i, part := range parts { + id, err := b.findFile(ctx, parentID, part) + if err != nil { + return "", err + } + if id == "" { + return "", fmt.Errorf("%s: %w", key, errNotFound) + } + if i < len(parts)-1 { + parentID = id + } else { + return id, nil + } + } + return "", fmt.Errorf("%s: %w", key, errNotFound) +} + +func (b *GDriveBackend) findFile(ctx context.Context, parentID, name string) (string, error) { + q := fmt.Sprintf("'%s' in parents and name = '%s' and trashed = false", escapeDriveQuery(parentID), escapeDriveQuery(name)) + list, err := b.srv.Files.List(). + Q(q). + Fields("files(id)"). + PageSize(1). + Context(ctx). + Do() + if err != nil { + return "", fmt.Errorf("find %s: %w", name, err) + } + if len(list.Files) == 0 { + return "", nil + } + return list.Files[0].Id, nil +} + +func (b *GDriveBackend) ensureFolderPath(ctx context.Context, parentID string, parts []string) (string, error) { + current := parentID + for _, part := range parts { + id, err := b.findFile(ctx, current, part) + if err != nil { + return "", err + } + if id == "" { + f, err := b.srv.Files.Create(&drive.File{ + Name: part, + Parents: []string{current}, + MimeType: "application/vnd.google-apps.folder", + }).Context(ctx).Fields("id").Do() + if err != nil { + return "", fmt.Errorf("create folder %s: %w", part, err) + } + id = f.Id + } + current = id + } + return current, nil +} + +func (b *GDriveBackend) resolveFolderPath(ctx context.Context, parentID string, parts []string) (string, error) { + current := parentID + for _, part := range parts { + id, err := b.findFile(ctx, current, part) + if err != nil { + return "", err + } + if id == "" { + return "", fmt.Errorf("folder %s: %w", part, errNotFound) + } + current = id + } + return current, nil +} + +func (b *GDriveBackend) listRecursive(ctx context.Context, folderID, prefix string) ([]string, error) { + var keys []string + q := fmt.Sprintf("'%s' in parents and trashed = false", escapeDriveQuery(folderID)) + err := b.srv.Files.List(). + Q(q). + Fields("files(id, name, mimeType)"). + Context(ctx). + Pages(ctx, func(list *drive.FileList) error { + for _, f := range list.Files { + path := prefix + if path != "" && !strings.HasSuffix(path, "/") { + path += "/" + } + path += f.Name + + if f.MimeType == "application/vnd.google-apps.folder" { + sub, err := b.listRecursive(ctx, f.Id, path) + if err != nil { + return err + } + keys = append(keys, sub...) + } else { + keys = append(keys, path) + } + } + return nil + }) + return keys, err +} + +func FindOrCreateDriveFolder(ctx context.Context, httpClient *http.Client, folderName string) (string, error) { + srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient)) + if err != nil { + return "", fmt.Errorf("create drive service: %w", err) + } + + q := fmt.Sprintf("name = '%s' and mimeType = 'application/vnd.google-apps.folder' and trashed = false", escapeDriveQuery(folderName)) + list, err := srv.Files.List().Q(q).Fields("files(id)").PageSize(1).Context(ctx).Do() + if err != nil { + return "", fmt.Errorf("search for folder: %w", err) + } + if len(list.Files) > 0 { + return list.Files[0].Id, nil + } + + f, err := srv.Files.Create(&drive.File{ + Name: folderName, + MimeType: "application/vnd.google-apps.folder", + }).Context(ctx).Fields("id").Do() + if err != nil { + return "", fmt.Errorf("create folder: %w", err) + } + return f.Id, nil +} + +// escapeDriveQuery escapes single quotes for Drive API query strings. +func escapeDriveQuery(s string) string { + return strings.ReplaceAll(s, "'", "\\'") +} + +var _ Backend = (*GDriveBackend)(nil) + diff --git a/service/backup/manifest.go b/service/backup/manifest.go new file mode 100644 index 00000000..47a90d01 --- /dev/null +++ b/service/backup/manifest.go @@ -0,0 +1,87 @@ +package backup + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "time" +) + +type Manifest struct { + Version int `json:"version"` + ID string `json:"id"` + Timestamp time.Time `json:"timestamp"` + Hostname string `json:"hostname"` + Files map[string]ManifestFile `json:"files"` +} + +type ManifestFile struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + CompressedSize int64 `json:"compressed_size"` +} + +func NewManifest(hostname string) *Manifest { + now := time.Now().UTC() + suffix := make([]byte, 4) + rand.Read(suffix) + return &Manifest{ + Version: 1, + ID: now.Format("20060102T150405Z") + "-" + hex.EncodeToString(suffix), + Timestamp: now, + Hostname: hostname, + Files: make(map[string]ManifestFile), + } +} + +func LoadManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return &Manifest{Files: make(map[string]ManifestFile)}, nil + } + return nil, fmt.Errorf("read manifest: %w", err) + } + var m Manifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parse manifest: %w", err) + } + if m.Files == nil { + m.Files = make(map[string]ManifestFile) + } + return &m, nil +} + +func SaveManifest(path string, m *Manifest) error { + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("marshal manifest: %w", err) + } + return os.WriteFile(path, data, 0600) +} + +type DiffResult struct { + Changed []string // files that are new or have different hashes + Removed []string // files in old manifest but not in new scan +} + +func DiffManifest(old *Manifest, current map[string]string) DiffResult { + var result DiffResult + + for path, hash := range current { + prev, ok := old.Files[path] + if !ok || prev.Hash != hash { + result.Changed = append(result.Changed, path) + } + } + + for path := range old.Files { + if _, ok := current[path]; !ok { + result.Removed = append(result.Removed, path) + } + } + + return result +} diff --git a/service/backup/r2.go b/service/backup/r2.go new file mode 100644 index 00000000..a45ebf62 --- /dev/null +++ b/service/backup/r2.go @@ -0,0 +1,98 @@ +package backup + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type R2Backend struct { + client *minio.Client + bucket string +} + +func NewR2Backend(endpoint, accessKey, secretKey, bucket string) (*R2Backend, error) { + endpoint = strings.TrimPrefix(endpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: true, + Region: "auto", + }) + if err != nil { + return nil, fmt.Errorf("create R2 client: %w", err) + } + + return &R2Backend{client: client, bucket: bucket}, nil +} + +func (b *R2Backend) Put(ctx context.Context, key string, r io.Reader, size int64) error { + _, err := b.client.PutObject(ctx, b.bucket, key, r, size, minio.PutObjectOptions{}) + if err != nil { + return fmt.Errorf("put %s: %w", key, err) + } + return nil +} + +func (b *R2Backend) Get(ctx context.Context, key string) (io.ReadCloser, error) { + obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("get %s: %w", key, err) + } + return obj, nil +} + +func (b *R2Backend) Head(ctx context.Context, key string) (bool, error) { + _, err := b.client.StatObject(ctx, b.bucket, key, minio.StatObjectOptions{}) + if err != nil { + resp := minio.ToErrorResponse(err) + if resp.Code == "NoSuchKey" { + return false, nil + } + return false, fmt.Errorf("head %s: %w", key, err) + } + return true, nil +} + +func (b *R2Backend) List(ctx context.Context, prefix string) ([]string, error) { + var keys []string + for obj := range b.client.ListObjects(ctx, b.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + return nil, fmt.Errorf("list %s: %w", prefix, obj.Err) + } + keys = append(keys, obj.Key) + } + return keys, nil +} + +func (b *R2Backend) Delete(ctx context.Context, key string) error { + if err := b.client.RemoveObject(ctx, b.bucket, key, minio.RemoveObjectOptions{}); err != nil { + return fmt.Errorf("delete %s: %w", key, err) + } + return nil +} + +func ValidateR2(ctx context.Context, endpoint, accessKey, secretKey, bucket string) error { + backend, err := NewR2Backend(endpoint, accessKey, secretKey, bucket) + if err != nil { + return err + } + exists, err := backend.client.BucketExists(ctx, bucket) + if err != nil { + return fmt.Errorf("check bucket: %w", err) + } + if !exists { + return fmt.Errorf("bucket %q does not exist", bucket) + } + return nil +} + +var _ Backend = (*R2Backend)(nil) diff --git a/service/backup/resolve.go b/service/backup/resolve.go new file mode 100644 index 00000000..bca82841 --- /dev/null +++ b/service/backup/resolve.go @@ -0,0 +1,87 @@ +package backup + +import ( + "context" + "fmt" + "net/http" + "os" +) + +// CredentialResolver loads a credential by reference (e.g. "keychain:obk/r2-access-key") +// and falls back to the given environment variable. +type CredentialResolver func(ref, envVar string) (string, error) + +// GoogleClientFactory creates an authenticated HTTP client for Google APIs. +type GoogleClientFactory func(ctx context.Context, cfg GoogleClientConfig) (*http.Client, error) + +type GoogleClientConfig struct { + CredentialsFile string + TokenDBPath string + Scopes []string +} + +// ResolveBackendOpts holds the dependencies needed to resolve a backup backend. +type ResolveBackendOpts struct { + ResolveCred CredentialResolver + GoogleClient GoogleClientFactory + BackupDest string + R2Bucket string + R2Endpoint string + R2AccessRef string + R2SecretRef string + GDriveFolderID string +} + +// ResolveBackend creates the appropriate backend from config. +func ResolveBackend(ctx context.Context, opts ResolveBackendOpts) (Backend, error) { + switch opts.BackupDest { + case "r2": + return resolveR2(ctx, opts) + case "gdrive": + return resolveGDrive(ctx, opts) + default: + return nil, fmt.Errorf("unknown backup destination: %q", opts.BackupDest) + } +} + +func resolveR2(_ context.Context, opts ResolveBackendOpts) (Backend, error) { + if opts.R2Bucket == "" || opts.R2Endpoint == "" { + return nil, fmt.Errorf("R2 config missing") + } + + accessKey, err := opts.ResolveCred(opts.R2AccessRef, "OBK_R2_ACCESS_KEY") + if err != nil { + return nil, fmt.Errorf("resolve R2 access key: %w", err) + } + secretKey, err := opts.ResolveCred(opts.R2SecretRef, "OBK_R2_SECRET_KEY") + if err != nil { + return nil, fmt.Errorf("resolve R2 secret key: %w", err) + } + + endpoint := opts.R2Endpoint + if e := os.Getenv("OBK_R2_ENDPOINT"); e != "" { + endpoint = e + } + bucket := opts.R2Bucket + if b := os.Getenv("OBK_R2_BUCKET"); b != "" { + bucket = b + } + + return NewR2Backend(endpoint, accessKey, secretKey, bucket) +} + +func resolveGDrive(ctx context.Context, opts ResolveBackendOpts) (Backend, error) { + if opts.GDriveFolderID == "" { + return nil, fmt.Errorf("Google Drive folder ID not configured") + } + if opts.GoogleClient == nil { + return nil, fmt.Errorf("Google client factory not provided") + } + + httpClient, err := opts.GoogleClient(ctx, GoogleClientConfig{Scopes: []string{"https://www.googleapis.com/auth/drive.file"}}) + if err != nil { + return nil, fmt.Errorf("get Drive client: %w", err) + } + + return NewGDriveBackend(ctx, httpClient, opts.GDriveFolderID) +} diff --git a/service/backup/restore.go b/service/backup/restore.go new file mode 100644 index 00000000..48d23590 --- /dev/null +++ b/service/backup/restore.go @@ -0,0 +1,113 @@ +package backup + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/klauspost/compress/zstd" +) + +type RestoreResult struct { + Restored int +} + +func (s *Service) Restore(ctx context.Context, snapshotID string) (*RestoreResult, error) { + manifestKey := fmt.Sprintf("snapshots/%s.json", snapshotID) + rc, err := s.backend.Get(ctx, manifestKey) + if err != nil { + return nil, fmt.Errorf("download manifest: %w", err) + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("read manifest: %w", err) + } + + var manifest Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("parse manifest: %w", err) + } + + var result RestoreResult + for rel, mf := range manifest.Files { + objectKey := objectKeyFromHash(mf.Hash) + objRC, err := s.backend.Get(ctx, objectKey) + if err != nil { + return nil, fmt.Errorf("download %s: %w", rel, err) + } + + compressed, err := io.ReadAll(objRC) + objRC.Close() + if err != nil { + return nil, fmt.Errorf("read %s: %w", rel, err) + } + + decompressed, err := decompressData(compressed) + if err != nil { + return nil, fmt.Errorf("decompress %s: %w", rel, err) + } + + destPath := filepath.Join(s.baseDir, rel) + if err := os.MkdirAll(filepath.Dir(destPath), 0700); err != nil { + return nil, fmt.Errorf("create dir for %s: %w", rel, err) + } + if err := os.WriteFile(destPath, decompressed, 0600); err != nil { + return nil, fmt.Errorf("write %s: %w", rel, err) + } + result.Restored++ + } + + return &result, nil +} + +func (s *Service) ListSnapshots(ctx context.Context) ([]string, error) { + keys, err := s.backend.List(ctx, "snapshots/") + if err != nil { + return nil, fmt.Errorf("list snapshots: %w", err) + } + + var ids []string + for _, key := range keys { + name := filepath.Base(key) + ext := filepath.Ext(name) + if ext == ".json" { + ids = append(ids, name[:len(name)-len(ext)]) + } + } + return ids, nil +} + +func (s *Service) GetManifest(ctx context.Context, snapshotID string) (*Manifest, error) { + manifestKey := fmt.Sprintf("snapshots/%s.json", snapshotID) + rc, err := s.backend.Get(ctx, manifestKey) + if err != nil { + return nil, fmt.Errorf("download manifest: %w", err) + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("read manifest: %w", err) + } + + var manifest Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("parse manifest: %w", err) + } + return &manifest, nil +} + +func decompressData(compressed []byte) ([]byte, error) { + r, err := zstd.NewReader(bytes.NewReader(compressed)) + if err != nil { + return nil, err + } + defer r.Close() + return io.ReadAll(r) +} diff --git a/service/backup/scanner.go b/service/backup/scanner.go new file mode 100644 index 00000000..44479a03 --- /dev/null +++ b/service/backup/scanner.go @@ -0,0 +1,117 @@ +package backup + +import ( + "io/fs" + "path/filepath" + "strings" +) + +var includePatterns = []string{ + "*/data.db", + "whatsapp/session.db", + "config.yaml", + "env", + "ngrok.yml", + "providers/**/*.json", + "models/*.json", + "applenotes/config.json", + "applecontacts/config.json", + "learnings/**/*.md", + "skills/**", +} + +var excludePatterns = []string{ + "*.db-wal", + "*.db-shm", + "daemon.log", + "server.log", + "bin/", + "scratch/", + "jobs.db", + "*.lock", + "backup/", +} + +func ScanFiles(baseDir string) ([]string, error) { + var files []string + err := filepath.WalkDir(baseDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if d.Type()&fs.ModeSymlink != 0 { + return nil + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(baseDir, path) + if err != nil { + return nil + } + rel = filepath.ToSlash(rel) + + if isExcluded(rel) { + return nil + } + if isIncluded(rel) { + files = append(files, rel) + } + return nil + }) + return files, err +} + +func isIncluded(rel string) bool { + for _, pattern := range includePatterns { + if matchGlob(pattern, rel) { + return true + } + } + return false +} + +func isExcluded(rel string) bool { + for _, pattern := range excludePatterns { + name := filepath.Base(rel) + if matchGlob(pattern, rel) || matchGlob(pattern, name) { + return true + } + if strings.HasSuffix(pattern, "/") { + dir := strings.TrimSuffix(pattern, "/") + if strings.HasPrefix(rel, dir+"/") || rel == dir { + return true + } + } + } + return false +} + +func matchGlob(pattern, name string) bool { + if strings.Contains(pattern, "**") { + prefix := strings.Split(pattern, "**")[0] + suffix := strings.Split(pattern, "**")[1] + suffix = strings.TrimPrefix(suffix, "/") + + if prefix != "" && !strings.HasPrefix(name, prefix) { + return false + } + if suffix == "" { + return strings.HasPrefix(name, prefix) + } + rest := strings.TrimPrefix(name, prefix) + parts := strings.Split(rest, "/") + for i := range parts { + tail := strings.Join(parts[i:], "/") + if matched, _ := filepath.Match(suffix, tail); matched { + return true + } + } + return false + } + if strings.Contains(pattern, "/") { + matched, _ := filepath.Match(pattern, name) + return matched + } + matched, _ := filepath.Match(pattern, filepath.Base(name)) + return matched +} diff --git a/service/backup/vacuum.go b/service/backup/vacuum.go new file mode 100644 index 00000000..16a68e4f --- /dev/null +++ b/service/backup/vacuum.go @@ -0,0 +1,36 @@ +package backup + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + + _ "modernc.org/sqlite" +) + +// VacuumInto creates a consistent snapshot of a SQLite database. +// relPath is the path relative to ~/.obk (e.g. "gmail/data.db") used to +// preserve directory structure in the staging dir and avoid collisions. +func VacuumInto(dbPath, stagingDir, relPath string) (string, error) { + destPath := filepath.Join(stagingDir, relPath) + if err := os.MkdirAll(filepath.Dir(destPath), 0700); err != nil { + return "", fmt.Errorf("create staging dir: %w", err) + } + + os.Remove(destPath) + + db, err := sql.Open("sqlite", dbPath) + if err != nil { + return "", fmt.Errorf("open %s: %w", dbPath, err) + } + defer db.Close() + + escaped := strings.ReplaceAll(destPath, "'", "''") + if _, err := db.Exec(fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil { + return "", fmt.Errorf("vacuum into %s: %w", destPath, err) + } + + return destPath, nil +} diff --git a/settings/registry.go b/settings/registry.go index c59ec35b..21e3dba1 100644 --- a/settings/registry.go +++ b/settings/registry.go @@ -18,6 +18,7 @@ func BuildTree(svc *Service) []Node { {Category: channelsCategory()}, {Category: dataSourcesCategory()}, {Category: integrationsCategory()}, + {Category: backupCategory(svc)}, {Category: advancedCategory()}, } } @@ -715,6 +716,253 @@ func integrationsCategory() *Category { } } +func backupCategory(svc *Service) *Category { + children := []Node{ + {Field: &Field{ + Key: "backup.enabled", + Label: "Enabled", + Type: TypeBool, + Get: func(c *config.Config) string { + if c.Backup == nil { + return "false" + } + return strconv.FormatBool(c.Backup.Enabled) + }, + Set: func(c *config.Config, v string) error { + b, err := strconv.ParseBool(v) + if err != nil { + return fmt.Errorf("invalid boolean: %w", err) + } + if b { + if err := validateBackupReady(c); err != nil { + return err + } + } + ensureBackup(c) + c.Backup.Enabled = b + return nil + }, + ReadOnly: func(c *config.Config) bool { + if c.Backup != nil && c.Backup.Enabled { + return false + } + return !isBackupDestinationConfigured(c) + }, + }}, + {Field: &Field{ + Key: "backup.destination", + Label: "Destination", + Type: TypeSelect, + Options: []Option{ + {"(not set)", ""}, + {"Cloudflare R2", "r2"}, + {"Google Drive", "gdrive"}, + }, + Get: func(c *config.Config) string { + if c.Backup == nil { + return "" + } + return c.Backup.Destination + }, + Set: func(c *config.Config, v string) error { + ensureBackup(c) + if c.Backup.Destination != v { + c.Backup.Enabled = false + } + c.Backup.Destination = v + return nil + }, + }}, + {Field: &Field{ + Key: "backup.schedule", + Label: "Schedule", + Type: TypeSelect, + Options: []Option{ + {"Every 6 hours", "6h"}, + {"Every 12 hours", "12h"}, + {"Daily", "24h"}, + {"Manual only", ""}, + }, + Get: func(c *config.Config) string { + if c.Backup == nil { + return "" + } + return c.Backup.Schedule + }, + Set: func(c *config.Config, v string) error { + ensureBackup(c) + c.Backup.Schedule = v + return nil + }, + ReadOnly: func(c *config.Config) bool { + return c.Backup == nil || !c.Backup.Enabled + }, + }}, + } + + // Only show R2 sub-category when destination is R2. + if BackupDest(svc.cfg) == "r2" { + children = append(children, Node{Category: &Category{ + Key: "backup.r2", + Label: "Cloudflare R2", + Children: []Node{ + {Field: &Field{ + Key: "backup.r2.bucket", + Label: "Bucket", + Description: "Cloudflare Dashboard → R2 Object Storage → your bucket", + Type: TypeString, + Get: func(c *config.Config) string { + if c.Backup == nil || c.Backup.R2 == nil { + return "" + } + return c.Backup.R2.Bucket + }, + Set: func(c *config.Config, v string) error { + ensureBackupR2(c) + c.Backup.R2.Bucket = v + return nil + }, + }}, + {Field: &Field{ + Key: "backup.r2.endpoint", + Label: "Endpoint", + Description: "Bucket → Settings → S3 API → endpoint URL", + Type: TypeString, + Get: func(c *config.Config) string { + if c.Backup == nil || c.Backup.R2 == nil { + return "" + } + return c.Backup.R2.Endpoint + }, + Set: func(c *config.Config, v string) error { + ensureBackupR2(c) + c.Backup.R2.Endpoint = v + return nil + }, + }}, + {Field: &Field{ + Key: "backup.r2.access_key", + Label: "Access Key ID", + Description: "R2 → Manage R2 API Tokens → Create API Token", + Type: TypePassword, + Get: func(c *config.Config) string { + if c.Backup == nil || c.Backup.R2 == nil || c.Backup.R2.AccessKeyRef == "" { + return "not configured" + } + return maskCredential(svc, c.Backup.R2.AccessKeyRef) + }, + Set: func(c *config.Config, v string) error { + if v == "" { + return nil + } + ensureBackupR2(c) + ref := "keychain:obk/r2-access-key" + if err := svc.StoreCredential(ref, v); err != nil { + return fmt.Errorf("store credential: %w", err) + } + c.Backup.R2.AccessKeyRef = ref + return nil + }, + }}, + {Field: &Field{ + Key: "backup.r2.secret_key", + Label: "Secret Access Key", + Description: "Shown once when you create the API token", + Type: TypePassword, + Get: func(c *config.Config) string { + if c.Backup == nil || c.Backup.R2 == nil || c.Backup.R2.SecretKeyRef == "" { + return "not configured" + } + return maskCredential(svc, c.Backup.R2.SecretKeyRef) + }, + Set: func(c *config.Config, v string) error { + if v == "" { + return nil + } + ensureBackupR2(c) + ref := "keychain:obk/r2-secret-key" + if err := svc.StoreCredential(ref, v); err != nil { + return fmt.Errorf("store credential: %w", err) + } + c.Backup.R2.SecretKeyRef = ref + return nil + }, + }}, + }, + }}) + } + + return &Category{ + Key: "backup", + Label: "Backup", + Children: children, + } +} + +func BackupDest(c *config.Config) string { + if c.Backup == nil { + return "" + } + return c.Backup.Destination +} + +func isR2Configured(c *config.Config) bool { + if c.Backup == nil || c.Backup.R2 == nil { + return false + } + r := c.Backup.R2 + return r.Bucket != "" && r.Endpoint != "" && r.AccessKeyRef != "" && r.SecretKeyRef != "" +} + +func isGDriveConfigured(c *config.Config) bool { + if c.Backup == nil || c.Backup.GDrive == nil { + return false + } + return c.Backup.GDrive.FolderID != "" +} + +func isBackupDestinationConfigured(c *config.Config) bool { + switch BackupDest(c) { + case "r2": + return isR2Configured(c) + case "gdrive": + return isGDriveConfigured(c) + default: + return false + } +} + +func validateBackupReady(c *config.Config) error { + dest := BackupDest(c) + if dest == "" { + return fmt.Errorf("select a destination first") + } + switch dest { + case "r2": + if !isR2Configured(c) { + return fmt.Errorf("configure R2 bucket, endpoint, and credentials first") + } + case "gdrive": + if !isGDriveConfigured(c) { + return fmt.Errorf("configure Google Drive folder ID first") + } + } + return nil +} + +func ensureBackup(c *config.Config) { + if c.Backup == nil { + c.Backup = &config.BackupConfig{} + } +} + +func ensureBackupR2(c *config.Config) { + ensureBackup(c) + if c.Backup.R2 == nil { + c.Backup.R2 = &config.R2Config{} + } +} + func advancedCategory() *Category { return &Category{ Key: "advanced", diff --git a/settings/settings.go b/settings/settings.go index db737e9e..b39f14e0 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -53,6 +53,9 @@ type Service struct { storeCred func(ref, value string) error loadCred func(ref string) (string, error) verifyProvider func(name string, cfg config.ModelProviderConfig) error + verifyBackup func(dest string, cfg *config.Config) error + setupGDrive func(cfg *config.Config, folderName string) (string, error) + triggerBackup func(cfg *config.Config) error } type ServiceOption func(*Service) @@ -73,6 +76,18 @@ func WithVerifyProvider(fn func(name string, cfg config.ModelProviderConfig) err return func(s *Service) { s.verifyProvider = fn } } +func WithVerifyBackup(fn func(dest string, cfg *config.Config) error) ServiceOption { + return func(s *Service) { s.verifyBackup = fn } +} + +func WithSetupGDrive(fn func(cfg *config.Config, folderName string) (string, error)) ServiceOption { + return func(s *Service) { s.setupGDrive = fn } +} + +func WithTriggerBackup(fn func(cfg *config.Config) error) ServiceOption { + return func(s *Service) { s.triggerBackup = fn } +} + func New(cfg *config.Config, opts ...ServiceOption) *Service { s := &Service{ cfg: cfg, @@ -136,6 +151,38 @@ func (s *Service) VerifyProvider(name string, cfg config.ModelProviderConfig) er return s.verifyProvider(name, cfg) } +func (s *Service) VerifyBackup(dest string, cfg *config.Config) error { + if s.verifyBackup == nil { + return nil + } + return s.verifyBackup(dest, cfg) +} + +func (s *Service) SetupGDrive(cfg *config.Config, folderName string) (string, error) { + if s.setupGDrive == nil { + return "", fmt.Errorf("Google Drive setup not available — run 'obk setup' instead") + } + return s.setupGDrive(cfg, folderName) +} + +func (s *Service) TriggerBackup() error { + if s.triggerBackup == nil { + return nil + } + return s.triggerBackup(s.cfg) +} + +// IsBackupDestConfigured returns true if the given destination has credentials configured. +func (s *Service) IsBackupDestConfigured(dest string) bool { + switch dest { + case "r2": + return isR2Configured(s.cfg) + case "gdrive": + return isGDriveConfigured(s.cfg) + } + return false +} + // ResolvedOptions returns the options for a field, using OptionsFunc if set. func (s *Service) ResolvedOptions(f *Field) []Option { if f.OptionsFunc != nil { diff --git a/settings/settings_test.go b/settings/settings_test.go index c9f2b0fa..136f2501 100644 --- a/settings/settings_test.go +++ b/settings/settings_test.go @@ -198,11 +198,11 @@ func TestTreeStructure(t *testing.T) { svc := testService(cfg) tree := svc.Tree() - if len(tree) != 6 { - t.Fatalf("tree has %d top-level nodes, want 6", len(tree)) + if len(tree) != 7 { + t.Fatalf("tree has %d top-level nodes, want 7", len(tree)) } - labels := []string{"General", "LLM Models", "Channels", "Data Sources", "Integrations", "Advanced"} + labels := []string{"General", "LLM Models", "Channels", "Data Sources", "Integrations", "Backup", "Advanced"} for i, n := range tree { if n.Category == nil { t.Errorf("tree[%d] is not a category", i) @@ -742,6 +742,339 @@ func findField(svc *Service, key string) *Field { return findFieldInNodes(svc.Tree(), key) } +func TestBackupEnabledRequiresDestination(t *testing.T) { + cfg := config.Default() + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + if field == nil { + t.Fatal("backup.enabled field not found") + } + + // Enabling without any destination should fail. + err := svc.SetValue(field, "true") + if err == nil { + t.Fatal("expected error enabling backup without destination") + } + if !strings.Contains(err.Error(), "select a destination") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestBackupEnabledRequiresR2Credentials(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Destination: "r2", + R2: &config.R2Config{}, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + + // Enabling with destination=r2 but no credentials should fail. + err := svc.SetValue(field, "true") + if err == nil { + t.Fatal("expected error enabling backup without R2 credentials") + } + if !strings.Contains(err.Error(), "configure R2") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestBackupEnabledSucceedsWithR2Configured(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Destination: "r2", + R2: &config.R2Config{ + Bucket: "test-bucket", + Endpoint: "https://example.r2.cloudflarestorage.com", + AccessKeyRef: "keychain:obk/r2-access-key", + SecretKeyRef: "keychain:obk/r2-secret-key", + }, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + if err := svc.SetValue(field, "true"); err != nil { + t.Fatalf("expected success, got: %v", err) + } + if !cfg.Backup.Enabled { + t.Error("backup should be enabled") + } +} + +func TestBackupEnabledRequiresGDriveFolderID(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Destination: "gdrive", + GDrive: &config.GDriveConfig{}, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + + err := svc.SetValue(field, "true") + if err == nil { + t.Fatal("expected error enabling backup without GDrive folder ID") + } + if !strings.Contains(err.Error(), "Google Drive folder ID") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestBackupEnabledSucceedsWithGDriveConfigured(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Destination: "gdrive", + GDrive: &config.GDriveConfig{FolderID: "1abc"}, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + if err := svc.SetValue(field, "true"); err != nil { + t.Fatalf("expected success, got: %v", err) + } + if !cfg.Backup.Enabled { + t.Error("backup should be enabled") + } +} + +func TestBackupDisableAlwaysAllowed(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{Enabled: true, Destination: "r2"} + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + if err := svc.SetValue(field, "false"); err != nil { + t.Fatalf("disabling should always work: %v", err) + } + if cfg.Backup.Enabled { + t.Error("backup should be disabled") + } +} + +func TestBackupR2FieldsHiddenWhenNotR2(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{Destination: "gdrive"} + svc := testService(cfg) + + for _, key := range []string{"backup.r2.bucket", "backup.r2.endpoint", "backup.r2.access_key", "backup.r2.secret_key"} { + field := findFieldInNodes(svc.Tree(), key) + if field != nil { + t.Errorf("%s should not be in tree when destination is gdrive", key) + } + } +} + +func TestBackupR2FieldsVisibleWhenR2(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{Destination: "r2"} + svc := testService(cfg) + + for _, key := range []string{"backup.r2.bucket", "backup.r2.endpoint", "backup.r2.access_key", "backup.r2.secret_key"} { + field := findFieldInNodes(svc.Tree(), key) + if field == nil { + t.Fatalf("%s should be in tree when destination is r2", key) + } + } +} + +func TestBackupScheduleReadOnlyWhenNotEnabled(t *testing.T) { + cfg := config.Default() + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.schedule") + if field == nil { + t.Fatal("backup.schedule field not found") + } + if !field.ReadOnly(cfg) { + t.Error("schedule should be read-only when backup is not enabled") + } + + // Even with destination configured but not enabled, schedule is locked. + cfg.Backup = &config.BackupConfig{ + Destination: "r2", + R2: &config.R2Config{ + Bucket: "b", + Endpoint: "e", + AccessKeyRef: "keychain:obk/r2-access-key", + SecretKeyRef: "keychain:obk/r2-secret-key", + }, + } + if !field.ReadOnly(cfg) { + t.Error("schedule should be read-only when destination is configured but backup not enabled") + } +} + +func TestBackupScheduleEditableWhenEnabled(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Enabled: true, + Destination: "r2", + R2: &config.R2Config{ + Bucket: "b", + Endpoint: "e", + AccessKeyRef: "keychain:obk/r2-access-key", + SecretKeyRef: "keychain:obk/r2-secret-key", + }, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.schedule") + if field.ReadOnly(cfg) { + t.Error("schedule should be editable when backup is enabled") + } +} + +func TestBackupEnabledReadOnlyWithoutDestination(t *testing.T) { + cfg := config.Default() + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + if field == nil { + t.Fatal("backup.enabled field not found") + } + if field.ReadOnly == nil { + t.Fatal("backup.enabled should have ReadOnly") + } + if !field.ReadOnly(cfg) { + t.Error("enabled should be read-only when no destination is configured") + } +} + +func TestBackupEnabledEditableWhenDestConfigured(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Destination: "r2", + R2: &config.R2Config{ + Bucket: "b", + Endpoint: "e", + AccessKeyRef: "keychain:obk/r2-access-key", + SecretKeyRef: "keychain:obk/r2-secret-key", + }, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + if field.ReadOnly(cfg) { + t.Error("enabled should be editable when destination is fully configured") + } +} + +func TestBackupEnabledEditableWhenAlreadyEnabled(t *testing.T) { + // Even if destination becomes unconfigured, user can still toggle OFF. + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Enabled: true, + Destination: "gdrive", + GDrive: &config.GDriveConfig{}, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.enabled") + if field.ReadOnly(cfg) { + t.Error("enabled should be editable when already enabled (so user can disable)") + } +} + +func TestBackupDestinationChangeResetsEnabled(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Enabled: true, + Destination: "r2", + R2: &config.R2Config{ + Bucket: "b", + Endpoint: "e", + AccessKeyRef: "keychain:obk/r2-access-key", + SecretKeyRef: "keychain:obk/r2-secret-key", + }, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.destination") + if err := svc.SetValue(field, "gdrive"); err != nil { + t.Fatal(err) + } + if cfg.Backup.Enabled { + t.Error("changing destination should reset enabled to false") + } +} + +func TestBackupDestinationSameValueKeepsEnabled(t *testing.T) { + cfg := config.Default() + cfg.Backup = &config.BackupConfig{ + Enabled: true, + Destination: "r2", + R2: &config.R2Config{ + Bucket: "b", + Endpoint: "e", + AccessKeyRef: "keychain:obk/r2-access-key", + SecretKeyRef: "keychain:obk/r2-secret-key", + }, + } + svc := testService(cfg) + + field := findFieldInNodes(svc.Tree(), "backup.destination") + if err := svc.SetValue(field, "r2"); err != nil { + t.Fatal(err) + } + if !cfg.Backup.Enabled { + t.Error("setting same destination should keep enabled=true") + } +} + +func TestIsBackupDestConfiguredR2(t *testing.T) { + cfg := config.Default() + svc := testService(cfg) + + // Not configured at all. + if svc.IsBackupDestConfigured("r2") { + t.Error("r2 should not be configured when backup is nil") + } + + // Partially configured. + cfg.Backup = &config.BackupConfig{ + Destination: "r2", + R2: &config.R2Config{Bucket: "b"}, + } + if svc.IsBackupDestConfigured("r2") { + t.Error("r2 should not be configured when only bucket is set") + } + + // Fully configured. + cfg.Backup.R2 = &config.R2Config{ + Bucket: "b", + Endpoint: "e", + AccessKeyRef: "ref-ak", + SecretKeyRef: "ref-sk", + } + if !svc.IsBackupDestConfigured("r2") { + t.Error("r2 should be configured when all fields are set") + } +} + +func TestIsBackupDestConfiguredGDrive(t *testing.T) { + cfg := config.Default() + svc := testService(cfg) + + if svc.IsBackupDestConfigured("gdrive") { + t.Error("gdrive should not be configured when backup is nil") + } + + cfg.Backup = &config.BackupConfig{ + Destination: "gdrive", + GDrive: &config.GDriveConfig{}, + } + if svc.IsBackupDestConfigured("gdrive") { + t.Error("gdrive should not be configured when folder_id is empty") + } + + cfg.Backup.GDrive.FolderID = "abc123" + if !svc.IsBackupDestConfigured("gdrive") { + t.Error("gdrive should be configured when folder_id is set") + } +} + func findFieldInNodes(nodes []Node, key string) *Field { for _, n := range nodes { if n.Field != nil && n.Field.Key == key { diff --git a/website/public/architecture.png b/website/public/architecture.png index 604dd798..d717c58f 100644 Binary files a/website/public/architecture.png and b/website/public/architecture.png differ