Skip to content

Commit 889105f

Browse files
authored
feat: restore job history from saved files on startup (#440)
## Summary - Adds automatic job history restoration from saved JSON files on startup - New config options: `restore-history` (default: true) and `restore-history-max-age` (default: 24h) - Respects per-job `history-limit` settings ## Problem When Ofelia restarts (container recreation, host reboot, etc.), job history displayed in the web UI is lost because it's stored only in memory. However, if `save-folder` is configured, execution data is already persisted to disk as JSON files but never read back. ## Solution On startup, if `save-folder` is configured: 1. Scan the save folder for `*.json` files 2. Parse JSON to extract job name and execution data 3. Filter by age (files older than `restore-history-max-age` are skipped) 4. Match executions to configured jobs by name 5. Populate job's in-memory history (respecting `history-limit`) ## New Configuration ```ini [global] restore-history = true # default: true (when save-folder is set) restore-history-max-age = 24h # default: 24h ``` ## Test plan - [x] Unit tests for restoration logic - [x] Tests for edge cases (invalid JSON, missing jobs, max age filtering) - [x] Tests for config option defaults - [x] All existing tests pass - [x] Linter passes Closes #439 Relates to discussion #437
2 parents e1c606a + e2732da commit 889105f

File tree

5 files changed

+574
-2
lines changed

5 files changed

+574
-2
lines changed

cli/daemon.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"time"
1111

1212
"github.com/netresearch/ofelia/core"
13+
"github.com/netresearch/ofelia/middlewares"
1314
"github.com/netresearch/ofelia/web"
1415
)
1516

@@ -98,6 +99,9 @@ func (c *DaemonCommand) boot() (err error) {
9899
// Re-apply CLI/environment options so they override Docker labels
99100
c.applyOptions(config)
100101
c.scheduler = config.sh
102+
103+
// Restore job history from saved files if configured
104+
c.restoreJobHistory(config)
101105
c.dockerHandler = config.dockerHandler
102106
c.config = config
103107

@@ -355,6 +359,18 @@ func (c *DaemonCommand) applyServerDefaults(config *Config) {
355359
}
356360
}
357361

362+
// restoreJobHistory restores job history from saved files if configured.
363+
func (c *DaemonCommand) restoreJobHistory(config *Config) {
364+
if !config.Global.SaveConfig.RestoreHistoryEnabled() {
365+
return
366+
}
367+
saveFolder := config.Global.SaveConfig.SaveFolder
368+
maxAge := config.Global.SaveConfig.GetRestoreHistoryMaxAge()
369+
if err := middlewares.RestoreHistory(saveFolder, maxAge, c.scheduler.Jobs, c.Logger); err != nil {
370+
c.Logger.Warningf("Failed to restore job history: %v", err)
371+
}
372+
}
373+
358374
func waitForServerWithErrChan(ctx context.Context, addr string, errChan <-chan error) error {
359375
ticker := time.NewTicker(50 * time.Millisecond)
360376
defer ticker.Stop()

config/validator.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,7 @@ func (cv *Validator2) isOptionalField(path string) bool {
447447
optionalFields := []string{
448448
"smtp-user", "smtp-password", "email-to", "email-from",
449449
"slack-webhook", "slack-channel", "save-folder",
450+
"restore-history", "restore-history-max-age",
450451
"container", "service", "image", "user", "network",
451452
"environment", "secrets", "volumes", "working_dir",
452453
"log-level", // Has default value "info"

middlewares/restore.go

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
package middlewares
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"os"
7+
"path/filepath"
8+
"sort"
9+
"strings"
10+
"time"
11+
12+
"github.com/netresearch/ofelia/core"
13+
)
14+
15+
// savedExecution represents the JSON structure saved by the Save middleware.
16+
// The JSON field names match the Save middleware's output format (PascalCase).
17+
//
18+
//nolint:tagliatelle // JSON format is defined by the Save middleware, must match exactly
19+
type savedExecution struct {
20+
Job struct {
21+
Name string `json:"Name"`
22+
} `json:"Job"`
23+
Execution struct {
24+
ID string `json:"ID"`
25+
Date time.Time `json:"Date"`
26+
Duration time.Duration `json:"Duration"`
27+
IsRunning bool `json:"IsRunning"`
28+
Failed bool `json:"Failed"`
29+
Skipped bool `json:"Skipped"`
30+
} `json:"Execution"`
31+
}
32+
33+
// restoredEntry holds a parsed execution ready for restoration.
34+
type restoredEntry struct {
35+
JobName string
36+
Execution *core.Execution
37+
}
38+
39+
// RestoreHistory restores job history from saved JSON files in the save folder.
40+
// It populates the in-memory history of jobs that support SetLastRun.
41+
// Only files newer than maxAge are restored.
42+
func RestoreHistory(saveFolder string, maxAge time.Duration, jobs []core.Job, logger core.Logger) error {
43+
if saveFolder == "" {
44+
return nil
45+
}
46+
47+
// Validate save folder - skip silently if invalid
48+
if DefaultSanitizer.ValidateSaveFolder(saveFolder) != nil {
49+
return nil //nolint:nilerr // Intentionally skip invalid folders
50+
}
51+
52+
// Check if folder exists
53+
info, err := os.Stat(saveFolder)
54+
if os.IsNotExist(err) {
55+
logger.Debugf("Save folder %q does not exist, skipping history restoration", saveFolder)
56+
return nil
57+
}
58+
if err != nil {
59+
return fmt.Errorf("stat save folder: %w", err)
60+
}
61+
if !info.IsDir() {
62+
return nil
63+
}
64+
65+
// Build job lookup map
66+
jobsByName := make(map[string]core.Job)
67+
for _, job := range jobs {
68+
jobsByName[job.GetName()] = job
69+
}
70+
71+
// Find and parse JSON files
72+
cutoff := time.Now().Add(-maxAge)
73+
entries, err := parseHistoryFiles(saveFolder, cutoff, logger)
74+
if err != nil {
75+
logger.Warningf("Error scanning save folder: %v", err)
76+
return nil // Don't fail startup for restore errors
77+
}
78+
79+
if len(entries) == 0 {
80+
logger.Debugf("No history files found to restore")
81+
return nil
82+
}
83+
84+
// Group entries by job name and sort by date
85+
entriesByJob := make(map[string][]*restoredEntry)
86+
for _, entry := range entries {
87+
entriesByJob[entry.JobName] = append(entriesByJob[entry.JobName], entry)
88+
}
89+
90+
restoredCount := 0
91+
restoredJobCount := 0
92+
for jobName, jobEntries := range entriesByJob {
93+
job, exists := jobsByName[jobName]
94+
if !exists {
95+
logger.Debugf("Skipping history for unknown job %q", jobName)
96+
continue
97+
}
98+
99+
// Sort by date ascending (oldest first) so SetLastRun works correctly
100+
sort.Slice(jobEntries, func(i, j int) bool {
101+
return jobEntries[i].Execution.Date.Before(jobEntries[j].Execution.Date)
102+
})
103+
104+
// Check if job supports SetLastRun
105+
setter, ok := job.(interface{ SetLastRun(*core.Execution) })
106+
if !ok {
107+
logger.Debugf("Job %q does not support history restoration", jobName)
108+
continue
109+
}
110+
111+
// Restore each execution
112+
for _, entry := range jobEntries {
113+
setter.SetLastRun(entry.Execution)
114+
restoredCount++
115+
}
116+
restoredJobCount++
117+
}
118+
119+
if restoredCount > 0 {
120+
logger.Noticef("Restored %d history entries for %d job(s) from saved files", restoredCount, restoredJobCount)
121+
}
122+
123+
return nil
124+
}
125+
126+
// parseHistoryFiles scans the save folder for JSON files and parses them.
127+
func parseHistoryFiles(saveFolder string, cutoff time.Time, logger core.Logger) ([]*restoredEntry, error) {
128+
var entries []*restoredEntry
129+
130+
// Resolve save folder to absolute path for containment check
131+
absSaveFolder, err := filepath.Abs(saveFolder)
132+
if err != nil {
133+
return nil, fmt.Errorf("resolve save folder: %w", err)
134+
}
135+
136+
err = filepath.Walk(saveFolder, func(path string, info os.FileInfo, walkErr error) error {
137+
if walkErr != nil {
138+
return nil //nolint:nilerr // Intentionally skip inaccessible files
139+
}
140+
141+
// Only process .json files
142+
if info.IsDir() || !strings.HasSuffix(info.Name(), ".json") {
143+
return nil
144+
}
145+
146+
// Skip files older than cutoff based on modification time (quick filter)
147+
if info.ModTime().Before(cutoff) {
148+
return nil
149+
}
150+
151+
// Verify path is within save folder (defense in depth for G304)
152+
absPath, absErr := filepath.Abs(path)
153+
if absErr != nil || !strings.HasPrefix(absPath, absSaveFolder) {
154+
return nil //nolint:nilerr // Intentionally skip invalid paths
155+
}
156+
157+
// Parse the JSON file
158+
entry, err := parseHistoryFile(absPath)
159+
if err != nil {
160+
logger.Debugf("Skipping invalid history file %q: %v", path, err)
161+
return nil
162+
}
163+
164+
// Skip if execution date is too old
165+
if entry.Execution.Date.Before(cutoff) {
166+
return nil
167+
}
168+
169+
entries = append(entries, entry)
170+
return nil
171+
})
172+
if err != nil {
173+
return entries, fmt.Errorf("walk save folder: %w", err)
174+
}
175+
return entries, nil
176+
}
177+
178+
// parseHistoryFile reads and parses a single JSON history file.
179+
// The path is validated by parseHistoryFiles to be within the save folder.
180+
func parseHistoryFile(path string) (*restoredEntry, error) {
181+
data, err := os.ReadFile(path) //#nosec G304 -- path is validated to be within save folder
182+
if err != nil {
183+
return nil, fmt.Errorf("read file: %w", err)
184+
}
185+
186+
var saved savedExecution
187+
if err := json.Unmarshal(data, &saved); err != nil {
188+
return nil, fmt.Errorf("parse json: %w", err)
189+
}
190+
191+
// Validate required fields
192+
if saved.Job.Name == "" {
193+
return nil, os.ErrInvalid
194+
}
195+
196+
// Convert to core.Execution
197+
exec := &core.Execution{
198+
ID: saved.Execution.ID,
199+
Date: saved.Execution.Date,
200+
Duration: saved.Execution.Duration,
201+
IsRunning: false, // Never restore as running
202+
Failed: saved.Execution.Failed,
203+
Skipped: saved.Execution.Skipped,
204+
}
205+
206+
return &restoredEntry{
207+
JobName: saved.Job.Name,
208+
Execution: exec,
209+
}, nil
210+
}

0 commit comments

Comments
 (0)