feat: add stuck job detection for run and await commands

Detect potentially stuck jobs and return early while the job continues
running in the background. This prevents indefinite waits when a job
hangs without producing output.

- Timeout: avg successful duration + 1 min (or 5 min if no history)
- Triggers when elapsed > timeout AND no output for 1 minute
- Shows helpful commands: gob stdout, gob await, gob stop
- Also applies to start -f and restart -f

Author: juanibiapina

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **Stuck job detection**: `gob run` and `gob await` now detect potentially stuck jobs and return early
+  - Timeout: average successful duration + 1 minute (or 5 minutes if no historical data)
+  - Triggers when: elapsed time exceeds timeout AND no output for 1 minute
+  - Displays stuck detection timeout when starting a job (e.g., "Stuck detection: timeout after 5m")
+  - When triggered, shows helpful commands: `gob stdout`, `gob await`, `gob stop`
+  - Job continues running in background - only the wait is aborted
+  - Also applies to `gob start -f` and `gob restart -f`
+
 ### Changed
 
 - **Stats by outcome**: `gob run` and `gob add` now show separate expected durations for successful and failed runs

README.md

@@ -32,6 +32,7 @@ No more "can you check if that's still running?" No more copy-pasting logs throu
 - **Reliable shutdowns** - Stop, restart, and shutdown verify every child process in the tree is gone
 - **Job persistence** - Jobs survive daemon restarts with SQLite-backed state
 - **Run history** - Track execution history and statistics for repeated commands
+- **Stuck detection** - Automatically detects jobs that may be stuck and returns early, while the job continues running
 
 ## Installation
 
@@ -184,9 +185,17 @@ Do NOT use `gob` for:
 - `gob await <job_id>` - Wait for job to finish, stream output
 - `gob await-any` - Wait for whichever job finishes first
 - `gob list` - List jobs with IDs, status, and descriptions
+- `gob stdout <job_id>` - View current output (useful if job may be stuck)
 - `gob stop <job_id>` - Graceful stop
 - `gob restart <job_id>` - Stop + start
 
+### Stuck Detection
+
+`gob run` and `gob await` automatically detect potentially stuck jobs:
+- Timeout: avg duration + 1 min (or 5 min if no history), triggers if no output for 1 min
+- Job continues running in background
+- Use `gob stdout <id>` to check output, `gob await <id>` to continue waiting
+
 ### Examples
 
 Servers and long-running:

cmd/await.go

@@ -61,15 +61,32 @@ Exit codes:
 		commandStr := strings.Join(job.Command, " ")
 
 		if job.Status == "running" {
+			// Fetch stats for stuck detection
+			var avgDurationMs int64
+			stats, err := client.Stats(jobID)
+			if err == nil && stats != nil && stats.SuccessCount >= 3 {
+				avgDurationMs = stats.AvgDurationMs
+			}
+			stuckTimeout := CalculateStuckTimeout(avgDurationMs)
+
 			fmt.Printf("Awaiting job %s: %s\n", job.ID, commandStr)
+			fmt.Printf("  Stuck detection: timeout after %s\n", formatDuration(stuckTimeout))
 
 			// Follow the output until completion
-			completed, err := followJob(job.ID, job.PID, job.StdoutPath)
+			followResult, err := followJob(job.ID, job.PID, job.StdoutPath, avgDurationMs)
 			if err != nil {
 				return err
 			}
 
-			if !completed {
+			if followResult.PossiblyStuck {
+				fmt.Printf("\nJob %s possibly stuck (no output for 1m)\n", job.ID)
+				fmt.Printf("  gob stdout %s   # check current output\n", job.ID)
+				fmt.Printf("  gob await %s    # continue waiting with output\n", job.ID)
+				fmt.Printf("  gob stop %s     # stop the job\n", job.ID)
+				return nil
+			}
+
+			if !followResult.Completed {
 				fmt.Printf("\nJob %s continues running in background\n", job.ID)
 				return nil
 			}

cmd/follow.go

@@ -12,10 +12,22 @@ import (
 	"github.com/juanibiapina/gob/internal/tail"
 )
 
-// followJob follows a job's output until it completes or is interrupted
-// Returns true if job completed, false if interrupted
+// DefaultStuckTimeoutMs is the timeout when no historical data (5 minutes)
+const DefaultStuckTimeoutMs int64 = 5 * 60 * 1000
+
+// NoOutputWindowMs is the constant "no output" window (1 minute)
+const NoOutputWindowMs int64 = 60 * 1000
+
+// FollowResult represents the result of following a job
+type FollowResult struct {
+	Completed     bool // job finished running
+	PossiblyStuck bool // job may be stuck (timed out without output)
+}
+
+// followJob follows a job's output until it completes, is interrupted, or is detected as possibly stuck
+// avgDurationMs is the average duration of successful runs (0 if no history)
 // stdoutPath is the full path to the stdout log file
-func followJob(jobID string, pid int, stdoutPath string) (bool, error) {
+func followJob(jobID string, pid int, stdoutPath string, avgDurationMs int64) (FollowResult, error) {
 	// Derive stderr path from stdout path
 	stderrPath := strings.Replace(stdoutPath, ".stdout.log", ".stderr.log", 1)
 
@@ -31,12 +43,26 @@ func followJob(jobID string, pid int, stdoutPath string) (bool, error) {
 
 	// Check if log files exist
 	if _, err := os.Stat(stdoutPath); os.IsNotExist(err) {
-		return false, fmt.Errorf("stdout log file not found: %s", stdoutPath)
+		return FollowResult{}, fmt.Errorf("stdout log file not found: %s", stdoutPath)
 	}
 	if _, err := os.Stat(stderrPath); os.IsNotExist(err) {
-		return false, fmt.Errorf("stderr log file not found: %s", stderrPath)
+		return FollowResult{}, fmt.Errorf("stderr log file not found: %s", stderrPath)
 	}
 
+	// Calculate stuck detection threshold
+	// No data: 5 minutes
+	// Has data: avg + 1 minute
+	// Trigger: elapsed > threshold AND no output for 1 minute
+	var stuckTimeoutMs int64
+	if avgDurationMs == 0 {
+		stuckTimeoutMs = DefaultStuckTimeoutMs
+	} else {
+		stuckTimeoutMs = avgDurationMs + NoOutputWindowMs
+	}
+
+	stuckTimeout := time.Duration(stuckTimeoutMs) * time.Millisecond
+	noOutputWindow := time.Duration(NoOutputWindowMs) * time.Millisecond
+
 	// Create follower
 	follower := tail.NewFollower(os.Stdout)
 
@@ -52,23 +78,37 @@ func followJob(jobID string, pid int, stdoutPath string) (bool, error) {
 	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
 
 	// Track completion status
-	completed := false
+	result := FollowResult{}
+	startTime := time.Now()
 
-	// Monitor for process completion or signal
+	// Monitor for process completion, signal, or stuck condition
 	done := make(chan struct{})
 	go func() {
 		for {
 			select {
 			case <-done:
 				return
 			default:
+				// Check if process completed
 				if !process.IsProcessRunning(pid) {
 					// Give a moment for any final output to be written
 					time.Sleep(200 * time.Millisecond)
-					completed = true
+					result.Completed = true
+					follower.Stop()
+					return
+				}
+
+				// Check for stuck condition
+				// Trigger: elapsed > timeout AND no output for 1 minute
+				elapsed := time.Since(startTime)
+				timeSinceOutput := time.Since(follower.LastOutputTime())
+
+				if elapsed > stuckTimeout && timeSinceOutput > noOutputWindow {
+					result.PossiblyStuck = true
 					follower.Stop()
 					return
 				}
+
 				time.Sleep(100 * time.Millisecond)
 			}
 		}
@@ -83,5 +123,15 @@ func followJob(jobID string, pid int, stdoutPath string) (bool, error) {
 
 	follower.Wait()
 
-	return completed, nil
+	return result, nil
+}
+
+// CalculateStuckTimeout returns the stuck detection timeout based on average duration
+// No data: 5 minutes
+// Has data: avg + 1 minute
+func CalculateStuckTimeout(avgDurationMs int64) time.Duration {
+	if avgDurationMs == 0 {
+		return time.Duration(DefaultStuckTimeoutMs) * time.Millisecond
+	}
+	return time.Duration(avgDurationMs+NoOutputWindowMs) * time.Millisecond
 }

cmd/restart.go

@@ -74,11 +74,25 @@ Exit codes:
 
 		// If follow flag is set, follow the output
 		if restartFollow {
-			completed, err := followJob(jobID, job.PID, job.StdoutPath)
+			// Fetch stats for stuck detection
+			var avgDurationMs int64
+			stats, statsErr := client.Stats(jobID)
+			if statsErr == nil && stats != nil && stats.SuccessCount >= 3 {
+				avgDurationMs = stats.AvgDurationMs
+			}
+			stuckTimeout := CalculateStuckTimeout(avgDurationMs)
+			fmt.Printf("  Stuck detection: timeout after %s\n", formatDuration(stuckTimeout))
+
+			followResult, err := followJob(jobID, job.PID, job.StdoutPath, avgDurationMs)
 			if err != nil {
 				return err
 			}
-			if completed {
+			if followResult.PossiblyStuck {
+				fmt.Printf("\nJob %s possibly stuck (no output for 1m)\n", jobID)
+				fmt.Printf("  gob stdout %s   # check current output\n", jobID)
+				fmt.Printf("  gob await %s    # continue waiting with output\n", jobID)
+				fmt.Printf("  gob stop %s     # stop the job\n", jobID)
+			} else if followResult.Completed {
 				fmt.Printf("\nJob %s completed\n", jobID)
 			} else {
 				fmt.Printf("\nJob %s continues running in background\n", jobID)

cmd/run.go

@@ -118,11 +118,19 @@ Exit codes:
 
 		commandStr := strings.Join(commandArgs, " ")
 
+		// Determine average duration for stuck detection
+		var avgDurationMs int64
+		if result.Stats != nil && result.Stats.SuccessCount >= 3 {
+			avgDurationMs = result.Stats.AvgDurationMs
+		}
+		stuckTimeout := CalculateStuckTimeout(avgDurationMs)
+
 		// Print message based on action
 		if result.Action == "already_running" {
 			startedAt, _ := time.Parse(time.RFC3339, result.Job.StartedAt)
 			duration := formatDuration(time.Since(startedAt))
 			fmt.Printf("Job %s already running (since %s ago), attaching...\n", result.Job.ID, duration)
+			fmt.Printf("  Stuck detection: timeout after %s\n", formatDuration(stuckTimeout))
 		} else {
 			fmt.Printf("Running job %s: %s\n", result.Job.ID, commandStr)
 
@@ -139,15 +147,24 @@ Exit codes:
 						formatDuration(time.Duration(result.Stats.FailureAvgDurationMs)*time.Millisecond))
 				}
 			}
+			fmt.Printf("  Stuck detection: timeout after %s\n", formatDuration(stuckTimeout))
 		}
 
 		// Follow the output until completion
-		completed, err := followJob(result.Job.ID, result.Job.PID, result.Job.StdoutPath)
+		followResult, err := followJob(result.Job.ID, result.Job.PID, result.Job.StdoutPath, avgDurationMs)
 		if err != nil {
 			return err
 		}
 
-		if !completed {
+		if followResult.PossiblyStuck {
+			fmt.Printf("\nJob %s possibly stuck (no output for 1m)\n", result.Job.ID)
+			fmt.Printf("  gob stdout %s   # check current output\n", result.Job.ID)
+			fmt.Printf("  gob await %s    # continue waiting with output\n", result.Job.ID)
+			fmt.Printf("  gob stop %s     # stop the job\n", result.Job.ID)
+			return nil
+		}
+
+		if !followResult.Completed {
 			fmt.Printf("\nJob %s continues running in background\n", result.Job.ID)
 			fmt.Printf("  gob await %s   # wait for completion with live output\n", result.Job.ID)
 			fmt.Printf("  gob stop %s    # stop the job\n", result.Job.ID)

cmd/start.go

@@ -63,11 +63,25 @@ Exit codes:
 
 		// If follow flag is set, follow the output
 		if startFollow {
-			completed, err := followJob(jobID, job.PID, job.StdoutPath)
+			// Fetch stats for stuck detection
+			var avgDurationMs int64
+			stats, statsErr := client.Stats(jobID)
+			if statsErr == nil && stats != nil && stats.SuccessCount >= 3 {
+				avgDurationMs = stats.AvgDurationMs
+			}
+			stuckTimeout := CalculateStuckTimeout(avgDurationMs)
+			fmt.Printf("  Stuck detection: timeout after %s\n", formatDuration(stuckTimeout))
+
+			followResult, err := followJob(jobID, job.PID, job.StdoutPath, avgDurationMs)
 			if err != nil {
 				return err
 			}
-			if completed {
+			if followResult.PossiblyStuck {
+				fmt.Printf("\nJob %s possibly stuck (no output for 1m)\n", jobID)
+				fmt.Printf("  gob stdout %s   # check current output\n", jobID)
+				fmt.Printf("  gob await %s    # continue waiting with output\n", jobID)
+				fmt.Printf("  gob stop %s     # stop the job\n", jobID)
+			} else if followResult.Completed {
 				fmt.Printf("\nJob %s completed\n", jobID)
 			} else {
 				fmt.Printf("\nJob %s continues running in background\n", jobID)

internal/tail/follow.go

@@ -60,13 +60,14 @@ type FileSource struct {
 
 // Follower manages following multiple files with support for dynamic source addition
 type Follower struct {
-	w       io.Writer
-	mu      sync.Mutex
-	sources map[string]bool // tracks which paths are already being followed
-	errCh   chan error
-	wg      sync.WaitGroup
-	done    chan struct{}
-	stopped bool
+	w              io.Writer
+	mu             sync.Mutex
+	sources        map[string]bool // tracks which paths are already being followed
+	errCh          chan error
+	wg             sync.WaitGroup
+	done           chan struct{}
+	stopped        bool
+	lastOutputTime time.Time // time of last output written
 }
 
 // SystemLogTag is the prefix used for system log messages (same length as job IDs)
@@ -86,13 +87,26 @@ func (f *Follower) SystemLog(format string, args ...interface{}) {
 // NewFollower creates a new Follower that writes to the given writer
 func NewFollower(w io.Writer) *Follower {
 	return &Follower{
-		w:       w,
-		sources: make(map[string]bool),
-		errCh:   make(chan error, 100),
-		done:    make(chan struct{}),
+		w:              w,
+		sources:        make(map[string]bool),
+		errCh:          make(chan error, 100),
+		done:           make(chan struct{}),
+		lastOutputTime: time.Now(), // initialize to now so we don't immediately trigger stuck detection
 	}
 }
 
+// LastOutputTime returns the time of the last output written (thread-safe)
+func (f *Follower) LastOutputTime() time.Time {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.lastOutputTime
+}
+
+// updateLastOutputTime updates the last output time (must be called with mu held)
+func (f *Follower) updateLastOutputTime() {
+	f.lastOutputTime = time.Now()
+}
+
 // AddSource adds a new file source to follow. If the source is already being
 // followed, this is a no-op.
 func (f *Follower) AddSource(source FileSource) {
@@ -107,7 +121,7 @@ func (f *Follower) AddSource(source FileSource) {
 	f.wg.Add(1)
 	go func() {
 		defer f.wg.Done()
-		err := followWithPrefix(source.Path, source.Prefix, f.w, &f.mu, f.done)
+		err := followWithPrefix(source.Path, source.Prefix, f.w, &f.mu, f.done, f.updateLastOutputTime)
 		if err != nil {
 			f.errCh <- err
 		}
@@ -149,7 +163,8 @@ func FollowMultiple(sources []FileSource, w io.Writer) error {
 }
 
 // followWithPrefix follows a file and prefixes each line with the given prefix
-func followWithPrefix(filePath string, prefix string, w io.Writer, mu *sync.Mutex, done <-chan struct{}) error {
+// onOutput is called (with mu held) each time output is written
+func followWithPrefix(filePath string, prefix string, w io.Writer, mu *sync.Mutex, done <-chan struct{}, onOutput func()) error {
 	file, err := os.Open(filePath)
 	if err != nil {
 		return err
@@ -192,6 +207,9 @@ func followWithPrefix(filePath string, prefix string, w io.Writer, mu *sync.Mute
 						w.Write([]byte(prefix))
 					}
 					w.Write(line)
+					if onOutput != nil {
+						onOutput()
+					}
 					mu.Unlock()
 
 					data = data[idx+1:]