cleanup progress

Author: davidnewhall

.golangci.yml

@@ -35,6 +35,7 @@ linters:
             - golift.io
             - golang.org/x
             - gopkg.in/yaml.v3
+            - code.cloudfoundry.org/bytefmt
 
   exclusions:
     generated: lax

examples/docker-compose.yml

@@ -137,4 +137,4 @@ services:
     - UN_CMDHOOK_0_EXCLUDE_1=lidarr
     - UN_CMDHOOK_0_TIMEOUT=10s
 
-## => Content Auto Generated, 20 APR 2025 20:24 UTC
+## => Content Auto Generated, 21 APR 2025 05:15 UTC

examples/unpackerr.conf.example

@@ -306,4 +306,4 @@ dir_mode = "0755"
 ## You can adjust how long to wait for the command to run.
 # timeout = "10s"
 
-## => Content Auto Generated, 20 APR 2025 20:24 UTC
+## => Content Auto Generated, 21 APR 2025 05:15 UTC

go.mod

@@ -25,6 +25,7 @@ require (
 )
 
 require (
+	code.cloudfoundry.org/bytefmt v0.37.0 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bodgit/plumbing v1.3.0 // indirect

go.sum

@@ -14,6 +14,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+code.cloudfoundry.org/bytefmt v0.37.0 h1:GR5rZgr/6QLV/U2/xgORHUY4lu4EEBgN4/cz7SD3GqM=
+code.cloudfoundry.org/bytefmt v0.37.0/go.mod h1:u3/LRyPLBYFtn8h9CnzTeupRMD+76qCovDU0vND81lU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=

pkg/unpackerr/folder.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"code.cloudfoundry.org/bytefmt"
 	"github.com/fsnotify/fsnotify"
 	"github.com/radovskyb/watcher"
 	"golift.io/cnfg"
@@ -278,7 +279,7 @@ func (u *Unpackerr) extractTrackedItem(name string, folder *Folder, now time.Tim
 	}
 
 	// create a queue counter in the main history; add to u.Map and send webhook for a new folder.
-	u.updateQueueStatus(&newStatus{Name: name, Status: QUEUED}, u.folders.Folders[name].updated, true)
+	item := u.updateQueueStatus(&newStatus{Name: name, Status: QUEUED}, u.folders.Folders[name].updated, true)
 	u.updateHistory(FolderString + ": " + name)
 
 	var exclude []string
@@ -297,7 +298,7 @@ func (u *Unpackerr) extractTrackedItem(name string, folder *Folder, now time.Tim
 		DeleteOrig:       false,
 		CBChannel:        u.folders.Updates,
 		CBFunction:       nil,
-		Progress:         u.progressUpdateCallback(name),
+		Progress:         u.progressUpdateCallback(item),
 		LogFile:          !folder.config.DisableLog,
 		DisableRecursion: folder.config.DisableRecursion,
 	})
@@ -332,17 +333,15 @@ func getFileList(path string) []os.FileInfo {
 func (u *Unpackerr) folderXtractrCallback(resp *xtractr.Response) {
 	folder, ok := u.folders.Folders[resp.X.Name]
 
-	switch {
-	case !ok:
+	switch item := u.Map[resp.X.Name]; {
+	case !ok, item == nil:
 		// It doesn't exist? weird. delete it and bail out.
+		delete(u.folders.Folders, resp.X.Name)
 		delete(u.Map, resp.X.Name)
+
 		return
 	case !resp.Done:
-		if u.Map[resp.X.Name] != nil && u.Map[resp.X.Name].Progress != nil {
-			// Update the total archive count in the progress status.
-			u.Map[resp.X.Name].Progress.Archives = resp.Archives.Count()
-		}
-
+		item.XProg.Archives = resp.Archives.Count() + resp.Extras.Count()
 		folder.status = EXTRACTING
 		u.Printf("[Folder] Extraction Started: %s, retries: %d, items in queue: %d", resp.X.Name, folder.retries, resp.Queued)
 	case errors.Is(resp.Error, xtractr.ErrNoCompressedFiles):
@@ -356,9 +355,9 @@ func (u *Unpackerr) folderXtractrCallback(resp *xtractr.Response) {
 	default: // this runs in a go routine
 		u.updateMetrics(resp, FolderString, folder.config.Path)
 		u.Printf("[Folder] Extraction Finished: %s => elapsed: %v, archives: %d, "+
-			"extra archives: %d, files extracted: %d, written: %dMiB",
+			"extra archives: %d, files extracted: %d, written: %sB",
 			resp.X.Name, resp.Elapsed.Round(time.Second), resp.Archives.Count(),
-			resp.Extras.Count(), len(resp.NewFiles), resp.Size/mebiByte)
+			resp.Extras.Count(), len(resp.NewFiles), bytefmt.ByteSize(resp.Size))
 
 		folder.archives = resp.Archives
 		folder.status = EXTRACTED
@@ -557,7 +556,7 @@ type newStatus struct {
 // updateQueueStatus for an on-going tracked extraction.
 // This is called from a channel callback to update status in a single go routine.
 // This is used by apps and Folders in a few other places as well.
-func (u *Unpackerr) updateQueueStatus(data *newStatus, now time.Time, sendHook bool) {
+func (u *Unpackerr) updateQueueStatus(data *newStatus, now time.Time, sendHook bool) *Extract {
 	if _, ok := u.Map[data.Name]; !ok {
 		// This is a new Folder being queued for extraction.
 		// Arr apps do not land here. They create their own queued items in u.Map.
@@ -569,11 +568,13 @@ func (u *Unpackerr) updateQueueStatus(data *newStatus, now time.Time, sendHook b
 			IDs:     map[string]any{"title": data.Name}, // required or webhook may break.
 		}
 
+		u.Map[data.Name].XProg = &ExtractProgress{Extract: u.Map[data.Name]}
+
 		if sendHook {
 			u.runAllHooks(u.Map[data.Name])
 		}
 
-		return
+		return u.Map[data.Name]
 	}
 
 	if data.Resp != nil {
@@ -586,4 +587,6 @@ func (u *Unpackerr) updateQueueStatus(data *newStatus, now time.Time, sendHook b
 	if sendHook {
 		u.runAllHooks(u.Map[data.Name])
 	}
+
+	return u.Map[data.Name]
 }

pkg/unpackerr/handlers.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"code.cloudfoundry.org/bytefmt"
 	"golift.io/cnfg"
 	"golift.io/starr"
 	"golift.io/xtractr"
@@ -25,7 +26,7 @@ type Extract struct {
 	Status      ExtractStatus
 	IDs         map[string]any
 	Resp        *xtractr.Response
-	Progress    *Progress
+	XProg       *ExtractProgress
 }
 
 // Shared config items for all starr apps.
@@ -75,7 +76,7 @@ func (u *Unpackerr) checkQueueChanges(now time.Time) {
 		}
 
 		u.Printf("[%s] Status: %s (%v, elapsed: %v) %s", data.App, name, data.Status.Desc(),
-			now.Sub(data.Updated).Round(time.Second), data.Progress)
+			now.Sub(data.Updated).Round(time.Second), data.XProg)
 	}
 }
 
@@ -134,7 +135,7 @@ func (u *Unpackerr) extractCompletedDownload(name string, now time.Time, item *E
 		TempFolder: false,
 		DeleteOrig: false,
 		CBChannel:  u.updates,
-		Progress:   u.progressUpdateCallback(name),
+		Progress:   u.progressUpdateCallback(item),
 	})
 
 	u.logQueuedDownload(queueSize, item, files)
@@ -208,10 +209,8 @@ func (u *Unpackerr) checkExtractDone(now time.Time) {
 func (u *Unpackerr) handleXtractrCallback(resp *xtractr.Response) {
 	if item := u.Map[resp.X.Name]; resp.Done && item != nil {
 		u.updateMetrics(resp, item.App, item.URL)
-
-		if item.Progress != nil {
-			item.Progress.Archives = resp.Archives.Count()
-		}
+	} else if item != nil {
+		item.XProg.Archives = resp.Archives.Count() + resp.Extras.Count()
 	}
 
 	switch now := resp.Started.Add(resp.Elapsed); {
@@ -224,8 +223,8 @@ func (u *Unpackerr) handleXtractrCallback(resp *xtractr.Response) {
 	default:
 		files := fileList(resp.X.Path)
 		u.Printf("Extraction Finished: %s => elapsed: %v, archives: %d, extra archives: %d, "+
-			"files extracted: %d, wrote: %dMiB", resp.X.Name, resp.Elapsed.Round(time.Second),
-			resp.Archives.Count(), resp.Extras.Count(), len(resp.NewFiles), resp.Size/mebiByte)
+			"files extracted: %d, wrote: %sB", resp.X.Name, resp.Elapsed.Round(time.Second),
+			resp.Archives.Count(), resp.Extras.Count(), len(resp.NewFiles), bytefmt.ByteSize(resp.Size))
 		u.Debugf("Extraction Finished: %d files in path: %s", len(files), files)
 		u.updateQueueStatus(&newStatus{Name: resp.X.Name, Status: EXTRACTED, Resp: resp}, now, true)
 	}

pkg/unpackerr/lidarr.go

@@ -104,6 +104,7 @@ func (u *Unpackerr) checkLidarrQueue(now time.Time) {
 						"reason":     buildStatusReason(record.Status, record.StatusMessages),
 					},
 				}
+				u.Map[record.Title].XProg = &ExtractProgress{Extract: u.Map[record.Title]}
 
 				fallthrough
 			default:

pkg/unpackerr/progress.go

@@ -3,28 +3,33 @@ package unpackerr
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
+	"code.cloudfoundry.org/bytefmt"
 	"golift.io/xtractr"
 )
 
 const (
-	minimumProgressInterval = 2 * time.Second
+	minimumProgressInterval = time.Second
 	defaultProgressInterval = 15 * time.Second
 )
 
-type Progress struct {
+// ExtractProgress holds the progress for an entire Extract.
+// An Extract is "a new item in a watch folder" or "a download in a starr app".
+// Either may produce multiple xtractr.XFile structs (extractable archives).
+type ExtractProgress struct {
 	*xtractr.Progress
-	// Name of this item in the Map.
-	Name string
+	// Extract extract that exists in the map.
+	*Extract
 	// Number of archives in this Xtract.
 	Archives int
 	// Number of archives extracted from this Xtract.
 	Extracted int
 }
 
-func (p *Progress) String() string {
-	if p == nil {
+func (p *ExtractProgress) String() string {
+	if p == nil || p.Progress == nil {
 		return "no progress yet"
 	}
 
@@ -36,32 +41,26 @@ func (p *Progress) String() string {
 		wrote, total = p.Read, p.Compressed
 	}
 
-	return fmt.Sprintf("extracted %d/%d archives, current: %d/%d bytes (%.0f%%): %s",
-		p.Extracted, p.Archives, wrote, total, p.Percent(), filepath.Base(p.XFile.FilePath))
+	return fmt.Sprintf("on archive: %d/%d @ %sB/%sB (%.0f%%): %s",
+		p.Extracted+1, p.Archives, bytefmt.ByteSize(wrote), bytefmt.ByteSize(total),
+		p.Percent(), strings.TrimLeft(strings.TrimPrefix(p.XFile.FilePath, p.Path), string(filepath.Separator)))
 }
 
-func (u *Unpackerr) progressUpdateCallback(name string) func(xtractr.Progress) {
-	return func(prog xtractr.Progress) {
-		u.progress <- &Progress{Progress: &prog, Name: name} // ends up in u.handleProgress() (below)
+func (u *Unpackerr) progressUpdateCallback(item *Extract) func(xtractr.Progress) {
+	return func(prog xtractr.Progress) { // sends update to u.handleProgress() (below)
+		u.progChan <- &ExtractProgress{Progress: &prog, Extract: item}
 	}
 }
 
-func (u *Unpackerr) handleProgress(prog *Progress) {
-	if item := u.Map[prog.Name]; item != nil {
-		if item.Progress == nil {
-			item.Progress = prog
-		} else {
-			item.Progress.Progress = prog.Progress
-		}
-
-		if prog.Done {
-			item.Progress.Extracted++
-		}
-
-		if item.Resp != nil {
-			prog.Archives = item.Resp.Archives.Count()
-		}
+// prog = what just came in, it's ephemeral.
+// prog.XProg = what is saved in the map, update this one.
+// prog.Progress = also what just came in, we just set it here.
+func (u *Unpackerr) handleProgress(prog *ExtractProgress) {
+	if prog.XProg.Progress != nil && prog.XProg.XFile != prog.XFile {
+		prog.XProg.Extracted++
 	}
+
+	prog.XProg.Progress = prog.Progress
 }
 
 func (u *Unpackerr) printProgress(now time.Time) {
@@ -70,7 +69,9 @@ func (u *Unpackerr) printProgress(now time.Time) {
 			continue
 		}
 
-		u.Printf("[%s] Status: %s (%v, elapsed: %v) %s", data.App, name, data.Status.Desc(),
-			now.Sub(data.Updated).Round(time.Second), data.Progress)
+		if prog := data.XProg.String(); prog != "no progress yet" {
+			u.Printf("[%s] Status: %s (%v, elapsed: %v) %s", data.App, name, data.Status.Desc(),
+				now.Sub(data.Updated).Round(time.Second), prog)
+		}
 	}
 }

pkg/unpackerr/radarr.go

@@ -103,6 +103,7 @@ func (u *Unpackerr) checkRadarrQueue(now time.Time) {
 						"reason":     buildStatusReason(record.Status, record.StatusMessages),
 					},
 				}
+				u.Map[record.Title].XProg = &ExtractProgress{Extract: u.Map[record.Title]}
 
 				fallthrough
 			default: