Skip to content

Commit 1b36f41

Browse files
committed
add progress support
1 parent e36aace commit 1b36f41

File tree

6 files changed

+94
-13
lines changed

6 files changed

+94
-13
lines changed

go.mod

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
golift.io/rotatorr v0.0.0-20240723172740-cb73b9c4894c
2121
golift.io/starr v1.1.0
2222
golift.io/version v0.0.2
23-
golift.io/xtractr v0.2.3-0.20240710043203-2d7c8a38d931
23+
golift.io/xtractr v0.2.3-0.20250420160016-6e4a403547d5
2424
gopkg.in/yaml.v3 v3.0.1
2525
)
2626

@@ -30,6 +30,8 @@ require (
3030
github.com/bodgit/plumbing v1.3.0 // indirect
3131
github.com/bodgit/sevenzip v1.6.0 // indirect
3232
github.com/bodgit/windows v1.0.1 // indirect
33+
github.com/cavaliergopher/cpio v1.0.1 // indirect
34+
github.com/cavaliergopher/rpm v1.3.0 // indirect
3335
github.com/cespare/xxhash/v2 v2.3.0 // indirect
3436
github.com/getlantern/context v0.0.0-20220418194847-3d5e7a086201 // indirect
3537
github.com/getlantern/errors v1.0.4 // indirect
@@ -50,6 +52,7 @@ require (
5052
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
5153
github.com/nwaples/rardecode v1.1.3 // indirect
5254
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
55+
github.com/peterebden/ar v0.0.0-20241106141004-20dc11b778e8 // indirect
5356
github.com/pierrec/lz4/v4 v4.1.22 // indirect
5457
github.com/pkg/errors v0.9.1 // indirect
5558
github.com/prometheus/client_model v0.6.2 // indirect
@@ -65,6 +68,7 @@ require (
6568
go.uber.org/multierr v1.11.0 // indirect
6669
go.uber.org/zap v1.27.0 // indirect
6770
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
71+
golang.org/x/crypto v0.37.0 // indirect
6872
golang.org/x/net v0.39.0 // indirect
6973
golang.org/x/sys v0.32.0 // indirect
7074
golang.org/x/text v0.24.0 // indirect

go.sum

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A
3030
github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc=
3131
github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
3232
github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
33+
github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM=
34+
github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc=
35+
github.com/cavaliergopher/rpm v1.3.0 h1:UHX46sasX8MesUXXQ+UbkFLUX4eUWTlEcX8jcnRBIgI=
36+
github.com/cavaliergopher/rpm v1.3.0/go.mod h1:vEumo1vvtrHM1Ov86f6+k8j7zNKOxQfHDCAIcR/36ZI=
3337
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
3438
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
3539
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -162,6 +166,8 @@ github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9l
162166
github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
163167
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw=
164168
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
169+
github.com/peterebden/ar v0.0.0-20241106141004-20dc11b778e8 h1:27L3dHkYbeWGU3/5NasAzVDgXG9QzlfKCvcl4cdNW6c=
170+
github.com/peterebden/ar v0.0.0-20241106141004-20dc11b778e8/go.mod h1:hpFkyhCgB5Rm8FK+ISypOE+9UyrCuL6MNcjPMB1s1ec=
165171
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
166172
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
167173
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -242,6 +248,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
242248
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
243249
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
244250
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
251+
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
252+
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
245253
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
246254
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
247255
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -383,8 +391,8 @@ golift.io/starr v1.1.0 h1:KTAecOEne/zKQvrh8mcfEA7YlQ39FnvjZRIhJSIvxL4=
383391
golift.io/starr v1.1.0/go.mod h1:WnLkyfF7X2q676mXriGMZQrBA3wGt1BjA2qdxMmA/wg=
384392
golift.io/version v0.0.2 h1:i0gXRuSDHKs4O0sVDUg4+vNIuOxYoXhaxspftu2FRTE=
385393
golift.io/version v0.0.2/go.mod h1:76aHNz8/Pm7CbuxIsDi97jABL5Zui3f2uZxDm4vB6hU=
386-
golift.io/xtractr v0.2.3-0.20240710043203-2d7c8a38d931 h1:3VYISfgN0VCI6qsW/131UaqYuT6glyk1/Eu9Y6ndPMU=
387-
golift.io/xtractr v0.2.3-0.20240710043203-2d7c8a38d931/go.mod h1:TrvZlQlpYaZC8kgu/vjvynnNW4rtRqKPbhKF74WoBGk=
394+
golift.io/xtractr v0.2.3-0.20250420160016-6e4a403547d5 h1:j4ayxll0tBJ2ssScu9/DwXiZpuBoOlBreVI0Ogz4cQo=
395+
golift.io/xtractr v0.2.3-0.20250420160016-6e4a403547d5/go.mod h1:invEOYfyBnFtegY2V2n+9K5bUEHB8pGZng1BK0U2r38=
388396
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
389397
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
390398
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=

pkg/unpackerr/folder.go

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,19 +93,19 @@ func (u *Unpackerr) logFolders() {
9393
epath = ", extract to: " + folder.ExtractPath
9494
}
9595

96-
u.Printf(" => Folder Config: 1 path: %s%s (delete_after:%v, delete_orig:%v, delete_files:%v"+
97-
"log_file:%v, move_back:%v, isos:%v, event_buffer:%d)",
96+
u.Printf(" => Folder Config: 1 path: %s%s; delete_after:%v delete_orig:%v delete_files:%v "+
97+
"log_file:%v move_back:%v isos:%v event_buffer:%d",
9898
folder.Path, epath, folder.DeleteAfter, folder.DeleteOrig, folder.DeleteFiles,
9999
!folder.DisableLog, folder.MoveBack, folder.ExtractISOs, u.Folder.Buffer)
100100
} else {
101101
u.Printf(" => Folder Config: %d paths, event_buffer:%d ", count, u.Folder.Buffer)
102102

103103
for _, folder := range u.Folders {
104104
if epath = ""; folder.ExtractPath != "" {
105-
epath = ", extract to: " + folder.ExtractPath
105+
epath = " extract to: " + folder.ExtractPath
106106
}
107107

108-
u.Printf(" => Path: %s%s (delete_after:%v, delete_orig:%v, delete_files:%v, log_file:%v, move_back:%v, isos:%v)",
108+
u.Printf(" => Path: %s%s; delete_after:%v delete_orig:%v delete_files:%v log_file:%v move_back:%v isos:%v",
109109
folder.Path, epath, folder.DeleteAfter, folder.DeleteOrig, folder.DeleteFiles,
110110
!folder.DisableLog, folder.MoveBack, folder.ExtractISOs)
111111
}
@@ -297,6 +297,7 @@ func (u *Unpackerr) extractTrackedItem(name string, folder *Folder, now time.Tim
297297
DeleteOrig: false,
298298
CBChannel: u.folders.Updates,
299299
CBFunction: nil,
300+
Updates: u.handleProgressUpdate(name),
300301
LogFile: !folder.config.DisableLog,
301302
DisableRecursion: folder.config.DisableRecursion,
302303
})
@@ -337,6 +338,11 @@ func (u *Unpackerr) folderXtractrCallback(resp *xtractr.Response) {
337338
delete(u.Map, resp.X.Name)
338339
return
339340
case !resp.Done:
341+
if u.Map[resp.X.Name] != nil && u.Map[resp.X.Name].Progress != nil {
342+
// Update the total archive count in the progress status.
343+
u.Map[resp.X.Name].Progress.Archives = len(resp.Archives)
344+
}
345+
340346
folder.status = EXTRACTING
341347
u.Printf("[Folder] Extraction Started: %s, retries: %d, items in queue: %d", resp.X.Name, folder.retries, resp.Queued)
342348
case errors.Is(resp.Error, xtractr.ErrNoCompressedFiles):
@@ -520,7 +526,6 @@ func (u *Unpackerr) checkFolderStats(now time.Time) {
520526
//nolint:wsl
521527
func (u *Unpackerr) deleteAfterReached(name string, now time.Time, folder *Folder) {
522528
var webhook bool
523-
524529
// Folder reached delete delay (after extraction), nuke it.
525530
if folder.config.DeleteFiles && !folder.config.MoveBack {
526531
u.delChan <- &fileDeleteReq{Paths: []string{strings.TrimRight(name, `/\`) + suffix}}

pkg/unpackerr/handlers.go

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type Extract struct {
2525
Status ExtractStatus
2626
IDs map[string]any
2727
Resp *xtractr.Response
28+
Progress *Progress
2829
}
2930

3031
// Shared config items for all starr apps.
@@ -73,11 +74,28 @@ func (u *Unpackerr) checkQueueChanges(now time.Time) {
7374
data.Updated = now
7475
}
7576

76-
u.Printf("[%s] Status: %s (%v, elapsed: %v)", data.App, name, data.Status.Desc(),
77-
now.Sub(data.Updated).Round(time.Second))
77+
u.Printf("[%s] Status: %s (%v, elapsed: %v) %s", data.App, name, data.Status.Desc(),
78+
now.Sub(data.Updated).Round(time.Second), data.Progress)
7879
}
7980
}
8081

82+
func (p *Progress) String() string {
83+
if p == nil {
84+
return ""
85+
}
86+
87+
var wrote, total uint64
88+
89+
if p.Total > 0 {
90+
wrote, total = p.Wrote, p.Total
91+
} else if p.Compressed > 0 {
92+
wrote, total = p.Read, p.Compressed
93+
}
94+
95+
return fmt.Sprintf("archive %d/%d current: %d/%d bytes (%.0f)",
96+
p.Extracted+1, p.Archives, wrote, total, p.Percent())
97+
}
98+
8199
// extractCompletedDownloads process each download and checks if it needs to be extracted.
82100
// This is called from the main go routine in start.go and it only processes starr apps, not folders.
83101
func (u *Unpackerr) extractCompletedDownloads(now time.Time) {
@@ -126,18 +144,54 @@ func (u *Unpackerr) extractCompletedDownload(name string, now time.Time, item *E
126144
Name: name,
127145
Filter: xtractr.Filter{
128146
Path: item.Path,
129-
ExcludeSuffix: xtractr.AllExcept([]string{
147+
ExcludeSuffix: xtractr.AllExcept(
130148
".rar", ".r00", ".zip", ".7z", ".7z.001", ".gz", ".tgz", ".tar", ".tar.gz", ".bz2", ".tbz2",
131-
}),
149+
),
132150
},
133151
TempFolder: false,
134152
DeleteOrig: false,
135153
CBChannel: u.updates,
154+
Updates: u.handleProgressUpdate(name),
136155
})
137156

138157
u.logQueuedDownload(queueSize, item, files)
139158
}
140159

160+
type Progress struct {
161+
xtractr.Progress
162+
// Name of this item in the Map.
163+
Name string
164+
// Number of archives in this Xtract.
165+
Archives int
166+
// Number of archives extracted from this Xtract.
167+
Extracted int
168+
}
169+
170+
func (u *Unpackerr) handleProgressUpdate(name string) chan xtractr.Progress {
171+
chn := make(chan xtractr.Progress)
172+
prog := &Progress{Name: name}
173+
174+
go func() {
175+
for p := range chn {
176+
if prog.Progress = p; p.Done {
177+
prog.Extracted++
178+
}
179+
180+
u.progress <- prog // ends up in u.handleProgress() (below)
181+
}
182+
}()
183+
184+
return chn
185+
}
186+
187+
func (u *Unpackerr) handleProgress(prog *Progress) {
188+
if item := u.Map[prog.Name]; item != nil {
189+
if item.Progress = prog; item.Resp != nil {
190+
prog.Archives = len(item.Resp.Archives)
191+
}
192+
}
193+
}
194+
141195
func (u *Unpackerr) logQueuedDownload(queueSize int, item *Extract, files xtractr.ArchiveList) {
142196
count := fmt.Sprint("1 archive: ", files.Random()[0])
143197
if fileCount := files.Count(); fileCount > 1 {
@@ -206,6 +260,10 @@ func (u *Unpackerr) checkExtractDone(now time.Time) {
206260
func (u *Unpackerr) handleXtractrCallback(resp *xtractr.Response) {
207261
if item := u.Map[resp.X.Name]; resp.Done && item != nil {
208262
u.updateMetrics(resp, item.App, item.URL)
263+
264+
if item.Progress != nil {
265+
item.Progress.Archives = len(resp.Archives)
266+
}
209267
}
210268

211269
switch now := resp.Started.Add(resp.Elapsed); {
@@ -218,6 +276,7 @@ func (u *Unpackerr) handleXtractrCallback(resp *xtractr.Response) {
218276
default:
219277
files := fileList(resp.X.Path)
220278

279+
close(resp.X.Updates)
221280
u.Printf("Extraction Finished: %s => elapsed: %v, archives: %d, extra archives: %d, "+
222281
"files extracted: %d, wrote: %dMiB", resp.X.Name, resp.Elapsed.Round(time.Second),
223282
resp.Archives.Count(), resp.Extras.Count(), len(resp.NewFiles), resp.Size/mebiByte)

pkg/unpackerr/start.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ type Unpackerr struct {
5959
folders *Folders
6060
sigChan chan os.Signal
6161
updates chan *xtractr.Response
62+
progress chan *Progress
6263
hookChan chan *hookQueueItem
6364
delChan chan *fileDeleteReq
6465
workChan chan []func()
@@ -99,6 +100,7 @@ func New() *Unpackerr {
99100
workChan: make(chan []func(), 1),
100101
History: &History{Map: make(map[string]*Extract)},
101102
updates: make(chan *xtractr.Response, updateChanBuf),
103+
progress: make(chan *Progress),
102104
menu: make(map[string]ui.MenuItem),
103105
Config: &Config{
104106
KeepHistory: defaultHistory,
@@ -306,6 +308,9 @@ func (u *Unpackerr) Run() {
306308
case now := <-logger.C:
307309
// Log/print current queue counts once in a while.
308310
u.logCurrentQueue(now)
311+
case prog := <-u.progress:
312+
// Progress update for starr app extraction.
313+
u.handleProgress(prog)
309314
}
310315
}
311316
}

pkg/unpackerr/webhooks_templates.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ type XtractPayload struct {
4141
File []string `json:"file,omitempty"` // list of all files extracted
4242
Start time.Time `json:"start,omitempty"` // start time of extraction
4343
Output string `json:"output,omitempty"` // temporary items folder
44-
Bytes int64 `json:"bytes,omitempty"` // Bytes written
44+
Bytes uint64 `json:"bytes,omitempty"` // Bytes written
4545
Elapsed cnfg.Duration `json:"elapsed,omitempty"` // Duration as a string: 5m32s
4646
Queue int `json:"queue,omitempty"` // Extraction Queue Size
4747
}

0 commit comments

Comments
 (0)