Skip to content

Commit ac36d0f

Browse files
garzjgarzj
committed
Add progress bars
1 parent b704243 commit ac36d0f

File tree

8 files changed

+168
-24
lines changed

8 files changed

+168
-24
lines changed

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
"d4sd": "lib/cli.js"
1919
},
2020
"devDependencies": {
21+
"@types/cli-progress": "^3.11.5",
2122
"@types/cookie": "^0.6.0",
2223
"@types/inquirer": "^8.2.2",
2324
"@types/minimatch": "^5.1.2",
@@ -28,9 +29,9 @@
2829
"typescript": "^5.3.3"
2930
},
3031
"dependencies": {
32+
"cli-progress": "^3.12.0",
3133
"cmd-ts": "^0.13.0",
3234
"cookie": "^0.6.0",
33-
"defaults-deep-ts": "^0.2.3",
3435
"dotenv": "^16.3.1",
3536
"inquirer": "^8.2.2",
3637
"minimatch": "^9.0.3",

‎src/cli.ts‎

Lines changed: 67 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ import { hasOwnProperty } from './util/object';
2424
import { ItemRef } from './item/ItemRef';
2525
import { ScookShelf } from './shelf/ScookShelf';
2626
import { DigiShelf } from './shelf/DigiShelf';
27+
import * as cliProgress from 'cli-progress';
28+
import { DownloadOptions, DownloadProgress } from './item/download-options';
29+
import { Book } from './item/Book';
30+
import { ItemGroup } from './item/ItemGroup';
31+
import { Item } from './item/Item';
2732

2833
const { version } = JSON.parse(
2934
readFileSync(join(__dirname, '../package.json')).toString()
@@ -177,16 +182,73 @@ const cmd = command({
177182
}
178183

179184
console.log(`Downloading "${itemRef.title}"...`);
185+
186+
const multiBar = new cliProgress.MultiBar(
187+
{
188+
format:
189+
' {bar} | {value}/{total} | {percentage}% | ETA: {eta}s | {title}',
190+
},
191+
cliProgress.Presets.shades_classic
192+
);
193+
const bars = new Map<Item, cliProgress.Bar>();
194+
const barsUpdater = setInterval(
195+
() => bars.forEach((bar) => bar.updateETA()),
196+
1000
197+
);
198+
199+
const options: DownloadOptions = {
200+
...args,
201+
format: args.format as PaperFormat | undefined,
202+
onStart(progress) {
203+
let bar: cliProgress.SingleBar | null = null;
204+
if (progress.item instanceof Book) {
205+
bar = multiBar.create(
206+
(progress as DownloadProgress<Book>).pageCount,
207+
0,
208+
{
209+
title: `${progress.item.constructor.name}: ${progress.item.title}`,
210+
}
211+
);
212+
} else if (progress.item instanceof ItemGroup) {
213+
bar = multiBar.create(
214+
(progress as DownloadProgress<ItemGroup>).items.length,
215+
0,
216+
{ title: `Group: ${progress.item.title}` }
217+
);
218+
}
219+
bar && bars.set(progress.item, bar);
220+
},
221+
onProgress(progress) {
222+
const bar = bars.get(progress.item)!;
223+
if (progress.item instanceof Book) {
224+
bar.update(
225+
(progress as DownloadProgress<Book>).downloadedPages
226+
);
227+
} else if (progress.item instanceof ItemGroup) {
228+
bar.update(
229+
(progress as DownloadProgress<ItemGroup>).downloadedItems
230+
.length
231+
);
232+
}
233+
},
234+
};
235+
236+
let err: unknown = null;
180237
try {
181-
await item.download(args.outDir, {
182-
...args,
183-
format: args.format as PaperFormat,
184-
});
238+
await item.download(args.outDir, options);
185239
} catch (e) {
186-
console.error(e);
240+
err = e;
241+
}
242+
243+
clearInterval(barsUpdater);
244+
multiBar.stop();
245+
246+
if (err) {
247+
console.error(err);
187248
console.error(`Failed to download "${itemRef.title}!"`);
188249
continue;
189250
}
251+
190252
console.log(`Successfully downloaded "${itemRef.title}"!`);
191253
}
192254
}

‎src/item/BiBoxBook.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,14 @@ export class BiBoxBook extends Book {
4747

4848
// Page download pool
4949
const pageCount = pageUrls.length;
50+
let downloadedPages = 0;
51+
const getProgress = () => ({
52+
item: this,
53+
percentage: downloadedPages / pageCount,
54+
downloadedPages,
55+
pageCount,
56+
});
57+
options.onStart(getProgress());
5058

5159
await promisePool(
5260
async (i) => {
@@ -67,6 +75,8 @@ export class BiBoxBook extends Book {
6775
...(await getPdfOptions(page, options)),
6876
path: pdfFile,
6977
});
78+
79+
options.onProgress(getProgress());
7080
} finally {
7181
await page.close();
7282
}

‎src/item/DigiBook.ts‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { URL } from 'url';
44
import { Book } from './Book';
55
import { defDownloadOptions, DownloadOptions } from './download-options';
66
import { SizeAttributes, getPdfOptions } from './get-pdf-options';
7+
import { ScrapeError } from '../error/ScrapeError';
78

89
export class DigiBook extends Book {
910
async download(outDir: string, _options?: DownloadOptions) {
@@ -14,6 +15,7 @@ export class DigiBook extends Book {
1415
const checkPage = await this.shelf.browser.newPage();
1516
let page1Url: string;
1617
let sizeHint: SizeAttributes;
18+
let pageCount: number;
1719
try {
1820
await checkPage.goto(new URL(`?page=1`, this.url).toString(), {
1921
waitUntil: 'networkidle2',
@@ -26,12 +28,23 @@ export class DigiBook extends Book {
2628
{ width: obj.width, height: obj.height },
2729
]
2830
);
31+
pageCount = await checkPage.$$eval(
32+
'#thumbnailPanel a.thumbnail',
33+
(elms) => elms.length
34+
);
2935
} finally {
3036
await checkPage.close();
3137
}
3238

3339
// Page download pool
34-
let pageCount = 0;
40+
let downloadedPages = 0;
41+
const getProgress = () => ({
42+
item: this,
43+
percentage: downloadedPages / pageCount,
44+
downloadedPages,
45+
pageCount,
46+
});
47+
options.onStart(getProgress());
3548

3649
await promisePool(async (i, stop) => {
3750
const pageNo = i + 1;
@@ -63,13 +76,20 @@ export class DigiBook extends Book {
6376
path: pdfFile,
6477
});
6578

66-
pageCount++;
79+
downloadedPages++;
80+
options.onProgress(getProgress());
6781
} finally {
6882
await page.close();
6983
}
7084
}, options.concurrency);
7185

86+
if (downloadedPages != pageCount) {
87+
throw new ScrapeError(
88+
`A page count of ${pageCount} was parsed, but ${downloadedPages} were downloaded. Please report this issue.`
89+
);
90+
}
91+
7292
// Merge pdf pages
73-
options.mergePdfs && (await this.mergePdfPages(dir, pageCount));
93+
options.mergePdfs && (await this.mergePdfPages(dir, downloadedPages));
7494
}
7595
}

‎src/item/ItemGroup.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,16 +33,33 @@ export abstract class ItemGroup extends Item {
3333
}
3434
}
3535

36+
const downloadedItems: Item[] = [];
37+
const getProgress = () => ({
38+
item: this,
39+
percentage: downloadedItems.length / items.length,
40+
downloadedItems,
41+
items,
42+
});
43+
options.onStart(getProgress());
44+
3645
// Document download pool
3746
await promisePool(
38-
(i) => docs[i].download(dir),
47+
async (i) => {
48+
await docs[i].download(dir);
49+
50+
downloadedItems.push(docs[i]);
51+
options.onProgress(getProgress());
52+
},
3953
options.concurrency,
4054
docs.length
4155
);
4256

4357
// Download other stuff (books / folders) individually
4458
for (let other of others) {
45-
await other.download(dir);
59+
await other.download(dir, options);
60+
61+
downloadedItems.push(other);
62+
options.onProgress(getProgress());
4663
}
4764
}
4865

‎src/item/ScookBook.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,15 @@ export class ScookBook extends Book {
6464
}
6565

6666
// Page download pool
67+
let downloadedPages = 0;
68+
const getProgress = () => ({
69+
item: this,
70+
percentage: downloadedPages / pageCount,
71+
downloadedPages,
72+
pageCount,
73+
});
74+
options.onStart(getProgress());
75+
6776
await promisePool(
6877
async (i) => {
6978
const pageNo = i + 1;
@@ -88,6 +97,9 @@ export class ScookBook extends Book {
8897
...(await getPdfOptions(page, options)),
8998
path: pdfFile,
9099
});
100+
101+
downloadedPages++;
102+
options.onProgress(getProgress());
91103
} finally {
92104
await page.close();
93105
}

‎src/item/download-options.ts‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,31 @@
1-
import defaultsDeep from 'defaults-deep-ts';
21
import { PaperFormat } from 'puppeteer';
2+
import { Item } from './Item';
3+
import { ItemGroup } from './ItemGroup';
4+
import { Book } from './Book';
5+
6+
export type DownloadProgress<I extends Item = Item> = {
7+
percentage: number;
8+
item: I;
9+
} & (I extends Book
10+
? { downloadedPages: number; pageCount: number }
11+
: I extends ItemGroup
12+
? { downloadedItems: Item[]; items: Item[] }
13+
: {});
314

415
export interface DownloadOptions {
516
concurrency?: number;
617
mergePdfs?: boolean;
718
format?: PaperFormat;
19+
onStart?: (progress: DownloadProgress) => void;
20+
onProgress?: (progress: DownloadProgress) => void;
821
}
922

10-
const defaultOptions: DownloadOptions = {
11-
concurrency: 10,
12-
mergePdfs: true,
13-
};
14-
1523
export function defDownloadOptions(_options?: DownloadOptions) {
16-
const options = _options ?? {};
17-
return defaultsDeep(options, defaultOptions);
24+
return (<T extends DownloadOptions>(x: T) => x)({
25+
..._options,
26+
concurrency: _options?.concurrency ?? 10,
27+
mergePdfs: _options?.mergePdfs ?? true,
28+
onStart: _options?.onStart ?? (() => {}),
29+
onProgress: _options?.onProgress ?? (() => {}),
30+
});
1831
}

‎yarn.lock‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,13 @@
298298
"@tufjs/canonical-json" "1.0.0"
299299
minimatch "^9.0.0"
300300

301+
"@types/cli-progress@^3.11.5":
302+
version "3.11.5"
303+
resolved "https://registry.yarnpkg.com/@types/cli-progress/-/cli-progress-3.11.5.tgz#9518c745e78557efda057e3f96a5990c717268c3"
304+
integrity sha512-D4PbNRbviKyppS5ivBGyFO29POlySLmA2HyUFE4p5QGazAMM3CwkKWcvTl8gvElSuxRh6FPKL8XmidX873ou4g==
305+
dependencies:
306+
"@types/node" "*"
307+
301308
"@types/cookie@^0.6.0":
302309
version "0.6.0"
303310
resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.6.0.tgz#eac397f28bf1d6ae0ae081363eca2f425bedf0d5"
@@ -755,6 +762,13 @@ cli-cursor@^3.1.0:
755762
dependencies:
756763
restore-cursor "^3.1.0"
757764

765+
cli-progress@^3.12.0:
766+
version "3.12.0"
767+
resolved "https://registry.yarnpkg.com/cli-progress/-/cli-progress-3.12.0.tgz#807ee14b66bcc086258e444ad0f19e7d42577942"
768+
integrity sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==
769+
dependencies:
770+
string-width "^4.2.3"
771+
758772
cli-spinners@^2.5.0:
759773
version "2.9.2"
760774
resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-2.9.2.tgz#1773a8f4b9c4d6ac31563df53b3fc1d79462fe41"
@@ -940,11 +954,6 @@ deep-extend@^0.6.0:
940954
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac"
941955
integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==
942956

943-
defaults-deep-ts@^0.2.3:
944-
version "0.2.3"
945-
resolved "https://registry.yarnpkg.com/defaults-deep-ts/-/defaults-deep-ts-0.2.3.tgz#50ba6487323dbe2cb23bea7061a4cd6fdf4541ae"
946-
integrity sha512-GRqwQy00m3fUjSqQZ+GWlrlE7yDXPnB1Pg9+045EerY5mmrE/7U4J8Uqhbscwf083Y7DvkWte1LnTF0++/TWOw==
947-
948957
defaults@^1.0.3:
949958
version "1.0.4"
950959
resolved "https://registry.yarnpkg.com/defaults/-/defaults-1.0.4.tgz#b0b02062c1e2aa62ff5d9528f0f98baa90978d7a"

0 commit comments

Comments
 (0)