Skip to content

Commit d4a3ff3

Browse files
Xen0XysXen0Xys
authored
Merge pull request #140 from Open-Webtoon-Reader/staging
2 parents a5c4d40 + 93ca668 commit d4a3ff3

File tree

7 files changed

+677
-404
lines changed

7 files changed

+677
-404
lines changed

‎bun.lock‎

Lines changed: 376 additions & 251 deletions
Large diffs are not rendered by default.

‎package.json‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -12,46 +12,46 @@
1212
"lint:fix": "eslint . --fix"
1313
},
1414
"dependencies": {
15-
"@fastify/helmet": "^13.0.1",
16-
"@fastify/static": "^8.2.0",
17-
"@nestjs/common": "^11.1.3",
15+
"@fastify/helmet": "^13.0.2",
16+
"@fastify/static": "^8.3.0",
17+
"@nestjs/common": "^11.1.10",
1818
"@nestjs/config": "^4.0.2",
19-
"@nestjs/cli": "^11.0.7",
20-
"@nestjs/core": "^11.1.3",
21-
"@nestjs/jwt": "^11.0.0",
19+
"@nestjs/cli": "^11.0.14",
20+
"@nestjs/core": "^11.1.10",
21+
"@nestjs/jwt": "^11.0.2",
2222
"@nestjs/passport": "^11.0.5",
23-
"@nestjs/platform-fastify": "^11.1.3",
24-
"@nestjs/platform-socket.io": "^11.1.3",
25-
"@nestjs/schedule": "^6.0.0",
26-
"@nestjs/swagger": "^11.2.0",
27-
"@nestjs/throttler": "^6.4.0",
28-
"@nestjs/websockets": "^11.1.3",
29-
"@prisma/client": "6.9.0",
30-
"axios": "^1.9.0",
23+
"@nestjs/platform-fastify": "^11.1.10",
24+
"@nestjs/platform-socket.io": "^11.1.10",
25+
"@nestjs/schedule": "^6.1.0",
26+
"@nestjs/swagger": "^11.2.3",
27+
"@nestjs/throttler": "^6.5.0",
28+
"@nestjs/websockets": "^11.1.10",
29+
"@prisma/client": "6.19.1",
30+
"axios": "^1.13.2",
3131
"class-transformer": "^0.5.1",
32-
"class-validator": "^0.14.2",
33-
"fastify": "5.3.3",
34-
"jsdom": "^26.1.0",
32+
"class-validator": "^0.14.3",
33+
"fastify": "5.6.2",
34+
"jsdom": "^27.3.0",
3535
"jszip": "^3.10.1",
36-
"minio": "^8.0.5",
36+
"minio": "^8.0.6",
3737
"passport-jwt": "^4.0.1",
38-
"sharp": "^0.34.2",
39-
"socket.io": "^4.8.1",
38+
"sharp": "^0.34.5",
39+
"socket.io": "^4.8.2",
4040
"swagger-themes": "^1.4.3",
41-
"uuid": "^11.1.0"
41+
"uuid": "^13.0.0"
4242
},
4343
"devDependencies": {
44-
"@nestjs/schematics": "^11.0.5",
45-
"@stylistic/eslint-plugin": "^4.4.1",
46-
"@types/bun": "^1.2.15",
47-
"@types/jsdom": "^21.1.7",
48-
"@types/node": "^22.15.30",
44+
"@nestjs/schematics": "^11.0.9",
45+
"@stylistic/eslint-plugin": "^5.6.1",
46+
"@types/bun": "^1.3.5",
47+
"@types/jsdom": "^27.0.0",
48+
"@types/node": "^25.0.3",
4949
"@types/passport-jwt": "^4.0.1",
50-
"@typescript-eslint/parser": "^8.33.1",
51-
"eslint": "^9.28.0",
52-
"prisma": "^6.9.0",
50+
"@typescript-eslint/parser": "^8.50.1",
51+
"eslint": "^9.39.2",
52+
"prisma": "^6.19.1",
5353
"source-map-support": "^0.5.21",
54-
"typescript": "^5.8.3"
54+
"typescript": "^5.9.3"
5555
},
5656
"prisma": {
5757
"seed": "bun prisma/seed.ts"

‎src/modules/misc/misc.service.ts‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,21 @@ export class MiscService{
5353
return this.axiosInstance;
5454
}
5555

56+
async axiosWithHardTimeout<T>(
57+
// eslint-disable-next-line @/no-unused-vars
58+
promiseFactory: (signal: AbortSignal) => Promise<T>,
59+
ms: number,
60+
): Promise<T>{
61+
const controller = new AbortController();
62+
const timer = setTimeout(() => controller.abort(), ms);
63+
64+
try{
65+
return await promiseFactory(controller.signal);
66+
}finally{
67+
clearTimeout(timer);
68+
}
69+
}
70+
5671
randomInt(min: number, max: number): number{
5772
return Math.floor(Math.random() * (max - min + 1)) + min;
5873
}
@@ -101,12 +116,17 @@ export class MiscService{
101116
}
102117

103118
async downloadImage(url: string, referer: string = "https://www.webtoons.com/fr/"): Promise<Buffer>{
104-
const response = await this.getAxiosInstance().get(url, {
105-
responseType: "arraybuffer",
106-
headers: {
107-
Referer: referer,
108-
},
109-
});
119+
const response = await this.axiosWithHardTimeout(
120+
signal =>
121+
this.getAxiosInstance().get(url, {
122+
signal,
123+
responseType: "arraybuffer",
124+
headers: {
125+
Referer: referer,
126+
},
127+
}),
128+
20000,
129+
);
110130
return response.data as Buffer;
111131
}
112132

‎src/modules/webtoon/webtoon/download-manager.service.ts‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,28 @@ export class DownloadManagerService{
9797
if(!this.downloadQueue.getCurrentDownload()) // If current download is cleared, stop downloading
9898
break;
9999
this.downloadGatewayService.onDownloadProgress(i / epList.length * 100);
100-
const epImageLinks: string[] = await this.webtoonParserService.getEpisodeLinks(this.downloadQueue.getCurrentDownload(), epList[i]);
101-
const episodeData: EpisodeDataModel = await this.webtoonDownloaderService.downloadEpisode(epList[i], epImageLinks);
100+
let epImageLinks: string[];
101+
let fetched = false;
102+
while(!fetched){
103+
try{
104+
epImageLinks = await this.webtoonParserService.getEpisodeLinks(this.downloadQueue.getCurrentDownload(), epList[i]);
105+
fetched = true;
106+
}catch (_: any){
107+
this.logger.warn(`Error fetching episode ${epList[i].number} of ${this.downloadQueue.getCurrentDownload().title}. Retrying...`);
108+
await new Promise(resolve => setTimeout(resolve, 3000));
109+
}
110+
}
111+
let downloaded = false;
112+
let episodeData: EpisodeDataModel;
113+
while(!downloaded){
114+
try{
115+
episodeData = await this.webtoonDownloaderService.downloadEpisode(epList[i], epImageLinks);
116+
downloaded = true;
117+
}catch(_: any){
118+
this.logger.warn(`Error downloading episode ${epList[i].number} of ${this.downloadQueue.getCurrentDownload().title}. Retrying...`);
119+
await new Promise(resolve => setTimeout(resolve, 3000));
120+
}
121+
}
102122
await this.webtoonDatabaseService.saveEpisode(currentDownload, epList[i], episodeData, i + 1);
103123
}
104124
}

‎src/modules/webtoon/webtoon/providers/webtoon-canvas.provider.ts‎

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -50,22 +50,41 @@ export class WebtoonCanvasProvider{
5050
return this.removeDuplicateWebtoons(languageWebtoons);
5151
}
5252

53-
private async getPageCountFromGenre(language: string, genre: string): Promise<number>{
53+
private async getPageCountFromGenre(
54+
language: string,
55+
genre: string,
56+
): Promise<number>{
5457
const url = `https://www.webtoons.com/${language}/canvas/list?genreTab=${genre.toUpperCase()}&page=999999999`;
55-
const response = await this.miscService.getAxiosInstance().get(url);
58+
const response = await this.miscService.axiosWithHardTimeout(
59+
signal =>
60+
this.miscService.getAxiosInstance().get(url, {
61+
signal,
62+
}),
63+
20000,
64+
);
5665
const document = new JSDOM(response.data).window.document;
57-
// Check if page is ALL
58-
const header = document.querySelector("ul#_genreTabList").querySelectorAll("li")[1].querySelector("a");
59-
if(header.ariaCurrent !== "false")
60-
return 0;
61-
// Fetch page count
62-
const lastLink = document.querySelector("div.paginate").querySelector("a:last-of-type");
63-
if(!lastLink) throw new NotFoundException(`No pagination found for genre: ${genre}`);
64-
const lastSpan = lastLink.querySelector("span");
65-
if(!lastSpan) throw new NotFoundException(`No span found in the last link for genre: ${genre}`);
66-
const pageCount = parseInt(lastSpan.textContent?.trim() || "", 10);
67-
if(isNaN(pageCount)) throw new NotFoundException(`Invalid page count for genre: ${genre}`);
68-
return pageCount;
66+
67+
// Check if genre page exists or is just "all"
68+
if(document.querySelector("ul.snb")?.querySelectorAll("li")[1]?.className.includes("is_selected"))
69+
return -1;
70+
71+
const paginate = document.querySelector("div.paginate");
72+
if(!paginate){
73+
throw new NotFoundException(`No pagination found for genre: ${genre}`);
74+
}
75+
76+
const pageNumbers = Array.from(
77+
paginate.querySelectorAll("a span"),
78+
)
79+
.map(span => span.textContent?.trim())
80+
.filter(text => text && /^\d+$/.test(text))
81+
.map(text => Number(text));
82+
83+
if(pageNumbers.length === 0){
84+
throw new NotFoundException(`No page numbers found for genre: ${genre}`);
85+
}
86+
87+
return Math.max(...pageNumbers);
6988
}
7089

7190
private async getWebtoonsFromGenre(language: string, genre: string): Promise<CachedWebtoonModel[]>{
@@ -75,8 +94,13 @@ export class WebtoonCanvasProvider{
7594
error = undefined;
7695
try{
7796
pageCount = await this.getPageCountFromGenre(language, genre);
97+
if(pageCount === -1){
98+
this.logger.warn(`(Webtoon Canvas) [${language}] Genre page does not exist (redirected to All): ${genre}`);
99+
return [];
100+
}
78101
}catch(e){
79102
error = e;
103+
this.logger.error(`(Webtoon Canvas) [${language}] Error while getting page count from genre: ${genre} - ${e.message}`, e.stack);
80104
await new Promise(resolve => setTimeout(resolve, 5000));
81105
}
82106
}while(error);
@@ -108,7 +132,13 @@ export class WebtoonCanvasProvider{
108132

109133
private async getWebtoonsFromGenrePage(language: string, genre: string, page: number): Promise<CachedWebtoonModel[]>{
110134
const url = `https://www.webtoons.com/${language}/canvas/list?genreTab=${genre.toUpperCase()}&sortOrder=LIKEIT&page=${page}`;
111-
const response = await this.miscService.getAxiosInstance().get(url);
135+
const response = await this.miscService.axiosWithHardTimeout(
136+
signal =>
137+
this.miscService.getAxiosInstance().get(url, {
138+
signal,
139+
}),
140+
20000,
141+
);
112142
const document = new JSDOM(response.data).window.document;
113143
const cards = document.querySelector("div.challenge_lst")?.querySelector("ul")?.querySelectorAll("li");
114144
if(!cards) throw new NotFoundException(`No cards found for genre: ${genre}`);

0 commit comments

Comments
 (0)