Merge branch 'version-2.0.0'

Author: dr-nyt

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "NovelScraper",
-	"version": "2.0.5",
+	"version": "2.0.6",
 	"description": "App for downloading novels from pirate sites.",
 	"homepage": "https://github.com/HanaDigital/NovelScraper",
 	"author": {

src/app/resources/sourceList.ts

@@ -3,7 +3,7 @@ import { sourcesList } from "./types"
 export const sources: sourcesList = [
 	{ name: "NovelFull", link: "https://novelfull.com", icon: "assets/img/sources/novelfull-logo.png" },
 	{ name: "BoxNovel", link: "https://boxnovel.com", icon: "assets/img/sources/boxnovel-logo.png" },
-	{ name: "ReadLightNovel", link: "https://www.readlightnovel.org", icon: "assets/img/sources/readlightnovel-logo.png" },
+	{ name: "ReadLightNovel", link: "https://www.readlightnovel.me", icon: "assets/img/sources/readlightnovel-logo.png" },
 ];
 
 export const deprecatedSources: sourcesList = [

src/app/services/novel-factory.service.ts

@@ -9,6 +9,7 @@ const epubGen = require("nodepub");
 import mime from "mime";
 import { writeFileSync, createWriteStream, writeFile } from "fs";
 import request from "request";
+import { async } from "@angular/core/testing";
 
 @Injectable({
 	providedIn: "root",
@@ -38,6 +39,18 @@ export class NovelFactoryService {
 		// Download the cover and get its path
 		const coverPath = await this.downloadCover(novel.cover, novelFolder);
 
+		// If the novel doesn't have a genre we default "Unknown"; empty or null will crash the epub creation
+		if (novel.genre == null || novel.genre == "") {
+			novel.genre = 'unknown'
+		}
+
+		// If the novel doesnt have a author we default "Unknown"; empty or null will crash the epub creation
+		if (novel.author == null || novel.author == "") {
+			novel.author = 'unknown'
+		}
+		console.log("Creating Epub with metadata");
+		console.log(novel)
+
 		// Set some meta data for the epub file
 		const metadata = {
 			id: "0000-0000-0001",
@@ -59,13 +72,14 @@ export class NovelFactoryService {
 		for (const chapter of chapters)
 			epub.addSection(chapter.title, chapter.data);
 
-		setTimeout(async () => {
-			await epub.writeEPUB(novelFolder, novelFile);
-
-			this.database.updateDownloading(novel.link, false);
-			this.database.updateDownloaded(novel.link, true);
-			this.database.updateIsUpdated(novel.link, true);
-			this.database.setDownloaded(downloadID);
+		setTimeout(() => {
+			(async () => {
+				await epub.writeEPUB(novelFolder, novelFile);
+				this.database.updateDownloading(novel.link, false);
+				this.database.updateDownloaded(novel.link, true);
+				this.database.updateIsUpdated(novel.link, true);
+				this.database.setDownloaded(downloadID);
+			})();
 		}, 2000);
 	}
 

src/app/services/sources/boxnovel.service.ts

@@ -35,11 +35,20 @@ export class BoxnovelService extends sourceService {
 		}
 
 		try {
+
+
 			const html = await this.getHtml(link); // Get HTML from the link
 
 			// Link
 			if (!updatingInfo) novel.link = link;
 
+			//boxnovel uses some lower level ajax call now to store the chapters.
+			const chapter_link = link + (link.slice(-1) == '/' ? "" : "/") + "ajax/chapters/";
+
+			const chapters = await this.postHtml(chapter_link) // we get the subpage listing all the chapters.
+
+
+
 			// Source
 			if (!updatingInfo) novel.source = source;
 
@@ -56,18 +65,23 @@ export class BoxnovelService extends sourceService {
 			novel.name = title.textContent.trim();
 
 			// LatestChapter
-			novel.latestChapter = html
-				.getElementsByClassName("wp-manga-chapter")[0]
-				.getElementsByTagName("a")[0]
-				.innerText.trim();
+			try {
+				novel.latestChapter = chapters
+					.getElementsByClassName("wp-manga-chapter")[0]
+					.getElementsByTagName("a")[0]
+					.innerText.trim();
+			} catch (error) {
+				console.log("Unknown find latest chapter");
+				novel.latestChapter = "Unknown"
+			}
 
 			// Cover
 			novel.cover = html
 				.getElementsByClassName("summary_image")[0]
 				.getElementsByTagName("img")[0].src;
 
 			// TotalChapters
-			novel.totalChapters = html.getElementsByClassName(
+			novel.totalChapters = chapters.getElementsByClassName(
 				"wp-manga-chapter"
 			).length;
 
@@ -238,9 +252,10 @@ export class BoxnovelService extends sourceService {
 		let downloadedChapters: chapterObj[] = []; // List of download chapters
 
 		try {
-			const html = await this.getHtml(novel.link);
+			const chapter_link = novel.link + (novel.link.slice(-1) == '/' ? "" : "/") + "ajax/chapters/"
+			const chapter_html = await this.postHtml(chapter_link) // we get the sub-page listing all the chapters.
 
-			const chapters = html.getElementsByClassName("wp-manga-chapter");
+			const chapters = chapter_html.getElementsByClassName("wp-manga-chapter");
 
 			// Get chapter links and names
 			let chapterLinks = [];

src/app/services/sources/novelfull.service.ts

@@ -90,14 +90,17 @@ export class NovelfullService extends sourceService {
 			// FIXME: Genre(s)
 			const genres = html
 				.getElementsByClassName("info")[0]
-				.getElementsByTagName("div")[1]
+				.getElementsByTagName("div")[2]
 				.getElementsByTagName("a");
 			let genre = "";
 			for (let i = 0; i < genres.length; i++) {
 				genre += genres[i].innerText + ", ";
 			}
 			novel.genre = genre.slice(0, -2);
 
+			// defaulting if none is found.
+			if (!novel.genre) novel.genre = "Unknown"
+
 			// FIXME: Summary
 			const summaryList = html
 				.getElementsByClassName("desc-text")[0]
@@ -113,6 +116,9 @@ export class NovelfullService extends sourceService {
 				console.log(error);
 			}
 
+			// defaulting if none is found.
+			if (!novel.summary) novel.summary = "Unknown"
+
 			//////////////////////// YOUR CODE ENDS HERE /////////////////////////////////
 
 			this.pushOrUpdateNovel(novel, updatingInfo);
@@ -314,7 +320,6 @@ export class NovelfullService extends sourceService {
 					);
 					chapterBody = chapterBody.replace(/<script.*><\/script>/g, "");
 					chapterBody = chapterBody.replace(/<ins.*<\/ins>/g, "");
-					console.log(chapterBody);
 
 					const chapter = this.prepChapter(novel, downloadID, chapterTitle, chapterBody, downloadedChapters.length, totalLength);
 					downloadedChapters.push(chapter);

src/app/services/sources/readlightnovel-service.service.ts

@@ -47,7 +47,11 @@ export class ReadlightnovelService extends sourceService {
 
 			// LatestChapter
 			try {
-				novel.latestChapter = html.getElementsByClassName('wp-manga-chapter')[0].getElementsByTagName('a')[0].innerText.trim();
+				const chapters_section = html.getElementsByClassName('tab-content')[0].getElementsByClassName("tab-pane")
+				const latest_chapter_section = chapters_section[chapters_section.length - 2].getElementsByTagName('a') // latest element is always empty.
+				const latest_chapter = latest_chapter_section[latest_chapter_section.length - 1]
+
+				novel.latestChapter = latest_chapter.innerText.trim();
 			} catch (error) {
 				novel.latestChapter = "N/A";
 				console.log(error);
@@ -170,13 +174,23 @@ export class ReadlightnovelService extends sourceService {
 				// Download each chapter at a time
 				for (let i = 0; i < chapterLinks.length; i++) {
 					if (this.database.isCanceled(downloadID)) {
-						this.database.updateDownloading(novel.link, false);
 						console.log('Download canceled!');
 						canceled = true;
 						break;
 					}
 
-					const html = await this.getHtml(chapterLinks[i]);
+					let html = null;
+					try {
+						html = await this.getHtml(chapterLinks[i]);
+					} catch (error) {
+						if (error.statusCode == 404) {
+							novel.totalChapters -= 1
+							this.database.updateTotalChapters(novel.link, novel.totalChapters);
+							continue;
+						}
+						canceled = true;
+						break;
+					}
 
 					//////////////////////// YOUR CODE STARTS HERE ///////////////////////////////
 

src/app/services/sources/sourceService.ts

@@ -103,7 +103,21 @@ export class sourceService {
 
 	async getHtml(link: string): Promise<HTMLHtmlElement> {
 		const stringHtml: string = await cloudscraper(
-			{ method: "GET", url: link },
+			{ method: "GET", url: link, rejectUnauthorized: false },
+			(error, response, novelHtmlString) => {
+				return novelHtmlString;
+			}
+		);
+
+		const html = document.createElement("html");
+		html.innerHTML = stringHtml;
+		return html;
+	}
+
+
+	async postHtml(link: string): Promise<HTMLHtmlElement> {
+		const stringHtml: string = await cloudscraper(
+			{ method: "POST", url: link, rejectUnauthorized: false },
 			(error, response, novelHtmlString) => {
 				return novelHtmlString;
 			}