class & Idol parse fixed

Author: raczak

lib/modules/global-parser/global-parser.js

@@ -5,10 +5,19 @@ import getItems from './getItems';
 import { gameUrlListSwitcher } from './services/utils';
 import { Spinner } from 'clui';
 import errorHandler from '../../services/errorHandler';
-import { mongoImport, requestOpts } from './services/utils';
+import { mongoImport, requestOpts, parseEachPageLinksClass, maxItemControl } from './services/utils';
 
-let url = null, globalUrl, itemCategory, game, maxItem = 1, all = true;
-const NB_ITEM_PER_PAGE = 96, links = [];
+const NB_ITEM_PER_PAGE = 96;
+const initData = {
+	url: null,
+	globalUrl: null,
+	itemCategory: null,
+	game: null,
+	maxItem: null,
+	all: true,
+	links: [],
+	$: null
+};
 
 /**
  * Function which initialize the parsing process (global var, urls, category to parse...)
@@ -18,31 +27,33 @@ function parserInit(cmdResponse) {
 	const countdown = new Spinner('Crawler in progress... It could take some time ', ['⣾', '⣽', '⣻', '⢿', '⡿', '⣟', '⣯', '⣷']);
 	countdown.start();
 
-	game = cmdResponse.game;
-	maxItem = cmdResponse.maxItem;
-	all = cmdResponse.all;
-	itemCategory = cmdResponse.category;
-	url = gameUrlListSwitcher(cmdResponse);
-	globalUrl = url.substring(0, url.indexOf('.com/') + 4);
-	requestOpts.url = url;
+	initData.game = cmdResponse.game;
+	initData.maxItem = cmdResponse.maxItem;
+	initData.all = cmdResponse.all;
+	initData.itemCategory = cmdResponse.category;
+	initData.url = gameUrlListSwitcher(cmdResponse);
+	initData.globalUrl = initData.url.substring(0, initData.url.indexOf('.com/') + 4);
+	requestOpts.url = initData.url;
 }
 
 /**
  * Function responsible for fetching all item's links and put them in a file
  */
 async function getPagesLinks() {
-	const $ = await request(requestOpts).catch(err => new errorHandler(err));
-	// In order to prevent from empty value of realMaxItem when there is only 1 page
-	const realMaxItem = Number($('div.ak-list-info > strong').text()) === 0 ? $('div.ak-container.ak-panel.main-object-list.ak-nocontentpadding tbody > tr').length : Number($('div.ak-list-info > strong').text());
-	if (maxItem >= realMaxItem || all === true) maxItem = realMaxItem; // security line to avoid overpasing the real amont of items & to take care about 'all' value if true
-
-	const nbPageToParse = Math.ceil(maxItem / NB_ITEM_PER_PAGE);
-	for (let page = 1; page <= nbPageToParse; page++) {
-		requestOpts.url = url + '&page=' + page;
-		const $ = await request(requestOpts).catch(err => new errorHandler(err));
-		parseEachPageLinks($);
+	initData.$ = await request(requestOpts).catch(err => new errorHandler(err));
+	initData.maxItem = maxItemControl(initData); // In order to prevent from empty value of realMaxItem when there is only 1 page & to avoird overpassing real maxitem value
+	if (initData.itemCategory == 'classe') {
+		initData.links = initData.links.concat(parseEachPageLinksClass(initData));
+	}
+	else {
+		const nbPageToParse = Math.ceil(initData.maxItem / NB_ITEM_PER_PAGE);
+		for (let page = 1; page <= nbPageToParse; page++) {
+			requestOpts.url = initData.url + '&page=' + page;
+			const $ = await request(requestOpts).catch(err => new errorHandler(err));
+			parseEachPageLinks($);
+		}
 	}
-	writeFileSync('./data/links/' + itemCategory + '_links.json', JSON.stringify(links));
+	writeFileSync('./data/links/' + initData.itemCategory + '_links.json', JSON.stringify(initData.links));
 	console.log('\x1b[36m%s\x1b[0m', '\n SUCCESS : all item(s) links crawled.');
 	console.log('\x1b[36m%s\x1b[0m', '\n START of item(s) crawling.');
 }
@@ -52,20 +63,11 @@ async function getPagesLinks() {
  * @param {String} $ html page to analize
  */
 function parseEachPageLinks($) {
-	// warning, classe is not a classic encyclopedia
-	if (itemCategory == 'classe') {
-		$('.ak-content-sections').find('.ak-section').each(function (i, div) {
-			if (!all) if (links.length >= maxItem) return false;
-			const link = globalUrl + $(this).find('a').attr('href');
-			links.push(link);
-		});
-	} else {
-		$('tbody').find('tr').each(function (i, tr) {
-			if (!all) if (links.length >= maxItem) return false;
-			const link = globalUrl + $(this).find('td').eq(1).find('a').attr('href');
-			links.push(link);
-		});
-	}
+	$('tbody').find('tr').each(function (i, tr) {
+		if (!initData.all) if (initData.links.length >= initData.maxItem) return false;
+		const link = initData.globalUrl + $(this).find('td').eq(1).find('a').attr('href');
+		initData.links.push(link);
+	});
 }
 
 /**