Skip to content

Commit d0bdf16

Browse files
author
Zakaria RACHEDI
committed
New parsing way added to "getSet" + no more full item in sets, only keys
1 parent 31ab794 commit d0bdf16

File tree

5 files changed

+116
-79
lines changed

5 files changed

+116
-79
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ In case you wan't directly the dofus JSON files, I oftenly upload up-to-date enc
3838
An item result example
3939
```json
4040
{
41-
"item_identifiant":"item ID",
41+
"_id":"item ID",
4242
"name":"item name",
4343
"description":"item description.",
4444
"lvl":"item lvl",

‎lib/getItems.js‎

Lines changed: 7 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -54,46 +54,25 @@ var getItems = exports.getItems = function(category, links, game, back) {
5454
function getData(url, back, callback) {
5555
var categoryPromise;
5656
switch (true) {
57-
case /\barmes\b/gi.test(url):
57+
case /\b(weapons|armes)\b/gi.test(url):
5858
categoryPromise = gw.getWeapons(url);
5959
break;
60-
case /\bweapons\b/gi.test(url):
61-
categoryPromise = gw.getWeapons(url);
62-
break;
63-
case /\bequipment\b/gi.test(url):
64-
categoryPromise = ge.getEquipments(url);
65-
break;
66-
case /\bequipements\b/gi.test(url):
60+
case /\b(equipment|equipements)\b/gi.test(url):
6761
categoryPromise = ge.getEquipments(url);
6862
break;
69-
case /\bsets\b/gi.test(url):
63+
case /\b(sets|panoplies)\b/gi.test(url):
7064
categoryPromise = gs.getSets(url);
7165
break;
72-
case /\bpanoplies\b/gi.test(url):
73-
categoryPromise = gs.getSets(url);
74-
break;
75-
case /\bmounts\b/gi.test(url):
76-
categoryPromise = gm.getMounts(url);
77-
break;
78-
case /\bmontures\b/gi.test(url):
66+
case /\b(mounts|monture)\b/gi.test(url):
7967
categoryPromise = gm.getMounts(url);
8068
break;
81-
case /\bpets\b/gi.test(url):
82-
categoryPromise = gp.getPets(url);
83-
break;
84-
case /\bfamiliers\b/gi.test(url):
69+
case /\b(pets|familiers)\b/gi.test(url):
8570
categoryPromise = gp.getPets(url);
8671
break;
87-
case /\bresources\b/gi.test(url):
72+
case /\b(resources|ressources)\b/gi.test(url):
8873
categoryPromise = gr.getResources(url);
8974
break;
90-
case /\bressources\b/gi.test(url):
91-
categoryPromise = gr.getResources(url);
92-
break;
93-
case /\bconsumables\b/gi.test(url):
94-
categoryPromise = gc.getConsumables(url);
95-
break;
96-
case /\bconsommables\b/gi.test(url):
75+
case /\b(consumables|consommables)\b/gi.test(url):
9776
categoryPromise = gc.getConsumables(url);
9877
break;
9978
default:

‎lib/item-crawler/getSets.js‎

Lines changed: 24 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,46 @@
11
var request = require('request-promise');
22
var cheerio = require('cheerio');
3-
var { getId, getElement } = require('./helpers');
3+
var { getId, getElement, sanatizer } = require('./helpers');
4+
var { effectParse, recipeParse, descriptionParse, getCategoryType } = require('./parsing-service/helpers');
45

56
var requestOpts = {
67
url: '',
78
transform: function (body) {
89
return cheerio.load(body);
910
}
1011
};
12+
var set = {};
13+
var body = '';
1114

1215
var getSets = exports.getSets = function (url) {
1316
requestOpts.url = url;
1417
return request(requestOpts).then(function ($) {
15-
var itemId = getId(url);
16-
var type = $('div.ak-encyclo-detail-right').find('div.ak-encyclo-detail-type').text().split(':')[1].trim();
17-
var name = $('h1.ak-return-link').text().trim();
18-
var lvl = $('div.ak-encyclo-detail-right').find('div.ak-encyclo-detail-level.col-xs-6.text-right').text().trim().replace(/\D/g,'');
19-
var imgUrl = $('div.ak-encyclo-detail-illu').find('img').attr('src').replace('dofus/ng/img/../../../', '');
20-
21-
var set = {
22-
id: itemId,
23-
name: name,
24-
lvl: lvl,
25-
type: type,
26-
imgUrl: imgUrl,
27-
url: url
28-
}
29-
18+
/////// Global initializations ///////
19+
body = $.html();
20+
21+
/////// Description parse ///////
22+
set = descriptionParse(body, url);
23+
24+
/////// Tabs initializations ///////
3025
set["bonus"] = [];
31-
set["items"] = [];
32-
$('div.ak-nocontentpadding').find('div.ak-item-list-preview').find('a').each(function(i, element){
33-
var item = {
34-
id: $(this).attr('href').replace(/\D/g,''),
35-
// name: $(this).attr('href').substr($(this).attr('href').lastIndexOf('/') + 1).replace(/[0-9]/g, '').replace('-', ' ').trim(),
36-
url: $(this).attr('href')
37-
}
38-
set["items"].push(item);
26+
27+
/////// Linked Items parse ///////
28+
set['equipments'] = [];
29+
set['weapons'] = [];
30+
$('div.ak-container.ak-panel.ak-set-composition.ak-nocontentpadding').find('div.ak-panel-content').find('tr').each(function(i, element){
31+
var id = getId($(this).find('td').eq(1).find('a').attr('href'));
32+
var glType = getCategoryType($(this).find('td').eq(1).find('div.ak-item-type-info').text().trim().toLowerCase().normalize('NFD').replace(/[\u0300-\u036f]/g, ""));
33+
set[glType].push(id);
3934
});
40-
// var currentNbBonus = 0;
35+
if (set['equipments'].length == 0) delete set['equipments'];
36+
if (set['weapons'].length == 0) delete set['weapons'];
37+
38+
/////// Bonus Set parse ///////
4139
$('div.ak-encyclo-detail-right').find('div.set-bonus-list').each(function(i, element){
42-
var stats = []
43-
$(this).find('div.ak-list-element').each(function(i, element){
44-
// currentNbBonus ++;
45-
var bonus = $(this).find( "div.ak-title" ).text().trim();
46-
var bonusToTest = bonus.toLowerCase();
47-
if (bonusToTest.includes('title') || bonusToTest.includes('titre') || bonusToTest.includes('attitude') || bonusToTest.includes('emote')) {
48-
if (bonusToTest.includes('title') || bonusToTest.includes('titre')) stats.push({'title': bonus.split(':')[1].trim()});
49-
else if (bonusToTest.includes('attitude') || bonusToTest.includes('emote')) stats.push({'emote': bonus});
50-
}else {
51-
var element = getElement(bonus);
52-
element = element.charAt(0).toUpperCase() + element.slice(1);
53-
var numbers = [];
54-
bonus.replace(/(-?\d[\d\.]*)/g, function( x ) {
55-
var n = Number(x); if (x == n) { numbers.push(x); }
56-
});
57-
var groupeElement = {[element]: {'from': numbers[0], 'to': numbers[1]}};
58-
stats.push(groupeElement);
59-
}
60-
});
6140
var nbBonus = {
6241
number: i + 1,
6342
url: $(this).attr('href'),
64-
stats: stats
43+
stats: effectParse($(this).html())
6544
}
6645
set["bonus"].push(nbBonus);
6746
});

‎lib/item-crawler/helpers.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,4 @@ module.exports = {
1616
sanatizer: (string) => {
1717
return string.replace(/(\\n)|(\(|\))/g, '').replace(/(\r\n|\n|\r)/gm," ").replace(/\s\s+/g, ' ').trim();
1818
}
19-
2019
};

‎lib/item-crawler/parsing-service/helpers.js‎

Lines changed: 84 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,21 +58,101 @@ module.exports = {
5858
descriptionParse: (body, url) => {
5959
var $ = cheerio.load(body);
6060
var itemId = getId(url);
61-
var type = $('div.ak-encyclo-detail-right.ak-nocontentpadding').find('div.ak-encyclo-detail-type.col-xs-6').find('span').text().trim();
61+
var type = $('div.ak-encyclo-detail-right').find('div.ak-encyclo-detail-type').find('span').text().trim();
62+
if(typeof type !== 'undefined') type = $('div.ak-encyclo-detail-right').find('div.ak-encyclo-detail-type').text().split(':')[1].trim();
6263
var name = $('h1.ak-return-link').text().trim();
63-
var description = $('div.ak-encyclo-detail-right.ak-nocontentpadding').find('div.ak-container.ak-panel').first().find('div.ak-panel-content').text().trim();
64-
var lvl = $('div.ak-encyclo-detail-right.ak-nocontentpadding').find('div.ak-encyclo-detail-level.col-xs-6.text-right').text().trim().replace(/\D/g,'');
64+
$descriptor = $('div.ak-encyclo-detail-right.ak-nocontentpadding').find('div.ak-container.ak-panel').first().find('div.ak-panel-content');
65+
if(typeof $descriptor !== 'undefined') var description = $descriptor.text().trim();
66+
var lvl = $('div.ak-encyclo-detail-right').find('div.ak-encyclo-detail-level.col-xs-6.text-right').text().trim().replace(/\D/g,'');
6567
var imgUrl = $('div.ak-encyclo-detail-illu').find('img').attr('src').replace('dofus/ng/img/../../../', '');
6668

6769
var item = {
6870
_id: itemId,
6971
name: name,
70-
description: description,
7172
lvl: lvl,
7273
type: type,
7374
imgUrl: imgUrl,
7475
url: url
7576
}
77+
if($descriptor.html() !== null) item.description = description;
7678
return item;
79+
},
80+
81+
getCategoryType: (type) => {
82+
var glType = null;
83+
switch (true) {
84+
case /\b(chapeau|hat)\b/gi.test(type):
85+
glType = 'equipments';
86+
break;
87+
case /\b(cloak|cape)\b/gi.test(type):
88+
glType = 'equipments';
89+
break;
90+
case /\b(amulet|amulette)\b/gi.test(type):
91+
glType = 'equipments';
92+
break;
93+
case /\b(boots|bottes)\b/gi.test(type):
94+
glType = 'equipments';
95+
break;
96+
case /\b(ring|anneau)\b/gi.test(type):
97+
glType = 'equipments';
98+
break;
99+
case /\b(belt|ceinture)\b/gi.test(type):
100+
glType = 'equipments';
101+
break;
102+
case /\b(backpack|sac a dos)\b/gi.test(type):
103+
glType = 'equipments';
104+
break;
105+
case /\b(shield|bouclier)\b/gi.test(type):
106+
glType = 'equipments';
107+
break;
108+
case /\b(trophy|trophee)\b/gi.test(type):
109+
glType = 'equipments';
110+
break;
111+
case /\b(pet|familier)\b/gi.test(type):
112+
glType = 'equipments';
113+
break;
114+
case /\bdofus\b/gi.test(type):
115+
glType = 'equipments';
116+
break;
117+
case /\b(sword|epee)\b/gi.test(type):
118+
glType = 'weapons';
119+
break;
120+
case /\b(dagger|dague)\b/gi.test(type):
121+
glType = 'weapons';
122+
break;
123+
case /\b(axe|hache)\b/gi.test(type):
124+
glType = 'weapons';
125+
break;
126+
case /\b(bow|arc)\b/gi.test(type):
127+
glType = 'weapons';
128+
break;
129+
case /\b(hammer|marteau)\b/gi.test(type):
130+
glType = 'weapons';
131+
break;
132+
case /\b(pickaxe|pioche)\b/gi.test(type):
133+
glType = 'weapons';
134+
break;
135+
case /\b(scythe|faux)\b/gi.test(type):
136+
glType = 'weapons';
137+
break;
138+
case /\b(shovel|pelle)\b/gi.test(type):
139+
glType = 'weapons';
140+
break;
141+
case /\b(soul stone|pierre d'ame)\b/gi.test(type):
142+
glType = 'weapons';
143+
break;
144+
case /\b(staff|baton)\b/gi.test(type):
145+
glType = 'weapons';
146+
break;
147+
case /\b(tool|outil)\b/gi.test(type):
148+
glType = 'weapons';
149+
break;
150+
case /\b(wand|baguette)\b/gi.test(type):
151+
glType = 'weapons';
152+
break;
153+
default:
154+
console.log('\x1b[31m%s\x1b[0m' ,'Sorry, we are out of ' + type + '.');
155+
}
156+
return glType;
77157
}
78158
};

0 commit comments

Comments
 (0)