feat(scraper): add ArcaneScraper and transformer for arcanes (#714)

Author: justsomelegs

build/parser.mjs

@@ -161,6 +161,7 @@ class Parser {
     const result = [];
 
     const bar = new Progress(`Parsing ${category}`, items.length);
+
     if (!items.length) {
       bar?.interrupt?.(`No ${category}`);
       return [];
@@ -192,7 +193,7 @@ class Parser {
     const result = cloneDeep(original);
 
     if (result.rewardName) result.uniqueName = result.rewardName;
-    this.addType(result);
+    this.addType(result, data);
     this.addDamage(result);
     this.sanitize(result);
     this.addImageName(result, data.manifest);
@@ -229,7 +230,7 @@ class Parser {
     const result = cloneDeep(original);
 
     if (result.rewardName) result.uniqueName = result.rewardName;
-    this.addType(result);
+    this.addType(result, data);
     this.sanitize(result);
     this.addImageName(result, data.manifest, previous);
     this.addCategory(result, category);
@@ -445,17 +446,25 @@ class Parser {
    * saved as /Lotus/Powersuits/*, meaning that Archwing has to be looked for
    * first, otherwise it would be considered a Warframe.
    * @param {Partial<Item>} item to have type adjusted on
+   * @param {RawItemData} data raw context data
    */
-  addType(item) {
+  addType(item, data) {
     if (item.parent) return;
-    // eslint-disable-next-line no-restricted-syntax
-    for (const type of types) {
-      const contains = type.regex ? new RegExp(type.id, 'ig').test(item.uniqueName) : item.uniqueName.includes(type.id);
-      if (contains) {
-        if (type.append) item.type = `${item.type}${type.name}`;
-        else item.type = type.name;
-        // if (item.type !== type.name) console.error(`${item.name} didn't update types`)
-        break;
+    const arcane = data.wikia.arcanes.find((entry) => entry.name === item.name);
+    if (arcane) {
+      item.type = `${arcane.type} Arcane`;
+    } else {
+      // eslint-disable-next-line no-restricted-syntax
+      for (const type of types) {
+        const contains = type.regex
+          ? new RegExp(type.id, 'ig').test(item.uniqueName)
+          : item.uniqueName.includes(type.id);
+        if (contains) {
+          if (type.append) item.type = `${item.type}${type.name}`;
+          else item.type = type.name;
+          // if (item.type !== type.name) console.error(`${item.name} didn't update types`)
+          break;
+        }
       }
     }
 
@@ -917,6 +926,9 @@ class Parser {
       case 'upgrades':
         this.addModWikiaData(item, wikiaItem);
         break;
+      case 'arcanes':
+        this.addArcaneWikiaData(item, wikiaItem);
+        break;
       default:
         break;
     }
@@ -987,6 +999,19 @@ class Parser {
     if (!wikiaItem.thumbnail) warnings.missingWikiThumb.push(item.name);
   }
 
+  /**
+   * Add additional data for mods from the wiki
+   * @param {Item} item mod to append wikia data to
+   * @param {WikiaArcane} wikiaItem to pull data from
+   */
+  addArcaneWikiaData(item, wikiaItem) {
+    item.wikiaThumbnail = wikiaItem.thumbnail;
+    item.wikiaUrl = wikiaItem.url;
+    item.transmutable = wikiaItem.transmutable;
+    item.type = wikiaItem.type;
+    if (!wikiaItem.thumbnail) warnings.missingWikiThumb.push(item.name);
+  }
+
   /**
    * Normalize Ogg vault dates to be technically ISO-8601 dates
    * @param {OggDateStamp} vaultDate vault date to normalize

build/scraper.mjs

@@ -4,6 +4,7 @@ import { Generator as RelicGenerator } from '@wfcd/relics';
 import patchlogs from 'warframe-patchlogs';
 
 import Progress from './progress.mjs';
+import ArcaneScraper from './wikia/scrapers/ArcaneScraper.mjs';
 import ArchwingScraper from './wikia/scrapers/ArchwingScraper.mjs';
 import CompanionScraper from './wikia/scrapers/CompanionScraper.mjs';
 import ModScraper from './wikia/scrapers/ModScraper.mjs';
@@ -207,13 +208,14 @@ class Scraper {
    * @property {Array<WikiaMods>} mods
    * @property {Array<WikiaVersions>} versions
    * @property {Array<WikiaDucats>} ducats
+   * @property {Array<WikiaArcanes>} arcanes
    */
   /**
    * Get additional data from wikia if it's not provided in the API
    * @returns {WikiaData}
    */
   async fetchWikiaData() {
-    const bar = new Progress('Fetching Wikia Data', 7);
+    const bar = new Progress('Fetching Wikia Data', 8);
     const ducats = [];
     const ducatsWikia = await get('https://wiki.warframe.com/w/Ducats/Prices/All', true);
     const $ = load(ducatsWikia);
@@ -231,6 +233,8 @@ class Scraper {
     bar.tick();
     const mods = await new ModScraper().scrape();
     bar.tick();
+    const arcanes = await new ArcaneScraper().scrape();
+    bar.tick();
     const versions = await new VersionScraper().scrape();
     bar.tick();
     const archwings = await new ArchwingScraper().scrape();
@@ -246,6 +250,7 @@ class Scraper {
       ducats,
       archwings,
       companions,
+      arcanes,
     };
   }
 

build/tradable.mjs

@@ -1,6 +1,21 @@
 const builtUntradable = ['Warframe', 'Throwing', 'Shotgun', 'Rifle', 'Pistol', 'Melee', 'Sword And Shield'];
 const tradableConditions = (item) => !(builtUntradable.includes(item.type) && item.name.match(/Prime/gi));
 
+const tradableArcanes = [
+  'Arcane',
+  'Primary Arcane',
+  'Secondary Arcane',
+  'Melee Arcane',
+  'Amp Arcane',
+  'Zaw Arcane',
+  'Kitgun Arcane',
+  'Shotgun Arcane',
+  'Sniper Arcane',
+  'Operator Arcane',
+  'Bow Arcane',
+  'Warframe Arcane',
+];
+
 const tradableMods = [
   'Arch-Melee Mod',
   'Archwing Mod',
@@ -17,7 +32,16 @@ const tradableMods = [
   'Stance Mod',
   'Warframe Mod',
 ];
-const tradableTypes = ['Arcane', 'Captura', 'Cut Gem', 'Fish', 'Focus Lens', 'Relic', 'Upgrades', ...tradableMods];
+const tradableTypes = [
+  'Captura',
+  'Cut Gem',
+  'Fish',
+  'Focus Lens',
+  'Relic',
+  'Upgrades',
+  ...tradableArcanes,
+  ...tradableMods,
+];
 const untradableTypes = [
   'Color Palette',
   'Exalted Weapon',

build/wikia/WikiaDataScraper.mjs

@@ -180,7 +180,6 @@ export default class WikiaDataScraper {
           things.push(transformedThing);
         })
       );
-
       things.sort(nameCompare);
     } catch (e) {
       console.error(e.stack);

build/wikia/scrapers/ArcaneScraper.mjs

@@ -0,0 +1,8 @@
+import WikiaDataScraper from '../WikiaDataScraper.mjs';
+import transformArcanes from '../transformers/transformArcanes.mjs';
+
+export default class ArcaneScraper extends WikiaDataScraper {
+  constructor() {
+    super('https://wiki.warframe.com/w/Module:Arcane/data?action=edit', 'Arcane', transformArcanes);
+  }
+}

build/wikia/transformers/transformArcanes.mjs

@@ -0,0 +1,23 @@
+export default async (oldArcane, imageUrls) => {
+  let newArcane;
+  if (!oldArcane || !oldArcane.Name) {
+    return undefined;
+  }
+
+  try {
+    const { Image, Name, Transmutable, Introduced, Type } = oldArcane;
+
+    newArcane = {
+      name: Name,
+      url: `https://wiki.warframe.com/w/${encodeURIComponent(Name.replace(/\s/g, '_'))}`,
+      transmutable: Transmutable,
+      introduced: Introduced,
+      type: Type,
+      thumbnail: imageUrls?.[Image],
+    };
+  } catch (error) {
+    console.error(`Error parsing ${oldArcane.Name}`);
+    console.error(error);
+  }
+  return newArcane;
+};

index.d.ts

@@ -249,7 +249,7 @@ declare module 'warframe-items' {
     patchlogs?: PatchLog[];
     rarity?: Rarity;
     tradable: true;
-    type: 'Arcane';
+    type: 'Arcane' | `${ArcaneType} Arcane`;
   }
   interface StanceMod extends Omit<Mod, 'levelStats'> {
     type: 'Stance Mod';
@@ -299,6 +299,20 @@ declare module 'warframe-items' {
     | 'Peculiar'
     | 'Plexus'
     | 'Posture';
+
+  type ArcaneType =
+    | 'Primary'
+    | 'Secondary'
+    | 'Melee'
+    | 'Warframe'
+    | 'Amp'
+    | 'Operator'
+    | 'Zaw'
+    | 'Kitgun'
+    | 'Bow'
+    | 'Shotgun'
+    | 'Sniper';
+
   interface Mod extends MinimalItem, WikiaItem, Droppable {
     baseDrain: number;
     category: 'Mods';