Scrap Web API from MDN

Author: clochix

scraper/webapi-mdn.js

@@ -0,0 +1,111 @@
+var requirejs = require('requirejs');
+
+requirejs([
+  'spider',
+  'underscore',
+  'cheerio',
+  '../models/sectionscrape',
+  'path',
+  'fs'
+], function(spider, _, cheerio, SectionScrape, path, fs) {
+
+  var results = [];
+
+  var spidey = spider();
+
+  // use this to visit all links on a page
+  var visitLinks = function($) {
+    $('a:not([class*=new])').each(function() {
+      var href = $(this).attr('href');
+      if (href && href.substr(0, 4) !== 'http') {
+        href = 'https://developer.mozilla.org' + href;
+      }
+      if (href && href.indexOf('$') === -1 && href.indexOf('?') === -1 && /Web.?API/.exec(href.split('=')[0]) !== null) {
+        spidey.get(href);
+      }
+    });
+  };
+
+  // file where we'll dump the json
+  var filename = path.dirname(__filename) + '/../static/data/webapi-mdn.json';
+  console.log('dumping to ' + filename);
+  var file = fs.openSync(filename,'w');
+
+  // main index of mdn's Web API docs
+  spidey.route('developer.mozilla.org', '/en-US/docs/tag/WebAPI', function ($) {
+    visitLinks($);
+  });
+
+  var blacklist = [
+  ];
+
+  // some urls redirect to other pages w/o changing the url (for example: https://developer.mozilla.org/en/CSS/-moz-scrollbars-none)
+  // so in addition to not visiting the same url twice, keep this list to prevent visiting the same title twice
+  var titles = [];
+
+  spidey.route('developer.mozilla.org', /\/en\-US\/docs\/Web.?API\/*/, function ($, url) {
+    if ( _.include(blacklist,url) ) return;
+    visitLinks($);
+
+    console.log('---------');
+    console.log('scraping:',url);
+
+    var title = $('h1').text().trim();
+    if ( title === '' || title === null ) {
+      console.log('ERROR: could not get title, skipping');
+      return;
+    } else if ( _.include(titles,title) ) {
+      console.log('WARNING: already scraped something with this title, skipping');
+      return;
+    }
+
+    console.log('title:',title);
+
+    var scrapeData = new SectionScrape();
+    scrapeData['title'] = title;
+    scrapeData['url'] = url;
+    scrapeData['sectionNames'] = [];
+    scrapeData['sectionHTMLs'] = [];
+
+    // get all section ids
+    var ids = _.map($('article[id]'), function(div) { return div.attribs.id } );
+    if ( ids.length === 0 ) {
+      console.log('WARNING: no sections...');
+      return;
+    }
+
+    for ( var i = 0; i < ids.length; i++ ) {
+      // load the section html as its own jquery object
+      var $section = cheerio.load($('[id^="' + ids[i] + '"]').html());
+
+      // strip scripts
+      $section('script').remove();
+      var sectionName = "";
+
+      // TODO find relative hrefs and turn them into absolute hrefs
+
+      // find the title of the section--mdn isn't very consistent with what size headers they use
+      _.each([1,2,3,4],function(h) {
+        var headers = $section('h' + h);
+        if ( sectionName === "" && headers.length > 0 ) {
+          sectionName = headers.text();
+        }
+      });
+
+      scrapeData['sectionNames'].push(sectionName);
+      scrapeData['sectionHTMLs'].push($section.html());
+    }
+
+    results.push(scrapeData.toJSON());
+    titles.push(title);
+  });
+
+  // start 'er up
+  spidey.get('https://developer.mozilla.org/en-US/docs/tag/WebAPI').log('info');
+
+  process.on('exit', function () {
+    fs.writeSync(file,JSON.stringify(results,null,'\t'));
+    console.log('DONE');
+  });
+  return;
+});

static/dochub.appcache

@@ -73,3 +73,4 @@ data/php-ext.json
 data/python.json
 data/python3.json
 data/xslt-w3.json
+data/webapi-mdn.json

static/js/collections/mozdevapiprops.js

@@ -0,0 +1,39 @@
+define([
+  'jQuery',
+  'Underscore',
+  'Backbone',
+  'models/sectionscrape'
+], function($, _, Backbone, SectionScrape) {
+
+  // Handle the following cases:
+  //  -xyz
+  //  ::-xyz
+  //  ::xyz
+  //  :xyz
+  //  @xyz
+  //  <xyz>
+  var apiPropsPattern = new RegExp("^(\\W*)(.+)$");
+
+  var MozDevAPIProps = Backbone.Collection.extend({
+    url: 'data/webapi-mdn.json',
+    model: SectionScrape,
+
+    comparator: function(model) {
+      var title = model.get('title');
+      var results = apiPropsPattern.exec(title);
+      var prefix = results[1];
+      var name   = results[2];
+
+      if (prefix) {
+        return '2' + title;
+      } else if (name[0] === name[0].toLowerCase()) {
+        return '0' + name;
+      } else {
+        return '1' + name;
+      }
+    },
+
+  });
+
+  return MozDevAPIProps;
+});

static/js/router.js

@@ -5,7 +5,7 @@ define([
   'jQuery',
   'Underscore',
   'Backbone',
-  
+
   // Settings
   'settings',
 
@@ -20,6 +20,7 @@ define([
   'collections/mozdevcssprops',
   'collections/mdnhtmlelements',
   'collections/mdnjsobjs',
+  'collections/mozdevapiprops',
   'collections/mdndomobjs',
   'collections/phpexts',
   'collections/jqentries',
@@ -31,13 +32,13 @@ define([
   // Templates
   'text!templates/mdnpage.html',
 ], function(doc, $, _, Backbone,
-	    Settings,
+            Settings,
             TopNavView, JQuerySearchResultsView, LanguageView, PageScrapedLanguageView,
             FullWindowView,
-            MozDevCSSPropCollection, MDNHtmlElementsCollection, MDNJsObjsCollection,
+            MozDevCSSPropCollection, MDNHtmlElementsCollection, MDNJsObjsCollection, MozDevAPIPropCollection,
             MDNDomObjsCollection, PHPExtensionsCollection, JQEntriesCollection,
-            XSLTPagesCollection, PythonPagesCollection, 
-	    Python3PagesCollection, NodejsPagesCollection,
+            XSLTPagesCollection, PythonPagesCollection,
+            Python3PagesCollection, NodejsPagesCollection,
             MDNPage) {
 
   var DocHub = Backbone.Router.extend({
@@ -116,7 +117,14 @@ define([
           debounceTime: 200,
           minQueryLength: 3,
         }),
-	
+        'webapi' : new LanguageView({
+          languageName: 'WebAPI',
+          resultsClassNames: 'webapi',
+          collection: new MozDevAPIPropCollection(),
+          placeholder: 'Type a Web API name',
+          mainResultTemplate: MDNPage,
+        }),
+
         // 'xslt' : new PageScrapedLanguageView({
         //   languageName: 'XSLT',
         //   resultsClassNames: 'w3',
@@ -133,7 +141,7 @@ define([
       this.renderTopNav = _.once(function() {
         self.topNavView = new TopNavView({
           el: $('.navbar'),
-	  settings: Settings
+          settings: Settings
         });
         self.topNavView.render();
         self.topNavView.bind('changeLanguage', self.changeLanguage);

static/js/settings.js

@@ -9,17 +9,18 @@ define([], function(){
       { lang: 'php', name: 'PHP' },
       { lang: 'python', name: 'Python' },
       { lang: 'python3', name: 'Python3' },
-      { lang: 'nodejs', name: 'Node.js' }
+      { lang: 'nodejs', name: 'Node.js' },
+      { lang: 'webapi', name: 'WebAPI' }
     ]
   }
-  
+
   Settings.languages.sort(function(a, b){
     var nameA=a.name.toLowerCase(), nameB=b.name.toLowerCase()
     //sort string ascending
     if (nameA < nameB) { return -1 }
     if (nameA > nameB) { return 1 }
     return 0 //default return value (no sorting)
   });
-  
+
   return Settings;
 });