Merge pull request #353 from ilanbm/link_alternate_scrap

stereobooster · web-flow · commit f1a68a6912cf · 2019-04-01T18:47:03.000+02:00
Fix #293 Crawl `<link rel="alternate">`
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
@@ -108,7 +108,7 @@ const enableLogging = opt => {
 const getLinks = async opt => {
   const { page } = opt;
   const anchors = await page.evaluate(() =>
-    Array.from(document.querySelectorAll("a")).map(anchor => {
+    Array.from(document.querySelectorAll("a,link[rel='alternate']")).map(anchor => {
       if (anchor.href.baseVal) {
         const a = document.createElement("a");
         a.href = anchor.href.baseVal;
diff --git a/tests/examples/many-pages/index.html b/tests/examples/many-pages/index.html
@@ -2,6 +2,7 @@
 
 <head>
   <meta charset="utf-8">
+  <link rel="alternate" href='/5' />
 </head>
 
 <body>
diff --git a/tests/run.test.js b/tests/run.test.js
@@ -157,13 +157,14 @@ describe("many pages", () => {
   } = mockFs();
   beforeAll(() => snapRun(fs, { source }));
   test("crawls all links and saves as index.html in separate folders", () => {
-    expect(filesCreated()).toEqual(6);
+    expect(filesCreated()).toEqual(7);
     expect(names()).toEqual(
       expect.arrayContaining([
         `/${source}/1/index.html`, // without slash in the end
         `/${source}/2/index.html`, // with slash in the end
         `/${source}/3/index.html`, // ignores hash
-        `/${source}/4/index.html` // ignores query
+        `/${source}/4/index.html`, // ignores query
+        `/${source}/5/index.html`, // link rel="alternate"
       ])
     );
   });
