diff --git a/index.js b/index.js index c0d9f91a..ab01addd 100644 --- a/index.js +++ b/index.js @@ -19,6 +19,7 @@ const defaultOptions = { destination: null, concurrency: 4, include: ["/"], + exclude: [], userAgent: "ReactSnap", // 4 params below will be refactored to one: `puppeteer: {}` // https://github.com/stereobooster/react-snap/issues/120 diff --git a/package.json b/package.json index 8358c99f..2a2885b1 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "clean-css": "4.2.1", "express": "4.17.1", "express-history-api-fallback": "2.2.1", + "glob-to-regexp": "^0.4.0", "highland": "2.13.4", "html-minifier": "4.0.0", "minimalcss": "0.8.2", @@ -40,5 +41,10 @@ "run.js", "src/", "vendor/" - ] + ], + "reactSnap": { + "destination": "tmp", + "inlineCss": true, + "cacheAjaxRequests": true + } } diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js index 820cded0..aa7489be 100644 --- a/src/puppeteer_utils.js +++ b/src/puppeteer_utils.js @@ -1,6 +1,8 @@ const puppeteer = require("puppeteer"); const _ = require("highland"); const url = require("url"); +const glob = require("glob-to-regexp"); +// @ts-ignore const mapStackTrace = require("sourcemapped-stacktrace-node").default; const path = require("path"); const fs = require("fs"); @@ -140,6 +142,7 @@ const crawl = async opt => { publicPath, sourceDir } = opt; + const exclude = options.exclude.map(g => glob(g, { extended: true, globstar: true})); let shuttingDown = false; let streamClosed = false; @@ -173,7 +176,7 @@ const crawl = async opt => { * @returns {void} */ const addToQueue = newUrl => { - const { hostname, search, hash, port } = url.parse(newUrl); + const { hostname, search, hash, port, pathname } = url.parse(newUrl); newUrl = newUrl.replace(`${search || ""}${hash || ""}`, ""); // Ensures that only link on the same port are crawled @@ -183,7 +186,7 @@ const crawl = async opt => { // we are converting both to string to be sure // Port can be null, therefore we need the null check const isOnAppPort = port && port.toString() === options.port.toString(); - + if (exclude.filter(regex => regex.test(pathname)).length > 0) return; if (hostname === "localhost" && isOnAppPort && !uniqueUrls.has(newUrl) && !streamClosed) { uniqueUrls.add(newUrl); enqued++; diff --git a/yarn.lock b/yarn.lock index 5342afe6..a0a392c7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1452,6 +1452,10 @@ getpass@^0.1.1: resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" dependencies: assert-plus "^1.0.0" + +glob-to-regexp@^0.4.0: + version "0.4.0" + resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.0.tgz#49bd677b1671022bd10921c3788f23cdebf9c7e6" glob@^7.0.5, glob@^7.1.1, glob@^7.1.2: version "7.1.2"