Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ await scrape({
launchOptions: { headless: "new" }, /* optional */
gotoOptions: { waitUntil: "networkidle0" }, /* optional */
scrollToBottom: { timeout: 10000, viewportN: 10 }, /* optional */
blockNavigation: true, /* optional */
})
]
});
Expand All @@ -41,7 +40,6 @@ Puppeteer plugin constructor accepts next params:
* `scrollToBottom` - *(optional)* - in some cases, the page needs to be scrolled down to render its assets (lazyloading). Because some pages can be really endless, the scrolldown process can be interrupted before reaching the bottom when one or both of the bellow limitations are reached:
* `timeout` - in milliseconds
* `viewportN` - viewport height multiplier
* `blockNavigation` - *(optional)* - defines whether navigation away from the page is permitted or not. If it is set to true, then the page is locked to the current url and redirects with `location.replace(anotherPage)` will not pass. Defaults to `false`

## How it works
It starts Chromium in headless mode which just opens page and waits until page is loaded.
Expand Down
21 changes: 1 addition & 20 deletions lib/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,14 @@ class PuppeteerPlugin {
launchOptions = {},
gotoOptions = {},
scrollToBottom = null,
blockNavigation = false
} = {}) {
this.launchOptions = launchOptions;
this.gotoOptions = gotoOptions;
this.scrollToBottom = scrollToBottom;
this.blockNavigation = blockNavigation;
this.browser = null;
this.headers = {};

logger.info('init plugin', { launchOptions, scrollToBottom, blockNavigation });
logger.info('init plugin', { launchOptions, scrollToBottom });
}

apply (registerAction) {
Expand All @@ -43,10 +41,6 @@ class PuppeteerPlugin {
await page.setExtraHTTPHeaders(this.headers);
}

if (this.blockNavigation) {
await blockNavigation(page, url);
}

await page.goto(url, this.gotoOptions);

if (this.scrollToBottom) {
Expand Down Expand Up @@ -78,17 +72,4 @@ async function scrollToBottom (page, timeout, viewportN) {
await page.evaluate(scrollToBottomBrowser, timeout, viewportN);
}

async function blockNavigation (page, url) {
logger.info(`block navigation for puppeteer page from url ${url}`);

page.on('request', req => {
if (req.isNavigationRequest() && req.frame() === page.mainFrame() && req.url() !== url) {
req.abort('aborted');
} else {
req.continue();
}
});
await page.setRequestInterception(true);
}

export default PuppeteerPlugin;
25 changes: 0 additions & 25 deletions test/puppeteer-plugin.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -42,31 +42,6 @@ describe('Puppeteer plugin test', () => {
expect(content).to.contain('<div id="special-characters-test">7년 동안 한국에서 살았어요. Слава Україні!</div>');
});
});

describe('Block navigation', () => {
before('scrape website', async () => {
result = await scrape({
urls: [`http://localhost:${SERVE_WEBSITE_PORT}/navigation.html`],
directory: directory,
plugins: [
new PuppeteerPlugin({
launchOptions: { headless: "new" },
blockNavigation: true
})
]
});
});
before('get content from file', () => {
content = fs.readFileSync(`${directory}/${result[0].filename}`).toString();
});
after('delete dir', () => fs.removeSync(directory));

it('should render content (and not be redirected)', async () => {
expect(content).to.contain('<div id="root">Navigation blocked!</div>');
});
});


});

function startWebserver(port = 3000) {
Expand Down