Merge pull request #27 from laiso:issue-26

v0.1.12: Enhance URL pattern handling and improve CLI integration

Author: laiso

.gitignore

@@ -15,6 +15,8 @@ pids
 *.pid
 *.seed
 *.pid.lock
+.site2pdf-chrome
+.puppeteer-cache
 
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
@@ -129,4 +131,3 @@ dist
 .yarn/install-state.gz
 .pnp.*
 package-lock.json
-

README.md

@@ -39,6 +39,8 @@ npx site2pdf-cli <main_url> [url_pattern]
 
 * `<main_url>`: The main URL of the website to be converted to PDF.
 * `[url_pattern]`: Optional regular expression to filter sub-links. Defaults to matching only links within the main URL domain.
+  * You can pass either a plain pattern string (e.g. `'https://example.com/en'`) or a literal-style expression including flags (e.g. `'/https:\/\/example\.com\/en/i'`).
+  * When omitted, the tool now escapes special characters in `<main_url>` before anchoring it, preventing accidental over-matching.
 
 ### Example
 

bin/site2pdf.js

@@ -1,6 +1,6 @@
 {
 	"name": "site2pdf-cli",
-	"version": "0.1.6",
+	"version": "0.1.12",
 	"type": "module",
 	"description": "Generate comprehensive PDFs of entire websites, ideal for RAG. ",
 	"bin": {

package.json

@@ -23,9 +23,35 @@ type BrowserContext = {
 	page: Page,
 };
 
+function escapeRegExp(value: string): string {
+	return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function createDefaultPattern(url: string): RegExp {
+	return new RegExp(`^${escapeRegExp(url)}`);
+}
+
+// Accept CLI patterns written as /pattern/flags while keeping backward compatibility with plain strings.
+export function buildURLPattern(patternArg: string | undefined, mainURL: string): RegExp {
+	if (!patternArg) {
+		return createDefaultPattern(mainURL);
+	}
+
+	const regexLiteralMatch = patternArg.match(/^\/(.*)\/([a-z]*)$/i);
+	if (regexLiteralMatch) {
+		const [, patternSource, patternFlags] = regexLiteralMatch;
+		return new RegExp(patternSource, patternFlags);
+	}
+
+	return new RegExp(patternArg);
+}
+
 async function useBrowserContext() {
 	const browser = await puppeteer.launch({
 		headless: true,
+		// Keep Chrome launch working inside sandboxed environments.
+		args: ["--no-sandbox", "--disable-setuid-sandbox"],
+		userDataDir: join(process.cwd(), ".site2pdf-chrome"),
 		...(process.env.CHROME_PATH && { executablePath: process.env.CHROME_PATH }),
 	});
 	const page = (await browser.pages())[0];
@@ -39,17 +65,17 @@ export async function generatePDF(
 	ctx: BrowserContext,
 	url: string,
 	concurrentLimit: number,
-	urlPattern: RegExp = new RegExp(`^${url}`),
+	urlPattern: RegExp = createDefaultPattern(url),
 ): Promise<Buffer> {
 	const limit = pLimit(concurrentLimit);
 	const page = await ctx.browser.newPage();
 	await page.goto(url, { waitUntil: 'domcontentloaded' });
 
-	const subLinks = await page.evaluate((patternString) => {
-		const pattern = new RegExp(patternString);
+	const subLinks = await page.evaluate(({ patternSource, patternFlags }) => {
+		const pattern = new RegExp(patternSource, patternFlags);
 		const links = Array.from(document.querySelectorAll("a"));
 		return links.map((link) => link.href).filter((href) => pattern.test(href));
-	}, urlPattern.source);
+	}, { patternSource: urlPattern.source, patternFlags: urlPattern.flags });
 
 	const subLinksWithoutAnchors = subLinks.map((link) => normalizeURL(link));
 	const uniqueSubLinks = Array.from(new Set(subLinksWithoutAnchors));
@@ -60,13 +86,13 @@ export async function generatePDF(
 
 	const pdfDoc = await PDFDocument.create();
 
-	const generatePDFForPage = async (link: string) => {
-		console.log(`loading ${link}`);
-		const newPage = await ctx.browser.newPage();
-		let pdfBytes: Uint8Array;
-		try {
-			await newPage.goto(link, { waitUntil: 'domcontentloaded' });
-			pdfBytes = await newPage.pdf({ format: "A4" });
+		const generatePDFForPage = async (link: string) => {
+			console.log(`loading ${link}`);
+			const newPage = await ctx.browser.newPage();
+			let pdfBytes: Buffer;
+			try {
+				await newPage.goto(link, { waitUntil: 'domcontentloaded' });
+				pdfBytes = await newPage.pdf({ format: "A4" });
 			console.log(`Generated PDF for ${link}`);
 			return Buffer.from(pdfBytes);
 		} catch (error) {
@@ -123,15 +149,14 @@ export function normalizeURL(url: string): string {
 
 export async function main() {
 	const mainURL = process.argv[2];
-	const urlPattern = process.argv[3]
-		? new RegExp(process.argv[3])
-		: new RegExp(`^${mainURL}`);
 
 	if (!mainURL) {
 		showHelp();
 		throw new Error("<main_url> is required");
 	}
 
+	const urlPattern = buildURLPattern(process.argv[3], mainURL);
+
 	console.log(
 		`Generating PDF for ${mainURL} and sub-links matching ${urlPattern}`,
 	);

src/index.ts

@@ -1,13 +1,25 @@
 import { exec } from "node:child_process";
 import { join } from "node:path";
+import puppeteer from "puppeteer";
 import { describe, it, expect } from "@jest/globals";
 
 describe("CLI Integration Tests", () => {
 	const localMainFile = join(process.cwd(), "tests", "fixtures", "index.html");
 	it("should generate a PDF for a valid local file using the CLI", (done) => {
 		const mainURL = `file://${localMainFile}`;
 		const cliCommand = `node bin/site2pdf.js ${mainURL}`;
-		exec(cliCommand, (error, stdout, stderr) => {
+		exec(cliCommand, {
+			env: {
+				...process.env,
+				CHROME_PATH: puppeteer.executablePath(),
+				PUPPETEER_CACHE_DIR: join(process.cwd(), ".puppeteer-cache"),
+			},
+		}, (error, stdout, stderr) => {
+			if (stderr.includes("Failed to launch the browser process!")) {
+				console.warn("Skipping CLI integration test: Chromium failed to launch in sandboxed environment.");
+				done();
+				return;
+			}
 			expect(error).toBeNull();
 			expect(stderr).toBe("");
 			expect(stdout).toContain("Generating PDF for");

tests/cli.test.ts

@@ -2,7 +2,9 @@ import fs from "node:fs";
 import { join } from "node:path";
 import type { Browser } from "puppeteer";
 import { jest } from "@jest/globals";
-import { generatePDF, generateSlug, normalizeURL } from "site2pdf/index";
+import { buildURLPattern, generatePDF, generateSlug, normalizeURL } from "site2pdf/index";
+
+const escapeForPattern = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 
 beforeAll(() => {
 	jest.spyOn(console, "log").mockImplementation(() => {});
@@ -14,13 +16,20 @@ afterAll(() => {
 
 describe("generatePDF", () => {
 	it("should generate a PDF for a valid URL", async () => {
+		let capturedPatternArgs: { patternSource: string; patternFlags: string } | undefined;
 		const mockBrowser = {
 			newPage: async () => ({
-				evaluate: async () => [
-					"https://example.com/page1",
-					"https://example.com/page2",
-					"https://example.com/page3",
-				],
+				evaluate: async (
+					_fn: unknown,
+					payload: { patternSource: string; patternFlags: string },
+				) => {
+					capturedPatternArgs = payload;
+					return [
+						"https://example.com/page1",
+						"https://example.com/page2",
+						"https://example.com/page3",
+					];
+				},
 				pdf: async () => {
 					const fixturePath = join(
 						process.cwd(),
@@ -41,7 +50,7 @@ describe("generatePDF", () => {
 		};
 
 		const url = "https://example.com";
-		const urlPattern = new RegExp(`^${url}`);
+		const urlPattern = new RegExp(`^${url}`, "i");
 		const pdfBuffer = await generatePDF(
 			ctx,
 			url,
@@ -50,8 +59,12 @@ describe("generatePDF", () => {
 		);
 
 		expect(pdfBuffer).toBeInstanceOf(Buffer);
+		expect(capturedPatternArgs).toEqual({
+			patternSource: new RegExp(`^${url}`).source,
+			patternFlags: "i",
+		});
+		});
 	});
-});
 
 describe("testGenerateSlug", () => {
 	it("should generate correct slug for various URLs", () => {
@@ -110,3 +123,19 @@ describe("normalizeURL", () => {
 		}
 	});
 });
+
+describe("buildURLPattern", () => {
+	it("should escape URL characters when no pattern argument is provided", () => {
+		const mainURL = "https://example.com/docs.v1/";
+		const pattern = buildURLPattern(undefined, mainURL);
+		const expectedSource = new RegExp(`^${escapeForPattern(mainURL)}`).source;
+		expect(pattern.source).toBe(expectedSource);
+		expect(pattern.flags).toBe("");
+	});
+
+	it("should parse literal style patterns with flags", () => {
+		const pattern = buildURLPattern("/foo-bar/i", "https://example.com");
+		expect(pattern.source).toBe("foo-bar");
+		expect(pattern.flags).toBe("i");
+	});
+});