Is there any JS bundle of SingleFile?

[NobinPegasus]

I'm doing some archival. Is there any js bundle for SingleFile? I want the same engine that single-file-cli is using but want to run it in puppeteer context. What should be the ideal way to do it?

[gildas-lormeau]

You can get the script hookScript to inject in the top window from single-file-cli/lib/single-file-bundle.js. You can also inject it optionally in frames served from third-party domains. This code is built from single-file-core via this build script.

• Code running in the JS runtime (e.g. Node.js/Deno/Bun etc.)

single-file-cli/lib/cdp-client.js

Lines 354 to 357 in bcd62d3

	await Page.addScriptToEvaluateOnNewDocument({
	source: getHookScriptSource(),
	runImmediately: true
	});

single-file-cli/lib/single-file-script.js

Lines 105 to 107 in bcd62d3

	function getHookScriptSource() {
	return hookScript;
	}

single-file-cli/lib/single-file-script.js

Line 26 in bcd62d3

import { script, hookScript, zipScript } from "../lib/single-file-bundle.js";

Then, you have to call window.singlefile.getPageData in the top window of the page, see below for example.

• Code injected in the page (data is split in multiple parts to handle large files)

single-file-cli/lib/single-file-script.js

Lines 141 to 152 in bcd62d3

	return window.singlefile.getPageData(options).then(data => {
	if (data.content instanceof Uint8Array) {
	data.content = Array.from(data.content);
	}
	data = JSON.stringify(data);
	let indexData = 0;
	do {
	globalThis[SET_PAGE_DATA_FUNCTION_NAME](data.slice(indexData, indexData + MAX_CONTENT_SIZE));
	indexData += MAX_CONTENT_SIZE;
	} while (indexData < data.length);
	globalThis[SET_PAGE_DATA_FUNCTION_NAME]("");
	});

• Code running in the JS runtime

single-file-cli/lib/cdp-client.js

Lines 513 to 528 in bcd62d3

	let pageDataResponse = "";
	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ({ params }) => {
	if (params.name === SET_PAGE_DATA_FUNCTION_NAME) {
	const { payload } = params;
	if (payload.length) {
	pageDataResponse += payload;
	} else {
	logData(["Setting page data"], { options, debugMessages });
	const result = JSON.parse(pageDataResponse);
	if (result.content instanceof Array) {
	result.content = new Uint8Array(result.content);
	}
	resolve(result);
	}
	}
	});

You may need to disable web security, or need to pass a fetch function to window.singlefile.init allowing the injected code to download resources from third-party domains, see below for example.

• Code injected in the page

single-file-cli/lib/single-file-script.js

Lines 46 to 87 in bcd62d3

	const nativeFetch = globalThis.fetch;
	const pendingRequests = new Map();
	let lastRequestId = 0;
	const { RESOLVE_FETCH_FUNCTION_NAME, REJECT_FETCH_FUNCTION_NAME, FETCH_FUNCTION_NAME } = constants;
	globalThis[RESOLVE_FETCH_FUNCTION_NAME] = (requestId, result) => {
	const pendingRequest = pendingRequests.get(requestId);
	if (pendingRequest) {
	pendingRequests.delete(requestId);
	const binaryString = atob(result.data);
	const bytes = new Uint8Array(binaryString.length);
	for (let i = 0; i < binaryString.length; i++) {
	bytes[i] = binaryString.charCodeAt(i);
	}
	pendingRequest.resolve({
	status: result.status,
	headers: new Headers(result.headers),
	arrayBuffer: () => Promise.resolve(bytes.buffer)
	});
	}
	};
	globalThis[REJECT_FETCH_FUNCTION_NAME] = (requestId, error) => {
	const pendingRequest = pendingRequests.get(requestId);
	if (pendingRequest) {
	pendingRequests.delete(requestId);
	pendingRequest.reject(new Error(error.error));
	}
	};
	window.singlefile.init({
	fetch: async (url, options) => {
	try {
	return await nativeFetch(url, options);
	} catch {
	const requestId = lastRequestId++;
	return new Promise((resolve, reject) => {
	pendingRequests.set(requestId, { resolve, reject });
	globalThis[FETCH_FUNCTION_NAME](JSON.stringify({ requestId, url, options }));
	});
	}
	}
	});

• Code running in the JS runtime

single-file-cli/lib/cdp-client.js

Lines 540 to 545 in bcd62d3

	await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextId: contextId });
	Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, async ({ params }) => {
	if (params.name === FETCH_FUNCTION_NAME) {
	await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages });
	}
	});

single-file-cli/lib/cdp-client.js

Lines 606 to 627 in bcd62d3

	async function handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages }) {
	const { payload } = params;
	const { requestId, url, options: fetchOptions } = JSON.parse(payload);
	logData(["Fetching URL", url], { options, debugMessages });
	try {
	const response = await fetch(url, fetchOptions);
	const arrayBuffer = await response.arrayBuffer();
	const base64Data = arrayBufferToBase64(arrayBuffer);
	const result = {
	status: response.status,
	headers: Object.fromEntries(response.headers.entries()),
	data: base64Data
	};
	await callBrowserFunction({ Runtime }, contextId, RESOLVE_FETCH_FUNCTION_NAME, [requestId, result]);
	} catch (error) {
	const errorResult = {
	error: error.message,
	code: error.code
	};
	await callBrowserFunction({ Runtime }, contextId, REJECT_FETCH_FUNCTION_NAME, [requestId, errorResult]);
	}
	}

Finally, you can also have a look at:

• https://github.com/gildas-lormeau/SingleFile/wiki/Executing-within-Playwright-(headless-browser)
• https://github.com/gildas-lormeau/SingleFile/wiki/How-to-integrate-SingleFile-library-code-in-%22custom%22-environments%3F

[NobinPegasus]

Thanks. Seems like with puppeteer we can skip the complex cdp-client.

import puppeteer from "puppeteer";
import fs from "fs/promises";

import {
  script as SINGLEFILE_SCRIPT,
  hookScript as SINGLEFILE_HOOK,
  zipScript as SINGLEFILE_ZIP
} from "single-file-cli/lib/single-file-bundle.js";

async function capture(url) {
  const browser = await puppeteer.launch({
    headless: true,
    args: [
      "--disable-web-security",
      "--allow-running-insecure-content",
      "--no-sandbox"
    ]
  });

  const page = await browser.newPage();

  await page.evaluateOnNewDocument(SINGLEFILE_HOOK);
  await page.evaluateOnNewDocument(`
    ${SINGLEFILE_SCRIPT}
    window.singlefile = singlefile;
  `);

  await page.goto(url, { waitUntil: "networkidle2", timeout: 60000 });

  await page.evaluate(async () => {
    await new Promise(resolve => {
      let totalScrolled = 0;
      const step = 400;
      const timer = setInterval(() => {
        window.scrollBy(0, step);
        totalScrolled += step;
        if (totalScrolled >= document.body.scrollHeight - window.innerHeight) {
          clearInterval(timer);
          resolve();
        }
      }, 150);
    });
  });

  const title = await page.evaluate(() =>
    (
      document.querySelector("meta[property='og:title']")?.content ||
      document.title ||
      "saved_page"
    ).trim()
  );

  const filename = title.replace(/[^a-z0-9\-_]+/gi, "_") + ".html";

  const result = await page.evaluate(
    async zip => {
      return await window.singlefile.getPageData({
        compressContent: false,
        blockScripts: false,
        blockImages: false,
        blockVideos: false,
        zipScript: zip
      });
    },
    SINGLEFILE_ZIP
  );

  await fs.writeFile(filename, result.content);
  await browser.close();

  console.log("Saved:", filename);
}

const url = process.argv[2];
if (!url) {
  console.error("Usage: node src/capture.js <url>");
  process.exit(1);
}

capture(url).catch(error => {
  console.error("Error:", error.message);
});

You can share this in the wiki for using anyone wanting to use singleFile with puppeteer.

[gildas-lormeau]

Thank you! I've created a new page here: https://github.com/gildas-lormeau/SingleFile/wiki/Executing-within-Puppeteer-(headless-browser).

Regarding the cdp-clientcomplexity, it's unfortunately the price to pay in order to provide executable running in Deno (see releases).