feat: add clone command (#73)

* feat: add clone command

* feat: adding more generic rerouteLocal function to handle both document and asset type files

* chore: update readme

* chore: add changest

Author: heyMP

.changeset/large-suns-buy.md

@@ -0,0 +1,21 @@
+---
+"@heymp/scratchpad": minor
+---
+
+Add clone command to copy remote files locally
+
+```bash
+npx @heymp/scratchpad@next clone https://www.example.com/scripts.js
+```
+
+Added new generic helper to reroute both document and asset file types.
+
+scratchpad.config.ts
+```ts
+export default Scratchpad.defineConfig({
+  playwright: async (args) => {
+    const { page } = args;
+    Scratchpad.rerouteLocal(page, 'pages');
+  }
+});
+```

README.md

@@ -92,9 +92,48 @@ You can then reuse the session by using the `login` option when using the `run`
 npx @heymp/scratchpad@next run --login
 ```
 
+#### Reroute Local Files
+
+The `rerouteLocal` function allows you to replace the contents of a document or asset file with a local file from your system. This is incredibly useful for testing changes or developing components in the context of a live site without deploying your code.
+
+When used, `rerouteLocal` also watches for file for changes. If you save an update to the file, the Playwright page will automatically reload to reflect your latest edits, providing a fast feedback loop.
+
+**How it works:**
+
+* It intercepts requests made by the Playwright `page.
+* If the request is not of type `document` it will URL's path to a local file structure. 
+* If the request is of type `document` (e.g. `http://example.com/user/profile`) and you've set your local directory to `'./my-pages'`, the function will look for `'./my-pages/user/profile/index.html'`.
+
+**Usage:**
+
+You would typically use this function within the `playwright` async method in your `scratchpad.config.js`:
+
+`scratchpad.config.js`
+```javascript
+import { defineConfig, rerouteDocument } from '@heymp/scratchpad';
+
+export default defineConfig({
+  url: 'https://example.com', // The initial URL you are working with
+  playwright: async (args) => {
+    const { page } = args;
+
+    // Tell Scratchpad to serve local HTML files from the './pages' directory
+    // whenever a document is requested.
+    await rerouteLocal(page, './pages');
+
+    // Now, if you navigate to https://example.com/some/path,
+    // Scratchpad will try to serve './pages/some/path/index.html'.
+    // If that file doesn't exist, it will load the original page from the web.
+
+    // If there is a local file of `./assets/scripts.js` it will serve the contents
+    // of that local file.
+  }
+});
+```
+
 🚨 It is highly recommended to add the `.scratchpad` directory to your .gitignore file. Never commit or share your session `login.json` file!
 
-#### Reroute Documents
+#### Reroute Documents [deprecated]
 
 The `rerouteDocument` function allows you to replace the HTML content of any webpage with a local HTML file from your system. This is incredibly useful for testing changes or developing components in the context of a live site without deploying your code.
 

src/Processor.ts

@@ -1,20 +1,15 @@
 import fs from 'node:fs';
 import { join } from 'node:path';
 import { build } from 'esbuild';
+import type { Config } from './config.js';
 
 export class ProcessorChangeEvent extends Event {
   constructor() {
     super('change');
   }
 }
 
-export type ProcessorOpts = {
-  headless?: boolean;
-  devtools?: boolean;
-  tsWrite?: boolean;
-  login?: boolean;
-  url?: string;
-  playwright?: any;
+export type ProcessorOpts = Config & {
   file: string;
 }
 

src/browser.ts

@@ -4,6 +4,7 @@ import { join } from 'node:path'
 import fs from 'node:fs/promises';
 import type { Processor } from './Processor.js';
 import { getSession } from './login.js';
+import { rerouteLocal } from './lib/index.js';
 util.inspect.defaultOptions.maxArrayLength = null;
 util.inspect.defaultOptions.depth = null;
 
@@ -36,6 +37,10 @@ export async function browser(processor: Processor) {
   const page = await context.newPage();
   const playwrightConfig = processor.opts.playwright;
 
+  if (processor.opts.rerouteDir) {
+    await rerouteLocal(page, processor.opts.rerouteDir);
+  }
+
   // Exposed functions
   await context.exposeFunction('writeFile', writeFile);
   await context.exposeFunction('appendFile', appendFile);

src/cli.ts

@@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url';
 import { Command } from '@commander-js/extra-typings';
 import { runCommand } from './runCommand.js';
 import { generateCommand } from './generateCommand.js';
+import { cloneCommand } from './cloneCommand.js';
 
 // Get pkg info
 const __filename = fileURLToPath(import.meta.url);
@@ -18,4 +19,5 @@ program
   .version(pkg.version);
 program.addCommand(runCommand);
 program.addCommand(generateCommand);
+program.addCommand(cloneCommand);
 program.parse(process.argv);

src/cloneCommand.ts

@@ -0,0 +1,44 @@
+import { Command } from '@commander-js/extra-typings';
+import { writeFile, mkdir } from 'node:fs/promises';
+import { dirname, join, extname, basename } from 'node:path';
+import { URL } from 'node:url';
+import assert from 'node:assert';
+import { getConfig } from './config.js';
+
+export const cloneCommand = new Command('clone')
+  .description('Generates a local copy of a file from a url.')
+  .argument('<url>', 'url to copy.')
+  .option('--dir <string>', 'source directory where you want to save the document.')
+  .action(async (_url, opts) => {
+    const config = await getConfig();
+    const dir = opts.dir ?? config.rerouteDir;
+
+    // verify that we have a path
+    if (!dir) {
+      assert.fail(`Source directory not specified`)
+    }
+
+    const outputDir = join(process.cwd(), dir);
+    const url = new URL(_url);
+    const isDocument = !extname(url.pathname);
+
+    // get content
+    const res = await fetch(url);
+    if (!res.ok) {
+      assert.fail(`HTTP error! Status: ${res.status} - ${res.statusText} for URL: ${url}`)
+    }
+    const html = await res.text();
+
+    if (isDocument) {
+      const fileDir = join(outputDir, url.pathname.replace(/^\//, ''));
+      const filePath = join(fileDir, 'index.html');
+      await mkdir(fileDir, { recursive: true });
+      await writeFile(filePath, html, 'utf8');
+    }
+    else {
+      const fileDir = join(outputDir, dirname(url.pathname));
+      const filePath = join(fileDir, basename(url.pathname));
+      await mkdir(fileDir, { recursive: true });
+      await writeFile(filePath, html, 'utf8');
+    }
+  });

src/config.ts

@@ -15,6 +15,7 @@ export type Config = {
   tsWrite?: boolean;
   url?: string;
   login?: boolean;
+  rerouteDir?: string;
   playwright?: (page: PlaywrightConfig) => Promise<void>
 }
 

src/lib/index.ts

@@ -1,6 +1,6 @@
 import { readFile } from 'node:fs/promises';
 import fs from 'node:fs';
-import { join } from 'node:path';
+import { extname, join, basename } from 'node:path';
 import type { Page } from 'playwright';
 import type { RerouteUrlParams } from './types.js';
 import { Config } from '../config.js';
@@ -18,6 +18,51 @@ export function defineConfig(config: Config) {
 }
 
 /**
+ * Reroutes requests to local files within a specified directory. If the file exists locally,
+ * the contents of the local file with be used.
+ * @async
+ * @param {Page} context - The Playwright Page object to apply the rerouting to.
+ * @param {string} dir - The root directory where the local page or file (e.g., 'about/index.html', 'assets/script.js') are stored.
+ * @returns {Promise<void>} A promise that resolves when the routing has been set up.
+ */
+export async function rerouteLocal(context: Page, dir: string) {
+  // reload the page if the document has been changed
+  const watchCallback = () => {
+    context.reload();
+  }
+
+  await context.route('**', async route => {
+    const resourceType = route.request().resourceType();
+    const url = new URL(route.request().url())
+    const basepath = resourceType === 'document'
+      ? !extname(url.pathname)
+        ? join(url.pathname, 'index.html')
+        : url.pathname
+      : basename(url.pathname);
+    const pagePath = join(dir, basepath);
+
+    const content = await readFile(join(process.cwd(), pagePath), 'utf8')
+      .catch(() => null);
+
+    // watch the file for changes
+    watchFile(pagePath, watchCallback);
+
+    if (content) {
+      const response = await route.fetch();
+      await route.fulfill({
+        response,
+        body: content
+      });
+      console.log(`\x1b[33m 🚸 Page override:\x1b[0m ${pagePath}`);
+    } else {
+      await route.fallback();
+    }
+  });
+}
+
+/**
+ * @deprecated Use rerouteLocal
+ *
  * Reroutes document requests to local files within a specified directory.
  * It intercepts 'document' type requests and attempts to serve a corresponding
  * 'index.html' file from the local file system. It also watches the local file
@@ -39,22 +84,23 @@ export async function rerouteDocument(context: Page, dir: string) {
       await route.fallback();
       return;
     }
-    const pageName = new URL(route.request().url())
-      .pathname
-      .replace(/^\//, '');
-    const pagePath = join(dir, `${pageName}/index.html`);
+    const url = new URL(route.request().url())
+    const basepath = !extname(url.pathname)
+      ? join(url.pathname, 'index.html')
+      : url.pathname;
+    const pagePath = join(dir, basepath);
 
     // watch file for changes
     watchFile(pagePath, watchCallback);
 
-    const dashboardHtml = await readFile(join(process.cwd(), pagePath), 'utf8')
+    const content = await readFile(join(process.cwd(), pagePath), 'utf8')
       .catch(() => null);
 
-    if (dashboardHtml) {
+    if (content) {
       const response = await route.fetch();
       await route.fulfill({
         response,
-        body: dashboardHtml
+        body: content
       });
       console.log(`\x1b[33m 🚸 Page override:\x1b[0m ${pagePath}`);
     } else {

src/runCommand.ts

@@ -22,6 +22,7 @@ export const runCommand = new Command('run')
       url: typeof opts['url'] === 'string' ? opts['url'] : undefined,
       playwright: opts['playwright'],
       login: !!opts['login'],
+      rerouteDir: opts['rerouteDir'],
       file,
     });
     browser(processor);