Feat/generate document (#69)

* feat: add generate document command to fetch content from remote url

* feat: adding a login command to save browser session

* chore: add changeset

* chore: update readme

Author: heyMP

.changeset/blue-bobcats-push.md

@@ -0,0 +1,27 @@
+---
+"@heymp/scratchpad": minor
+---
+
+Add new generate commands
+
+**Generate document**
+
+Fetch HTML source of url and save it to a local file. This is helpful when using the `rerouteDocument` command.
+
+```bash
+npx @heymp/scratchpad@next generate document https://www.example.com pages
+```
+
+**Generate login**
+
+Launch a new browser that saves your session so it can be reused.
+
+```bash
+npx @heymp/scratchpad@next generate login
+```
+
+You can then reuse the session by using the `login` option when using the `run` command.
+
+```bash
+npx @heymp/scratchpad@next run --login
+```

README.md

@@ -26,7 +26,7 @@ Options:
 
 Commands:
   run [options] <file>  Execute a file in a browser.
-  generate              Generate files from templates.
+  generate              Generate files from templates or remote data.
   help [command]        display help for command
 ```
 
@@ -75,6 +75,20 @@ export default defineConfig({
 });
 ```
 
+#### Save browser session
+
+Using Playwright, you can launch a new session with the `generate login` command. This will launch a new browser session where you can authenticate to a website. After you have authenticated you can close the browser session. You session will be saved to a local file `.scratchpad/login.json`.
+
+```bash
+npx @heymp/scratchpad@next login
+```
+
+You can then reuse the session by using the `login` option when using the `run` command.
+
+```bash
+npx @heymp/scratchpad@next run --login
+```
+
 #### Reroute Documents
 
 The `rerouteDocument` function allows you to replace the HTML content of any webpage with a local HTML file from your system. This is incredibly useful for testing changes or developing components in the context of a live site without deploying your code.
@@ -163,9 +177,16 @@ export default Scratchpad.defineConfig({
 });
 ```
 
-## Logging
+## Generate files
+
+| Method | Description                                | Example   |
+|--------|--------------------------------------------|-----------|
+| document | Fetch HTML source of url and save it to a local file. This is helpful when using the `rerouteDocument` command. | `npx @heymp/scratchpad@next generate document https://www.example.com pages ` |
+| login  | Launch a new browser that saves your session so it can be reused. | `npx @heymp/scratchpad@next generate login` |
+
+## Default exposed functions 
 
-To send log information from the Chromium browser to node.js we expose the following functions.
+The following functions are exposed by default that can be used in the `run` commands target file.
 
 | Method | Description                                | Example   |
 |--------|--------------------------------------------|-----------|
@@ -176,7 +197,7 @@ To send log information from the Chromium browser to node.js we expose the follo
 | readFile  | read data from a local file. | `readFile('./log.txt', 'utf8')` |
 
 
-Example
+**Log example**
 
 ```js
 clear();

src/Processor.ts

@@ -12,6 +12,7 @@ export type ProcessorOpts = {
   headless?: boolean;
   devtools?: boolean;
   tsWrite?: boolean;
+  login?: boolean;
   url?: string;
   playwright?: any;
   file: string;
@@ -41,6 +42,13 @@ export class Processor extends EventTarget {
 
   watcher() {
     const file = this.opts.file;
+
+    if (this.opts.login) {
+      if (!fs.existsSync(join(process.cwd(), '.scratchpad', 'login.json'))) {
+        throw new Error(`Session file not found.`);
+      }
+    }
+
     if (!fs.existsSync(file)) {
       throw new Error(`${file} file not found.`);
     }

src/browser.ts

@@ -3,6 +3,7 @@ import util from 'node:util';
 import { join } from 'node:path'
 import fs from 'node:fs/promises';
 import type { Processor } from './Processor.js';
+import { getSession } from './login.js';
 util.inspect.defaultOptions.maxArrayLength = null;
 util.inspect.defaultOptions.depth = null;
 
@@ -23,12 +24,15 @@ function readFile(...args: Parameters<typeof fs.readFile>) {
 }
 
 export async function browser(processor: Processor) {
+  // Get session login session
   // Launch the browser
   const browser = await playwright['chromium'].launch({
     headless: !!processor.opts.headless,
     devtools: !!processor.opts.devtools
   });
-  const context = await browser.newContext();
+  const context = await browser.newContext({
+    storageState: processor.opts.login ? await getSession('.scratchpad/login.json') : undefined,
+  });
   const page = await context.newPage();
   const playwrightConfig = processor.opts.playwright;
 

src/config.ts

@@ -14,6 +14,7 @@ export type Config = {
   devtools?: boolean;
   tsWrite?: boolean;
   url?: string;
+  login?: boolean;
   playwright?: (page: PlaywrightConfig) => Promise<void>
 }
 

src/generateCommand.ts

@@ -1,9 +1,14 @@
 import { Command } from '@commander-js/extra-typings';
 import * as readline from 'node:readline/promises';
 import { stdin, stdout } from 'node:process';
-import { writeFile, readFile, stat } from 'node:fs/promises';
+import { writeFile, readFile, mkdir } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
+import { URL } from 'node:url';
 import { fileURLToPath } from 'node:url';
+import assert from 'node:assert';
+import { login } from './login.js';
+import { getConfig } from './config.js';
+import { Processor } from './Processor.js';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
@@ -27,6 +32,42 @@ ${outpath} [Y/n]: `);
     rl.close();
   });
 
+const documentCommand = new Command('document')
+  .description('Generates a local copy of a document from a url.')
+  .argument('<url>', 'url to copy.')
+  .argument('<dir>', 'source directory where you want to save the document.')
+  .action(async (_url, dir) => {
+    const outputDir = join(process.cwd(), dir);
+    const url = new URL(_url);
+
+    // get html
+    const res = await fetch(url);
+    if (!res.ok) {
+      assert.fail(`HTTP error! Status: ${res.status} - ${res.statusText} for URL: ${url}`)
+    }
+    const html = await res.text();
+
+    // save file
+    const fileDir = join(outputDir, url.pathname.replace(/^\//, ''));
+    const filePath = join(fileDir, 'index.html');
+    await mkdir(fileDir, { recursive: true });
+    await writeFile(filePath, html, 'utf8');
+  });
+
+const loginCommand = new Command('login')
+  .description('Launches a session and saves the browser session in a local file on termination.')
+  .action(async (...options) => {
+    const config = await getConfig();
+    const opts = { ...config, options };
+    login({
+      // type narrow the options
+      devtools: !!opts['devtools'],
+      url: typeof opts['url'] === 'string' ? opts['url'] : undefined,
+    });
+  });
+
 export const generateCommand = new Command('generate')
   .description('Generate files from templates.')
-  .addCommand(configCommand);
+  .addCommand(configCommand)
+  .addCommand(documentCommand)
+  .addCommand(loginCommand);

src/login.ts

@@ -0,0 +1,39 @@
+import playwright from 'playwright';
+import util from 'node:util';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { exists } from './utils.js';
+import type { Config } from './config.js';
+util.inspect.defaultOptions.maxArrayLength = null;
+util.inspect.defaultOptions.depth = null;
+
+export async function login(config: Config) {
+  const browser = await playwright['chromium'].launch({
+    headless: false,
+    devtools: !!config.devtools
+  });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  // Create a page
+  if (config.url) {
+    await page.goto(config.url);
+  }
+
+  page.on('close', async () => {
+    await page.context().storageState({ path: '.scratchpad/login.json' });
+    await browser.close();
+    console.log(`\x1b[33m 👻 Session saved\x1b[0m`);
+  });
+}
+
+export async function getSession(path: string): Promise<string | undefined> {
+  const filePath = join(process.cwd(), path);
+  const fileExists = await exists(filePath);
+  if (!fileExists) {
+    return undefined;
+  }
+  const sessionFile = await readFile(filePath, 'utf8');
+  const session = sessionFile ? JSON.parse(sessionFile) : undefined;
+  return session;
+}

src/runCommand.ts

@@ -10,6 +10,7 @@ export const runCommand = new Command('run')
   .option('--devtools [boolean]', 'open browser devtools automatically.')
   .option('--ts-write [boolean]', 'write the js output of the target ts file.')
   .option('--url [string]', 'specify a specific url to execute the code in.')
+  .option('--login [boolean]', `use previously saved session from 'generate login' command`)
   .action(async (file, options, command) => {
     const config = await getConfig();
     const opts = { ...config, ...options};
@@ -20,6 +21,7 @@ export const runCommand = new Command('run')
       tsWrite: !!opts['tsWrite'],
       url: typeof opts['url'] === 'string' ? opts['url'] : undefined,
       playwright: opts['playwright'],
+      login: !!opts['login'],
       file,
     });
     browser(processor);