Merge branch 'main' into multiple-files

Author: guillermoscript

.github/workflows/build.yml

@@ -20,4 +20,4 @@ jobs:
       - run: npm run build
       - uses: preactjs/compressed-size-action@v2
         with:
-          pattern: ".dist/**/*.{js,ts,json}"
+          pattern: ".dist/**/*.{js,ts,json}"

.github/workflows/release.yml

@@ -13,11 +13,11 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-            cache: npm
-            node-version: 18
+          cache: npm
+          node-version: 18
       - run: npm i
       - run: npm run build
       - run: npm run semantic-release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,18 @@
+name: Test workflow
+
+on: [push, pull_request]
+
+jobs:
+  prettier_check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: "20"
+      - name: Install Dependencies
+        run: npm ci
+      - name: Run prettier
+        run: npm run prettier:check

README.md

@@ -78,16 +78,16 @@ type Config = {
   maxPagesToCrawl: number;
   /** File name for the finished data */
   outputFileName: string;
-  /** Optional resources to exclude 
-   * 
+  /** Optional resources to exclude
+   *
    * @example
    * ['png','jpg','jpeg','gif','svg','css','js','ico','woff','woff2','ttf','eot','otf','mp4','mp3','webm','ogg','wav','flac','aac','zip','tar','gz','rar','7z','exe','dmg','apk','csv','xls','xlsx','doc','docx','pdf','epub','iso','dmg','bin','ppt','pptx','odt','avi','mkv','xml','json','yml','yaml','rss','atom','swf','txt','dart','webp','bmp','tif','psd','ai','indd','eps','ps','zipx','srt','wasm','m4v','m4a','webp','weba','m4b','opus','ogv','ogm','oga','spx','ogx','flv','3gp','3g2','jxr','wdp','jng','hief','avif','apng','avifs','heif','heic','cur','ico','ani','jp2','jpm','jpx','mj2','wmv','wma','aac','tif','tiff','mpg','mpeg','mov','avi','wmv','flv','swf','mkv','m4v','m4p','m4b','m4r','m4a','mp3','wav','wma','ogg','oga','webm','3gp','3g2','flac','spx','amr','mid','midi','mka','dts','ac3','eac3','weba','m3u','m3u8','ts','wpl','pls','vob','ifo','bup','svcd','drc','dsm','dsv','dsa','dss','vivo','ivf','dvd','fli','flc','flic','flic','mng','asf','m2v','asx','ram','ra','rm','rpm','roq','smi','smil','wmf','wmz','wmd','wvx','wmx','movie','wri','ins','isp','acsm','djvu','fb2','xps','oxps','ps','eps','ai','prn','svg','dwg','dxf','ttf','fnt','fon','otf','cab']
-  */
-  resourceExclusions?: string[];  
+   */
+  resourceExclusions?: string[];
   /** Optional maximum file size in megabytes to include in the output file */
   maxFileSize?: number,
   /** Optional maximum number tokens to include in the output file */
-  maxTokens?: number().,
+  maxTokens?: number,
 };
 ```
 

containerapp/data/config.ts

@@ -5,4 +5,4 @@ export const defaultConfig: Config = {
   match: "https://www.builder.io/c/docs/**",
   maxPagesToCrawl: 50,
   outputFileName: "../data/output.json",
-};
+};

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@builder.io/gpt-crawler",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "type": "module",
   "bin": {
     "gpt-crawler": "./dist/src/cli.js"
@@ -14,7 +14,6 @@
     "gpt-tokenizer": "^2.1.2",
     "inquirer": "^9.2.12",
     "playwright": "*",
-    "prettier": "^3.1.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {
@@ -23,6 +22,7 @@
     "@semantic-release/git": "^10.0.1",
     "@types/inquirer": "^9.0.7",
     "@types/node": "^20.0.0",
+    "prettier": "^3.1.0",
     "semantic-release": "^22.0.8",
     "ts-node": "^10.8.0",
     "typescript": "^5.0.0"
@@ -35,7 +35,8 @@
     "start:dev": "cross-env NODE_ENV=development npm run build && node dist/src/main.js",
     "start:prod": "node dist/src/main.js",
     "build": "tsc",
-    "fmt": "prettier --write ."
+    "fmt": "prettier --write .",
+    "prettier:check": "prettier --check ."
   },
   "author": "It's not you it's me",
   "license": "ISC"

package.json

@@ -1,4 +1,4 @@
-import { z } from 'zod';
+import { z } from "zod";
 
 import type { Page } from "playwright";
 
@@ -36,27 +36,30 @@ export const configSchema = z.object({
    */
   outputFileName: z.string(),
   /** Optional cookie to be set. E.g. for Cookie Consent */
-  cookie: z.object({
-    name: z.string(),
-    value: z.string(),
-  }).optional(),
+  cookie: z
+    .object({
+      name: z.string(),
+      value: z.string(),
+    })
+    .optional(),
   /** Optional function to run for each page found */
-  onVisitPage: z.function()
-      .args(z.object({
+  onVisitPage: z
+    .function()
+    .args(
+      z.object({
         page: Page,
-        pushData: z.function()
-            .args(z.any())
-            .returns(z.promise(z.void()))
-      }))
-      .returns(z.promise(z.void()))
-      .optional(),
+        pushData: z.function().args(z.any()).returns(z.promise(z.void())),
+      }),
+    )
+    .returns(z.promise(z.void()))
+    .optional(),
   /** Optional timeout for waiting for a selector to appear */
   waitForSelectorTimeout: z.number().int().nonnegative().optional(),
-  /** Optional resources to exclude 
-     * 
-     * @example
-     * ['png','jpg','jpeg','gif','svg','css','js','ico','woff','woff2','ttf','eot','otf','mp4','mp3','webm','ogg','wav','flac','aac','zip','tar','gz','rar','7z','exe','dmg','apk','csv','xls','xlsx','doc','docx','pdf','epub','iso','dmg','bin','ppt','pptx','odt','avi','mkv','xml','json','yml','yaml','rss','atom','swf','txt','dart','webp','bmp','tif','psd','ai','indd','eps','ps','zipx','srt','wasm','m4v','m4a','webp','weba','m4b','opus','ogv','ogm','oga','spx','ogx','flv','3gp','3g2','jxr','wdp','jng','hief','avif','apng','avifs','heif','heic','cur','ico','ani','jp2','jpm','jpx','mj2','wmv','wma','aac','tif','tiff','mpg','mpeg','mov','avi','wmv','flv','swf','mkv','m4v','m4p','m4b','m4r','m4a','mp3','wav','wma','ogg','oga','webm','3gp','3g2','flac','spx','amr','mid','midi','mka','dts','ac3','eac3','weba','m3u','m3u8','ts','wpl','pls','vob','ifo','bup','svcd','drc','dsm','dsv','dsa','dss','vivo','ivf','dvd','fli','flc','flic','flic','mng','asf','m2v','asx','ram','ra','rm','rpm','roq','smi','smil','wmf','wmz','wmd','wvx','wmx','movie','wri','ins','isp','acsm','djvu','fb2','xps','oxps','ps','eps','ai','prn','svg','dwg','dxf','ttf','fnt','fon','otf','cab']
-    */
+  /** Optional resources to exclude
+   *
+   * @example
+   * ['png','jpg','jpeg','gif','svg','css','js','ico','woff','woff2','ttf','eot','otf','mp4','mp3','webm','ogg','wav','flac','aac','zip','tar','gz','rar','7z','exe','dmg','apk','csv','xls','xlsx','doc','docx','pdf','epub','iso','dmg','bin','ppt','pptx','odt','avi','mkv','xml','json','yml','yaml','rss','atom','swf','txt','dart','webp','bmp','tif','psd','ai','indd','eps','ps','zipx','srt','wasm','m4v','m4a','webp','weba','m4b','opus','ogv','ogm','oga','spx','ogx','flv','3gp','3g2','jxr','wdp','jng','hief','avif','apng','avifs','heif','heic','cur','ico','ani','jp2','jpm','jpx','mj2','wmv','wma','aac','tif','tiff','mpg','mpeg','mov','avi','wmv','flv','swf','mkv','m4v','m4p','m4b','m4r','m4a','mp3','wav','wma','ogg','oga','webm','3gp','3g2','flac','spx','amr','mid','midi','mka','dts','ac3','eac3','weba','m3u','m3u8','ts','wpl','pls','vob','ifo','bup','svcd','drc','dsm','dsv','dsa','dss','vivo','ivf','dvd','fli','flc','flic','flic','mng','asf','m2v','asx','ram','ra','rm','rpm','roq','smi','smil','wmf','wmz','wmd','wvx','wmx','movie','wri','ins','isp','acsm','djvu','fb2','xps','oxps','ps','eps','ai','prn','svg','dwg','dxf','ttf','fnt','fon','otf','cab']
+   */
   resourceExclusions: z.array(z.string()).optional(),
 
   /** Optional maximum file size in megabytes to include in the output file
@@ -70,4 +73,3 @@ export const configSchema = z.object({
 });
 
 export type Config = z.infer<typeof configSchema>;
-

src/config.ts

@@ -2,7 +2,7 @@
 import { PlaywrightCrawler, downloadListOfUrls } from "crawlee";
 import { readFile, writeFile } from "fs/promises";
 import { glob } from "glob";
-import {Config, configSchema} from "./config.js";
+import { Config, configSchema } from "./config.js";
 import { Page } from "playwright";
 import {
   isWithinTokenLimit,
@@ -19,7 +19,7 @@ export function getPageHtml(page: Page, selector = "body") {
         document,
         null,
         XPathResult.ANY_TYPE,
-        null
+        null,
       );
       let result = elements.iterateNext();
       return result ? result.textContent || "" : "";
@@ -39,16 +39,16 @@ export async function waitForXPath(page: Page, xpath: string, timeout: number) {
         document,
         null,
         XPathResult.ANY_TYPE,
-        null
+        null,
       );
       return elements.iterateNext() !== null;
     },
     xpath,
-    { timeout }
+    { timeout },
   );
 }
 
-export async function crawl(config: Config) { 
+export async function crawl(config: Config) {
   configSchema.parse(config);
 
   if (process.env.NO_CRAWL !== "true") {
@@ -70,7 +70,7 @@ export async function crawl(config: Config) {
         const title = await page.title();
         pageCounter++;
         log.info(
-          `Crawling: Page ${pageCounter} / ${config.maxPagesToCrawl} - URL: ${request.loadedUrl}...`
+          `Crawling: Page ${pageCounter} / ${config.maxPagesToCrawl} - URL: ${request.loadedUrl}...`,
         );
 
         // Use custom handling for XPath selector
@@ -79,7 +79,7 @@ export async function crawl(config: Config) {
             await waitForXPath(
               page,
               config.selector,
-              config.waitForSelectorTimeout ?? 1000
+              config.waitForSelectorTimeout ?? 1000,
             );
           } else {
             await page.waitForSelector(config.selector, {
@@ -116,21 +116,25 @@ export async function crawl(config: Config) {
           if (RESOURCE_EXCLUSTIONS.length === 0) {
             return;
           }
-          await page.route(`**\/*.{${RESOURCE_EXCLUSTIONS.join()}}`, route => route.abort('aborted'));
-          log.info(`Aborting requests for as this is a resource excluded route`);
-        }
+          await page.route(`**\/*.{${RESOURCE_EXCLUSTIONS.join()}}`, (route) =>
+            route.abort("aborted"),
+          );
+          log.info(
+            `Aborting requests for as this is a resource excluded route`,
+          );
+        },
       ],
     });
 
     const SITEMAP_SUFFIX = "sitemap.xml";
     const isUrlASitemap = config.url.endsWith(SITEMAP_SUFFIX);
-  
+
     if (isUrlASitemap) {
       const listOfUrls = await downloadListOfUrls({ url: config.url });
-  
+
       // Add the initial URL to the crawling queue.
       await crawler.addRequests(listOfUrls);
-  
+
       // Run the crawler
       await crawler.run();
     } else {