push commit

Author: Aure7138

config.ts

@@ -6,4 +6,5 @@ export const defaultConfig: Config = {
   maxPagesToCrawl: 50,
   outputFileName: "output.json",
   maxTokens: 2000000,
+  // proxyUrls: ["http://username:password@proxyserver:port"], // socks5://username:password@proxyserver:port
 };

src/config.ts

@@ -85,6 +85,10 @@ export const configSchema = z.object({
    * @example 5000
    */
   maxTokens: z.number().int().positive().optional(),
+  /** Optional proxy server
+   * @example ['http://username:password@proxyserver:port', 'socks5://username:password@proxyserver:port']
+   */
+  proxyUrls: z.array(z.string()).optional(),
 });
 
 export type Config = z.infer<typeof configSchema>;

src/core.ts

@@ -1,5 +1,5 @@
 // For more information, see https://crawlee.dev/
-import { Configuration, PlaywrightCrawler, downloadListOfUrls } from "crawlee";
+import { Configuration, PlaywrightCrawler, ProxyConfiguration, downloadListOfUrls } from "crawlee";
 import { readFile, writeFile } from "fs/promises";
 import { glob } from "glob";
 import { Config, configSchema } from "./config.js";
@@ -54,8 +54,13 @@ export async function crawl(config: Config) {
   if (process.env.NO_CRAWL !== "true") {
     // PlaywrightCrawler crawls the web using a headless
     // browser controlled by the Playwright library.
+    const proxyConfiguration = new ProxyConfiguration({
+      proxyUrls: config.proxyUrls,
+    });
+
     crawler = new PlaywrightCrawler(
       {
+        proxyConfiguration,
         // Use the requestHandler to process each of the crawled pages.
         async requestHandler({ request, page, enqueueLinks, log, pushData }) {
           const title = await page.title();