min_vehicles global

Author: quicklywilliam

scrapers/carmax/scrape.js

@@ -1,5 +1,6 @@
 import * as cheerio from 'cheerio';
 import { BaseScraper } from '../lib/base-scraper.js';
+import { MIN_VEHICLES } from '../lib/config.js';
 
 class CarMaxScraper extends BaseScraper {
   constructor() {
@@ -54,7 +55,6 @@ class CarMaxScraper extends BaseScraper {
     const allListings = [];
     const seenIds = new Set();
     const searchUrl = buildSearchUrl(query.make, query.model);
-    const MIN_VEHICLES = 250;
 
     await this.page.goto(searchUrl, {
       waitUntil: 'networkidle2',

scrapers/carvana/scrape.js

@@ -1,5 +1,6 @@
 import * as cheerio from 'cheerio';
 import { BaseScraper } from '../lib/base-scraper.js';
+import { MIN_VEHICLES } from '../lib/config.js';
 
 class CarvanaScraper extends BaseScraper {
   constructor() {
@@ -45,7 +46,6 @@ class CarvanaScraper extends BaseScraper {
 
   async scrapeModel(query) {
     const allListings = [];
-    const MIN_VEHICLES = 250;
 
     // Use the search box - let Carvana's autocomplete handle regularization
     // This is more robust than constructing filter URLs

scrapers/lib/config.js

@@ -0,0 +1,8 @@
+/**
+ * Scraper configuration
+ */
+
+export const MIN_VEHICLES = 50; // Minimum vehicles to scrape per model before stopping
+
+// TODO: Change back to 250 after testing
+// export const MIN_VEHICLES = 250;

scrapers/plattauto/scrape.js

@@ -1,5 +1,6 @@
 import * as cheerio from 'cheerio';
 import { BaseScraper } from '../lib/base-scraper.js';
+import { MIN_VEHICLES } from '../lib/config.js';
 
 class PlattAutoScraper extends BaseScraper {
   constructor() {
@@ -45,8 +46,7 @@ class PlattAutoScraper extends BaseScraper {
 
   async scrapeModel(query) {
     const allListings = [];
-    const MIN_VEHICLES = 250;
-    const maxPages = 100; // High enough to get to 250 vehicles
+    const maxPages = 100; // High enough to get to MIN_VEHICLES
 
     for (let pageNum = 1; pageNum <= maxPages && allListings.length < MIN_VEHICLES; pageNum++) {
       const searchUrl = buildSearchUrl(query.make, query.model, pageNum);