fix: remove brand crawled at

Author: caiopizzol

initial.sql

@@ -3,8 +3,7 @@ CREATE TABLE IF NOT EXISTS reference_tables (
   code INTEGER UNIQUE NOT NULL,
   month INTEGER NOT NULL,
   year INTEGER NOT NULL,
-  crawled_at TIMESTAMP,
-  brands_crawled_at TIMESTAMP
+  crawled_at TIMESTAMP
 );
 
 CREATE TABLE IF NOT EXISTS brands (

src/crawler/processor.ts

@@ -94,24 +94,18 @@ export async function crawl(options: CrawlOptions = {}): Promise<void> {
       await repo.clearCrawlStatus(refRecord.id);
     }
 
-    // Phase 1: Crawl brands (if not already crawled)
-    const brandsCrawled = await repo.isBrandsCrawled(refRecord.id);
-    if (!brandsCrawled) {
-      log('  Phase 1: Crawling brands...');
-      const apiBrands = await fipeClient.getBrands(ref.Codigo);
-      const filteredBrands = options.brandCode
-        ? apiBrands.filter((b) => b.Value === options.brandCode)
-        : apiBrands;
-
-      for (const b of filteredBrands) {
-        const brand = await repo.upsertBrand(b.Value, b.Label);
-        await repo.upsertReferenceBrand(refRecord.id, brand.id);
-      }
-      await repo.markBrandsCrawled(refRecord.id);
-      log(`    Crawled ${filteredBrands.length} brands`);
-    } else {
-      log('  Phase 1: Brands already crawled');
+    // Phase 1: Crawl brands (always fetch - cheap API call)
+    log('  Phase 1: Crawling brands...');
+    const apiBrands = await fipeClient.getBrands(ref.Codigo);
+    const filteredBrands = options.brandCode
+      ? apiBrands.filter((b) => b.Value === options.brandCode)
+      : apiBrands;
+
+    for (const b of filteredBrands) {
+      const brand = await repo.upsertBrand(b.Value, b.Label);
+      await repo.upsertReferenceBrand(refRecord.id, brand.id);
     }
+    log(`    Crawled ${filteredBrands.length} brands`);
 
     // Phase 2: Crawl models for each uncrawled brand
     const uncrawledBrands = await repo.getUncrawledReferenceBrands(refRecord.id);
@@ -143,7 +137,10 @@ export async function crawl(options: CrawlOptions = {}): Promise<void> {
             }
           }
 
-          await repo.markReferenceBrandModelsCrawled(brand.id);
+          // Only mark as crawled if we fetched ALL models (no filter)
+          if (!options.modelCodes) {
+            await repo.markReferenceBrandModelsCrawled(brand.id);
+          }
         } catch {
           // Models fetch failed - leave uncrawled for retry
           log(`    Error crawling models for ${brand.name}`);

src/db/repository.ts

@@ -39,27 +39,7 @@ export async function getCrawledReferences(): Promise<number[]> {
   return rows.map((r) => r.code);
 }
 
-export async function isBrandsCrawled(referenceId: number): Promise<boolean> {
-  const [ref] = await db
-    .select({ brandsCrawledAt: referenceTables.brandsCrawledAt })
-    .from(referenceTables)
-    .where(eq(referenceTables.id, referenceId));
-  return !!ref?.brandsCrawledAt;
-}
-
-export async function markBrandsCrawled(referenceId: number) {
-  await db
-    .update(referenceTables)
-    .set({ brandsCrawledAt: new Date() })
-    .where(eq(referenceTables.id, referenceId));
-}
-
 export async function clearCrawlStatus(referenceId: number) {
-  await db
-    .update(referenceTables)
-    .set({ brandsCrawledAt: null })
-    .where(eq(referenceTables.id, referenceId));
-
   await db.delete(referenceModelYears).where(eq(referenceModelYears.referenceTableId, referenceId));
   await db.delete(referenceModels).where(eq(referenceModels.referenceTableId, referenceId));
   await db.delete(referenceBrands).where(eq(referenceBrands.referenceTableId, referenceId));

src/db/schema.ts

@@ -17,7 +17,6 @@ export const referenceTables = pgTable(
     month: integer('month').notNull(),
     year: integer('year').notNull(),
     crawledAt: timestamp('crawled_at'),
-    brandsCrawledAt: timestamp('brands_crawled_at'),
   },
   (table) => [index('idx_reference_year_month').on(table.year, table.month)],
 );