csv-processor.ts

/**
 * CSV Data Processor - Production-Ready
 *
 * Processes ANP (Agência Nacional do Petróleo) CSV data with:
 * - Robust error handling and validation
 * - Type safety throughout the pipeline
 * - Performance optimization with caching
 * - Proper encoding handling (windows-1252 → UTF-8)
 * - Defensive programming against malformed data
 */

import fs from 'fs/promises';
import path from 'path';
import { parse } from 'csv-parse/sync';
import iconv from 'iconv-lite';
import type {
  GasPriceRaw,
  GasStation,
} from '@/types/gas';
import { isValidGasPriceRaw } from '@/types/gas';

// ============================================================================
// CONSTANTS
// ============================================================================

/**
 * ANP (Agência Nacional do Petróleo) official data source URL
 * Always fetch live data from government portal
 */
const ANP_DATA_URL = 'https://www.gov.br/anp/pt-br/centrais-de-conteudo/dados-abertos/arquivos/shpc/qus/ultimas-4-semanas-glp.csv';

/**
 * Fallback: Local CSV file path (for development/testing only)
 */
const FALLBACK_CSV_PATH = path.join(
  process.cwd(),
  'data',
  'ultimas-4-semanas-glp.csv'
);

/**
 * Cache duration in milliseconds (30 minutes)
 * Live data is cached to reduce load on government servers
 */
const CACHE_DURATION_MS = 30 * 60 * 1000;

// ============================================================================
// TYPES
// ============================================================================

interface ProcessorCache {
  data: GasStation[];
  timestamp: number;
}

interface ProcessingStats {
  totalRecords: number;
  validRecords: number;
  invalidRecords: number;
  duplicatesRemoved: number;
  processingTimeMs: number;
}

// ============================================================================
// CACHE
// ============================================================================

let cache: ProcessorCache | null = null;

/**
 * Checks if cached data is still valid
 */
function isCacheValid(): boolean {
  if (!cache) return false;
  const now = Date.now();
  return (now - cache.timestamp) < CACHE_DURATION_MS;
}

/**
 * Clears the cache (useful for testing or manual refresh)
 */
export function clearCache(): void {
  cache = null;
}

// ============================================================================
// VALIDATION & TRANSFORMATION
// ============================================================================

/**
 * Parses a Brazilian-formatted price string to number
 *
 * @param priceStr - Price string like "115,00" or "115.00"
 * @returns Parsed number or NaN if invalid
 *
 * @example
 * ```typescript
 * parsePrice("115,00") // 115.00
 * parsePrice("115.50") // 115.50
 * parsePrice("invalid") // NaN
 * ```
 */
function parsePrice(priceStr: string): number {
  if (!priceStr || typeof priceStr !== 'string') {
    return NaN;
  }

  // Remove whitespace and replace comma with dot
  const normalized = priceStr.trim().replace(',', '.');
  const parsed = parseFloat(normalized);

  // Validate: must be positive, finite number
  if (isNaN(parsed) || !isFinite(parsed) || parsed <= 0) {
    return NaN;
  }

  return parsed;
}

/**
 * Parses a Brazilian date string (DD/MM/YYYY) to Date object
 *
 * @param dateStr - Date string in DD/MM/YYYY format
 * @returns Date object or current date if invalid
 *
 * @example
 * ```typescript
 * parseDate("22/07/2025") // Date object for July 22, 2025
 * parseDate("invalid") // Current date
 * ```
 */
function parseDate(dateStr: string): Date {
  if (!dateStr || typeof dateStr !== 'string') {
    return new Date();
  }

  const parts = dateStr.trim().split('/');
  if (parts.length !== 3) {
    return new Date();
  }

  const day = parseInt(parts[0] ?? '1', 10);
  const month = parseInt(parts[1] ?? '1', 10) - 1; // JS months are 0-indexed
  const year = parseInt(parts[2] ?? '2024', 10);

  const date = new Date(year, month, day);

  // Validate date is valid and not in the future
  if (isNaN(date.getTime()) || date > new Date()) {
    return new Date();
  }

  return date;
}

/**
 * Builds a formatted full address from address components
 *
 * @param raw - Raw CSV record
 * @returns Formatted address string
 */
function buildFullAddress(raw: GasPriceRaw): string {
  const parts: string[] = [];

  // Street name and number
  if (raw['Nome da Rua']) {
    const street = raw['Nome da Rua'].trim();
    const number = raw['Numero Rua']?.trim();
    parts.push(number ? `${street}, ${number}` : street);
  }

  // Neighborhood
  if (raw['Bairro']) {
    parts.push(raw['Bairro'].trim());
  }

  // CEP
  if (raw['Cep']) {
    parts.push(`CEP ${raw['Cep'].trim()}`);
  }

  return parts.join(' - ') || 'Endereço não informado';
}

/**
 * Transforms a raw CSV record into a typed GasStation object
 *
 * @param raw - Raw CSV record
 * @returns GasStation object or null if transformation fails
 */
function transformRawToStation(raw: GasPriceRaw): GasStation | null {
  try {
    // Validate input
    if (!isValidGasPriceRaw(raw)) {
      return null;
    }

    // Filter: Only process GLP product records
    if (raw['Produto']?.trim().toUpperCase() !== 'GLP') {
      return null;
    }

    // Parse and validate price
    const price = parsePrice(raw['Valor de Venda']);
    if (isNaN(price)) {
      return null;
    }

    // Parse date
    const lastUpdate = parseDate(raw['Data da Coleta']);

    // Build full address
    const fullAddress = buildFullAddress(raw);

    // Normalize city name (remove extra spaces, convert to title case)
    const city = raw['Municipio']
      .trim()
      .split(' ')
      .map((word) => {
        // Keep acronyms uppercase (e.g., "SAO PAULO" stays "SAO PAULO")
        return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
      })
      .join(' ');

    return {
      id: raw['CNPJ da Revenda'].trim(),
      name: raw['Revenda'].trim(),
      price,
      city,
      state: raw['Estado - Sigla'].trim().toUpperCase(),
      neighborhood: raw['Bairro']?.trim() || 'Não informado',
      flag: raw['Bandeira'].trim(),
      lastUpdate,
      fullAddress,
    };
  } catch (error) {
    // Silently skip invalid records
    console.warn('Failed to transform record:', error);
    return null;
  }
}

// ============================================================================
// CSV PARSING
// ============================================================================

/**
 * Fetches CSV data from ANP government URL
 *
 * @param url - URL to fetch CSV data from
 * @returns CSV content as string (decoded to UTF-8)
 * @throws {Error} If fetch fails
 */
async function fetchCSVFromURL(url: string): Promise<string> {
  try {
    const response = await fetch(url, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (compatible; GasMaisBarato/1.0)',
      },
      // Revalidate every 30 minutes (1800 seconds)
      // Note: CSV file is ~2.5MB, exceeds Next.js 2MB cache limit
      // Using our own in-memory cache instead (CACHE_DURATION_MS)
      next: {
        revalidate: 1800,
      },
    });

    if (!response.ok) {
      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
    }

    // ANP CSV comes in UTF-8 with BOM encoding
    const buffer = await response.arrayBuffer();
    const content = iconv.decode(Buffer.from(buffer), 'utf-8');

    return content;
  } catch (error) {
    if (error instanceof Error) {
      throw new Error(`Failed to fetch CSV from URL: ${error.message}`);
    }
    throw new Error('Failed to fetch CSV from URL: Unknown error');
  }
}

/**
 * Loads and parses the CSV file from local disk (fallback only)
 *
 * @param filePath - Absolute path to CSV file
 * @returns Array of raw CSV records
 * @throws {Error} If file cannot be read or parsed
 */
async function loadCSVFromFile(filePath: string): Promise<GasPriceRaw[]> {
  try {
    // Read file as buffer and decode from UTF-8 (with BOM)
    const buffer = await fs.readFile(filePath);
    const fileContent = iconv.decode(buffer, 'utf-8');

    // Parse CSV with csv-parse library
    const records = parse(fileContent, {
      columns: true, // Use first row as headers
      skip_empty_lines: true,
      trim: true,
      bom: true, // Handle BOM (Byte Order Mark)
      relaxColumnCount: true, // Allow inconsistent column counts
      delimiter: ';', // Semicolon delimiter (ANP format)
      relax_quotes: true, // Allow quotes to appear in unquoted fields
      escape: '"', // Use double quote as escape character
      quote: '"', // Quote character
    }) as GasPriceRaw[];

    return records;
  } catch (error) {
    if (error instanceof Error) {
      throw new Error(`Failed to load CSV file: ${error.message}`);
    }
    throw new Error('Failed to load CSV file: Unknown error');
  }
}

/**
 * Loads and parses CSV data from live ANP URL or fallback file
 *
 * @param useLiveData - If true, fetch from ANP URL; if false, use local file
 * @returns Array of raw CSV records
 * @throws {Error} If both live fetch and fallback fail
 */
async function loadCSV(useLiveData: boolean = true): Promise<GasPriceRaw[]> {
  try {
    let fileContent: string;

    if (useLiveData) {
      // Fetch live data from ANP government portal
      console.log('Fetching live data from ANP...');
      fileContent = await fetchCSVFromURL(ANP_DATA_URL);
      console.log('Live data fetched successfully');
    } else {
      // Fallback to local file
      console.log('Loading data from local file...');
      fileContent = await fs.readFile(FALLBACK_CSV_PATH, { encoding: 'utf-8' });
    }

    // Parse CSV with csv-parse library
    const records = parse(fileContent, {
      columns: true, // Use first row as headers
      skip_empty_lines: true,
      trim: true,
      bom: true, // Handle BOM (Byte Order Mark)
      relaxColumnCount: true, // Allow inconsistent column counts
      delimiter: ';', // Semicolon delimiter (ANP format)
      relax_quotes: true, // Allow quotes to appear in unquoted fields
      escape: '"', // Use double quote as escape character
      quote: '"', // Quote character
    }) as GasPriceRaw[];

    return records;
  } catch (error) {
    // If live data fails, try fallback
    if (useLiveData) {
      console.warn('Failed to fetch live data, trying fallback file...', error);
      try {
        return await loadCSVFromFile(FALLBACK_CSV_PATH);
      } catch (fallbackError) {
        console.error('Fallback also failed:', fallbackError);
        throw new Error(
          `Failed to load data from both live source and fallback: ${
            error instanceof Error ? error.message : 'Unknown error'
          }`
        );
      }
    }

    if (error instanceof Error) {
      throw new Error(`Failed to load CSV: ${error.message}`);
    }
    throw new Error('Failed to load CSV: Unknown error');
  }
}

// ============================================================================
// MAIN PROCESSING FUNCTION
// ============================================================================

/**
 * Fetches, processes, and caches gas station data from ANP live source
 *
 * This is the main entry point for data retrieval. It:
 * 1. Checks cache validity
 * 2. Fetches live CSV data from ANP government portal
 * 3. Validates and transforms each record
 * 4. Removes duplicates based on CNPJ
 * 5. Caches the result
 *
 * @param useLiveData - If true (default), fetch from ANP URL; if false, use local fallback
 * @returns Array of processed gas stations
 * @throws {Error} If CSV cannot be loaded or processed
 *
 * @example
 * ```typescript
 * // Fetch live data from ANP (default)
 * const stations = await fetchAndProcessData();
 *
 * // Use local fallback for testing
 * const stations = await fetchAndProcessData(false);
 * ```
 */
export async function fetchAndProcessData(
  useLiveData: boolean = true
): Promise<GasStation[]> {
  const startTime = Date.now();

  // Return cached data if valid
  if (isCacheValid() && cache) {
    console.log('Returning cached data');
    return cache.data;
  }

  console.log('Loading and processing CSV data...');

  try {
    // Load raw CSV records from live ANP source
    const rawRecords = await loadCSV(useLiveData);

    // Transform records with validation
    const stations: GasStation[] = [];
    let invalidCount = 0;

    for (const raw of rawRecords) {
      const station = transformRawToStation(raw);
      if (station) {
        stations.push(station);
      } else {
        invalidCount++;
      }
    }

    // Remove duplicates based on CNPJ (id)
    const uniqueStations = Array.from(
      new Map(stations.map((s) => [s.id, s])).values()
    );

    const duplicatesRemoved = stations.length - uniqueStations.length;

    // Update cache
    cache = {
      data: uniqueStations,
      timestamp: Date.now(),
    };

    // Log processing stats
    const stats: ProcessingStats = {
      totalRecords: rawRecords.length,
      validRecords: uniqueStations.length,
      invalidRecords: invalidCount,
      duplicatesRemoved,
      processingTimeMs: Date.now() - startTime,
    };

    console.log('Processing complete:', stats);

    return uniqueStations;
  } catch (error) {
    // Clear cache on error
    cache = null;

    if (error instanceof Error) {
      throw new Error(`Data processing failed: ${error.message}`);
    }
    throw new Error('Data processing failed: Unknown error');
  }
}

// ============================================================================
// UTILITY FUNCTIONS
// ============================================================================

/**
 * Gets unique city names from all stations
 *
 * @param stations - Array of gas stations
 * @returns Sorted array of unique city names
 */
export function getUniqueCities(stations: GasStation[]): string[] {
  const cities = new Set(stations.map((s) => s.city));
  return Array.from(cities).sort();
}

/**
 * Gets unique brand/flag names from all stations
 *
 * @param stations - Array of gas stations
 * @returns Sorted array of unique brands
 */
export function getUniqueBrands(stations: GasStation[]): string[] {
  const brands = new Set(stations.map((s) => s.flag));
  return Array.from(brands).sort();
}

/**
 * Validates CSV file exists and is readable (for fallback file only)
 *
 * @param filePath - Path to CSV file (defaults to FALLBACK_CSV_PATH)
 * @returns True if file exists and is readable
 */
export async function validateCSVFile(filePath: string = FALLBACK_CSV_PATH): Promise<boolean> {
  try {
    await fs.access(filePath, fs.constants.R_OK);
    return true;
  } catch {
    return false;
  }
}

/**
 * Tests connectivity to ANP live data source
 *
 * @returns True if ANP URL is accessible
 */
export async function testANPConnectivity(): Promise<boolean> {
  try {
    const response = await fetch(ANP_DATA_URL, {
      method: 'HEAD',
      headers: {
        'User-Agent': 'Mozilla/5.0 (compatible; GasMaisBarato/1.0)',
      },
    });
    return response.ok;
  } catch {
    return false;
  }
}