Facing Issue "Too Many request"

[austin495]

Hello, hope your doing well. I'm facing issue while fetching "trackdrive" API please help me to fix this issue below is the code.

Thanks

page.tsx:

'use client';
import { useState, useEffect, useCallback } from 'react';
import { Button } from "@/components/ui/button";
import { Table, TableBody, TableCaption, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import * as XLSX from "xlsx";
import DateTimePicker from "react-datetime-picker";
import 'react-datetime-picker/dist/DateTimePicker.css';
import 'react-calendar/dist/Calendar.css';
import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover';
import { CalendarIcon, Loader2 } from 'lucide-react';
import { ScrollArea, ScrollBar } from '@/components/ui/scroll-area';

interface TrafficSourceData {
  trafficSourceId: number;
  trafficSourceName: string;
  totalCalls: number;
  answeredCalls: number;
  convertedCalls: number;
  revenue: number;
  cost: number;
  revenuePerCall: number;
  profit: number;
}

function formatDateToLocalISOString(date: Date): string {
  const pad = (num: number) => num.toString().padStart(2, "0");
  return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
}

export default function Buyers() {
  const [data, setData] = useState<TrafficSourceData[]>([]);
  const [loading, setLoading] = useState(false);
  const [exportLoading, setExportLoading] = useState(false);
  const [error, setError] = useState<string | null>(null);
  const [startDate, setStartDate] = useState<Date | null>(null);
  const [endDate, setEndDate] = useState<Date | null>(null);

  const fetchData = useCallback(async () => {
    setLoading(true);
    setError(null);

    const created_at_from = startDate ? formatDateToLocalISOString(startDate) : '';
    const created_at_to = endDate ? formatDateToLocalISOString(endDate) : '';
    try {
      const response = await fetch(`/api/buyers?created_at_from=${created_at_from}&created_at_to=${created_at_to}`);
      const result = await response.json();
      if (response.ok) {
        setData(result.data);
      } else {
        setError(result.error || "Unknown error occurred");
      }
    } catch (err) {
      setError("Failed to fetch data");
    } finally {
      setLoading(false);
    }
  }, [startDate, endDate]);

  useEffect(() => {
    fetchData();
  }, [fetchData]);

  const handleFetchWithDate = () => {
    if (startDate && endDate) {
      fetchData(); // Fetch with date filter when both dates are selected
    } else {
      setError("Please select both start and end dates.");
    }
  };

  const exportToExcel = () => {
    setExportLoading(true);
    // Filter and format the logs data
    const simplifiedData = data.map(items => ({
      trafficSourceId: items.trafficSourceId,
      trafficSourceName: items.trafficSourceName,
      totalCalls: items.totalCalls,
      answeredCalls: items.answeredCalls,
      convertedCalls: items.convertedCalls,
      revenue: items.revenue,
      cost: items.cost,
      revenuePerCall: items.revenuePerCall,
      profit: items.profit,
    }));
  
    // Create worksheet and workbook
    const worksheet = XLSX.utils.json_to_sheet(simplifiedData);
    const workbook = XLSX.utils.book_new();
    XLSX.utils.book_append_sheet(workbook, worksheet, "Logs");
  
    // Write the Excel file
    XLSX.writeFile(workbook, "buyers_data.xlsx");
    setExportLoading(false);
  };

  return (
    <main className="flex flex-col gap-6 max-w-5xl m-auto py-10 max-w-[1360px] m-auto py-[50px]">
      <div className="flex gap-[10px]">
        <div className="w-[50%] flex gap-[10px]">
          <div>
            <Popover>
              <PopoverTrigger asChild>
                <Button variant="outline">
                  <CalendarIcon />
                  <span>Start Date</span>
                </Button>
              </PopoverTrigger>
              <PopoverContent className="w-auto p-0">
                <div style={{ width: "250px" }}>
                  <DateTimePicker
                    onChange={setStartDate}
                    value={startDate}
                    format="y-MM-dd h:mm a"
                  />
                </div>
              </PopoverContent>
            </Popover>
          </div>

          <div>
            <Popover>
              <PopoverTrigger asChild>
                <Button variant="outline">
                  <CalendarIcon />
                  <span>End Date</span>
                </Button>
              </PopoverTrigger>
              <PopoverContent className="w-auto p-0">
                <DateTimePicker
                  onChange={setEndDate}
                  value={endDate}
                  format="y-MM-dd h:mm a"
                />
              </PopoverContent>
            </Popover>
          </div>

          <div>
            <Button onClick={handleFetchWithDate}>Fetch Data</Button>
          </div>
        </div>

        <div className="w-[50%] flex gap-[10px] justify-end">
          <div>
            <Button onClick={exportToExcel}>
              {exportLoading ? (
                <>
                  <Loader2 className="mr-2 h-4 w-4 animate-spin" />
                  Please wait
                </>
              ) : (
                "Export as Excel"
              )}
            </Button>
          </div>
        </div>
      </div>

      <ScrollArea className="whitespace-nowrap rounded-md border">
          <Table>
            <TableCaption>A list of your recent Buyers Data.</TableCaption>
            <TableHeader>
              <TableRow>
                <TableHead>Buyers</TableHead>
                <TableHead>Total Calls</TableHead>
                <TableHead>Answered Calls</TableHead>
                <TableHead>Converted Calls</TableHead>
                <TableHead>Revenue</TableHead>
                <TableHead>Revenue Per Call</TableHead>
                <TableHead>Cost</TableHead>
                <TableHead>Profit</TableHead>
              </TableRow>
            </TableHeader>
            <TableBody>
              {loading ? (
                  <TableRow>
                    <TableCell colSpan={20}>Loading...</TableCell>
                  </TableRow>
                ) : error ? (
                  <TableRow>
                    <TableCell colSpan={20}>Error: {error}</TableCell>
                  </TableRow>
                ) : data.length === 0 ? (
                  <TableRow>
                    <TableCell colSpan={20}>No logs found.</TableCell>
                  </TableRow>
                ) : (
                  data.map((item) => (
                    <TableRow key={item.trafficSourceId}>
                      <TableCell>{item.trafficSourceName}</TableCell>
                      <TableCell>{item.totalCalls}</TableCell>
                      <TableCell>{item.answeredCalls}</TableCell>
                      <TableCell>{item.convertedCalls}</TableCell>
                      <TableCell>${item.revenue.toFixed(2)}</TableCell>
                      <TableCell>${item.revenuePerCall.toFixed(2)}</TableCell>
                      <TableCell>${item.cost.toFixed(2)}</TableCell>
                      <TableCell>${item.profit.toFixed(2)}</TableCell>
                    </TableRow>
                  ))
                )}
            </TableBody>
          </Table>
          <ScrollBar orientation="horizontal" />
        </ScrollArea>
    </main>
  );
}

route.ts:

export const dynamic = 'force-dynamic';
import { NextResponse } from 'next/server';

interface Call {
  status: string; // To check for 'finished' status
  disposition_key: string;
  buyer_converted: boolean | string | null; // To check for conversion
  buyer_revenue?: number | null;
  trackdrive_cost?: number | null;
}

interface BuyersData {
  id: number;
  name: string;
}

function getDateNDaysAgo(days: number): string {
  const date = new Date();
  date.setDate(date.getDate() - days);
  return date.toISOString().split('T')[0]; // Returns the date part in YYYY-MM-DD format
}

function delay(ms: number) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

export async function GET(req: Request) {
  const {
    TRACKDRIVE_PUBLIC_KEY,
    TRACKDRIVE_PRIVATE_KEY,
  } = process.env;

  const urlParams = new URL(req.url).searchParams;
  const defaultCreatedAtFrom = getDateNDaysAgo(1);
  const created_at_from = urlParams.get('created_at_from') || defaultCreatedAtFrom;
  const created_at_to = urlParams.get('created_at_to') || new Date().toISOString().split('T')[0];

  const authString = `${TRACKDRIVE_PUBLIC_KEY}:${TRACKDRIVE_PRIVATE_KEY}`;
  const encodedAuthString = Buffer.from(authString).toString("base64");

  try {
    // Step 1: Fetch Buyers
    const buyersResponse = await fetch(`https://conversion-io.trackdrive.com/api/v1/buyers?per_page=100`, {
      method: 'GET',
      headers: {
        'Authorization': `Basic ${encodedAuthString}`,
        'Content-Type': 'application/json',
      },
    });

    if (!buyersResponse.ok) {
      return NextResponse.json(
        { error: `Failed to fetch buyers: ${buyersResponse.statusText}` },
        { status: buyersResponse.status }
      );
    }

    const buyersData = await buyersResponse.json();
    const buyersSources: BuyersData[] = buyersData.buyers;

    // Step 2: Aggregate call data across all pages for each Buyers
    const aggregatedData = await Promise.all(buyersSources.map(async (source) => {
      let page = 1;
      const per_page = 500; // Fetch maximum items per page allowed by the API
      let totalCalls = 0;
      let answeredCalls = 0;
      let convertedCalls = 0;
      let revenue = 0;
      let cost = 0;

      // Loop through all pages to aggregate call data
      while (true) {
        await delay(500);
        const callsResponse = await fetch(`https://conversion-io.trackdrive.com/api/v1/calls?buyer_id=${source.id}&created_at_from=${created_at_from}&created_at_to=${created_at_to}&page=${page}&per_page=${per_page}`, {
          method: 'GET',
          headers: {
            'Authorization': `Basic ${encodedAuthString}`,
            'Content-Type': 'application/json',
          },
        });

        if (!callsResponse.ok) {
          throw new Error(`Failed to fetch calls for Buyers ${source.id}: ${callsResponse.statusText}`);
        }

        const callsData = await callsResponse.json();
        const calls: Call[] = callsData.calls || [];

        // If there are no calls in this page, we have reached the end
        if (calls.length === 0) break;

        // Aggregate data from the current page of calls
        totalCalls += calls.length;
        answeredCalls += calls.filter((call) => call.status === 'finished').length;
        convertedCalls += calls.filter((call) => call.buyer_converted === true || call.buyer_converted === 'Converted').length;
        revenue += calls.reduce((acc, call) => acc + (call.buyer_revenue ?? 0), 0);
        cost += calls.reduce((acc, call) => acc + (call.trackdrive_cost ?? 0), 0);

        // Move to the next page
        page += 1;
      }

      const revenuePerCall = totalCalls > 0 ? revenue / totalCalls : 0;
      const profit = revenue - cost;

      return {
        trafficSourceId: source.id,
        trafficSourceName: source.name,
        totalCalls,         // Total calls for the traffic source
        answeredCalls,      // Calls with 'finished' status
        convertedCalls,     // Calls marked as buyer_converted
        revenue,            // Total revenue from all calls
        cost,               // Total cost from all calls
        revenuePerCall,     // Average revenue per call
        profit,             // Total profit (revenue - cost)
      };
    }));

    return NextResponse.json({ data: aggregatedData });
  } catch (error) {
    console.error("Error fetching TrackDrive data:", error);
    return NextResponse.json(
      { error: (error as Error).message },
      { status: 500 }
    );
  }
}

[austin495]

@icyJoseph @leerob Bro can you please help me with that?

[austin495]

Yeah it is working better

[austin495]

I edited that answer, to also reduce the number of pages you, while true, through

I read it but I need to show all the data you can see here "https://nextjs-conversion-io.vercel.app/cpm-for-traffic-source"

[austin495]

So that's why I can not add pagination on front-end that's the issue

[icyJoseph]

Right! I just wanted to show that the problem is the number of calls...

They surely have a rate-limit.. better contact their support, to figure out this number, but I think it is probably in their docs, and headers in the responses.

[austin495]

Thank you very much man appreciated