feat: reimplemented chart data fetching

Author: alexcambose

apps/insights/src/app/api/pyth/get-history/route.ts

@@ -0,0 +1,64 @@
+import type { NextRequest } from "next/server";
+import { z } from "zod";
+
+import { getHistory } from "../../../../services/clickhouse";
+
+const queryParamsSchema = z.object({
+  symbol: z.string(),
+  range: z.enum(["1H", "1D", "1W", "1M"]),
+  cluster: z.enum(["pythnet", "pythtest-conformance"]),
+  from: z.string().transform(Number),
+  until: z.string().transform(Number),
+});
+
+export async function GET(req: NextRequest) {
+  const searchParams = req.nextUrl.searchParams;
+  
+  // Parse and validate query parameters
+  const symbol = searchParams.get("symbol");
+  const range = searchParams.get("range");
+  const cluster = searchParams.get("cluster");
+  const from = searchParams.get("from");
+  const until = searchParams.get("until");
+
+  if (!symbol || !range || !cluster) {
+    return new Response(
+      "Missing required parameters. Must provide `symbol`, `range`, and `cluster`",
+      { status: 400 }
+    );
+  }
+
+  try {
+    // Validate parameters using the schema
+    const validatedParams = queryParamsSchema.parse({
+      symbol,
+      range,
+      cluster,
+      from,
+      until,
+    });
+
+    const data = await getHistory({
+      symbol: validatedParams.symbol,
+      range: validatedParams.range,
+      cluster: validatedParams.cluster,
+      from: validatedParams.from,
+      until: validatedParams.until,
+    });
+
+    return Response.json(data);
+  } catch (error) {
+    if (error instanceof z.ZodError) {
+      return new Response(
+        `Invalid parameters: ${error.errors.map(e => e.message).join(", ")}`,
+        { status: 400 }
+      );
+    }
+    
+    console.error("Error fetching history data:", error);
+    return new Response(
+      "Internal server error",
+      { status: 500 }
+    );
+  }
+}

apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts

@@ -1 +1 @@
-export { ChartPageLoading as default } from "../../../../components/PriceFeed/chart-page";
+export { ChartPageLoading as default } from "../../../../components/PriceFeed/Chart/chart-page";

apps/insights/src/app/price-feeds/[slug]/(main)/page.ts

@@ -1,3 +1,3 @@
-export { ChartPage as default } from "../../../../components/PriceFeed/chart-page";
+export { ChartPage as default } from "../../../../components/PriceFeed/Chart/chart-page";
 
 export const revalidate = 3600;

apps/insights/src/components/PriceFeed/chart-page.module.scss renamed to apps/insights/src/components/PriceFeed/Chart/chart-page.module.scss

@@ -5,7 +5,8 @@ import { Spinner } from "@pythnetwork/component-library/Spinner";
 
 import { Chart } from "./chart";
 import styles from "./chart-page.module.scss";
-import { getFeed } from "./get-feed";
+import { getFeed } from "../get-feed";
+import { ChartToolbar } from "./chart-toolbar";
 
 type Props = {
   params: Promise<{
@@ -26,7 +27,7 @@ type ChartPageImplProps =
     });
 
 const ChartPageImpl = (props: ChartPageImplProps) => (
-  <Card title="Chart" className={styles.chartCard}>
+  <Card title="Chart" className={styles.chartCard} toolbar={<ChartToolbar />}>
     <div className={styles.chart}>
       {props.isLoading ? (
         <div className={styles.spinnerContainer}>

apps/insights/src/components/PriceFeed/chart-page.tsx renamed to apps/insights/src/components/PriceFeed/Chart/chart-page.tsx

@@ -0,0 +1,70 @@
+'use client'
+import { SingleToggleGroup } from "@pythnetwork/component-library/SingleToggleGroup";
+import { useLogger } from "@pythnetwork/component-library/useLogger";
+import { parseAsStringEnum, useQueryState } from 'nuqs';
+import { useCallback } from 'react';
+
+export enum Interval {
+  Live,
+  OneHour,
+  OneDay,
+  OneWeek,
+  OneMonth,
+}
+
+export const INTERVAL_NAMES = {
+  [Interval.Live]: "Live",
+  [Interval.OneHour]: "1H",
+  [Interval.OneDay]: "1D",
+  [Interval.OneWeek]: "1W",
+  [Interval.OneMonth]: "1M",
+} as const;
+
+export const toInterval = (name: (typeof INTERVAL_NAMES)[keyof typeof INTERVAL_NAMES]): Interval => {
+  switch (name) {
+    case "Live": {
+      return Interval.Live;
+    }
+    case "1H": {
+      return Interval.OneHour;
+    }
+    case "1D": {
+      return Interval.OneDay;
+    }
+    case "1W": {
+      return Interval.OneWeek;
+    }
+    case "1M": {
+      return Interval.OneMonth;
+    }
+  }
+};
+export const ChartToolbar = () => {
+  const logger = useLogger();
+  const [interval, setInterval] = useQueryState(
+    "interval",
+    parseAsStringEnum(Object.values(INTERVAL_NAMES)).withDefault("Live"),
+  );
+
+  const handleSelectionChange = useCallback((newValue: Interval) => {
+    setInterval(INTERVAL_NAMES[newValue]).catch((error: unknown) => {
+      logger.error("Failed to update interval", error);
+    });
+  }, [logger, setInterval]);
+
+  return (
+    <SingleToggleGroup
+      selectedKey={toInterval(interval)}
+      // @ts-expect-error - wrong param type
+      onSelectionChange={handleSelectionChange}
+      rounded
+      items={[
+        { id: Interval.Live, children: INTERVAL_NAMES[Interval.Live] },
+        { id: Interval.OneHour, children: INTERVAL_NAMES[Interval.OneHour] },
+        { id: Interval.OneDay, children: INTERVAL_NAMES[Interval.OneDay] },
+        { id: Interval.OneWeek, children: INTERVAL_NAMES[Interval.OneWeek] },
+        { id: Interval.OneMonth, children: INTERVAL_NAMES[Interval.OneMonth] },
+      ]}
+    />
+  );
+};

apps/insights/src/components/PriceFeed/Chart/chart-toolbar.tsx

@@ -10,9 +10,11 @@ import { useEffect, useRef, useCallback } from "react";
 import { z } from "zod";
 
 import styles from "./chart.module.scss";
-import { useLivePriceData } from "../../hooks/use-live-price-data";
-import { usePriceFormatter } from "../../hooks/use-price-formatter";
-import { Cluster } from "../../services/pyth";
+import { useLivePriceData } from "../../../hooks/use-live-price-data";
+import { usePriceFormatter } from "../../../hooks/use-price-formatter";
+import { Cluster } from "../../../services/pyth";
+import { parseAsStringEnum, useQueryState } from 'nuqs';
+import { INTERVAL_NAMES } from './chart-toolbar';
 
 type Props = {
   symbol: string;
@@ -37,6 +39,30 @@ const useChart = (symbol: string, feedId: string) => {
   useChartColors(chartContainerRef, chartRef);
   return chartContainerRef;
 };
+const historySchema = z.array(
+  z.object({
+    timestamp: z.number(),
+    openPrice: z.number(),
+    lowPrice: z.number(),
+    closePrice: z.number(),
+    highPrice: z.number(),
+    avgPrice: z.number(),
+    avgConfidence: z.number(),
+    avgEmaPrice: z.number(),
+    avgEmaConfidence: z.number(),
+    startSlot: z.string(),
+    endSlot: z.string(),
+  }));
+const fetchHistory = async ({ symbol, range, cluster, from, until }: { symbol: string, range: string, cluster: string, from: bigint, until: bigint }) => {
+  const url = new URL("/api/pyth/get-history", globalThis.location.origin);
+  url.searchParams.set("symbol", symbol);
+  url.searchParams.set("range", range);
+  url.searchParams.set("cluster", cluster);
+  url.searchParams.set("from", from.toString());
+  url.searchParams.set("until", until.toString());
+  console.log("fetching history", {from: new Date(Number(from) * 1000), until: new Date(Number(until) * 1000)}, url.toString());
+  return fetch(url).then(async (data) => historySchema.parse(await data.json()));
+}
 
 const useChartElem = (symbol: string, feedId: string) => {
   const logger = useLogger();
@@ -46,15 +72,17 @@ const useChartElem = (symbol: string, feedId: string) => {
   const earliestDateRef = useRef<bigint | undefined>(undefined);
   const isBackfilling = useRef(false);
   const priceFormatter = usePriceFormatter();
-
+const [interval] = useQueryState(
+    "interval",
+    parseAsStringEnum(Object.values(INTERVAL_NAMES)).withDefault("Live"),
+  );
   const backfillData = useCallback(() => {
     if (!isBackfilling.current && earliestDateRef.current) {
       isBackfilling.current = true;
-      const url = new URL("/historical-prices", globalThis.location.origin);
-      url.searchParams.set("symbol", symbol);
-      url.searchParams.set("until", earliestDateRef.current.toString());
-      fetch(url)
-        .then(async (data) => historicalDataSchema.parse(await data.json()))
+      // seconds to date
+      console.log("backfilling", new Date(Number(earliestDateRef.current) * 1000));
+      const range = interval === "Live" ? "1H" : interval;
+      fetchHistory({ symbol, range, cluster: "pythnet", from: earliestDateRef.current - 100n, until: earliestDateRef.current -2n })
         .then((data) => {
           const firstPoint = data[0];
           if (firstPoint) {
@@ -65,19 +93,30 @@ const useChartElem = (symbol: string, feedId: string) => {
             chartRef.current.resolution === Resolution.Tick
           ) {
             const convertedData = data.map(
-              ({ timestamp, price, confidence }) => ({
-                time: getLocalTimestamp(new Date(timestamp * 1000)),
-                price,
-                confidence,
+              ({ timestamp, avgPrice, avgConfidence }) => ({
+                time: getLocalTimestamp(new Date(timestamp*1000)),
+                price: avgPrice,
+                confidence: avgConfidence,
               }),
             );
+            console.log("convertedData",   
+              {current: chartRef?.current?.price.data().map(({ time, value }) => ({
+                time: new Date(time*1000),
+                value,
+              })),
+              converted: convertedData.map(({ time, price }) => ({
+                time: new Date(time*1000),
+                value: price,
+              }))
+            });
             chartRef.current.price.setData([
               ...convertedData.map(({ time, price }) => ({
                 time,
                 value: price,
               })),
               ...chartRef.current.price.data(),
             ]);
+
             chartRef.current.confidenceHigh.setData([
               ...convertedData.map(({ time, price, confidence }) => ({
                 time,
@@ -194,14 +233,6 @@ type ChartRefContents = {
     }
 );
 
-const historicalDataSchema = z.array(
-  z.strictObject({
-    timestamp: z.number(),
-    price: z.number(),
-    confidence: z.number(),
-  }),
-);
-
 const priceFormat = {
   type: "price",
   precision: 5,

apps/insights/src/components/PriceFeed/chart.module.scss renamed to apps/insights/src/components/PriceFeed/Chart/chart.module.scss

@@ -183,6 +183,8 @@ const _getYesterdaysPrices = async (symbols: string[]) =>
     },
   );
 
+
+
 const _getPublisherRankingHistory = async ({
   cluster,
   key,
@@ -369,13 +371,99 @@ export const getHistoricalPrices = async ({
     },
   );
 
+
+// Main history function
+export const getHistory = async ({
+  symbol,
+  range,
+  cluster,
+  from,
+  until,
+}: {
+  symbol: string;
+  range: "1H" | "1D" | "1W" | "1M";
+  cluster: "pythnet" | "pythtest-conformance";
+  from: number;
+  until: number;
+}) => {
+
+  // Calculate interval parameters based on range
+  const start = (range === "1H" || range === "1D") ? "1" : (range === "1W") ? "7" : "30";
+  const range_unit = range === "1H" ? "HOUR" : "DAY";
+  const interval_number = range === "1H" ? 5 : 1;
+  const interval_unit = range === "1H" ? "SECOND" : (range === "1D") ? "MINUTE" : "HOUR";
+
+  let additional_cluster_clause = "";
+  if (cluster === "pythtest-conformance") {
+    additional_cluster_clause = " OR (cluster = 'pythtest')";
+  }
+
+      const query = `
+      SELECT
+        toUnixTimestamp(toStartOfInterval(publishTime, INTERVAL {interval_number: UInt32} ${interval_unit})) AS timestamp,
+        argMin(price, slot) AS openPrice,
+        min(price) AS lowPrice,
+        argMax(price, slot) AS closePrice,
+        max(price) AS highPrice,
+        avg(price) AS avgPrice,
+        avg(confidence) AS avgConfidence,
+        avg(emaPrice) AS avgEmaPrice,
+        avg(emaConfidence) AS avgEmaConfidence,
+        min(slot) AS startSlot,
+        max(slot) AS endSlot
+    FROM prices
+    FINAL
+    WHERE ((symbol = {symbol: String}) OR (symbol = {base_quote_symbol: String}))
+    AND ((cluster = {cluster: String})${additional_cluster_clause})
+    AND (version = 2)
+    AND (publishTime > fromUnixTimestamp(toInt64({from: String})))
+    AND (publishTime <= fromUnixTimestamp(toInt64({until: String})))
+    AND (time > (fromUnixTimestamp(toInt64({from: String})) - INTERVAL 5 SECOND))
+    AND (time <= (fromUnixTimestamp(toInt64({until: String})) + INTERVAL 5 SECOND))
+    AND (publisher = {publisher: String})
+    AND (status = 1)
+    GROUP BY timestamp
+    ORDER BY timestamp ASC
+    SETTINGS do_not_merge_across_partitions_select_final=1
+    `;
+console.log(query)
+  const data = await safeQuery(z.array(
+  z.object({
+    timestamp: z.number(),
+    openPrice: z.number(),
+    lowPrice: z.number(),
+    closePrice: z.number(),
+    highPrice: z.number(),
+    avgPrice: z.number(),
+    avgConfidence: z.number(),
+    avgEmaPrice: z.number(),
+    avgEmaConfidence: z.number(),
+    startSlot: z.string(),
+    endSlot: z.string(),
+  })), {
+    query,
+    query_params: {
+      symbol,
+      cluster,
+      publisher: "",
+      base_quote_symbol: symbol.split(".")[-1],
+      from,
+      until,
+      start,
+      interval_number,
+    },
+  });
+  console.log("from", new Date(from * 1000), from, "until", new Date(until * 1000), until, data[0], data[data.length - 1]);
+
+  return data;
+};
+
 const safeQuery = async <Output, Def extends ZodTypeDef, Input>(
   schema: ZodSchema<Output, Def, Input>,
   query: Omit<Parameters<typeof client.query>[0], "format">,
 ) => {
   const rows = await client.query({ ...query, format: "JSON" });
   const result = await rows.json();
-
   return schema.parse(result.data);
 };