From 3c3af57c70802f33014addc48ec62cd472a01ade Mon Sep 17 00:00:00 2001 From: Alexandru Cambose Date: Thu, 21 Aug 2025 10:47:34 +0200 Subject: [PATCH 1/3] feat: reimplemented chart data fetching --- .../src/app/api/pyth/get-history/route.ts | 64 +++++++++++++ .../app/price-feeds/[slug]/(main)/loading.ts | 2 +- .../src/app/price-feeds/[slug]/(main)/page.ts | 2 +- .../{ => Chart}/chart-page.module.scss | 0 .../PriceFeed/{ => Chart}/chart-page.tsx | 5 +- .../PriceFeed/Chart/chart-toolbar.tsx | 70 +++++++++++++++ .../PriceFeed/{ => Chart}/chart.module.scss | 0 .../PriceFeed/{ => Chart}/chart.tsx | 73 ++++++++++----- apps/insights/src/services/clickhouse.ts | 90 ++++++++++++++++++- apps/insights/src/services/pyth/get-feeds.ts | 4 +- 10 files changed, 282 insertions(+), 28 deletions(-) create mode 100644 apps/insights/src/app/api/pyth/get-history/route.ts rename apps/insights/src/components/PriceFeed/{ => Chart}/chart-page.module.scss (100%) rename apps/insights/src/components/PriceFeed/{ => Chart}/chart-page.tsx (93%) create mode 100644 apps/insights/src/components/PriceFeed/Chart/chart-toolbar.tsx rename apps/insights/src/components/PriceFeed/{ => Chart}/chart.module.scss (100%) rename apps/insights/src/components/PriceFeed/{ => Chart}/chart.tsx (77%) diff --git a/apps/insights/src/app/api/pyth/get-history/route.ts b/apps/insights/src/app/api/pyth/get-history/route.ts new file mode 100644 index 0000000000..ab06debd23 --- /dev/null +++ b/apps/insights/src/app/api/pyth/get-history/route.ts @@ -0,0 +1,64 @@ +import type { NextRequest } from "next/server"; +import { z } from "zod"; + +import { getHistory } from "../../../../services/clickhouse"; + +const queryParamsSchema = z.object({ + symbol: z.string(), + range: z.enum(["1H", "1D", "1W", "1M"]), + cluster: z.enum(["pythnet", "pythtest-conformance"]), + from: z.string().transform(Number), + until: z.string().transform(Number), +}); + +export async function GET(req: NextRequest) { + const searchParams = req.nextUrl.searchParams; + + // Parse and validate query parameters + const symbol = searchParams.get("symbol"); + const range = searchParams.get("range"); + const cluster = searchParams.get("cluster"); + const from = searchParams.get("from"); + const until = searchParams.get("until"); + + if (!symbol || !range || !cluster) { + return new Response( + "Missing required parameters. Must provide `symbol`, `range`, and `cluster`", + { status: 400 } + ); + } + + try { + // Validate parameters using the schema + const validatedParams = queryParamsSchema.parse({ + symbol, + range, + cluster, + from, + until, + }); + + const data = await getHistory({ + symbol: validatedParams.symbol, + range: validatedParams.range, + cluster: validatedParams.cluster, + from: validatedParams.from, + until: validatedParams.until, + }); + + return Response.json(data); + } catch (error) { + if (error instanceof z.ZodError) { + return new Response( + `Invalid parameters: ${error.errors.map(e => e.message).join(", ")}`, + { status: 400 } + ); + } + + console.error("Error fetching history data:", error); + return new Response( + "Internal server error", + { status: 500 } + ); + } +} diff --git a/apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts b/apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts index a1085ea612..62f0c87e75 100644 --- a/apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts +++ b/apps/insights/src/app/price-feeds/[slug]/(main)/loading.ts @@ -1 +1 @@ -export { ChartPageLoading as default } from "../../../../components/PriceFeed/chart-page"; +export { ChartPageLoading as default } from "../../../../components/PriceFeed/Chart/chart-page"; diff --git a/apps/insights/src/app/price-feeds/[slug]/(main)/page.ts b/apps/insights/src/app/price-feeds/[slug]/(main)/page.ts index 165ad5abcd..10e4cc3c72 100644 --- a/apps/insights/src/app/price-feeds/[slug]/(main)/page.ts +++ b/apps/insights/src/app/price-feeds/[slug]/(main)/page.ts @@ -1,3 +1,3 @@ -export { ChartPage as default } from "../../../../components/PriceFeed/chart-page"; +export { ChartPage as default } from "../../../../components/PriceFeed/Chart/chart-page"; export const revalidate = 3600; diff --git a/apps/insights/src/components/PriceFeed/chart-page.module.scss b/apps/insights/src/components/PriceFeed/Chart/chart-page.module.scss similarity index 100% rename from apps/insights/src/components/PriceFeed/chart-page.module.scss rename to apps/insights/src/components/PriceFeed/Chart/chart-page.module.scss diff --git a/apps/insights/src/components/PriceFeed/chart-page.tsx b/apps/insights/src/components/PriceFeed/Chart/chart-page.tsx similarity index 93% rename from apps/insights/src/components/PriceFeed/chart-page.tsx rename to apps/insights/src/components/PriceFeed/Chart/chart-page.tsx index aa2797827d..a60cfe9506 100644 --- a/apps/insights/src/components/PriceFeed/chart-page.tsx +++ b/apps/insights/src/components/PriceFeed/Chart/chart-page.tsx @@ -5,7 +5,8 @@ import { Spinner } from "@pythnetwork/component-library/Spinner"; import { Chart } from "./chart"; import styles from "./chart-page.module.scss"; -import { getFeed } from "./get-feed"; +import { getFeed } from "../get-feed"; +import { ChartToolbar } from "./chart-toolbar"; type Props = { params: Promise<{ @@ -26,7 +27,7 @@ type ChartPageImplProps = }); const ChartPageImpl = (props: ChartPageImplProps) => ( - + }>
{props.isLoading ? (
diff --git a/apps/insights/src/components/PriceFeed/Chart/chart-toolbar.tsx b/apps/insights/src/components/PriceFeed/Chart/chart-toolbar.tsx new file mode 100644 index 0000000000..634774740a --- /dev/null +++ b/apps/insights/src/components/PriceFeed/Chart/chart-toolbar.tsx @@ -0,0 +1,70 @@ +'use client' +import { SingleToggleGroup } from "@pythnetwork/component-library/SingleToggleGroup"; +import { useLogger } from "@pythnetwork/component-library/useLogger"; +import { parseAsStringEnum, useQueryState } from 'nuqs'; +import { useCallback } from 'react'; + +export enum Interval { + Live, + OneHour, + OneDay, + OneWeek, + OneMonth, +} + +export const INTERVAL_NAMES = { + [Interval.Live]: "Live", + [Interval.OneHour]: "1H", + [Interval.OneDay]: "1D", + [Interval.OneWeek]: "1W", + [Interval.OneMonth]: "1M", +} as const; + +export const toInterval = (name: (typeof INTERVAL_NAMES)[keyof typeof INTERVAL_NAMES]): Interval => { + switch (name) { + case "Live": { + return Interval.Live; + } + case "1H": { + return Interval.OneHour; + } + case "1D": { + return Interval.OneDay; + } + case "1W": { + return Interval.OneWeek; + } + case "1M": { + return Interval.OneMonth; + } + } +}; +export const ChartToolbar = () => { + const logger = useLogger(); + const [interval, setInterval] = useQueryState( + "interval", + parseAsStringEnum(Object.values(INTERVAL_NAMES)).withDefault("Live"), + ); + + const handleSelectionChange = useCallback((newValue: Interval) => { + setInterval(INTERVAL_NAMES[newValue]).catch((error: unknown) => { + logger.error("Failed to update interval", error); + }); + }, [logger, setInterval]); + + return ( + + ); +}; diff --git a/apps/insights/src/components/PriceFeed/chart.module.scss b/apps/insights/src/components/PriceFeed/Chart/chart.module.scss similarity index 100% rename from apps/insights/src/components/PriceFeed/chart.module.scss rename to apps/insights/src/components/PriceFeed/Chart/chart.module.scss diff --git a/apps/insights/src/components/PriceFeed/chart.tsx b/apps/insights/src/components/PriceFeed/Chart/chart.tsx similarity index 77% rename from apps/insights/src/components/PriceFeed/chart.tsx rename to apps/insights/src/components/PriceFeed/Chart/chart.tsx index 478d1172cd..b62e248e2f 100644 --- a/apps/insights/src/components/PriceFeed/chart.tsx +++ b/apps/insights/src/components/PriceFeed/Chart/chart.tsx @@ -10,9 +10,11 @@ import { useEffect, useRef, useCallback } from "react"; import { z } from "zod"; import styles from "./chart.module.scss"; -import { useLivePriceData } from "../../hooks/use-live-price-data"; -import { usePriceFormatter } from "../../hooks/use-price-formatter"; -import { Cluster } from "../../services/pyth"; +import { useLivePriceData } from "../../../hooks/use-live-price-data"; +import { usePriceFormatter } from "../../../hooks/use-price-formatter"; +import { Cluster } from "../../../services/pyth"; +import { parseAsStringEnum, useQueryState } from 'nuqs'; +import { INTERVAL_NAMES } from './chart-toolbar'; type Props = { symbol: string; @@ -37,6 +39,30 @@ const useChart = (symbol: string, feedId: string) => { useChartColors(chartContainerRef, chartRef); return chartContainerRef; }; +const historySchema = z.array( + z.object({ + timestamp: z.number(), + openPrice: z.number(), + lowPrice: z.number(), + closePrice: z.number(), + highPrice: z.number(), + avgPrice: z.number(), + avgConfidence: z.number(), + avgEmaPrice: z.number(), + avgEmaConfidence: z.number(), + startSlot: z.string(), + endSlot: z.string(), + })); +const fetchHistory = async ({ symbol, range, cluster, from, until }: { symbol: string, range: string, cluster: string, from: bigint, until: bigint }) => { + const url = new URL("/api/pyth/get-history", globalThis.location.origin); + url.searchParams.set("symbol", symbol); + url.searchParams.set("range", range); + url.searchParams.set("cluster", cluster); + url.searchParams.set("from", from.toString()); + url.searchParams.set("until", until.toString()); + console.log("fetching history", {from: new Date(Number(from) * 1000), until: new Date(Number(until) * 1000)}, url.toString()); + return fetch(url).then(async (data) => historySchema.parse(await data.json())); +} const useChartElem = (symbol: string, feedId: string) => { const logger = useLogger(); @@ -46,15 +72,17 @@ const useChartElem = (symbol: string, feedId: string) => { const earliestDateRef = useRef(undefined); const isBackfilling = useRef(false); const priceFormatter = usePriceFormatter(); - +const [interval] = useQueryState( + "interval", + parseAsStringEnum(Object.values(INTERVAL_NAMES)).withDefault("Live"), + ); const backfillData = useCallback(() => { if (!isBackfilling.current && earliestDateRef.current) { isBackfilling.current = true; - const url = new URL("/historical-prices", globalThis.location.origin); - url.searchParams.set("symbol", symbol); - url.searchParams.set("until", earliestDateRef.current.toString()); - fetch(url) - .then(async (data) => historicalDataSchema.parse(await data.json())) + // seconds to date + console.log("backfilling", new Date(Number(earliestDateRef.current) * 1000)); + const range = interval === "Live" ? "1H" : interval; + fetchHistory({ symbol, range, cluster: "pythnet", from: earliestDateRef.current - 100n, until: earliestDateRef.current -2n }) .then((data) => { const firstPoint = data[0]; if (firstPoint) { @@ -65,12 +93,22 @@ const useChartElem = (symbol: string, feedId: string) => { chartRef.current.resolution === Resolution.Tick ) { const convertedData = data.map( - ({ timestamp, price, confidence }) => ({ - time: getLocalTimestamp(new Date(timestamp * 1000)), - price, - confidence, + ({ timestamp, avgPrice, avgConfidence }) => ({ + time: getLocalTimestamp(new Date(timestamp*1000)), + price: avgPrice, + confidence: avgConfidence, }), ); + console.log("convertedData", + {current: chartRef?.current?.price.data().map(({ time, value }) => ({ + time: new Date(time*1000), + value, + })), + converted: convertedData.map(({ time, price }) => ({ + time: new Date(time*1000), + value: price, + })) + }); chartRef.current.price.setData([ ...convertedData.map(({ time, price }) => ({ time, @@ -78,6 +116,7 @@ const useChartElem = (symbol: string, feedId: string) => { })), ...chartRef.current.price.data(), ]); + chartRef.current.confidenceHigh.setData([ ...convertedData.map(({ time, price, confidence }) => ({ time, @@ -194,14 +233,6 @@ type ChartRefContents = { } ); -const historicalDataSchema = z.array( - z.strictObject({ - timestamp: z.number(), - price: z.number(), - confidence: z.number(), - }), -); - const priceFormat = { type: "price", precision: 5, diff --git a/apps/insights/src/services/clickhouse.ts b/apps/insights/src/services/clickhouse.ts index c0b3b2521a..5ee0ac4267 100644 --- a/apps/insights/src/services/clickhouse.ts +++ b/apps/insights/src/services/clickhouse.ts @@ -183,6 +183,8 @@ const _getYesterdaysPrices = async (symbols: string[]) => }, ); + + const _getPublisherRankingHistory = async ({ cluster, key, @@ -369,13 +371,99 @@ export const getHistoricalPrices = async ({ }, ); + +// Main history function +export const getHistory = async ({ + symbol, + range, + cluster, + from, + until, +}: { + symbol: string; + range: "1H" | "1D" | "1W" | "1M"; + cluster: "pythnet" | "pythtest-conformance"; + from: number; + until: number; +}) => { + + // Calculate interval parameters based on range + const start = (range === "1H" || range === "1D") ? "1" : (range === "1W") ? "7" : "30"; + const range_unit = range === "1H" ? "HOUR" : "DAY"; + const interval_number = range === "1H" ? 5 : 1; + const interval_unit = range === "1H" ? "SECOND" : (range === "1D") ? "MINUTE" : "HOUR"; + + let additional_cluster_clause = ""; + if (cluster === "pythtest-conformance") { + additional_cluster_clause = " OR (cluster = 'pythtest')"; + } + + const query = ` + SELECT + toUnixTimestamp(toStartOfInterval(publishTime, INTERVAL {interval_number: UInt32} ${interval_unit})) AS timestamp, + argMin(price, slot) AS openPrice, + min(price) AS lowPrice, + argMax(price, slot) AS closePrice, + max(price) AS highPrice, + avg(price) AS avgPrice, + avg(confidence) AS avgConfidence, + avg(emaPrice) AS avgEmaPrice, + avg(emaConfidence) AS avgEmaConfidence, + min(slot) AS startSlot, + max(slot) AS endSlot + FROM prices + FINAL + WHERE ((symbol = {symbol: String}) OR (symbol = {base_quote_symbol: String})) + AND ((cluster = {cluster: String})${additional_cluster_clause}) + AND (version = 2) + AND (publishTime > fromUnixTimestamp(toInt64({from: String}))) + AND (publishTime <= fromUnixTimestamp(toInt64({until: String}))) + AND (time > (fromUnixTimestamp(toInt64({from: String})) - INTERVAL 5 SECOND)) + AND (time <= (fromUnixTimestamp(toInt64({until: String})) + INTERVAL 5 SECOND)) + AND (publisher = {publisher: String}) + AND (status = 1) + GROUP BY timestamp + ORDER BY timestamp ASC + SETTINGS do_not_merge_across_partitions_select_final=1 + `; +console.log(query) + const data = await safeQuery(z.array( + z.object({ + timestamp: z.number(), + openPrice: z.number(), + lowPrice: z.number(), + closePrice: z.number(), + highPrice: z.number(), + avgPrice: z.number(), + avgConfidence: z.number(), + avgEmaPrice: z.number(), + avgEmaConfidence: z.number(), + startSlot: z.string(), + endSlot: z.string(), + })), { + query, + query_params: { + symbol, + cluster, + publisher: "", + base_quote_symbol: symbol.split(".")[-1], + from, + until, + start, + interval_number, + }, + }); + console.log("from", new Date(from * 1000), from, "until", new Date(until * 1000), until, data[0], data[data.length - 1]); + + return data; +}; + const safeQuery = async ( schema: ZodSchema, query: Omit[0], "format">, ) => { const rows = await client.query({ ...query, format: "JSON" }); const result = await rows.json(); - return schema.parse(result.data); }; diff --git a/apps/insights/src/services/pyth/get-feeds.ts b/apps/insights/src/services/pyth/get-feeds.ts index 93cc9ff486..f513d3bc7f 100644 --- a/apps/insights/src/services/pyth/get-feeds.ts +++ b/apps/insights/src/services/pyth/get-feeds.ts @@ -22,8 +22,8 @@ const _getFeeds = async (cluster: Cluster) => { publisher: publisher.toBase58(), })) ?? [], }, - })); - return priceFeedsSchema.parse(filtered); + })) + return priceFeedsSchema.parse(filtered) }; export const getFeeds = redisCache.define("getFeeds", _getFeeds).getFeeds; From a3be596f0ffa936992c1d2ec2639f51763728e8f Mon Sep 17 00:00:00 2001 From: Alexandru Cambose Date: Thu, 21 Aug 2025 15:34:54 +0200 Subject: [PATCH 2/3] feat: chart data consistency --- .../src/components/PriceFeed/Chart/chart.tsx | 63 ++++++++++++++----- apps/insights/src/services/clickhouse.ts | 11 +--- apps/insights/src/services/pyth/get-feeds.ts | 4 +- .../src/services/pyth/get-metadata.ts | 6 +- .../pyth/get-publishers-for-cluster.ts | 4 +- 5 files changed, 56 insertions(+), 32 deletions(-) diff --git a/apps/insights/src/components/PriceFeed/Chart/chart.tsx b/apps/insights/src/components/PriceFeed/Chart/chart.tsx index b62e248e2f..6ab639e786 100644 --- a/apps/insights/src/components/PriceFeed/Chart/chart.tsx +++ b/apps/insights/src/components/PriceFeed/Chart/chart.tsx @@ -64,6 +64,35 @@ const fetchHistory = async ({ symbol, range, cluster, from, until }: { symbol: s return fetch(url).then(async (data) => historySchema.parse(await data.json())); } +// const checkPriceData = (data: {time: UTCTimestamp}[]) => { +// const chartData = [...data].sort((a, b) => a.time - b.time); +// if(chartData.length < 2) { +// return; +// } +// const firstElement = chartData.at(-2); +// const secondElement = chartData.at(-1); +// if(!firstElement || !secondElement ) { +// return; +// } +// const detectedInterval = secondElement.time - firstElement.time +// for(let i = 0; i < chartData.length - 1; i++) { +// const currentElement = chartData[i]; +// const nextElement = chartData[i + 1]; +// if(!currentElement || !nextElement) { +// return; +// } +// const interval = nextElement.time - currentElement.time +// if(interval !== detectedInterval) { +// console.warn("Price chartData is not consistent", { +// current: currentElement, +// next: nextElement, +// detectedInterval, +// }); +// } +// } +// return detectedInterval; +// } + const useChartElem = (symbol: string, feedId: string) => { const logger = useLogger(); const { current } = useLivePriceData(Cluster.Pythnet, feedId); @@ -80,9 +109,9 @@ const [interval] = useQueryState( if (!isBackfilling.current && earliestDateRef.current) { isBackfilling.current = true; // seconds to date - console.log("backfilling", new Date(Number(earliestDateRef.current) * 1000)); const range = interval === "Live" ? "1H" : interval; - fetchHistory({ symbol, range, cluster: "pythnet", from: earliestDateRef.current - 100n, until: earliestDateRef.current -2n }) + console.log("backfilling", new Date(Number(earliestDateRef.current) * 1000), {from: earliestDateRef.current - 100n, until: earliestDateRef.current}); + fetchHistory({ symbol, range, cluster: "pythnet", from: earliestDateRef.current - 100n, until: earliestDateRef.current }) .then((data) => { const firstPoint = data[0]; if (firstPoint) { @@ -109,28 +138,28 @@ const [interval] = useQueryState( value: price, })) }); - chartRef.current.price.setData([ - ...convertedData.map(({ time, price }) => ({ + const newPriceData = [...convertedData.map(({ time, price }) => ({ time, value: price, })), - ...chartRef.current.price.data(), - ]); - - chartRef.current.confidenceHigh.setData([ - ...convertedData.map(({ time, price, confidence }) => ({ + ...chartRef.current.price.data(),] + const newConfidenceHighData = [...convertedData.map(({ time, price, confidence }) => ({ time, value: price + confidence, })), - ...chartRef.current.confidenceHigh.data(), - ]); - chartRef.current.confidenceLow.setData([ - ...convertedData.map(({ time, price, confidence }) => ({ + ...chartRef.current.confidenceHigh.data(),] + const newConfidenceLowData = [...convertedData.map(({ time, price, confidence }) => ({ + time, + value: price - confidence, + })), ...chartRef.current.confidenceLow.data(),] + checkPriceData(convertedData.map(({ time, price }) => ({ time, - value: price - confidence, - })), - ...chartRef.current.confidenceLow.data(), - ]); + value: price, + }))); + console.log(newPriceData) + chartRef.current.price.setData(newPriceData); + chartRef.current.confidenceHigh.setData(newConfidenceHighData); + chartRef.current.confidenceLow.setData(newConfidenceLowData); } isBackfilling.current = false; }) diff --git a/apps/insights/src/services/clickhouse.ts b/apps/insights/src/services/clickhouse.ts index 5ee0ac4267..b38788f97e 100644 --- a/apps/insights/src/services/clickhouse.ts +++ b/apps/insights/src/services/clickhouse.ts @@ -388,9 +388,6 @@ export const getHistory = async ({ }) => { // Calculate interval parameters based on range - const start = (range === "1H" || range === "1D") ? "1" : (range === "1W") ? "7" : "30"; - const range_unit = range === "1H" ? "HOUR" : "DAY"; - const interval_number = range === "1H" ? 5 : 1; const interval_unit = range === "1H" ? "SECOND" : (range === "1D") ? "MINUTE" : "HOUR"; let additional_cluster_clause = ""; @@ -400,7 +397,7 @@ export const getHistory = async ({ const query = ` SELECT - toUnixTimestamp(toStartOfInterval(publishTime, INTERVAL {interval_number: UInt32} ${interval_unit})) AS timestamp, + toUnixTimestamp(toStartOfInterval(publishTime, INTERVAL 1 ${interval_unit})) AS timestamp, argMin(price, slot) AS openPrice, min(price) AS lowPrice, argMax(price, slot) AS closePrice, @@ -418,8 +415,8 @@ export const getHistory = async ({ AND (version = 2) AND (publishTime > fromUnixTimestamp(toInt64({from: String}))) AND (publishTime <= fromUnixTimestamp(toInt64({until: String}))) - AND (time > (fromUnixTimestamp(toInt64({from: String})) - INTERVAL 5 SECOND)) - AND (time <= (fromUnixTimestamp(toInt64({until: String})) + INTERVAL 5 SECOND)) + AND (time > (fromUnixTimestamp(toInt64({from: String})))) + AND (time <= (fromUnixTimestamp(toInt64({until: String})))) AND (publisher = {publisher: String}) AND (status = 1) GROUP BY timestamp @@ -449,8 +446,6 @@ console.log(query) base_quote_symbol: symbol.split(".")[-1], from, until, - start, - interval_number, }, }); console.log("from", new Date(from * 1000), from, "until", new Date(until * 1000), until, data[0], data[data.length - 1]); diff --git a/apps/insights/src/services/pyth/get-feeds.ts b/apps/insights/src/services/pyth/get-feeds.ts index f513d3bc7f..30caaca800 100644 --- a/apps/insights/src/services/pyth/get-feeds.ts +++ b/apps/insights/src/services/pyth/get-feeds.ts @@ -1,9 +1,9 @@ import { Cluster, priceFeedsSchema } from "."; -import { getPythMetadataCached } from "./get-metadata"; +import { getPythMetadata } from "./get-metadata"; import { redisCache } from "../../cache"; const _getFeeds = async (cluster: Cluster) => { - const unfilteredData = await getPythMetadataCached(cluster); + const unfilteredData = await getPythMetadata(cluster); const filtered = unfilteredData.symbols .filter( (symbol) => diff --git a/apps/insights/src/services/pyth/get-metadata.ts b/apps/insights/src/services/pyth/get-metadata.ts index a87dd0733f..b5cd814d89 100644 --- a/apps/insights/src/services/pyth/get-metadata.ts +++ b/apps/insights/src/services/pyth/get-metadata.ts @@ -1,11 +1,11 @@ import { clients, Cluster } from "."; import { memoryOnlyCache } from "../../cache"; -const getPythMetadata = async (cluster: Cluster) => { +const _getPythMetadata = async (cluster: Cluster) => { return clients[cluster].getData(); }; -export const getPythMetadataCached = memoryOnlyCache.define( +export const getPythMetadata = memoryOnlyCache.define( "getPythMetadata", - getPythMetadata, + _getPythMetadata, ).getPythMetadata; diff --git a/apps/insights/src/services/pyth/get-publishers-for-cluster.ts b/apps/insights/src/services/pyth/get-publishers-for-cluster.ts index 452a369156..0c6fa5e854 100644 --- a/apps/insights/src/services/pyth/get-publishers-for-cluster.ts +++ b/apps/insights/src/services/pyth/get-publishers-for-cluster.ts @@ -1,9 +1,9 @@ import { Cluster } from "."; -import { getPythMetadataCached } from "./get-metadata"; +import { getPythMetadata } from "./get-metadata"; import { redisCache } from "../../cache"; const _getPublishersForCluster = async (cluster: Cluster) => { - const data = await getPythMetadataCached(cluster); + const data = await getPythMetadata(cluster); const result: Record = {}; for (const key of data.productPrice.keys()) { const price = data.productPrice.get(key); From 22a185b350f188bcbe61bb06fde9d512d763609d Mon Sep 17 00:00:00 2001 From: Alexandru Cambose Date: Mon, 25 Aug 2025 16:00:45 +0200 Subject: [PATCH 3/3] feat: more chart changes --- .../src/components/PriceFeed/Chart/chart.tsx | 139 ++++++++++++------ apps/insights/src/historicalChart.ts | 10 ++ apps/insights/src/services/clickhouse.ts | 2 +- 3 files changed, 104 insertions(+), 47 deletions(-) create mode 100644 apps/insights/src/historicalChart.ts diff --git a/apps/insights/src/components/PriceFeed/Chart/chart.tsx b/apps/insights/src/components/PriceFeed/Chart/chart.tsx index 6ab639e786..d2ebf407c9 100644 --- a/apps/insights/src/components/PriceFeed/Chart/chart.tsx +++ b/apps/insights/src/components/PriceFeed/Chart/chart.tsx @@ -5,22 +5,23 @@ import { useResizeObserver, useMountEffect } from "@react-hookz/web"; import type { IChartApi, ISeriesApi, UTCTimestamp } from "lightweight-charts"; import { LineSeries, LineStyle, createChart } from "lightweight-charts"; import { useTheme } from "next-themes"; +import { parseAsStringEnum, useQueryState } from 'nuqs'; import type { RefObject } from "react"; import { useEffect, useRef, useCallback } from "react"; import { z } from "zod"; +import { INTERVAL_NAMES } from './chart-toolbar'; import styles from "./chart.module.scss"; import { useLivePriceData } from "../../../hooks/use-live-price-data"; import { usePriceFormatter } from "../../../hooks/use-price-formatter"; import { Cluster } from "../../../services/pyth"; -import { parseAsStringEnum, useQueryState } from 'nuqs'; -import { INTERVAL_NAMES } from './chart-toolbar'; type Props = { symbol: string; feedId: string; }; + export const Chart = ({ symbol, feedId }: Props) => { const chartContainerRef = useChart(symbol, feedId); @@ -64,34 +65,34 @@ const fetchHistory = async ({ symbol, range, cluster, from, until }: { symbol: s return fetch(url).then(async (data) => historySchema.parse(await data.json())); } -// const checkPriceData = (data: {time: UTCTimestamp}[]) => { -// const chartData = [...data].sort((a, b) => a.time - b.time); -// if(chartData.length < 2) { -// return; -// } -// const firstElement = chartData.at(-2); -// const secondElement = chartData.at(-1); -// if(!firstElement || !secondElement ) { -// return; -// } -// const detectedInterval = secondElement.time - firstElement.time -// for(let i = 0; i < chartData.length - 1; i++) { -// const currentElement = chartData[i]; -// const nextElement = chartData[i + 1]; -// if(!currentElement || !nextElement) { -// return; -// } -// const interval = nextElement.time - currentElement.time -// if(interval !== detectedInterval) { -// console.warn("Price chartData is not consistent", { -// current: currentElement, -// next: nextElement, -// detectedInterval, -// }); -// } -// } -// return detectedInterval; -// } +const checkPriceData = (data: {time: UTCTimestamp}[]) => { + const chartData = [...data].sort((a, b) => a.time - b.time); + if(chartData.length < 2) { + return; + } + const firstElement = chartData.at(-2); + const secondElement = chartData.at(-1); + if(!firstElement || !secondElement ) { + return; + } + const detectedInterval = secondElement.time - firstElement.time + for(let i = 0; i < chartData.length - 1; i++) { + const currentElement = chartData[i]; + const nextElement = chartData[i + 1]; + if(!currentElement || !nextElement) { + return; + } + const interval = nextElement.time - currentElement.time + if(interval !== detectedInterval) { + console.warn("Price chartData is not consistent", { + current: currentElement, + next: nextElement, + detectedInterval, + }); + } + } + return detectedInterval; +} const useChartElem = (symbol: string, feedId: string) => { const logger = useLogger(); @@ -101,17 +102,26 @@ const useChartElem = (symbol: string, feedId: string) => { const earliestDateRef = useRef(undefined); const isBackfilling = useRef(false); const priceFormatter = usePriceFormatter(); -const [interval] = useQueryState( + const [interval] = useQueryState( "interval", parseAsStringEnum(Object.values(INTERVAL_NAMES)).withDefault("Live"), ); + useEffect(() => { + if(interval !== "Live") { + backfillData(); + } + }, [interval]); + const backfillData = useCallback(() => { - if (!isBackfilling.current && earliestDateRef.current) { + console.log(earliestDateRef) + const { from, until } = backfillIntervals({ interval, earliestDate: earliestDateRef.current }); + + if (!isBackfilling.current) { isBackfilling.current = true; // seconds to date const range = interval === "Live" ? "1H" : interval; - console.log("backfilling", new Date(Number(earliestDateRef.current) * 1000), {from: earliestDateRef.current - 100n, until: earliestDateRef.current}); - fetchHistory({ symbol, range, cluster: "pythnet", from: earliestDateRef.current - 100n, until: earliestDateRef.current }) + console.log("backfilling", {from: new Date(Number(from) * 1000), until: new Date(Number(until) * 1000)}); + fetchHistory({ symbol, range, cluster: "pythnet", from, until }) .then((data) => { const firstPoint = data[0]; if (firstPoint) { @@ -128,16 +138,16 @@ const [interval] = useQueryState( confidence: avgConfidence, }), ); - console.log("convertedData", - {current: chartRef?.current?.price.data().map(({ time, value }) => ({ - time: new Date(time*1000), - value, - })), - converted: convertedData.map(({ time, price }) => ({ - time: new Date(time*1000), - value: price, - })) - }); + // console.log("convertedData", + // {current: chartRef?.current?.price.data().map(({ time, value }) => ({ + // time: new Date(time*1000), + // value, + // })), + // converted: convertedData.map(({ time, price }) => ({ + // time: new Date(time*1000), + // value: price, + // })) + // }); const newPriceData = [...convertedData.map(({ time, price }) => ({ time, value: price, @@ -156,7 +166,6 @@ const [interval] = useQueryState( time, value: price, }))); - console.log(newPriceData) chartRef.current.price.setData(newPriceData); chartRef.current.confidenceHigh.setData(newConfidenceHighData); chartRef.current.confidenceLow.setData(newConfidenceLowData); @@ -167,7 +176,7 @@ const [interval] = useQueryState( logger.error("Error fetching historical prices", error); }); } - }, [logger, symbol]); + }, [logger, symbol, interval]); useMountEffect(() => { const chartElem = chartContainerRef.current; @@ -217,6 +226,10 @@ const [interval] = useQueryState( }); useEffect(() => { + // don't live update if the interval is not set to live + if(interval !== "Live") { + return; + } if (current && chartRef.current) { earliestDateRef.current ??= current.timestamp; const { price, confidence } = current.aggregate; @@ -361,3 +374,37 @@ const getLocalTimestamp = (date: Date): UTCTimestamp => date.getSeconds(), date.getMilliseconds(), ) / 1000) as UTCTimestamp; + + const backfillIntervals = ({ interval, earliestDate } : { + interval: typeof INTERVAL_NAMES[keyof typeof INTERVAL_NAMES], + earliestDate: bigint | undefined, + }) => { + const until = earliestDate ?? BigInt(Math.floor(Date.now() / 1000)); + let from; + + switch (interval) { + case '1H': { + from = until - 3600n; + break; + } + case '1D': { + from = until - 86_400n; // seconds in a day + + break; + } + case '1W': { + from = until - 604_800n; // seconds in a week + + break; + } + case '1M': { + from = until - 2_592_000n; + + break; + } + default: { + from = until - 100n; + } + } + return { from, until }; + } \ No newline at end of file diff --git a/apps/insights/src/historicalChart.ts b/apps/insights/src/historicalChart.ts new file mode 100644 index 0000000000..d8fd41aa5f --- /dev/null +++ b/apps/insights/src/historicalChart.ts @@ -0,0 +1,10 @@ +import { INTERVAL_NAMES } from "./components/PriceFeed/Chart/chart-toolbar"; + +export const intervalToResolution = (interval: typeof INTERVAL_NAMES[keyof typeof INTERVAL_NAMES]) => { + switch (interval) { + case '1H': + return Resolution.Hour; + case '1D': + return Resolution.Day; + } +} \ No newline at end of file diff --git a/apps/insights/src/services/clickhouse.ts b/apps/insights/src/services/clickhouse.ts index b38788f97e..f66c190070 100644 --- a/apps/insights/src/services/clickhouse.ts +++ b/apps/insights/src/services/clickhouse.ts @@ -388,7 +388,7 @@ export const getHistory = async ({ }) => { // Calculate interval parameters based on range - const interval_unit = range === "1H" ? "SECOND" : (range === "1D") ? "MINUTE" : "HOUR"; + const interval_unit = range === "1H" ? "SECOND" : (range === "1D") ? "HOUR" : "DAY"; let additional_cluster_clause = ""; if (cluster === "pythtest-conformance") {