feat(insights): pyth metadata caching

apps/insights/next.config.js

@@ -3,7 +3,6 @@ const config = {
     useCache: true,
     reactCompiler: true,
   },
-
   reactStrictMode: true,
 
   pageExtensions: ["ts", "tsx", "mdx"],

apps/insights/package.json

@@ -28,10 +28,12 @@
     "@pythnetwork/known-publishers": "workspace:*",
     "@react-hookz/web": "catalog:",
     "@solana/web3.js": "catalog:",
+    "async-cache-dedupe": "^3.0.0",
     "bs58": "catalog:",
     "clsx": "catalog:",
     "cryptocurrency-icons": "catalog:",
     "dnum": "catalog:",
+    "ioredis": "^5.7.0",
     "lightweight-charts": "catalog:",
     "motion": "catalog:",
     "next": "catalog:",

apps/insights/src/app/api/pyth/get-feeds-for-publisher/[publisher]/route.ts

@@ -0,0 +1,30 @@
+import { NextResponse } from "next/server";
+import { stringify } from "superjson";
+
+import { getFeedsCached } from '../../../../../server/pyth/get-feeds';
+import { Cluster } from "../../../../../services/pyth";
+
+export const GET = async (request: Request, { params }: { params: Promise<{ publisher: string }> }) => {
+  const { publisher } = await params;
+  const { searchParams } = new URL(request.url);
+  const cluster = Number.parseInt(searchParams.get("cluster") ?? Cluster.Pythnet.toString()) as Cluster;
+
+  // check if cluster is valid
+  if (cluster && !Object.values(Cluster).includes(cluster)) {
+    return NextResponse.json({ error: "Invalid cluster" }, { status: 400 });
+  }
+
+  if (!publisher) {
+    return NextResponse.json({ error: "Publisher is required" }, { status: 400 });
+  } 
+
+  const feeds = await getFeedsCached(cluster);
+
+  const filteredFeeds = feeds.filter((feed) => feed.price.priceComponents.some((c) => c.publisher === publisher));
+
+  return new Response(stringify(filteredFeeds), {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  });
+};

apps/insights/src/app/api/pyth/get-feeds/[symbol]/route.ts

@@ -0,0 +1,25 @@
+import { NextResponse } from "next/server";
+import { stringify } from 'superjson';
+
+import { getFeedsCached } from "../../../../../server/pyth/get-feeds";
+import { Cluster } from "../../../../../services/pyth";
+
+export const GET = async (request: Request, { params }: { params: Promise<{ symbol: string }> }) => {
+  const { symbol } = await params;
+  const { searchParams } = new URL(request.url);
+  const cluster = Number.parseInt(searchParams.get("cluster") ?? Cluster.Pythnet.toString()) as Cluster;
+
+  // check if cluster is valid
+  if (cluster && !Object.values(Cluster).includes(cluster)) {
+    return NextResponse.json({ error: "Invalid cluster" }, { status: 400 });
+  }
+
+  const feeds = await getFeedsCached(cluster);
+  const feed = feeds.find((feed) => feed.symbol === symbol);
+
+  return new Response(stringify(feed), {
+    headers: {
+      'Content-Type': 'application/json',
+    }
+  });
+};

apps/insights/src/app/api/pyth/get-feeds/route.ts

@@ -0,0 +1,33 @@
+import { NextResponse } from "next/server";
+import { stringify } from 'superjson';
+import type { z } from 'zod';
+
+import { getFeedsCached } from "../../../../server/pyth/get-feeds";
+import { Cluster, priceFeedsSchema } from "../../../../services/pyth";
+
+export const GET = async (request: Request) => {
+  // get cluster from query params
+  const { searchParams } = new URL(request.url);
+  const excludePriceComponents = searchParams.get("excludePriceComponents") === "true";
+  const cluster = Number.parseInt(searchParams.get("cluster") ?? Cluster.Pythnet.toString()) as Cluster;
+
+  // check if cluster is valid
+  if (cluster && !Object.values(Cluster).includes(cluster)) {
+    return NextResponse.json({ error: "Invalid cluster" }, { status: 400 });
+  }
+
+  let feeds = await getFeedsCached(cluster) as Omit<z.infer<typeof priceFeedsSchema>[number], 'price'>[];
+
+  if(excludePriceComponents) {
+    feeds = feeds.map((feed) => ({
+      ...feed,
+      price: undefined,
+    }));
+  }
+
+  return new Response(stringify(feeds), {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  });
+};

apps/insights/src/app/api/pyth/get-publishers/[symbol]/route.ts

@@ -0,0 +1,24 @@
+import { NextResponse } from "next/server";
+
+import { getPublishersForClusterCached } from "../../../../../server/pyth/get-publishers-for-cluster";
+import { Cluster } from "../../../../../services/pyth";
+
+export const GET = async (request: Request, { params }: { params: Promise<{ symbol: string }> }) => {
+  const { symbol } = await params;
+  // get cluster from query params
+  const { searchParams } = new URL(request.url);
+  const cluster = Number.parseInt(searchParams.get("cluster") ?? Cluster.Pythnet.toString()) as Cluster;
+
+  // check if cluster is valid
+  if (cluster && !Object.values(Cluster).includes(cluster)) {
+    return NextResponse.json({ error: "Invalid cluster" }, { status: 400 });
+  }
+
+  if (!symbol) {
+    return NextResponse.json({ error: "Symbol is required" }, { status: 400 });
+  }
+
+  const map = await getPublishersForClusterCached(cluster);
+
+  return NextResponse.json(map[symbol] ?? []);
+};

apps/insights/src/app/price-feeds/[slug]/layout.ts

@@ -2,7 +2,8 @@ import type { Metadata } from "next";
 import { notFound } from "next/navigation";
 import type { ReactNode } from "react";
 
-import { Cluster, getFeeds } from "../../../services/pyth";
+import { getFeedsCached } from "../../../server/pyth/get-feeds";
+import { Cluster } from "../../../services/pyth";
 
 export { PriceFeedLayout as default } from "../../../components/PriceFeed/layout";
 
@@ -19,7 +20,7 @@ export const generateMetadata = async ({
 }: Props): Promise<Metadata> => {
   const [{ slug }, feeds] = await Promise.all([
     params,
-    getFeeds(Cluster.Pythnet),
+    getFeedsCached(Cluster.Pythnet),
   ]);
   const symbol = decodeURIComponent(slug);
   const feed = feeds.find((item) => item.symbol === symbol);

apps/insights/src/components/Overview/index.tsx

@@ -9,7 +9,8 @@ import PriceFeedsLight from "./price-feeds-light.svg";
 import PublishersDark from "./publishers-dark.svg";
 import PublishersLight from "./publishers-light.svg";
 import { TabList } from "./tab-list";
-import { Cluster, getFeeds } from "../../services/pyth";
+import { getFeedsCached } from "../../server/pyth/get-feeds";
+import { Cluster } from "../../services/pyth";
 import {
   totalVolumeTraded,
   activeChains,
@@ -23,8 +24,9 @@ import { FormattedDate } from "../FormattedDate";
 import { FormattedNumber } from "../FormattedNumber";
 
 export const Overview = async () => {
-  const priceFeeds = await getFeeds(Cluster.Pythnet);
+  const priceFeeds = await getFeedsCached(Cluster.Pythnet);
   const today = new Date();
+
   const feedCounts = [
     ...activeFeeds.map(({ date, numFeeds }) => ({
       x: date,
@@ -37,6 +39,7 @@ export const Overview = async () => {
       y: priceFeeds.length,
     },
   ];
+
   return (
     <div className={styles.overview}>
       <h1 className={styles.header}>Overview</h1>

apps/insights/src/components/PriceFeed/get-feed.tsx

@@ -1,20 +1,31 @@
 import { notFound } from "next/navigation";
+import { parse } from "superjson";
+import { z } from "zod";
 
-import { Cluster, getFeeds } from "../../services/pyth";
+import { PUBLIC_URL, VERCEL_AUTOMATION_BYPASS_SECRET } from '../../config/server';
+import { getFeedForSymbolCached } from '../../server/pyth';
+import { Cluster, priceFeedsSchema } from "../../services/pyth";
+import { DEFAULT_CACHE_TTL } from '../../utils/cache';
 
 export const getFeed = async (params: Promise<{ slug: string }>) => {
-  "use cache";
+  const data = await fetch(`${PUBLIC_URL}/api/pyth/get-feeds?cluster=${Cluster.Pythnet.toString()}&excludePriceComponents=true`, {
+    next: {
+      revalidate: DEFAULT_CACHE_TTL,
+    },
+    headers: {
+      'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
+    },
+  });
+  const dataJson = await data.text();
+  const feeds: z.infer<typeof priceFeedsSchema> = parse(dataJson);
 
-  const [{ slug }, feeds] = await Promise.all([params, getPythnetFeeds()]);
+  const { slug } = await params;
   const symbol = decodeURIComponent(slug);
+  const feed = await getFeedForSymbolCached({symbol, cluster: Cluster.Pythnet});
+
   return {
     feeds,
-    feed: feeds.find((item) => item.symbol === symbol) ?? notFound(),
+    feed: feed ?? notFound(),
     symbol,
   } as const;
 };
-
-const getPythnetFeeds = async () => {
-  "use cache";
-  return getFeeds(Cluster.Pythnet);
-};

apps/insights/src/components/PriceFeed/publishers.tsx

@@ -1,12 +1,11 @@
 import { lookup as lookupPublisher } from "@pythnetwork/known-publishers";
 import { notFound } from "next/navigation";
 
-import { getRankingsBySymbol } from "../../services/clickhouse";
+import { getRankingsBySymbolCached } from '../../server/clickhouse';
+import { getFeedForSymbolCached, getPublishersForFeedCached } from "../../server/pyth";
 import {
   Cluster,
   ClusterToName,
-  getFeeds,
-  getPublishersForFeed,
 } from "../../services/pyth";
 import { getStatus } from "../../status";
 import { PublisherIcon } from "../PublisherIcon";
@@ -19,24 +18,23 @@ type Props = {
   }>;
 };
 
+
 export const Publishers = async ({ params }: Props) => {
   const { slug } = await params;
   const symbol = decodeURIComponent(slug);
+
   const [
-    pythnetFeeds,
-    pythtestConformanceFeeds,
+    feed,
+    testFeed,
     pythnetPublishers,
     pythtestConformancePublishers,
   ] = await Promise.all([
-    getFeeds(Cluster.Pythnet),
-    getFeeds(Cluster.PythtestConformance),
+    getFeedForSymbolCached({symbol, cluster: Cluster.Pythnet}),
+    getFeedForSymbolCached({symbol, cluster: Cluster.PythtestConformance}),
     getPublishers(Cluster.Pythnet, symbol),
     getPublishers(Cluster.PythtestConformance, symbol),
   ]);
-  const feed = pythnetFeeds.find((feed) => feed.symbol === symbol);
-  const testFeed = pythtestConformanceFeeds.find(
-    (feed) => feed.symbol === symbol,
-  );
+
   const publishers = [...pythnetPublishers, ...pythtestConformancePublishers];
   const metricsTime = pythnetPublishers.find(
     (publisher) => publisher.ranking !== undefined,
@@ -86,13 +84,14 @@ export const Publishers = async ({ params }: Props) => {
 export const PublishersLoading = () => <PublishersCard isLoading />;
 
 const getPublishers = async (cluster: Cluster, symbol: string) => {
+
   const [publishers, rankings] = await Promise.all([
-    getPublishersForFeed(cluster, symbol),
-    getRankingsBySymbol(symbol),
+    getPublishersForFeedCached(cluster, symbol),
+    getRankingsBySymbolCached(symbol),
   ]);
 
   return (
-    publishers?.map((publisher) => {
+    publishers.map((publisher) => {
       const ranking = rankings.find(
         (ranking) =>
           ranking.publisher === publisher &&
@@ -106,6 +105,6 @@ const getPublishers = async (cluster: Cluster, symbol: string) => {
         cluster,
         knownPublisher: lookupPublisher(publisher),
       };
-    }) ?? []
+    })
   );
 };

apps/insights/src/components/PriceFeeds/index.tsx

@@ -19,7 +19,8 @@ import { AssetClassTable } from "./asset-class-table";
 import { ComingSoonList } from "./coming-soon-list";
 import styles from "./index.module.scss";
 import { PriceFeedsCard } from "./price-feeds-card";
-import { Cluster, getFeeds } from "../../services/pyth";
+import { getFeedsCached } from "../../server/pyth/get-feeds";
+import { Cluster } from "../../services/pyth";
 import { priceFeeds as priceFeedsStaticConfig } from "../../static-data/price-feeds";
 import { activeChains } from "../../static-data/stats";
 import { Cards } from "../Cards";
@@ -289,7 +290,7 @@ const FeaturedFeedsCard = <T extends ElementType>({
 );
 
 const getPriceFeeds = async () => {
-  const priceFeeds = await getFeeds(Cluster.Pythnet);
+  const priceFeeds = await getFeedsCached(Cluster.Pythnet);
   const activeFeeds = priceFeeds.filter((feed) => isActive(feed));
   const comingSoon = priceFeeds.filter((feed) => !isActive(feed));
   return { activeFeeds, comingSoon };

apps/insights/src/components/Publisher/get-price-feeds.tsx

@@ -1,12 +1,12 @@
-import { getRankingsByPublisher } from "../../services/clickhouse";
-import type { Cluster } from "../../services/pyth";
-import { ClusterToName, getFeedsForPublisher } from "../../services/pyth";
+import { getRankingsByPublisherCached } from '../../server/clickhouse';
+import { getFeedsForPublisherCached } from "../../server/pyth";
+import { Cluster, ClusterToName } from "../../services/pyth";
 import { getStatus } from "../../status";
 
 export const getPriceFeeds = async (cluster: Cluster, key: string) => {
   const [feeds, rankings] = await Promise.all([
-    getFeedsForPublisher(cluster, key),
-    getRankingsByPublisher(key),
+    getFeedsForPublisherCached(cluster, key),
+    getRankingsByPublisherCached(key),
   ]);
   return feeds.map((feed) => {
     const ranking = rankings.find(

apps/insights/src/components/Publisher/layout.tsx

@@ -16,10 +16,10 @@ import type { ReactNode } from "react";
 import { Suspense } from "react";
 
 import {
-  getPublisherRankingHistory,
-  getPublisherAverageScoreHistory,
-  getPublishers,
-} from "../../services/clickhouse";
+  getPublishersCached,
+  getPublisherRankingHistoryCached,
+  getPublisherAverageScoreHistoryCached,
+} from "../../server/clickhouse";
 import { getPublisherCaps } from "../../services/hermes";
 import { ClusterToName, parseCluster, Cluster } from "../../services/pyth";
 import { getPublisherPoolData } from "../../services/staking";
@@ -150,10 +150,7 @@ const RankingCard = async ({
   cluster: Cluster;
   publisherKey: string;
 }) => {
-  const rankingHistory = await getPublisherRankingHistory(
-    cluster,
-    publisherKey,
-  );
+  const rankingHistory = await getPublisherRankingHistoryCached({ cluster, key: publisherKey });
   return <RankingCardImpl rankingHistory={rankingHistory} />;
 };
 
@@ -234,10 +231,8 @@ const ScoreCard = async ({
   cluster: Cluster;
   publisherKey: string;
 }) => {
-  const averageScoreHistory = await getPublisherAverageScoreHistory(
-    cluster,
-    publisherKey,
-  );
+  const averageScoreHistory = await getPublisherAverageScoreHistoryCached({ cluster, key: publisherKey });
+
   return <ScoreCardImpl averageScoreHistory={averageScoreHistory} />;
 };
 
@@ -338,7 +333,7 @@ const ActiveFeedsCard = async ({
   publisherKey: string;
 }) => {
   const [publishers, priceFeeds] = await Promise.all([
-    getPublishers(cluster),
+    getPublishersCached(cluster),
     getPriceFeeds(cluster, publisherKey),
   ]);
   const publisher = publishers.find(