feat: added fetch routes

alexcambose · alexcambose · commit 0682c73ca432 · 2025-08-12T15:26:08.000+02:00
diff --git a/apps/insights/src/app/api/pyth/get-feeds/[symbol]/route.ts b/apps/insights/src/app/api/pyth/get-feeds/[symbol]/route.ts
@@ -0,0 +1,16 @@
+import { stringify } from 'superjson';
+
+import { getFeedsCached } from "../../../../../server/pyth/get-feeds";
+import { Cluster } from "../../../../../services/pyth";
+
+export const GET = async (_: Request, { params }: { params: Promise<{ symbol: string }> }) => {
+  const { symbol } = await params;
+
+  const feeds = await getFeedsCached(Cluster.Pythnet);
+  const feed = feeds.find((feed) => feed.symbol === symbol);
+  return new Response(stringify(feed), {
+    headers: {
+      'Content-Type': 'application/json',
+    }
+  });
+};
diff --git a/apps/insights/src/app/api/pyth/get-feeds/route.ts b/apps/insights/src/app/api/pyth/get-feeds/route.ts
@@ -0,0 +1,30 @@
+import { NextResponse } from "next/server";
+import { stringify } from 'superjson';
+import type { z } from 'zod';
+
+import { getFeedsCached } from "../../../../server/pyth/get-feeds";
+import { Cluster, priceFeedsSchema } from "../../../../services/pyth";
+
+export const GET = async (request: Request) => {
+  // get cluster from query params
+  const { searchParams } = new URL(request.url);
+  const excludePriceComponents = searchParams.get("excludePriceComponents") === "true";
+  const cluster = Number.parseInt(searchParams.get("cluster") ?? Cluster.Pythnet.toString()) as Cluster;
+  // check if cluster is valid
+  if (cluster && !Object.values(Cluster).includes(cluster)) {
+    return NextResponse.json({ error: "Invalid cluster" }, { status: 400 });
+  }
+
+  let feeds = await getFeedsCached(cluster) as Omit<z.infer<typeof priceFeedsSchema>[number], 'price'>[];
+  if(excludePriceComponents) {
+    feeds = feeds.map((feed) => ({
+      ...feed,
+      price: undefined,
+    }));
+  }
+  return new Response(stringify(feeds), {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  });
+};
diff --git a/apps/insights/src/app/api/pyth/get-publishers/route.ts b/apps/insights/src/app/api/pyth/get-publishers/route.ts
@@ -0,0 +1,21 @@
+import { NextResponse } from "next/server";
+import { stringify } from 'superjson';
+
+import { getPublishersForCluster } from "../../../../server/pyth/get-publishers-for-cluster";
+import { Cluster } from "../../../../services/pyth";
+
+export const GET = async (request: Request) => {
+  // get cluster from query params
+  const { searchParams } = new URL(request.url);
+  const cluster = Number.parseInt(searchParams.get("cluster") ?? Cluster.Pythnet.toString()) as Cluster;
+  // check if cluster is valid
+  if (cluster && !Object.values(Cluster).includes(cluster)) {
+    return NextResponse.json({ error: "Invalid cluster" }, { status: 400 });
+  }
+  const publishers = await getPublishersForCluster(cluster);
+  return new Response(stringify(publishers), {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  });
+};
diff --git a/apps/insights/src/components/Overview/index.tsx b/apps/insights/src/components/Overview/index.tsx
@@ -24,6 +24,8 @@ import { FormattedDate } from "../FormattedDate";
 import { FormattedNumber } from "../FormattedNumber";
 
 export const Overview = async () => {
+  // eslint-disable-next-line no-console
+  console.log('overview');
   const priceFeeds = await getFeeds(Cluster.Pythnet);
   const today = new Date();
   const feedCounts = [
diff --git a/apps/insights/src/components/PriceFeed/get-feed.tsx b/apps/insights/src/components/PriceFeed/get-feed.tsx
@@ -1,14 +1,37 @@
-import { notFound } from "next/navigation";
+import { parse } from "superjson";
+import { z } from "zod";
 
-import { getFeeds } from "../../server/pyth/get-feeds";
-import { Cluster } from "../../services/pyth";
+import { PUBLIC_URL, VERCEL_AUTOMATION_BYPASS_SECRET } from '../../config/server';
+import { Cluster, priceFeedsSchema } from "../../services/pyth";
 
 export const getFeed = async (params: Promise<{ slug: string }>) => {
-  const [{ slug }, feeds] = await Promise.all([params, getFeeds(Cluster.Pythnet)]);
+  const data = await fetch(`${PUBLIC_URL}/api/pyth/get-feeds?cluster=${Cluster.Pythnet.toString()}&excludePriceComponents=true`, {
+    next: {
+      revalidate: 1000 * 60 * 60 * 24,
+    },
+    headers: {
+      'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
+    },
+  });
+  const dataJson = await data.text();
+  const feeds: z.infer<typeof priceFeedsSchema> = parse(dataJson);
+
+  const { slug } = await params;
   const symbol = decodeURIComponent(slug);
+  const feed = await fetch(`${PUBLIC_URL}/api/pyth/get-feeds/${encodeURIComponent(symbol)}`, {
+    next: {
+      revalidate: 1000 * 60 * 60 * 24,
+    },
+     headers: {
+      'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
+    },
+  });
+  const feedJson = await feed.text();
+  const feedData: z.infer<typeof priceFeedsSchema>[0] = parse(feedJson);
+
   return {
     feeds,
-    feed: feeds.find((item) => item.symbol === symbol) ?? notFound(),
+    feed: feedData,
     symbol,
   } as const;
 };
diff --git a/apps/insights/src/components/PriceFeeds/index.tsx b/apps/insights/src/components/PriceFeeds/index.tsx
@@ -290,6 +290,8 @@ const FeaturedFeedsCard = <T extends ElementType>({
 );
 
 const getPriceFeeds = async () => {
+  // eslint-disable-next-line no-console
+  console.log('getPriceFeeds');
   const priceFeeds = await getFeeds(Cluster.Pythnet);
   const activeFeeds = priceFeeds.filter((feed) => isActive(feed));
   const comingSoon = priceFeeds.filter((feed) => !isActive(feed));
diff --git a/apps/insights/src/components/Root/index.tsx b/apps/insights/src/components/Root/index.tsx
@@ -75,6 +75,8 @@ const getPublishersForSearchDialog = async (cluster: Cluster) => {
 };
 
 const getFeedsForSearchDialog = async (cluster: Cluster) => {
+  // eslint-disable-next-line no-console
+  console.log('getFeedsForSearchDialog');
   const feeds = await getFeeds(cluster);
 
   return feeds.map((feed) => ({
diff --git a/apps/insights/src/config/server.ts b/apps/insights/src/config/server.ts
@@ -31,6 +31,7 @@ const getEnvOrDefault = (key: string, defaultValue: string) =>
  * Indicates that this server is the live customer-facing production server.
  */
 export const IS_PRODUCTION_SERVER = process.env.VERCEL_ENV === "production";
+export const IS_PREVIEW_SERVER = process.env.VERCEL_ENV === "preview";
 
 const defaultInProduction = IS_PRODUCTION_SERVER
   ? getEnvOrDefault
@@ -59,19 +60,35 @@ export const ENABLE_ACCESSIBILITY_REPORTING =
   !IS_PRODUCTION_SERVER && !process.env.DISABLE_ACCESSIBILITY_REPORTING;
 
 
-export async function getRedis(): Promise<Redis> {
+let redisClient: Redis | undefined;
+
+export function getRedis(): Redis {
   const host = process.env.REDIS_HOST;
   const port = process.env.REDIS_PORT;
   const password = process.env.REDIS_PASSWORD;
   if (!host || !port) {
     throw new Error('REDIS_HOST, and REDIS_PORT must be set');
   }
-  const client = new Redis({ 
+  redisClient ??= new Redis({ 
     username: 'default',
     password: password ?? '',
     host,
     port: Number.parseInt(port),
-  });
-  await client.set('test2', Math.random().toString());
-  return client;
-}
+    });
+  return redisClient;
+}
+
+export const PUBLIC_URL = (() => {
+  if (IS_PRODUCTION_SERVER) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion, turbo/no-undeclared-env-vars
+    return `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL!}`;
+  } else if (IS_PREVIEW_SERVER) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion, turbo/no-undeclared-env-vars
+    return `https://${process.env.VERCEL_URL!}`;
+  } else {
+    return `http://localhost:3003`;
+  }
+})();
+
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion, turbo/no-undeclared-env-vars
+export const VERCEL_AUTOMATION_BYPASS_SECRET = process.env.VERCEL_AUTOMATION_BYPASS_SECRET!;
diff --git a/apps/insights/src/server/clickhouse.ts b/apps/insights/src/server/clickhouse.ts
@@ -1,45 +1,22 @@
-
 import { getPublisherAverageScoreHistory, getPublisherRankingHistory, getPublishers, getRankingsByPublisher, getRankingsBySymbol } from "../services/clickhouse";
 import { Cluster } from "../services/pyth";
-import { createChunkedCacheFetcher, fetchAllChunks } from '../utils/cache';
-
-
-const _getRankingsBySymbol = createChunkedCacheFetcher(async (symbol: string) => {
-  return getRankingsBySymbol(symbol);
-}, 'getRankingsBySymbol');
 
 export const getRankingsBySymbolCached = async (symbol: string) => {
-    return fetchAllChunks<ReturnType<typeof getRankingsBySymbol>, [string]>(_getRankingsBySymbol, symbol);
+    return getRankingsBySymbol(symbol);
 };
 
-const _getRankingsByPublisher = createChunkedCacheFetcher(async (publisherKey: string) => {
-  return getRankingsByPublisher(publisherKey);
-}, 'getRankingsByPublisher');
-
 export const getRankingsByPublisherCached = async (publisherKey: string) => {
-    return fetchAllChunks<ReturnType<typeof getRankingsByPublisher>, [string]>(_getRankingsByPublisher, publisherKey);
+    return getRankingsByPublisher(publisherKey);
 };
 
-const _getPublisherAverageScoreHistory = createChunkedCacheFetcher(async (cluster: Cluster, key: string) => {
-  return getPublisherAverageScoreHistory(cluster, key);
-}, 'getPublisherAverageScoreHistory');
-
 export const getPublisherAverageScoreHistoryCached = async (cluster: Cluster, key: string) => {
-    return fetchAllChunks<ReturnType<typeof getPublisherAverageScoreHistory>, [Cluster, string]>(_getPublisherAverageScoreHistory, cluster, key);
+    return getPublisherAverageScoreHistory(cluster, key);
 };
 
-const _getPublisherRankingHistory = createChunkedCacheFetcher(async (cluster: Cluster, key: string) => {
-  return getPublisherRankingHistory(cluster, key);
-}, 'getPublisherRankingHistory');
-
 export const getPublisherRankingHistoryCached = async (cluster: Cluster, key: string) => {
-    return fetchAllChunks<ReturnType<typeof getPublisherRankingHistory>, [Cluster, string]>(_getPublisherRankingHistory, cluster, key);
+    return getPublisherRankingHistory(cluster, key);
 };
 
-const _getPublishers = createChunkedCacheFetcher(async (cluster: Cluster) => {
-  return getPublishers(cluster);
-}, 'getPublishers');
-
 export const getPublishersCached = async (cluster: Cluster) => {
-    return fetchAllChunks<ReturnType<typeof getPublishers>, [Cluster]>(_getPublishers, cluster);
+    return getPublishers(cluster);
 }
diff --git a/apps/insights/src/server/pyth.ts b/apps/insights/src/server/pyth.ts
@@ -15,6 +15,8 @@ export async function getFeedsForPublisherCached(
   cluster: Cluster,
   publisher: string
 ) {
+  // eslint-disable-next-line no-console
+  console.log('getFeedsForPublisherCached');
   const data = await getFeeds(cluster);
   return priceFeedsSchema.parse(
     data.filter(({ price }) =>
diff --git a/apps/insights/src/server/pyth/get-feeds.ts b/apps/insights/src/server/pyth/get-feeds.ts
@@ -1,6 +1,10 @@
+// disable file
+//  @typescript-eslint/no-unused-vars
+import { z } from 'zod';
+
 
 import { getPythMetadata } from './get-metadata';
-import { Cluster, priceFeedsSchema } from '../../services/pyth';
+import { Cluster, priceFeedsSchema } from "../../services/pyth";
 import { redisCache } from '../../utils/cache';
 
 
@@ -32,24 +36,33 @@ const _getFeeds = async (cluster: Cluster) => {
   return priceFeedsSchema.parse(filtered);
 };
 
-export const getFeeds = redisCache.define(
+
+export const getFeedsCached = redisCache.define(
   "getFeeds",
   {
     ttl: 1000 * 60 * 60 * 24,
-    references: (cluster: Cluster) => {
-      return[`pyth:feeds:${cluster.toString()}`];
-    },
   },
   _getFeeds,
 ).getFeeds;
 
 
+
+
+export const getFeeds = async (cluster: Cluster): Promise<z.infer<typeof priceFeedsSchema>> => {
+  // eslint-disable-next-line no-console
+  console.log('getFeeds function called');
+  return getFeedsCached(cluster);
+};
+
 // const _getFeedsBySymbol = async (cluster: Cluster, symbol: string) => {
 //   const feeds = await getFeeds(cluster);
 //   return feeds.find((feed) => feed.symbol === symbol);
 // };
 
 // export const getFeedsBySymbol = redisCache.define(
 //   "getFeedsBySymbol",
+//   {
+//     ttl: 1000 * 60 * 60 * 24,
+//   },
 //   _getFeedsBySymbol,
 // ).getFeedsBySymbol;
diff --git a/apps/insights/src/utils/cache.ts b/apps/insights/src/utils/cache.ts
@@ -1,68 +1,14 @@
 import type { Cache as ACDCache } from "async-cache-dedupe";
 import { createCache } from "async-cache-dedupe";
-import { unstable_cache } from "next/cache";
-import { stringify, parse } from "superjson";
 
 import { getRedis } from '../config/server';
 
-type CacheFetcher<Args extends unknown[]> = (...args: Args) => Promise<string>;
-
-/**
- * Wraps an async function that returns data of type T, and caches it in chunks.
- * @param keyCacheBase - base cache key prefix
- * @param fetchFullData - async function returning the full data T for given args except chunk number
- * @returns a new async function that fetches chunked cached data with the same args plus optional chunk number
- */
-export function createChunkedCacheFetcher<T, Args extends unknown[]>(
-  fetchFullData: (...args: Args) => Promise<T>,
-  key: string,
-): CacheFetcher<Args> {
-  return unstable_cache(
-    async (...args: Args) => {
-      const fullData = await fetchFullData(...args);
-      const serialized = stringify(fullData);
-
-
-      return serialized;
-    },
-    [key],
-    { revalidate: false }
-  );
-}
-
-/**
- * Utility function to fetch all chunks and combine them transparently
- * @param fetcher - cached fetcher returned by createChunkedCacheFetcher
- * @param args - args to the cached fetcher except chunk number
- */
-export async function fetchAllChunks<T, Args extends unknown[]>(
-  fetcher: CacheFetcher<Args>,
-  ...args: Args
-): Promise<T> {
-  const firstChunkData = await fetcher(...args);
-
-  const fullString = firstChunkData;
-  return parse(fullString);
-}
-
-
-export const timeFunction = async <T>(fn: () => Promise<T>, name: string) => {
-  const start = Date.now();
-  const result = await fn();
-  const end = Date.now();
-  // eslint-disable-next-line no-console
-  console.info(`${name} took ${(end - start).toString()}ms`);
-  return result;
-}
-
-
 // L2-backed cache: in-memory LRU (L1) + Redis (L2)
 export const redisCache: ACDCache = createCache({
   storage: {
     type: "redis",
     options: {
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-      client: await getRedis(),
+      client: getRedis(),
     },
   },
 });
diff --git a/apps/insights/turbo.json b/apps/insights/turbo.json
@@ -4,7 +4,9 @@
   "tasks": {
     "build:vercel": {
       "env": [
+        "VERCEL_AUTOMATION_BYPASS_SECRET",
         "VERCEL_ENV",
+        "VERCEL_URL",
         "GOOGLE_ANALYTICS_ID",
         "AMPLITUDE_API_KEY",
         "CLICKHOUSE_URL",
