chore: added logging

Author: alexcambose

apps/insights/src/server/clickhouse.ts

@@ -1,45 +1,55 @@
 
 import { getPublisherAverageScoreHistory, getPublisherRankingHistory, getPublishers, getRankingsByPublisher, getRankingsBySymbol } from "../services/clickhouse";
 import { Cluster } from "../services/pyth";
-import { createChunkedCacheFetcher, fetchAllChunks } from '../utils/cache';
+import { createChunkedCacheFetcher, fetchAllChunks, timeFunction } from '../utils/cache';
 
 
 const _getRankingsBySymbol = createChunkedCacheFetcher(async (symbol: string) => {
   return getRankingsBySymbol(symbol);
 }, 'getRankingsBySymbol');
 
 export const getRankingsBySymbolCached = async (symbol: string) => {
-  return fetchAllChunks<ReturnType<typeof getRankingsBySymbol>, [string]>(_getRankingsBySymbol, symbol);
+  return timeFunction(async () => {
+    return fetchAllChunks<ReturnType<typeof getRankingsBySymbol>, [string]>(_getRankingsBySymbol, symbol);
+  }, 'getRankingsBySymbolCached');
 };
 
 const _getRankingsByPublisher = createChunkedCacheFetcher(async (publisherKey: string) => {
   return getRankingsByPublisher(publisherKey);
 }, 'getRankingsByPublisher');
 
 export const getRankingsByPublisherCached = async (publisherKey: string) => {
-  return fetchAllChunks<ReturnType<typeof getRankingsByPublisher>, [string]>(_getRankingsByPublisher, publisherKey);
+  return timeFunction(async () => {
+    return fetchAllChunks<ReturnType<typeof getRankingsByPublisher>, [string]>(_getRankingsByPublisher, publisherKey);
+  }, 'getRankingsByPublisherCached');
 };
 
 const _getPublisherAverageScoreHistory = createChunkedCacheFetcher(async (cluster: Cluster, key: string) => {
   return getPublisherAverageScoreHistory(cluster, key);
 }, 'getPublisherAverageScoreHistory');
 
 export const getPublisherAverageScoreHistoryCached = async (cluster: Cluster, key: string) => {
-  return fetchAllChunks<ReturnType<typeof getPublisherAverageScoreHistory>, [Cluster, string]>(_getPublisherAverageScoreHistory, cluster, key);
+  return timeFunction(async () => {
+    return fetchAllChunks<ReturnType<typeof getPublisherAverageScoreHistory>, [Cluster, string]>(_getPublisherAverageScoreHistory, cluster, key);
+  }, 'getPublisherAverageScoreHistoryCached');
 };
 
 const _getPublisherRankingHistory = createChunkedCacheFetcher(async (cluster: Cluster, key: string) => {
   return getPublisherRankingHistory(cluster, key);
 }, 'getPublisherRankingHistory');
 
 export const getPublisherRankingHistoryCached = async (cluster: Cluster, key: string) => {
-  return fetchAllChunks<ReturnType<typeof getPublisherRankingHistory>, [Cluster, string]>(_getPublisherRankingHistory, cluster, key);
+  return timeFunction(async () => {
+    return fetchAllChunks<ReturnType<typeof getPublisherRankingHistory>, [Cluster, string]>(_getPublisherRankingHistory, cluster, key);
+  }, 'getPublisherRankingHistoryCached');
 };
 
 const _getPublishers = createChunkedCacheFetcher(async (cluster: Cluster) => {
   return getPublishers(cluster);
 }, 'getPublishers');
 
 export const getPublishersCached = async (cluster: Cluster) => {
-  return fetchAllChunks<ReturnType<typeof getPublishers>, [Cluster]>(_getPublishers, cluster);
+  return timeFunction(async () => {
+    return fetchAllChunks<ReturnType<typeof getPublishers>, [Cluster]>(_getPublishers, cluster);
+  }, 'getPublishersCached');
 }

apps/insights/src/server/pyth.ts

@@ -2,34 +2,52 @@ import type { PythHttpClientResult } from '@pythnetwork/client/lib/PythHttpClien
 import type { z } from 'zod';
 
 import { Cluster, clients, priceFeedsSchema } from "../services/pyth";
-import { createChunkedCacheFetcher, fetchAllChunks } from '../utils/cache';
+import { createChunkedCacheFetcher, fetchAllChunks, timeFunction } from '../utils/cache';
 
 
-type CachedData = {
-  data: PythHttpClientResult;
-  timestamp: number;
+type CacheEntry = {
+  data?: PythHttpClientResult;
+  timestamp?: number;
+  promise?: Promise<PythHttpClientResult>;
 };
 
-const dataCache = new Map<Cluster, CachedData>();
+const dataCache = new Map<Cluster, CacheEntry>();
 const CACHE_EXPIRY_MS = 24 * 60 * 60; // 1 day in seconds
 
 const getDataCached = async (cluster: Cluster) => {
   const now = Date.now();
   const cached = dataCache.get(cluster);
 
   // Check if cache exists and is not expired
-  if (cached && (now - cached.timestamp) < CACHE_EXPIRY_MS * 1000) {
+  if (cached?.data && cached.timestamp && (now - cached.timestamp) < CACHE_EXPIRY_MS * 1000) {
     return cached.data;
   }
 
-  // Fetch fresh data
-  const data = await clients[cluster].getData();
-  dataCache.set(cluster, { data, timestamp: now });
-  return data;
+  // Check if there's already a pending request
+  if (cached?.promise) {
+    return cached.promise;
+  }
+  
+  // eslint-disable-next-line no-console
+  console.log('fetching fresh FULL data');
+  
+  // Create a new promise for the request
+  const promise = clients[cluster].getData().then((data) => {
+    // Store the result in cache
+    dataCache.set(cluster, { data, timestamp: now });
+    return data;
+  });
+  
+  // Store the promise in cache to prevent duplicate requests
+  dataCache.set(cluster, { promise });
+  
+  return promise;
 };
 
 const fetchFeeds = createChunkedCacheFetcher(async (cluster: Cluster) => {
   const unfilteredData = await getDataCached(cluster);
+  // eslint-disable-next-line no-console
+  console.log('fetchFeeds called');
   const filteredData = unfilteredData.symbols
       .filter(
         (symbol) =>
@@ -54,6 +72,8 @@ const fetchFeeds = createChunkedCacheFetcher(async (cluster: Cluster) => {
 
 const fetchPublishers = createChunkedCacheFetcher(async (cluster: Cluster) => {
   const data = await getDataCached(cluster);
+  // eslint-disable-next-line no-console
+  console.log('fetchPublishers called');
   const result: Record<string, string[]> = {};
   for (const key of data.productPrice.keys()) {
     const price = data.productPrice.get(key);
@@ -63,19 +83,25 @@ const fetchPublishers = createChunkedCacheFetcher(async (cluster: Cluster) => {
 }, 'fetchPublishers');
 
 export const getFeedsCached = async (cluster: Cluster) => {
-  return fetchAllChunks<z.infer<typeof priceFeedsSchema>, [Cluster]>(fetchFeeds,  cluster)
+  return timeFunction(async () => {
+    return fetchAllChunks<z.infer<typeof priceFeedsSchema>, [Cluster]>(fetchFeeds,  cluster);
+  }, 'getFeedsCached');
 };
 
 export const getPublishersForFeedCached = async (cluster: Cluster, symbol: string) => {
-  const data = await fetchAllChunks<Record<string, string[]>, [Cluster]>(fetchPublishers, cluster);
+  const data = await timeFunction(async () => {
+    return fetchAllChunks<Record<string, string[]>, [Cluster]>(fetchPublishers, cluster);
+  }, 'getPublishersForFeedCached');
   return data[symbol]
 };
 
 export const getFeedsForPublisherCached = async (
   cluster: Cluster,
   publisher: string
 ) => {
-  const data = await getFeedsCached(cluster); 
+  const data = await timeFunction(async () => {
+    return getFeedsCached(cluster);
+  }, 'getFeedsForPublisherCached');
   return priceFeedsSchema.parse(
     data.filter(({ price }) =>
       price.priceComponents.some(

apps/insights/src/utils/cache.ts

@@ -75,3 +75,13 @@ export async function fetchAllChunks<T, Args extends unknown[]>(
     firstChunkData.chunk + otherChunks.map(({ chunk }) => chunk).join("");
   return superjson.parse(fullString);
 }
+
+
+export const timeFunction = async <T>(fn: () => Promise<T>, name: string) => {
+  const start = Date.now();
+  const result = await fn();
+  const end = Date.now();
+  // eslint-disable-next-line no-console
+  console.info(`${name} took ${(end - start).toString()}ms`);
+  return result;
+}