diff --git a/defi/l2/adapters/ada.ts b/defi/l2/adapters/ada.ts index f6f9144e1f..fa1da99afc 100644 --- a/defi/l2/adapters/ada.ts +++ b/defi/l2/adapters/ada.ts @@ -1,7 +1,7 @@ -import fetch from "node-fetch"; +import { cache } from "@defillama/sdk"; export async function fetchAdaTokens(): Promise { - const res = await fetch(`https://api.muesliswap.com/token-list`).then((r) => r.json()); + const res = await cache.cachedFetch({key: "muesliswap-token-list", endpoint: "https://api.muesliswap.com/token-list"}) const coins = res .filter((c: any) => c.supply.circulating != null) .map((c: any) => ({ diff --git a/defi/l2/adapters/index.ts b/defi/l2/adapters/index.ts index 946151338f..6044d8fe58 100644 --- a/defi/l2/adapters/index.ts +++ b/defi/l2/adapters/index.ts @@ -131,19 +131,19 @@ export const starknet = async (): Promise => { addresses.starknet = data.map((t: any) => t.l2_token_address?.toLowerCase()).filter((t: any) => t != null); return addresses.starknet; }; -export const era = async (): Promise => { - if (addresses.era) return addresses.era; - const { - data: { result: data }, - } = await axios.post("https://mainnet.era.zksync.io", { - method: "zks_getConfirmedTokens", - params: [0, 255], - id: 1, - jsonrpc: "2.0", - }); - addresses.era = data.map((d: any) => d.l2Address.toLowerCase()); - return addresses.era; -}; +// export const era = async (): Promise => { +// if (addresses.era) return addresses.era; +// const { +// data: { result: data }, +// } = await axios.post("https://mainnet.era.zksync.io", { +// method: "zks_getConfirmedTokens", +// params: [0, 255], +// id: 1, +// jsonrpc: "2.0", +// }); +// addresses.era = data.map((d: any) => d.l2Address.toLowerCase()); +// return addresses.era; +// }; export const tron = async (): Promise => { if (!("tron" in addresses)) addresses.tron = []; addresses.tron.push( @@ -177,19 +177,19 @@ export const mode = async (): Promise => { ); return addresses.mode; }; -export const zklink = async (): Promise => { - if (addresses.zklink) return addresses.zklink; - const allTokens = []; - let page = 1; - do { - const { items, meta } = await fetch(`https://explorer-api.zklink.io/tokens?limit=200&page=${page}&key=`); - allTokens.push(...items); - page++; - if (page >= meta.totalPages) break; - } while (page < 100); - addresses.zklink = allTokens.map((d: any) => d.l2Address.toLowerCase()); - return addresses.zklink; -}; +// export const zklink = async (): Promise => { +// if (addresses.zklink) return addresses.zklink; +// const allTokens = []; +// let page = 1; +// do { +// const { items, meta } = await fetch(`https://explorer-api.zklink.io/tokens?limit=200&page=${page}&key=`); +// allTokens.push(...items); +// page++; +// if (page >= meta.totalPages) break; +// } while (page < 100); +// addresses.zklink = allTokens.map((d: any) => d.l2Address.toLowerCase()); +// return addresses.zklink; +// }; export const manta = async (): Promise => { if (addresses.manta) return addresses.manta; const bridge = ( diff --git a/defi/l2/adapters/thirdParty.ts b/defi/l2/adapters/thirdParty.ts index b6828f5114..796033f0d2 100644 --- a/defi/l2/adapters/thirdParty.ts +++ b/defi/l2/adapters/thirdParty.ts @@ -3,16 +3,13 @@ import providers from "@defillama/sdk/build/providers.json"; import { Address } from "@defillama/sdk/build/types"; import { allChainKeys } from "../constants"; import { bridgedTvlMixedCaseChains } from "../../src/utils/shared/constants"; -import fetch from "node-fetch"; import { additional, excluded } from "./manual"; -import axios from "axios"; -import PromisePool from "@supercharge/promise-pool"; +import _ from "lodash"; +import { cachedFetch } from "@defillama/sdk/build/util/cache"; +import runInPromisePool from "@defillama/sdk/build/util/promisePool"; -let bridgePromises: { [bridge: string]: Promise } = {}; const addresses: { [chain: Chain]: Address[] } = {}; allChainKeys.map((c: string) => (addresses[c] = [])); -let doneAdapters: string[] = []; -let mappingDone: boolean = false; const chainMap: { [chain: string]: string } = { binance: "bsc", @@ -25,24 +22,18 @@ Object.keys(providers).map((c: string) => { }); const hyperlane = async (): Promise => { - const bridge = "hyperlane"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = fetch( - "https://raw.githubusercontent.com/Eclipse-Laboratories-Inc/gist/refs/heads/main/hyperlane-assets.json" - ).then((r) => r.json()); - const data = await bridgePromises[bridge]; + const data = await cachedFetch({ + key: "hyperlane-assets", + endpoint: "https://raw.githubusercontent.com/Eclipse-Laboratories-Inc/gist/refs/heads/main/hyperlane-assets.json", + }); + if (Object.keys(data).length == 0) throw new Error("No data or cache found for hyperlane third party"); if (!addresses.eclipse) addresses.eclipse = []; data.map(({ address }: any) => addresses.eclipse.push(address)); - doneAdapters.push(bridge); }; const axelar = async (): Promise => { - const bridge = "axelar"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = fetch("https://api.axelarscan.io/api/getAssets").then((r) => r.json()); - const data = await bridgePromises[bridge]; + const data = await cachedFetch({ key: "axelar-assets", endpoint: "https://api.axelarscan.io/api/getAssets" }); + if (Object.keys(data).length == 0) throw new Error("No data or cache found for axelar third party"); data.map((token: any) => { if (!token.addresses) return; Object.keys(token.addresses).map((chain: string) => { @@ -51,22 +42,18 @@ const axelar = async (): Promise => { if (!allChainKeys.includes(normalizedChain)) return; if (!("address" in token.addresses[chain] && "symbol" in token.addresses[chain])) return; if (!token.addresses[chain].symbol.startsWith("axl")) return; - addresses[normalizedChain].push(token.addresses[chain].address.toLowerCase()); + addresses[normalizedChain].push(token.addresses[chain].address); }); }); - doneAdapters.push(bridge); }; const wormhole = async (): Promise => { - const bridge = "wormhole"; - - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = axios.get( - "https://raw.githubusercontent.com/wormhole-foundation/wormhole-token-list/main/content/by_dest.csv" - ); + const data = await cachedFetch({ + key: "wormhole-token-list", + endpoint: "https://raw.githubusercontent.com/wormhole-foundation/wormhole-token-list/main/content/by_dest.csv", + }); + if (data.length == 0) throw new Error("No data or cache found for wormhole third party"); - const data = (await bridgePromises[bridge]).data; const chainMap: { [ticker: string]: string } = { sol: "solana", eth: "ethereum", @@ -95,43 +82,40 @@ const wormhole = async (): Promise => { if (!addresses[chain]) addresses[chain] = []; addresses[chain].push(rows[3]); }); - doneAdapters.push(bridge); }; const celer = async (): Promise => { - const bridge = "celer"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = fetch("https://cbridge-prod2.celer.app/v2/getTransferConfigsForAll").then((r) => r.json()); - const data = await bridgePromises[bridge]; + const data = await cachedFetch({ + key: "celer-transfer-configs", + endpoint: "https://cbridge-prod2.celer.app/v2/getTransferConfigsForAll", + }); + if (Object.keys(data).length == 0) throw new Error("No data or cache found for celer third party"); data.pegged_pair_configs.map((pp: any) => { const chain = chainIdMap[pp.org_chain_id]; let normalizedChain: string = chain; if (chain in chainMap) normalizedChain = chainMap[chain]; if (!allChainKeys.includes(normalizedChain)) return; if (!addresses[normalizedChain]) addresses[normalizedChain] = []; - addresses[normalizedChain].push(pp.pegged_token.token.address.toLowerCase()); + addresses[normalizedChain].push(pp.pegged_token.token.address); }); - doneAdapters.push(bridge); }; const layerzero = async (): Promise => { - const bridge = "layerzero"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) { - bridgePromises[bridge] = Promise.all([ - fetch( - "https://gist.githubusercontent.com/vrtnd/02b1125edf1afe2baddbf1027157aa31/raw/5cab2009357b1acb8982e6a80e66b64ab7ea1251/mappings.json" - ).then((r) => r.json()), - fetch("https://metadata.layerzero-api.com/v1/metadata").then((r) => r.json()), - ]); - } - const data = await bridgePromises[bridge]; + const data = await Promise.all([ + cachedFetch({ + key: "layerzero-mappings", + endpoint: + "https://gist.githubusercontent.com/vrtnd/02b1125edf1afe2baddbf1027157aa31/raw/5cab2009357b1acb8982e6a80e66b64ab7ea1251/mappings.json", + }), + cachedFetch({ key: "layerzero-metadata", endpoint: "https://metadata.layerzero-api.com/v1/metadata" }), + ]); + if (data[0].length == 0 || Object.keys(data[1]).length == 0) + throw new Error("No data or cache found for layerzero third party"); data[0].map(({ to }: any) => { const [chain, address] = to.split(":"); if (!(chain in addresses)) addresses[chain] = []; - if (!(address in addresses[chain])) addresses[chain].push(address.toLowerCase()); + if (!(address in addresses[chain])) addresses[chain].push(address); }); const nonEvmMapping: { [key: string]: string } = { @@ -173,18 +157,14 @@ const layerzero = async (): Promise => { if (!(chain in addresses)) addresses[chain] = []; addresses[chain].push(...staticTokens[chain]); }); - - doneAdapters.push(bridge); }; const flow = async (): Promise => { - const bridge = "flow"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = fetch( - "https://raw.githubusercontent.com/onflow/assets/refs/heads/main/tokens/outputs/mainnet/token-list.json" - ).then((r) => r.json()); - const data = await bridgePromises[bridge]; + const data = await cachedFetch({ + key: "flow-token-list", + endpoint: "https://raw.githubusercontent.com/onflow/assets/refs/heads/main/tokens/outputs/mainnet/token-list.json", + }); + if (Object.keys(data).length == 0) throw new Error("No data or cache found for flow third party"); data.tokens.map(({ chainId, address, tags }: any) => { const chain = chainIdMap[chainId]; if (!allChainKeys.includes(chain)) return; @@ -192,14 +172,9 @@ const flow = async (): Promise => { if (!(chain in addresses)) addresses[chain] = []; addresses[chain].push(address); }); - - doneAdapters.push(bridge); }; const unit = async (): Promise => { - const bridge = "unit"; - if (doneAdapters.includes(bridge)) return; - const staticTokens: { [chain: string]: string[] } = { hyperliquid: [ "0x9FDBdA0A5e284c32744D2f17Ee5c74B284993463", @@ -213,27 +188,23 @@ const unit = async (): Promise => { if (!(chain in addresses)) addresses[chain] = []; addresses[chain].push(...staticTokens[chain]); }); - - doneAdapters.push(bridge); }; const adapters = { axelar, wormhole, celer, hyperlane, layerzero, flow, unit }; const filteredAddresses: { [chain: Chain]: Address[] } = {}; -const tokenAddresses = async (): Promise<{ [chain: Chain]: Address[] }> => { - await PromisePool.withConcurrency(5) - .for(Object.entries(adapters)) - .process(async ([key, adapter]: any) => { - try { - await adapter(); - } catch (e: any) { - throw new Error(`${key} fails with ${e.message}`); - } - }); - - if (Object.keys(adapters).length == doneAdapters.length && mappingDone) return filteredAddresses; - +const tokenAddresses = _.once(async (): Promise<{ [chain: Chain]: Address[] }> => { + await runInPromisePool({ + items: Object.entries(adapters), + concurrency: 5, + processor: async ([key, adapter]: any) => { + await adapter().catch((e: any) => { + throw new Error(`${key} fails with ${e.message}`); + }); + }}) + + // remove excluded assets and add additional assets, normalize case Object.keys(addresses).map((chain: string) => { let chainAddresses = chain in excluded ? addresses[chain].filter((t: string) => !excluded[chain].includes(t)) : addresses[chain]; @@ -248,8 +219,7 @@ const tokenAddresses = async (): Promise<{ [chain: Chain]: Address[] }> => { filteredAddresses[chain] = [...new Set([...chainAddresses, ...additionalTokens])]; }); - mappingDone = true; return filteredAddresses; -}; +}); -export default tokenAddresses; \ No newline at end of file +export default tokenAddresses; diff --git a/defi/l2/constants.ts b/defi/l2/constants.ts index 5007b0f11a..752e7ec8bd 100644 --- a/defi/l2/constants.ts +++ b/defi/l2/constants.ts @@ -29,7 +29,7 @@ export const chainsWithoutCanonicalBridges: string[] = [ // "aurora", "berachain", "flow", - "somnia", + "somnia", "plasma" ]; @@ -57,7 +57,7 @@ export const canonicalBridgeIds: { [id: string]: Chain } = { "3785": "polygon_zkevm", "3786": "scroll", "3787": "starknet", - "3788": "era", + // "3788": "era", "3813": "alephium", "3861": "rsk", // "3866": "near", @@ -65,18 +65,17 @@ export const canonicalBridgeIds: { [id: string]: Chain } = { "3935": "boba", "3936": "zksync", "4032": "manta", - "4141": "BSquared", + "4141": "bsquared", "4236": "blast", "4237": "mode", - "4335": "zklink", + // "4335": "zklink", "4336": "kinto", "4384": "rss3_vsl", "4438": "degen", "4439": "pulsechain", "4440": "ronin", - "4690": "lorenzo", "4692": "taiko", - "4558": "BOB", + "4558": "bob", "4796": "sanko", "4797": "xai", "4124": "merlin", @@ -95,36 +94,27 @@ export const canonicalBridgeIds: { [id: string]: Chain } = { "5513": "ink", "5538": "swellchain", "5552": "cronos_zkevm", - "5564": "eclipse", + // "5564": "eclipse", "5565": "shape", "5566": "zora", - "5609": "sophon", "5624": "soneium", "5683": "sonic", "5691": "abstract", "5692": "ancient8", - "5693": "cyber", "5694": "fraxtal", "5695": "gravity", "5696": "karak", "5697": "kroma", - "5698": "orderly", "5699": "rari", "5700": "redstone", "5701": "wc", "5702": "zero_network", // "5703": "zkfair", "5732": "sxr", - "5735": "sorare", "5772": "unichain", // "5833": "formnetwork", - "5854": "hemi-l2", "6063": "mxczkevm", "6148": "lens", - "6149": "openzk", - "6150": "treasure", - "6151": "zkcandy", - "6284": "ao", "6424": "soon", "6468": "btnx", "6498": "eventum", @@ -140,8 +130,18 @@ export const protocolBridgeIds: { [chain: string]: Chain } = { "344": "zkswap", "5130": "polynomial", // "5323": "exSat", + "5609": "sophon", + "5693": "cyber", + "5735": "sorare", + "5854": "hemi-l2", "4947": "ignition-fbtc", + "4690": "lorenzo", "4702": "immutable zkevm", + "5698": "orderly", + "6149": "openzk", + "6150": "treasure", + "6151": "zkcandy", + "6284": "ao", "6401": "embr", "6414": "xion", "6438": "echelon_initia", diff --git a/defi/l2/excluded.ts b/defi/l2/excluded.ts index 0778e1c71b..0cd6c3497c 100644 --- a/defi/l2/excluded.ts +++ b/defi/l2/excluded.ts @@ -1,4 +1,4 @@ -import { getPrices } from "./utils"; +import { coins } from "@defillama/sdk"; import { excludedTvlId } from "./constants"; import { multiCall } from "@defillama/sdk/build/abi/abi2"; import { getBlock } from "@defillama/sdk/build/util/blocks"; @@ -54,7 +54,7 @@ export async function getExcludedTvl(timestamp: number) { block: block.number, withMetadata: true, }), - getPrices( + coins.getPrices( uniqueTokens.map((token) => `${chain}:${token}`), timestamp ), diff --git a/defi/l2/incoming.ts b/defi/l2/incoming.ts index f1764ea1d7..15e87eb8b5 100644 --- a/defi/l2/incoming.ts +++ b/defi/l2/incoming.ts @@ -3,7 +3,8 @@ import { Chain } from "@defillama/sdk/build/general"; import BigNumber from "bignumber.js"; import { DollarValues, TokenTvlData } from "./types"; import { geckoSymbols, zero } from "./constants"; -import { fetchBridgeTokenList, fetchSupplies, getPrices } from "./utils"; +import { fetchBridgeTokenList, fetchSupplies } from "./utils"; +import { coins } from "@defillama/sdk"; export async function fetchIncoming(params: { canonical: TokenTvlData; @@ -23,7 +24,7 @@ export async function fetchIncoming(params: { return; } - const prices = await getPrices( + const prices = await coins.getPrices( tokens.map((t: string) => `${chain}:${t}`), timestamp ); diff --git a/defi/l2/index.ts b/defi/l2/index.ts index de91263723..e296897c90 100644 --- a/defi/l2/index.ts +++ b/defi/l2/index.ts @@ -2,6 +2,7 @@ import chainAssets from "../l2/tvl"; import { storeR2JSONString } from "../src/utils/r2"; import { getCurrentUnixTimestamp } from "../src/utils/date"; import storeHistorical from "../l2/storeToDb"; +import { storeChainAssetsV2 } from "./v2"; export default async function storeChainAssets(override: boolean) { const res: any = await chainAssets(override); @@ -11,5 +12,7 @@ export default async function storeChainAssets(override: boolean) { await storeR2JSONString("chainAssets", JSON.stringify(res)); await storeHistorical(res); console.log("chain assets stored"); + await storeChainAssetsV2(override); + console.log("chain assets v2 stored"); process.exit(); } diff --git a/defi/l2/native.ts b/defi/l2/native.ts index 801dda7f5f..b26c2ac4f4 100644 --- a/defi/l2/native.ts +++ b/defi/l2/native.ts @@ -1,15 +1,16 @@ import { getCurrentUnixTimestamp } from "../src/utils/date"; import { fetchAllTokens } from "../src/utils/shared/bridgedTvlPostgres"; -import { McapData, TokenTvlData, DollarValues, CoinsApiData } from "./types"; +import { McapData, TokenTvlData, DollarValues } from "./types"; import { Chain } from "@defillama/sdk/build/general"; import BigNumber from "bignumber.js"; import { Address } from "@defillama/sdk/build/types"; import { geckoSymbols, ownTokens, zero } from "./constants"; -import { getMcaps, getPrices, fetchBridgeTokenList, fetchSupplies } from "./utils"; +import { fetchBridgeTokenList, fetchSupplies } from "./utils"; import { fetchAdaTokens } from "./adapters/ada"; import { nativeWhitelist } from "./adapters/manual"; import { withTimeout } from "../src/utils/shared/withTimeout"; import PromisePool from "@supercharge/promise-pool"; +import { coins } from "@defillama/sdk"; export async function fetchMinted(params: { chains: TokenTvlData; @@ -51,7 +52,7 @@ export async function fetchMinted(params: { console.log(`DBUG start for ${chain}`); // do these in order to lighten rpc, rest load - const prices = await getPrices( + const prices = await coins.getPrices( storedTokens.map((t: string) => (t.startsWith("coingecko:") ? t : `${chain}:${t}`)), timestamp ); @@ -61,7 +62,7 @@ export async function fetchMinted(params: { Object.keys(prices).map((p: string) => { if (p.startsWith("coingecko:")) prices[p].decimals = 0; }); - const mcaps = await getMcaps(Object.keys(prices), timestamp); + const mcaps = await coins.getMcaps(Object.keys(prices), timestamp); console.log(`DBUG mcaps done for ${chain}`); diff --git a/defi/l2/tvl.ts b/defi/l2/tvl.ts index c9134dd8d4..629f8edeed 100644 --- a/defi/l2/tvl.ts +++ b/defi/l2/tvl.ts @@ -5,12 +5,12 @@ import { ChainData, DollarValues, FinalData } from "./types"; import BigNumber from "bignumber.js"; import { allChainKeys, ownTokens, tokenFlowCategories, zero } from "./constants"; import { Chain } from "@defillama/sdk/build/general"; -import { getMcaps } from "./utils"; import { getCurrentUnixTimestamp } from "../src/utils/date"; import { getChainDisplayName } from "../src/utils/normalizeChain"; -import { verifyChanges } from "./test"; +import { verifyChanges } from "./verifyChanges"; import { getExcludedTvl } from "./excluded"; import { saveRawBridgedTvls } from "./raw"; +import { coins } from "@defillama/sdk"; export default async function main(override?: boolean, timestamp?: number) { let symbolMap: { [pk: string]: string | null } = {}; @@ -86,7 +86,7 @@ async function translateToChainData( : `${chain}:${ownTokens[chain].address}` ); const nativeTokenSymbols = Object.keys(ownTokens).map((chain: string) => ownTokens[chain].ticker); - const mcapsPromise = getMcaps(nativeTokenKeys, timestamp); + const mcapsPromise = coins.getMcaps(nativeTokenKeys, timestamp); const nativeTokenTotalValues: any = {}; let translatedData: any = {}; diff --git a/defi/l2/utils.ts b/defi/l2/utils.ts index 570dcc29ad..4eeb9b6f4f 100644 --- a/defi/l2/utils.ts +++ b/defi/l2/utils.ts @@ -1,5 +1,5 @@ import BigNumber from "bignumber.js"; -import { AllProtocols, CoinsApiData, McapsApiData, TokenTvlData } from "./types"; +import { AllProtocols, TokenTvlData } from "./types"; import { canonicalBridgeIds, excludedTvlKeys, geckoSymbols, protocolBridgeIds, zero } from "./constants"; import fetch from "node-fetch"; import { bridgedTvlMixedCaseChains } from "../src/utils/shared/constants"; @@ -60,113 +60,6 @@ async function restCallWrapper(request: () => Promise, retries: number = 8, } throw new Error(`couldnt work ${name} call after retries!`); } -export async function getPrices( - readKeys: string[], - timestamp: number | "now" -): Promise<{ [address: string]: CoinsApiData }> { - if (!readKeys.length) return {}; - const bodies: string[] = []; - for (let i = 0; i < readKeys.length; i += 100) { - const body = { - coins: readKeys.slice(i, i + 100), - } as any; - if (timestamp !== "now") { - body.timestamp = timestamp; - } - bodies.push(JSON.stringify(body)); - } - - const tokenData: any[] = []; - await PromisePool.withConcurrency(10) - .for(bodies) - .process(async (body) => { - const res = await restCallWrapper( - () => - fetch( - `https://coins.llama.fi/prices?source=internal${ - process.env.COINS_KEY ? `?apikey=${process.env.COINS_KEY}` : "" - }`, - { - method: "POST", - body, - headers: { "Content-Type": "application/json" }, - } - ) - .then((r) => r.json()) - .then((r) => - Object.entries(r.coins).map(([PK, value]) => ({ - ...(value as any), - PK, - })) - ), - undefined, - "coin prices" - ); - tokenData.push(res); - }) - .catch((e) => { - throw new Error(`coin prices call failed with ${e}`); - }); - - const aggregatedRes: { [address: string]: CoinsApiData } = {}; - const normalizedReadKeys = readKeys.map((k: string) => k.toLowerCase()); - tokenData.map((batch: CoinsApiData[]) => { - batch.map((a: CoinsApiData) => { - if (!a.PK) return; - const i = normalizedReadKeys.indexOf(a.PK.toLowerCase()); - aggregatedRes[readKeys[i]] = a; - }); - }); - - return aggregatedRes; -} -export async function getMcaps( - readKeys: string[], - timestamp: number | "now" -): Promise<{ [address: string]: McapsApiData }> { - if (!readKeys.length) return {}; - const bodies: string[] = []; - for (let i = 0; i < readKeys.length; i += 100) { - const body = { - coins: readKeys.slice(i, i + 100), - } as any; - if (timestamp !== "now") { - body.timestamp = timestamp; - } - bodies.push(JSON.stringify(body)); - } - - const tokenData: any[] = []; - await PromisePool.withConcurrency(10) - .for(bodies) - .process(async (body) => { - const res = await restCallWrapper( - () => - fetch(`https://coins.llama.fi/mcaps${process.env.COINS_KEY ? `?apikey=${process.env.COINS_KEY}` : ""}`, { - method: "POST", - body, - headers: { "Content-Type": "application/json" }, - }).then((r) => r.json()), - undefined, - "mcaps" - ); - tokenData.push(res); - }) - .catch((e) => { - throw new Error(`coin mcaps call failed with ${e}`); - }); - - const aggregatedRes: { [address: string]: any } = {}; - const normalizedReadKeys = readKeys.map((k: string) => k.toLowerCase()); - tokenData.map((batch: { [address: string]: McapsApiData }[]) => { - Object.keys(batch).map((a: any) => { - if (!batch[a].mcap) return; - const i = normalizedReadKeys.indexOf(a.toLowerCase()); - aggregatedRes[readKeys[i]] = batch[a]; - }); - }); - return aggregatedRes; -} async function getOsmosisSupplies(tokens: string[], timestamp?: number): Promise<{ [token: string]: number }> { if (timestamp) throw new Error(`timestamp incompatible with Osmosis adapter!`); const supplies: { [token: string]: number } = {}; @@ -332,15 +225,15 @@ async function getEVMSupplies( })), abi: "erc20:totalSupply", permitFailure: true, - block: block.block, + block: block?.block, }); contracts.slice(i, i + step).map((c: Address, i: number) => { if (res[i]) supplies[`${chain}:${bridgedTvlMixedCaseChains.includes(chain) ? c : c.toLowerCase()}`] = res[i]; }); - } catch { + } catch (e) { try { process.env.TRON_RPC = process.env.TRON_RPC?.substring(process.env.TRON_RPC.indexOf(",") + 1); - await PromisePool.withConcurrency(2) + await PromisePool.withConcurrency(5) .for(contracts.slice(i, i + step)) .process(async (target) => { const res = await call({ @@ -349,7 +242,7 @@ async function getEVMSupplies( abi: "erc20:totalSupply", block, }).catch(async (e) => { - await sleep(2000); + await sleep(1000); if (chain == "tron") console.log(`${target}:: \t ${e.message}`); }); if (res) @@ -380,9 +273,8 @@ export async function fetchSupplies( } } export async function fetchBridgeTokenList(chain: Chain): Promise { - const j = Object.keys(incomingAssets).indexOf(chain); try { - const tokens: Address[] = j == -1 ? [] : await Object.values(incomingAssets)[j](); + const tokens: Address[] = incomingAssets[chain as keyof typeof incomingAssets] ? await incomingAssets[chain as keyof typeof incomingAssets]() : [] tokens.push(...((await fetchThirdPartyTokenList())[chain] ?? [])); let filteredTokens: Address[] = chain in excluded ? tokens.filter((t: string) => !excluded[chain].includes(t)) : tokens; diff --git a/defi/l2/v2/index.ts b/defi/l2/v2/index.ts index cf62ef7d93..e7e089cc4a 100644 --- a/defi/l2/v2/index.ts +++ b/defi/l2/v2/index.ts @@ -1,31 +1,57 @@ import { canonicalBridgeIds, excludedTvlKeys, protocolBridgeIds, zero, ownTokens, allChainKeys } from "../constants"; -import getTVLOfRecordClosestToTimestamp from "../../src/utils/shared/getRecordClosestToTimestamp"; import { getCurrentUnixTimestamp } from "../../src/utils/date"; -import { fetchAllTokens } from "../../src/utils/shared/bridgedTvlPostgres"; import { Chain } from "@defillama/sdk/build/types"; -import { getMcaps, getPrices, fetchBridgeTokenList, fetchSupplies } from "../utils"; +import { fetchBridgeTokenList, fetchSupplies } from "../utils"; import { fetchAdaTokens } from "../adapters/ada"; +import { fetchAllTokens as fetchAllTokensFromDB } from "../../src/utils/shared/bridgedTvlPostgres"; import { withTimeout } from "../../src/utils/shared/withTimeout"; -import PromisePool from "@supercharge/promise-pool"; import { CoinsApiData, FinalData, FinalChainData } from "../types"; import { McapsApiData } from "../types"; import { getBlock } from "@defillama/sdk/build/util/blocks"; import { multiCall } from "@defillama/sdk/build/abi/abi2"; import BigNumber from "bignumber.js"; -import { bridgedTvlMixedCaseChains, chainsThatShouldNotBeLowerCased } from "../../src/utils/shared/constants"; +import { + bridgedTvlMixedCaseChains, + chainsThatShouldNotBeLowerCased, + chainsWithCaseSensitiveDataProviders, +} from "../../src/utils/shared/constants"; import { getR2JSONString, storeR2JSONString } from "../../src/utils/r2"; import { additional, excluded } from "../adapters/manual"; import { storeHistoricalToDB } from "./storeToDb"; import { stablecoins } from "../../src/getProtocols"; import { metadata as rwaMetadata } from "../../src/rwa/protocols"; -import { verifyChanges } from "../test"; +import { verifyChanges } from "../verifyChanges"; +import { initializePriceQueryFilter, whitelistedTokenSetRawPids } from "../../src/storeTvlInterval/computeTVL"; +import { getClosestProtocolItem, getLatestProtocolItems, initializeTVLCacheDB } from "../../src/api2/db"; +import { hourlyRawTokensTvl } from "../../src/utils/getLastRecord"; +import { Balances } from "@defillama/sdk"; +import runInPromisePool from "@defillama/sdk/build/util/promisePool"; +import { cachedFetch } from "@defillama/sdk/build/util/cache"; +import { coins } from "@defillama/sdk"; const searchWidth = 10800; // 3hr +const allTokens: { [chain: Chain]: string[] } = {}; -async function fetchNativeAndMcaps( - timestamp: number, - override: boolean = false -): Promise<{ +// fetch a list of all token addresses from ES +async function fetchAllTokens() { + await initializePriceQueryFilter(); + + whitelistedTokenSetRawPids.forEach((t) => { + const seperater = t.indexOf(":"); + const chain = t.substring(0, seperater); + + const address = t.substring(seperater + 1); + if (address == "undefined") return; + + if (!allTokens[chain]) allTokens[chain] = []; + allTokens[chain].push(address); + }); + + return allTokens; +} + +// find the prices, mcaps and supplies of all tokens on all chains +async function fetchNativeAndMcaps(timestamp: number): Promise<{ chainData: { [chain: Chain]: { prices: { [token: string]: CoinsApiData }; @@ -47,71 +73,77 @@ async function fetchNativeAndMcaps( const allPrices: { [token: string]: CoinsApiData } = {}; const allMcaps: { [token: string]: McapsApiData } = {}; - await PromisePool.withConcurrency(5) - .for(allChainKeys) - .process(async (chain: Chain) => { + await runInPromisePool({ + items: allChainKeys, + concurrency: 5, + processor: async (chain: Chain) => { + // protocol bridge IDs are closed and have no native tvl if (Object.values(protocolBridgeIds).includes(chain)) return; - await withTimeout(1000 * 60 * (override ? 120 : 120), minted(chain)).catch(() => { + await withTimeout(1000 * 60 * 20, minted(chain)).catch(() => { throw new Error(`fetchMinted() timed out for ${chain}`); }); async function minted(chain: Chain) { try { const start = new Date().getTime(); - let storedTokens = await fetchAllTokens(chain); - if (chain == "cardano") storedTokens = await fetchAdaTokens(); + const storedTokens = + chain == "cardano" + ? await fetchAdaTokens() + : [...chainsThatShouldNotBeLowerCased, ...chainsWithCaseSensitiveDataProviders].includes(chain) + ? await fetchAllTokensFromDB(chain) + : allTokens[chain]; const ownTokenCgid: string | undefined = ownTokens[chain]?.address.startsWith("coingecko:") ? ownTokens[chain].address : undefined; if (ownTokenCgid) storedTokens.push(ownTokenCgid); - let prices: { [token: string]: CoinsApiData } = {}; - try { - prices = await getR2JSONString(`prices/${chain}.json`); - } catch (e) { - console.log(`${chain} prices not cached, fetching`); - prices = await getPrices( - storedTokens.map((t: string) => normalizeKey(t.startsWith("coingecko:") ? t : `${chain}:${t}`)), - timestamp - ); - await storeR2JSONString(`prices/${chain}.json`, JSON.stringify(prices)); - } + // let prices: { [token: string]: CoinsApiData } = {}; + // try { + // prices = await getR2JSONString(`prices/${chain}.json`); + // } catch (e) { + // console.log(`${chain} prices not cached, fetching`); + const prices = await coins.getPrices( + storedTokens.map((t: string) => normalizeKey(t.startsWith("coingecko:") ? t : `${chain}:${t}`)), + timestamp + ); + // await storeR2JSONString(`prices/${chain}.json`, JSON.stringify(prices)); + // } Object.keys(prices).map((p: string) => { if (p.startsWith("coingecko:")) prices[p].decimals = 0; allPrices[p] = prices[p]; }); - let mcaps: { [token: string]: McapsApiData } = {}; - try { - mcaps = await getR2JSONString(`mcaps/${chain}.json`); - } catch (e) { - console.log(`${chain} mcaps not cached, fetching`); - mcaps = await getMcaps(Object.keys(prices), timestamp); - await storeR2JSONString(`mcaps/${chain}.json`, JSON.stringify(mcaps)); - } + // let mcaps: { [token: string]: McapsApiData } = {}; + // try { + // mcaps = await getR2JSONString(`mcaps/${chain}.json`); + // } catch (e) { + // console.log(`${chain} mcaps not cached, fetching`); + const mcaps = await coins.getMcaps(Object.keys(prices), timestamp); + // await storeR2JSONString(`mcaps/${chain}.json`, JSON.stringify(mcaps)); + // } Object.keys(mcaps).map((m: string) => { allMcaps[m] = mcaps[m]; }); - let supplies; - try { - supplies = await getR2JSONString(`supplies/${chain}.json`); - if (ownTokenCgid && mcaps[ownTokenCgid]) - supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; - } catch (e) { - console.log(`${chain} supplies not cached, fetching`); - supplies = await fetchSupplies( - chain, - Object.keys(prices).map((t: string) => t.substring(t.indexOf(":") + 1)), - timestamp - ); - if (ownTokenCgid && mcaps[ownTokenCgid]) - supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; - await storeR2JSONString(`supplies/${chain}.json`, JSON.stringify(supplies)); - } + // let supplies; + // try { + // supplies = await getR2JSONString(`supplies/${chain}.json`); + // if (ownTokenCgid && mcaps[ownTokenCgid]) + // supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; + // } catch (e) { + // console.log(`${chain} supplies not cached, fetching`); + const supplies = await fetchSupplies( + chain, + Object.keys(prices).map((t: string) => t.substring(t.indexOf(":") + 1)), + timestamp + ); + if (ownTokenCgid && mcaps[ownTokenCgid]) + supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; + // await storeR2JSONString(`supplies/${chain}.json`, JSON.stringify(supplies)); + // } chainData[chain] = { prices, mcaps, supplies }; @@ -119,23 +151,29 @@ async function fetchNativeAndMcaps( const time = end - start; if (time > 60 * 1000) console.log(`${chain}: ${time}`); } catch (e) { - console.error(`fetchNativeAndMcaps() failed for ${chain} with ${e}`); + throw new Error(`fetchNativeAndMcaps() failed for ${chain} with ${e}`); } } - }); + }, + }); return { chainData, allPrices, allMcaps }; } +// incoming asset lists are fetched from canonical bridge token mappings async function fetchIncomingAssetsList(): Promise<{ [chain: Chain]: string[] }> { const incomingAssets: { [chain: Chain]: string[] } = {}; - await PromisePool.withConcurrency(5) - .for(allChainKeys) - .process(async (chain: Chain) => { + // fetch all canonical bridge incoming assets + await runInPromisePool({ + items: allChainKeys, + concurrency: 5, + processor: async (chain: Chain) => { incomingAssets[chain] = await fetchBridgeTokenList(chain); - }); + }, + }); + // add any additional hard coded Object.keys(additional).map((chain) => { if (!incomingAssets[chain]) incomingAssets[chain] = []; const additionalTokens = bridgedTvlMixedCaseChains.includes(chain) @@ -144,6 +182,7 @@ async function fetchIncomingAssetsList(): Promise<{ [chain: Chain]: string[] }> incomingAssets[chain].push(...additionalTokens); }); + // exclude any unwanted assets const filteredIncomingAssets: { [chain: Chain]: string[] } = {}; Object.keys(incomingAssets).filter((chain: Chain) => { filteredIncomingAssets[chain] = @@ -155,48 +194,52 @@ async function fetchIncomingAssetsList(): Promise<{ [chain: Chain]: string[] }> return incomingAssets; } +// outgoing amounts from chains are derived from balances of canonical bridges of each chain async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{ sourceChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } }; protocolAmounts: { [protocolSlug: string]: { [token: string]: BigNumber } }; destinationChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } }; }> { const ids: string[] = [...Object.keys(canonicalBridgeIds), ...Object.keys(protocolBridgeIds)]; - const tvls: any[] = await Promise.all( - ids.map((i: string) => - getTVLOfRecordClosestToTimestamp( - `hourlyRawTokensTvl#${i}`, - timestamp == 0 ? getCurrentUnixTimestamp() : timestamp, - searchWidth - ) - ) - ); + + await initializeTVLCacheDB(); + const tvls = + timestamp == 0 + ? await getLatestProtocolItems(hourlyRawTokensTvl, { filterLast24Hours: true }) + : await runInPromisePool({ + items: ids, + concurrency: 5, + processor: (i: string) => + getClosestProtocolItem(hourlyRawTokensTvl, i, getCurrentUnixTimestamp(), { searchWidth }), + }); const sourceChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; const protocolAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; const destinationChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; - tvls.map((b: any, i: number) => { - Object.keys(b).map((chain: string) => { + tvls.map(({ data, id }: any) => { + if (!ids.includes(id)) return; + Object.keys(data).map((chain: string) => { if (excludedTvlKeys.includes(chain)) return; if (!sourceChainAmounts[chain]) sourceChainAmounts[chain] = {}; - Object.keys(b[chain]).map((token: string) => { + Object.keys(data[chain]).map((token: string) => { const key = normalizeKey(token); if (!sourceChainAmounts[chain][key]) sourceChainAmounts[chain][key] = zero; - sourceChainAmounts[chain][key] = sourceChainAmounts[chain][key].plus(b[chain][token]); + sourceChainAmounts[chain][key] = sourceChainAmounts[chain][key].plus(data[chain][token]); - const protocolId = b.PK.substring(b.PK.indexOf("#") + 1); - if (Object.keys(protocolBridgeIds).includes(protocolId)) { - const protocolSlug = protocolBridgeIds[protocolId]; + if (Object.keys(protocolBridgeIds).includes(id)) { + const protocolSlug = protocolBridgeIds[id]; if (!protocolAmounts[protocolSlug]) protocolAmounts[protocolSlug] = {}; if (!protocolAmounts[protocolSlug][key]) protocolAmounts[protocolSlug][key] = zero; - protocolAmounts[protocolSlug][key] = protocolAmounts[protocolSlug][key].plus(b[chain][token]); + protocolAmounts[protocolSlug][key] = protocolAmounts[protocolSlug][key].plus(data[chain][token]); return; } - const destinationChain = [...Object.values(canonicalBridgeIds), ...Object.values(protocolBridgeIds)][i]; + const destinationChain = canonicalBridgeIds[id]; + if (!destinationChain) return; if (!destinationChainAmounts[destinationChain]) destinationChainAmounts[destinationChain] = {}; if (!destinationChainAmounts[destinationChain][key]) destinationChainAmounts[destinationChain][key] = zero; destinationChainAmounts[destinationChain][key] = destinationChainAmounts[destinationChain][key].plus( - b[chain][token] + data[chain][token] ); }); }); @@ -205,6 +248,7 @@ async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{ return { sourceChainAmounts, protocolAmounts, destinationChainAmounts }; } +// fetch amounts in hardcoded excluded wallets - eg pre mines etc async function fetchExcludedAmounts(timestamp: number) { const excludedTokensAndOwners: { [chain: string]: [string, string][] } = { base: [ @@ -223,9 +267,11 @@ async function fetchExcludedAmounts(timestamp: number) { ], }; - const excludedAmounts: { [chain: string]: { [token: string]: BigNumber } } = {}; - await Promise.all( - Object.keys(excludedTokensAndOwners).map(async (chain: string) => { + const excludedAmounts: { [chain: string]: Balances } = {}; + await runInPromisePool({ + items: Object.keys(excludedTokensAndOwners), + concurrency: 5, + processor: async (chain: string) => { const block = await getBlock(chain, (timestamp == 0 ? getCurrentUnixTimestamp() : timestamp) - 10); const balances = await multiCall({ @@ -235,55 +281,64 @@ async function fetchExcludedAmounts(timestamp: number) { block: block.number, }); + excludedAmounts[chain] = new Balances({ chain }); excludedTokensAndOwners[chain].map(([token, _], i) => { - if (!excludedAmounts[chain]) excludedAmounts[chain] = {}; - if (!excludedAmounts[chain][token]) excludedAmounts[chain][token] = zero; - excludedAmounts[chain][token] = excludedAmounts[chain][token].plus(balances[i]); + excludedAmounts[chain].add(token, BigNumber(balances[i])); }); - }) - ); + }, + }); return excludedAmounts; } +// fetch stablecoin symbols async function fetchStablecoinSymbols() { - const { peggedAssets } = await fetch("https://stablecoins.llama.fi/stablecoins").then((r) => r.json()); + const { peggedAssets } = await cachedFetch({ + key: "stablecoin-symbols", + endpoint: "https://stablecoins.llama.fi/stablecoins", + }); const symbols = peggedAssets.map((s: any) => s.symbol); const allSymbols = [...new Set([...symbols, ...stablecoins].map((t) => t.toUpperCase()))]; return allSymbols; } +// fetch lst symbols async function fetchLstSymbols() { - const assets = await fetch("https://yields.llama.fi/lsdRates").then((r) => r.json()); + const assets = await cachedFetch({ key: "lst-symbols", endpoint: "https://yields.llama.fi/lsdRates" }); const symbols = assets.map((s: any) => s.symbol.toUpperCase()); return symbols; } +// fetch rwa symbols function fetchRwaSymbols() { const allSymbols: { [symbol: string]: boolean } = {}; Object.values(rwaMetadata).map(({ matchExact, symbols }: { matchExact: boolean; symbols: string[] }) => { - symbols.map((symbol) => allSymbols[symbol] = matchExact) + symbols.map((symbol) => (allSymbols[symbol] = matchExact)); }); return allSymbols; } +// check if a symbol is an rwa symbol function isRwaSymbol(symbol: string, rwaSymbols: { [symbol: string]: boolean }) { if (rwaSymbols[symbol]) return true; Object.keys(rwaSymbols).map((s) => { if (!rwaSymbols[s] && symbol.startsWith(s)) return true; }); - return false + return false; } +// normalize a key to a standard format function normalizeKey(key: string) { if (key.startsWith("0x")) return `ethereum:${key.toLowerCase()}`; const [chain, address] = key.split(":"); if (!address) return `coingecko:${key}`; if (chainsThatShouldNotBeLowerCased.includes(chain)) return key; + if (chainsWithCaseSensitiveDataProviders.includes(chain)) return key; return key.toLowerCase(); } +// check if a symbol is an own token function isOwnToken(chain: string, symbol: string) { const ownToken = ownTokens[chain]; if (!ownToken) return false; @@ -296,8 +351,22 @@ function isOwnToken(chain: string, symbol: string) { return false; } -async function main() { +// create a new chain assets object +const newChainAssets = () => ({ + canonical: { breakdown: {} }, + thirdParty: { breakdown: {} }, + native: { breakdown: {} }, + ownTokens: { breakdown: {} }, + total: { breakdown: {} }, + rwa: { breakdown: {} }, + lst: { breakdown: {} }, + stablecoins: { breakdown: {} }, +}); + +// main function +export async function storeChainAssetsV2(override: boolean = false) { const timestamp = 0; + await fetchAllTokens(); const { sourceChainAmounts, protocolAmounts, destinationChainAmounts } = await fetchOutgoingAmountsFromDB(timestamp); const incomingAssets = await fetchIncomingAssetsList(); const excludedAmounts = await fetchExcludedAmounts(timestamp); @@ -306,6 +375,7 @@ async function main() { const lstSymbols = await fetchLstSymbols(); const rwaSymbols = fetchRwaSymbols(); + // adjust native asset balances by excluded and outgoing amounts const nativeDataAfterDeductions: { [chain: Chain]: { [token: string]: BigNumber } } = {}; allChainKeys.map((chain: Chain) => { if (Object.values(protocolBridgeIds).includes(chain)) return; @@ -317,7 +387,7 @@ async function main() { const key = normalizeKey(token); const coinData = allPrices[key]; if (!coinData || !coinData.price) return; - const excludedAmount = excludedAmounts[chain]?.[key] ?? zero; + const excludedAmount = excludedAmounts[chain]?._balances[key] ?? zero; const sourceChainAmount = sourceChainAmounts[chain]?.[key] ?? zero; const nativeAmount = BigNumber(supplies[token]); @@ -342,6 +412,7 @@ async function main() { } }); + // adjust balances for mcaps const nativeDataAfterMcaps: { [chain: Chain]: { [token: string]: BigNumber } } = {}; Object.keys(nativeDataAfterDeductions).map((chain: Chain) => { nativeDataAfterMcaps[chain] = {}; @@ -352,18 +423,10 @@ async function main() { }); }); + // split all chain balances into sections (canonical, thirdParty, native, ownTokens, stablecoins, rwa, lst) const rawData: FinalData = {}; allChainKeys.map((chain: Chain) => { - rawData[chain] = { - canonical: { breakdown: {} }, - thirdParty: { breakdown: {} }, - native: { breakdown: {} }, - ownTokens: { breakdown: {} }, - total: { breakdown: {} }, - rwa: { breakdown: {} }, - lst: { breakdown: {} }, - stablecoins: { breakdown: {} }, - }; + rawData[chain] = newChainAssets(); if (Object.values(protocolBridgeIds).includes(chain)) { const protocolAmount = protocolAmounts[chain]; if (!protocolAmount) { @@ -376,71 +439,68 @@ async function main() { if (!coinData || !coinData.price) return; const amount = BigNumber(coinData.price).times(protocolAmount[token]).div(BigNumber(10).pow(coinData.decimals)); - const symbol = allPrices[key].symbol.toUpperCase(); // filter for rwa, lst, stablecoins let section = "canonical"; - if (isOwnToken(chain, symbol)) section = "ownTokens"; - else if (stablecoinSymbols.includes(symbol)) section = "stablecoins"; + if (isOwnToken(chain, symbol)) { + if (ownTokens[chain].address.startsWith("coingecko:") && !key.startsWith("coingecko:")) return; + section = "ownTokens"; + } else if (stablecoinSymbols.includes(symbol)) section = "stablecoins"; else if (isRwaSymbol(symbol, rwaSymbols)) section = "rwa"; else if (lstSymbols.includes(symbol)) section = "lst"; rawData[chain][section as keyof FinalChainData].breakdown[key] = amount; if (!isOwnToken(chain, symbol)) rawData[chain].total.breakdown[key] = amount; }); + } else { + Object.keys(nativeDataAfterMcaps[chain]).map((key: string) => { + if (!allPrices[key]) return; + const symbol = allPrices[key].symbol.toUpperCase(); - return; + let section = "native"; + if (isOwnToken(chain, symbol)) { + if (ownTokens[chain].address.startsWith("coingecko:") && !key.startsWith("coingecko:")) return; + section = "ownTokens"; + } else if (stablecoinSymbols.includes(symbol)) section = "stablecoins"; + else if (isRwaSymbol(symbol, rwaSymbols)) section = "rwa"; + else if (lstSymbols.includes(symbol)) section = "lst"; + else if (incomingAssets[chain] && incomingAssets[chain].includes(key.substring(key.indexOf(":") + 1))) + section = "thirdParty"; + else if (!key.startsWith("coingecko:") && !key.startsWith(chain)) section = "canonical"; + const amount = nativeDataAfterMcaps[chain][key]; + rawData[chain][section as keyof FinalChainData].breakdown[key] = amount; + if (section != "ownTokens") rawData[chain].total.breakdown[key] = amount; + }); } - - Object.keys(nativeDataAfterMcaps[chain]).map((key: string) => { - if (!allPrices[key]) return; - const symbol = allPrices[key].symbol.toUpperCase(); - - let section = "native"; - if (isOwnToken(chain, symbol)) section = "ownTokens"; - else if (stablecoinSymbols.includes(symbol)) section = "stablecoins"; - else if (isRwaSymbol(symbol, rwaSymbols)) section = "rwa"; - else if (lstSymbols.includes(symbol)) section = "lst"; - else if (incomingAssets[chain] && incomingAssets[chain].includes(key.substring(key.indexOf(":") + 1))) - section = "thirdParty"; - else if (!key.startsWith("coingecko:") && !key.startsWith(chain)) section = "canonical"; - const amount = nativeDataAfterMcaps[chain][key]; - rawData[chain][section as keyof FinalChainData].breakdown[key] = amount; - if (section != "ownTokens") rawData[chain].total.breakdown[key] = amount; - }); }); + // create a symbol map and symbol data for human readable data const symbolMap: { [key: string]: string } = (await getR2JSONString("chainAssetsSymbolMap")) ?? {}; const symbolData: FinalData = {}; Object.keys(rawData).map((chain: Chain) => { - symbolData[chain] = { - canonical: { breakdown: {} }, - thirdParty: { breakdown: {} }, - native: { breakdown: {} }, - ownTokens: { breakdown: {} }, - total: { breakdown: {} }, - rwa: { breakdown: {} }, - lst: { breakdown: {} }, - stablecoins: { breakdown: {} }, - }; + symbolData[chain] = newChainAssets(); Object.keys(rawData[chain]).map((section: string) => { Object.keys(rawData[chain][section as keyof FinalChainData].breakdown).map((key: string) => { const symbol = allPrices[key].symbol.toUpperCase(); - symbolData[chain][section as keyof FinalChainData].breakdown[symbol] = - rawData[chain][section as keyof FinalChainData].breakdown[key]; + if (!symbolData[chain][section as keyof FinalChainData].breakdown[symbol]) + symbolData[chain][section as keyof FinalChainData].breakdown[symbol] = zero; + symbolData[chain][section as keyof FinalChainData].breakdown[symbol] = symbolData[chain][ + section as keyof FinalChainData + ].breakdown[symbol].plus(rawData[chain][section as keyof FinalChainData].breakdown[key]); if (!symbolMap[key]) symbolMap[key] = symbol; }); }); }); + // create symbol key data const symbolMapPromise = storeR2JSONString("chainAssetsSymbolMap", JSON.stringify(symbolMap)); [rawData, symbolData].map((allData) => { Object.keys(allData).map((chain: Chain) => { let totalTotal = zero; Object.keys(allData[chain]).map((section: string) => { - if (section == "total") return; const amounts = Object.values(allData[chain][section as keyof FinalChainData].breakdown); const total = amounts.length ? (amounts.reduce((p: any, c: any) => c.plus(p), zero) as BigNumber) : zero; allData[chain][section as keyof FinalChainData].total = total; + if (["ownTokens", "total"].includes(section)) return; totalTotal = totalTotal.plus(total); }); @@ -448,35 +508,29 @@ async function main() { }); }); + // sort symbol data for manual verifcation const sortedSymbolData: FinalData = {}; Object.keys(symbolData).map((chain: Chain) => { - sortedSymbolData[chain] = { - canonical: { breakdown: {} }, - thirdParty: { breakdown: {} }, - native: { breakdown: {} }, - ownTokens: { breakdown: {} }, - total: { breakdown: {} }, - rwa: { breakdown: {} }, - lst: { breakdown: {} }, - stablecoins: { breakdown: {} }, - }; + sortedSymbolData[chain] = newChainAssets(); Object.keys(symbolData[chain]).map((section: string) => { - const a = Object.entries(symbolData[chain][section as keyof FinalChainData].breakdown).sort((a: any, b: any) => - b[1].minus(a[1]) + const orderedTokenAmounts = Object.entries(symbolData[chain][section as keyof FinalChainData].breakdown).sort( + (a: any, b: any) => b[1].minus(a[1]) ); - const b = a.slice(0, 100); - const c: { [key: string]: string } = {}; - b.map(([key, value]: any) => { - c[key] = value.toString(); + + const top100Tokens = orderedTokenAmounts.slice(0, 100); + const topTokensObject: { [key: string]: string } = {}; + top100Tokens.map(([key, value]: any) => { + topTokensObject[key] = value.toString(); }); - sortedSymbolData[chain][section as keyof FinalChainData].breakdown = c; + + sortedSymbolData[chain][section as keyof FinalChainData].breakdown = topTokensObject; sortedSymbolData[chain][section as keyof FinalChainData].total = symbolData[chain][section as keyof FinalChainData].total.toString(); sortedSymbolData[chain].total.total = symbolData[chain].total.total.toString(); }); }); - await verifyChanges(symbolData); + if (!override) await verifyChanges(symbolData); await Promise.all([ symbolMapPromise, @@ -484,8 +538,3 @@ async function main() { storeR2JSONString("chainAssets2", JSON.stringify({ timestamp: getCurrentUnixTimestamp(), value: symbolData })), ]); } - -main().catch((e) => { - console.error(e); - process.exit(1); -}).then(() => process.exit(0)); // ts-node defi/l2/v2/index.ts diff --git a/defi/l2/test.ts b/defi/l2/verifyChanges.ts similarity index 100% rename from defi/l2/test.ts rename to defi/l2/verifyChanges.ts diff --git a/defi/src/getProtocols.ts b/defi/src/getProtocols.ts index f52ab52610..98d6a0e1ee 100644 --- a/defi/src/getProtocols.ts +++ b/defi/src/getProtocols.ts @@ -51,6 +51,8 @@ const majors = [ "mETH", ].map((t) => t.toUpperCase()); export const stablecoins = [ + "USD₮0", + "USD₮", "USDT", "USDC", "DAI", diff --git a/defi/src/storeTvlInterval/computeTVL.ts b/defi/src/storeTvlInterval/computeTVL.ts index 117806d72b..a19bae69d1 100644 --- a/defi/src/storeTvlInterval/computeTVL.ts +++ b/defi/src/storeTvlInterval/computeTVL.ts @@ -227,9 +227,10 @@ interface Counter { const priceQueryFilterCoins = !!process.env.PRICE_QUERY_FILTER_FOR_KNOWN_COINS const whitelistedTokenSet = new Set() as Set; +export const whitelistedTokenSetRawPids = new Set() as Set; let priceQueryFilterInitializedPromise: any -async function initializePriceQueryFilter() { +export async function initializePriceQueryFilter() { if (!priceQueryFilterInitializedPromise) priceQueryFilterInitializedPromise = _initializePriceQueryFilter() return priceQueryFilterInitializedPromise async function _initializePriceQueryFilter() { @@ -251,6 +252,7 @@ async function initializePriceQueryFilter() { while (response.hits.hits.length) { response.hits.hits.map((i: any) => { + whitelistedTokenSetRawPids.add(`${i._source.chain}:${i._source.address}`); whitelistedTokenSet.add(normalizeCoinId(i._source.pid)); }) sdk.log(`Fetched ${whitelistedTokenSet.size} records, ${response.hits.hits.length} in batch`);