build tvl artifact

Author: g1nt0ki

.github/workflows/build-modules.yml

@@ -0,0 +1,33 @@
+name: Build tvlModules and upload artifact
+
+on:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  build-and-upload:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run buildImports script
+        run: node scripts/buildImports.js
+
+      - name: Publish tvlModules.json as "latest" release
+        uses: ncipollo/release-action@v1
+        with:
+          tag: latest
+          name: Latest tvlModules
+          artifacts: scripts/tvlModules.json
+          allowUpdates: true

.gitignore

@@ -10,4 +10,6 @@ yarn.lock
 .vscode
 projects/binance/data.csv
 
-*.log
+*.log
+
+scripts/tvlModules.json

package.json

@@ -7,6 +7,7 @@
     "test": "echo \"Error: no test specified\" && exit 1",
     "weeklyChanges": "git pull && git diff $(git log -1 --before=@{7.days.ago} --format=%H) --stat | grep -E \"projects/\" | cut -d / -f 2 | cut -d \" \" -f 1 | uniq | wc -l",
     "dev": "babel-watch curve.js",
+    "build": "node scripts/buildImports.js",
     "lint": "eslint -c .eslintrc.js .",
     "eslint:github-action": "eslint -c .eslintrc.js .",
     "test-interactive": "node utils/testInteractive",

projects/helper/cache.js

@@ -15,12 +15,12 @@ function getLink(project, chain) {
   return `https://${Bucket}.s3.eu-central-1.amazonaws.com/${getKey(project, chain)}`
 }
 
-async function getCache(project, chain, { _ } = {}) {
+async function getCache(project, chain, { skipCompression } = {}) {
   const Key = getKey(project, chain)
   const fileKey = getFileKey(project, chain)
 
   try {
-    const json = await sdk.cache.readCache(fileKey)
+    const json = await sdk.cache.readCache(fileKey, { skipCompression})
     if (!json || Object.keys(json).length === 0) throw new Error('Invalid data')
     return json
   } catch (e) {
@@ -36,11 +36,13 @@ async function getCache(project, chain, { _ } = {}) {
   }
 }
 
-async function setCache(project, chain, cache, { _ } = {}) {
+async function setCache(project, chain, cache, { skipCompression } = {}) {
   const Key = getKey(project, chain)
 
   try {
-    await sdk.cache.writeCache(getFileKey(project, chain), cache)
+    await sdk.cache.writeCache(getFileKey(project, chain), cache, {
+      skipCompression,
+    })
   } catch (e) {
     sdk.log('failed to write data to s3 bucket: ', Key)
     sdk.log(e)

scripts/buildImports.js

@@ -0,0 +1,85 @@
+const fs = require("fs")
+const { get } = require("../projects/helper/http")
+// const { setCache, getCache } = require("../projects/helper/cache")
+
+async function run() {
+  // await getCache('defi-configs', 'tvlModules')
+  const configs = await get('https://api.llama.fi/_fe/static/configs')
+
+  const moduleMap = {}
+  const protocols = configs.protocols.concat(configs.treasuries).concat(configs.entities)
+
+  console.log('# of protocols/treasuries/entities:', protocols.length)
+
+  for (const protocol of protocols) {
+    try {
+
+      if (moduleMap[protocol.module]) continue;  // already imported
+
+      const modulePath = `../projects/${protocol.module}`
+      const importedModule = mockFunctions(require(modulePath))
+
+      if (importedModule.hallmarks)
+        importedModule.hallmarks = convertHallmarkStrings(importedModule.hallmarks)
+
+      moduleMap[protocol.module] = importedModule
+    } catch (e) {
+      console.error(`Error importing ${protocol.module}:`, e)
+    }
+  }
+
+  fs.writeFileSync('scripts/tvlModules.json', JSON.stringify(moduleMap))
+  // await setCache('defi-configs', 'tvlModules', moduleMap, {
+  //   skipCompression: true,
+  // })
+
+  process.exit(0)
+}
+
+run().catch((e) => {
+  console.error(e)
+  process.exit(1)
+})
+
+
+function convertHallmarkStrings(hallmarks) {
+  if (!Array.isArray(hallmarks)) return hallmarks
+  return hallmarks.map((item) => {
+    if (typeof item?.[0] === 'string') {
+      item[0] = dateStringToTimestamp(item[0])
+    }
+    if (Array.isArray(item?.[0])) {
+      item[0].forEach((subItem, index) => {
+        if (typeof subItem === 'string') {
+          item[0][index] = dateStringToTimestamp(subItem)
+        }
+      })
+    }
+    return item
+  }).filter((item) => {
+    if (typeof item?.[0] === 'number') return true
+    // if it is a range hallmark
+    if (Array.isArray(item?.[0] && typeof item[0][0] === 'number' && typeof item[0][1] === 'number')) {
+      return true
+    }
+    return false
+  })
+}
+
+//Replace all fuctions with mock functions in an object all the way down
+function mockFunctions(obj) {
+  if (typeof obj === "function") {
+    return '_f'  // llamaMockedTVLFunction
+  } else if (typeof obj === "object") {
+    Object.keys(obj).forEach((key) => obj[key] = mockFunctions(obj[key]))
+  }
+  return obj
+}
+
+function dateStringToTimestamp(dateString) {
+
+  let timestamp = Math.floor(+new Date(dateString) / 1e3)
+  if (!isNaN(timestamp))
+    return timestamp
+  return dateString
+}