Change store API and add migrations

Ferossgp · Ferossgp · commit b51a32024364 · 2025-08-16T15:48:58.000+02:00
diff --git a/.changeset/old-signs-lay.md b/.changeset/old-signs-lay.md
@@ -0,0 +1,5 @@
+---
+'@3loop/transaction-decoder': minor
+---
+
+BREAKING! This version changes the public API for ABI store. If you use built in stores evrything should work out of the box. When using SQL store ensure that migrations complete on start.
diff --git a/packages/transaction-decoder/src/abi-store.ts b/packages/transaction-decoder/src/abi-store.ts
@@ -9,30 +9,34 @@ export interface AbiParams {
   signature?: string | undefined
 }
 
-export interface ContractAbiSuccess {
-  status: 'success'
-  result: ContractABI
-}
+export type CachedContractABIStatus = 'success' | 'invalid' | 'not-found'
 
-export interface ContractAbiNotFound {
-  status: 'not-found'
-  result: null
+export type CachedContractABI = ContractABI & {
+  id: string
+  source?: string
+  status: CachedContractABIStatus
+  timestamp?: string
 }
 
-export interface ContractAbiEmpty {
-  status: 'empty'
-  result: null
+export type CacheContractABIParam = ContractABI & {
+  source?: string
+  status: CachedContractABIStatus
+  timestamp?: string
 }
 
-export type ContractAbiResult = ContractAbiSuccess | ContractAbiNotFound | ContractAbiEmpty
+export type ContractAbiResult = CachedContractABI[]
 
 type ChainOrDefault = number | 'default'
 
 export interface AbiStore {
   readonly strategies: Record<ChainOrDefault, readonly ContractAbiResolverStrategy[]>
-  readonly set: (key: AbiParams, value: ContractAbiResult) => Effect.Effect<void, never>
+  readonly set: (key: AbiParams, value: CacheContractABIParam) => Effect.Effect<void, never>
   readonly get: (arg: AbiParams) => Effect.Effect<ContractAbiResult, never>
   readonly getMany?: (arg: Array<AbiParams>) => Effect.Effect<Array<ContractAbiResult>, never>
+  readonly updateStatus?: (
+    id: string | number,
+    status: 'success' | 'invalid' | 'not-found',
+  ) => Effect.Effect<void, never>
   readonly circuitBreaker: CircuitBreaker.CircuitBreaker<unknown>
   readonly requestPool: RequestPool.RequestPool
 }
diff --git a/packages/transaction-decoder/src/in-memory/abi-store.ts b/packages/transaction-decoder/src/in-memory/abi-store.ts
@@ -2,51 +2,59 @@ import { Effect, Layer } from 'effect'
 import * as AbiStore from '../abi-store.js'
 import { ContractABI } from '../abi-strategy/request-model.js'
 
+// Keyed by composite: kind|key|source to allow per-strategy replacement
 const abiCache = new Map<string, ContractABI>()
 
 export const make = (strategies: AbiStore.AbiStore['strategies']) =>
   Layer.scoped(
     AbiStore.AbiStore,
     AbiStore.make({
       strategies,
-      set: (_key, value) =>
+      set: (_key, abi) =>
         Effect.sync(() => {
-          if (value.status === 'success') {
-            if (value.result.type === 'address') {
-              abiCache.set(value.result.address, value.result)
-            } else if (value.result.type === 'event') {
-              abiCache.set(value.result.event, value.result)
-            } else if (value.result.type === 'func') {
-              abiCache.set(value.result.signature, value.result)
-            }
+          const source = abi.source ?? 'unknown'
+          if (abi.type === 'address') {
+            abiCache.set(`addr|${abi.address}|${source}`.toLowerCase(), abi)
+          } else if (abi.type === 'event') {
+            abiCache.set(`event|${abi.event}|${source}`, abi)
+          } else if (abi.type === 'func') {
+            abiCache.set(`sig|${abi.signature}|${source}`, abi)
           }
         }),
       get: (key) =>
         Effect.sync(() => {
-          if (abiCache.has(key.address)) {
-            return {
-              status: 'success',
-              result: abiCache.get(key.address)!,
-            }
-          }
+          const results: ContractABI[] = []
 
-          if (key.event && abiCache.has(key.event)) {
-            return {
-              status: 'success',
-              result: abiCache.get(key.event)!,
-            }
-          }
+          // If a specific strategy is requested via mark on keys in future, we return union of all strategies for that key
+          const prefixAddr = `addr|${key.address}|`.toLowerCase()
+          const prefixSig = key.signature ? `sig|${key.signature}|` : undefined
+          const prefixEvt = key.event ? `event|${key.event}|` : undefined
 
-          if (key.signature && abiCache.has(key.signature)) {
-            return {
-              status: 'success',
-              result: abiCache.get(key.signature)!,
+          for (const [k, v] of abiCache.entries()) {
+            if (
+              k.startsWith(prefixAddr) ||
+              (prefixSig && k.startsWith(prefixSig)) ||
+              (prefixEvt && k.startsWith(prefixEvt))
+            ) {
+              results.push(v)
             }
           }
 
-          return {
-            status: 'empty',
-            result: null,
+          return results
+        }),
+      updateStatus: (id, status) =>
+        Effect.sync(() => {
+          // For in-memory store, we need to find the ABI by ID and update its status
+          // Since we don't have ID-based lookup in memory, we'll iterate through cache
+          for (const [key, abi] of abiCache.entries()) {
+            if (abi.id === id) {
+              // Create a new ABI object with updated status
+              // Note: For in-memory, we can't actually change the status of the result
+              // since it's used in ContractAbiResult. This is a limitation of the in-memory approach.
+              // In practice, you'd want to remove invalid ABIs from cache or mark them differently.
+              abiCache.delete(key)
+              break
+            }
           }
         }),
     }),
diff --git a/packages/transaction-decoder/src/sql/abi-store.ts b/packages/transaction-decoder/src/sql/abi-store.ts
@@ -1,6 +1,7 @@
 import * as AbiStore from '../abi-store.js'
 import { Effect, Layer } from 'effect'
 import { SqlClient } from '@effect/sql'
+import { runMigrations, migration } from './migrations.js'
 
 // Utility function to build query conditions for a single key
 const buildQueryForKey = (
@@ -21,39 +22,20 @@ const buildQueryForKey = (
     : sql.or([addressQuery, signatureQuery, eventQuery].filter(Boolean))
 }
 
-// Convert database items to result format
+// Convert database items to result format - returns all ABIs with their individual status
 const createResult = (items: readonly any[], address: string, chainID: number): AbiStore.ContractAbiResult => {
-  const successItems = items.filter((item) => item.status === 'success')
-
-  const item =
-    successItems.find((item) => {
-      // Prioritize address over fragments
-      return item.type === 'address'
-    }) ?? successItems[0]
-
-  if (item != null) {
-    return {
-      status: 'success',
-      result: {
-        type: item.type,
-        event: item.event,
-        signature: item.signature,
-        address,
-        chainID,
-        abi: item.abi,
-      },
-    } as AbiStore.ContractAbiResult
-  } else if (items[0] != null && items[0].status === 'not-found') {
-    return {
-      status: 'not-found',
-      result: null,
-    }
-  }
-
-  return {
-    status: 'empty',
-    result: null,
-  }
+  return items.map((item) => ({
+    type: item.type,
+    event: item.event,
+    signature: item.signature,
+    address,
+    chainID,
+    abi: item.abi,
+    id: item.id,
+    source: item.source || 'unknown',
+    status: item.status as 'success' | 'invalid' | 'not-found',
+    timestamp: item.timestamp,
+  }))
 }
 
 // Build single lookup map with prefixed keys
@@ -94,77 +76,113 @@ export const make = (strategies: AbiStore.AbiStore['strategies']) =>
     Effect.gen(function* () {
       const sql = yield* SqlClient.SqlClient
 
-      const table = sql('_loop_decoder_contract_abi_v2')
+      const tableV3 = sql('_loop_decoder_contract_abi_v3')
+      const tableV2 = sql('_loop_decoder_contract_abi_v2')
       const id = sql.onDialectOrElse({
         sqlite: () => sql`id INTEGER PRIMARY KEY AUTOINCREMENT,`,
         pg: () => sql`id SERIAL PRIMARY KEY,`,
         mysql: () => sql`id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id),`,
         orElse: () => sql``,
       })
 
-      // TODO; add timestamp to the table
-      yield* sql`
-        CREATE TABLE IF NOT EXISTS ${table} (
-          ${id}
-          type TEXT NOT NULL,
-          address TEXT,
-          event TEXT,
-          signature TEXT,
-          chain INTEGER,
-          abi TEXT,
-          status TEXT NOT NULL,
-          timestamp TEXT DEFAULT CURRENT_TIMESTAMP
-        )
-      `.pipe(
-        Effect.tapError(Effect.logError),
-        Effect.catchAll(() => Effect.dieMessage('Failed to create contractAbi table')),
-      )
+      // TODO: Allow skipping migrations if users want to apply it manually
+      // Run structured migrations (idempotent, transactional)
+      yield* runMigrations([
+        migration('001_create_contract_abi_v3', (q) =>
+          Effect.gen(function* () {
+            yield* q`CREATE TABLE IF NOT EXISTS ${tableV3} (
+            ${id}
+            type TEXT NOT NULL,
+            address TEXT,
+            event TEXT,
+            signature TEXT,
+            chain INTEGER,
+            abi TEXT,
+            status TEXT NOT NULL,
+            timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
+            source TEXT DEFAULT 'unknown'
+          )`
+
+            const tsCoalesce = q.onDialectOrElse({
+              sqlite: () => q`COALESCE(timestamp, CURRENT_TIMESTAMP)`,
+              pg: () => q`COALESCE(timestamp, CURRENT_TIMESTAMP)`,
+              mysql: () => q`IFNULL(timestamp, CURRENT_TIMESTAMP)`,
+              orElse: () => q`CURRENT_TIMESTAMP`,
+            })
+
+            yield* q`
+              INSERT INTO ${tableV3} (type, address, chain, abi, status, timestamp, source)
+              SELECT 'address' as type, v.address, v.chain, v.abi, v.status, ${tsCoalesce} as timestamp, 'unknown' as source
+              FROM ${tableV2} as v
+              WHERE v.type = 'address'
+                AND v.address IS NOT NULL AND v.chain IS NOT NULL
+                AND NOT EXISTS (
+                  SELECT 1 FROM ${tableV3} t
+                  WHERE t.type = 'address' AND t.address = v.address AND t.chain = v.chain
+                )
+            `.pipe(Effect.catchAll(Effect.logError))
+
+            yield* q`
+              INSERT INTO ${tableV3} (type, signature, abi, status, timestamp, source)
+              SELECT 'func' as type, v.signature, v.abi, v.status, ${tsCoalesce} as timestamp, 'unknown' as source
+              FROM ${tableV2} as v
+              WHERE v.type = 'func' AND v.signature IS NOT NULL
+                AND NOT EXISTS (
+                  SELECT 1 FROM ${tableV3} t
+                  WHERE t.type = 'func' AND t.signature = v.signature
+                )
+            `.pipe(Effect.catchAll(Effect.logError))
+
+            yield* q`
+              INSERT INTO ${tableV3} (type, event, abi, status, timestamp, source)
+              SELECT 'event' as type, v.event, v.abi, v.status, ${tsCoalesce} as timestamp, 'unknown' as source
+              FROM ${tableV2} as v
+              WHERE v.type = 'event' AND v.event IS NOT NULL
+                AND NOT EXISTS (
+                  SELECT 1 FROM ${tableV3} t
+                  WHERE t.type = 'event' AND t.event = v.event
+                )
+            `.pipe(Effect.catchAll(Effect.logError))
+          }),
+        ),
+      ])
+
+      const table = tableV3
 
       return yield* AbiStore.make({
         strategies,
-        set: (key, value) =>
+        set: (key, abi) =>
           Effect.gen(function* () {
             const normalizedAddress = key.address.toLowerCase()
-            if (value.status === 'success' && value.result.type === 'address') {
-              const result = value.result
-              yield* sql`
-              INSERT INTO ${table}
-               ${sql.insert([
-                 {
-                   type: result.type,
-                   address: normalizedAddress,
-                   chain: key.chainID,
-                   abi: result.abi,
-                   status: 'success',
-                 },
-               ])}
-            `
-            } else if (value.status === 'success') {
-              const result = value.result
+
+            if (abi.type === 'address') {
               yield* sql`
-              INSERT INTO ${table}
-              ${sql.insert([
-                {
-                  type: result.type,
-                  event: 'event' in result ? result.event : null,
-                  signature: 'signature' in result ? result.signature : null,
-                  abi: result.abi,
-                  status: 'success',
-                },
-              ])}
-            `
+                INSERT INTO ${table}
+                ${sql.insert([
+                  {
+                    type: abi.type,
+                    address: normalizedAddress,
+                    chain: key.chainID,
+                    abi: abi.abi,
+                    status: abi.status,
+                    source: abi.source || 'unknown',
+                  },
+                ])}
+              `
             } else {
               yield* sql`
-              INSERT INTO ${table}
-              ${sql.insert([
-                {
-                  type: 'address',
-                  address: normalizedAddress,
-                  chain: key.chainID,
-                  status: 'not-found',
-                },
-              ])}
-            `
+                INSERT INTO ${table}
+                ${sql.insert([
+                  {
+                    type: abi.type,
+                    event: 'event' in abi ? abi.event : null,
+                    signature: 'signature' in abi ? abi.signature : null,
+                    abi: abi.abi,
+                    status: abi.status,
+                    source: abi.source || 'unknown',
+                  },
+                ])}
+              `
             }
           }).pipe(
             Effect.tapError(Effect.logError),
@@ -226,6 +244,18 @@ export const make = (strategies: AbiStore.AbiStore['strategies']) =>
               return createResult(keyItems, address, chainID)
             })
           }),
+
+        updateStatus: (id, status) =>
+          Effect.gen(function* () {
+            yield* sql`
+              UPDATE ${table}
+              SET status = ${status}
+              WHERE id = ${id}
+            `.pipe(
+              Effect.tapError(Effect.logError),
+              Effect.catchAll(() => Effect.succeed(null)),
+            )
+          }),
       })
     }),
   )
diff --git a/packages/transaction-decoder/src/sql/migrations.ts b/packages/transaction-decoder/src/sql/migrations.ts
diff --git a/packages/transaction-interpreter/interpreters/moxie.ts b/packages/transaction-interpreter/interpreters/moxie.ts

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +'@3loop/transaction-decoder': minor
 +---
++
 +BREAKING! This version changes the public API for ABI store. If you use built in stores evrything should work out of the box. When using SQL store ensure that migrations complete on start.